S3 snapshot manager #4109

.all-contributorsrc

@@ -202,12 +202,22 @@
  "contributions": [
  "code"
  ]
+ },
+ {
+ "login": "erare-humanum",
+ "name": "erare-humanum",
+ "avatar_url": "https://avatars.githubusercontent.com/u/116254494?v=4",
+ "profile": "https://github.com/erare-humanum",
+ "contributions": [
+ "code"
+ ]
  }
  ],
  "contributorsPerLine": 7,
  "projectName": "qdrant",
  "projectOwner": "qdrant",
  "repoType": "github",
  "repoHost": "https://github.com",
- "skipCi": true
+ "skipCi": true,
+ "commitConvention": "angular"
 }

.codespellrc

@@ -0,0 +1,6 @@
+[codespell]
+# openapi.json is typically a released version and could carry old typos
+skip = .git,*.pdf,*.svg,openapi.json,key.pem,Cargo.lock
+# hel - Query token
+# generall - the boss
+ignore-words-list = crate,hel,generall

.config/nextest.toml

@@ -0,0 +1,12 @@
+[profile.ci]
+# Retry tests before failing them.
+retries = { backoff = "exponential", count = 2, delay = "2s" }
+# Do not cancel the test run on the first failure.
+fail-fast = false
+# Print out output for failing tests as soon as they fail, and also at the end
+# of the run (for easy scrollability).
+failure-output = "immediate-final"
+
+# save junit report
+[profile.ci.junit]
+path = "junit.xml"

.dockerignore

@@ -6,5 +6,6 @@ lib/segment/target
 lib/api/target
 lib/collection/target
 lib/storage/target
-openapi/tests/
 *.tar
+.git
+.github/

.gitattributes

@@ -1 +1,4 @@
 tests/storage-compat/storage.tar.bz2 filter=lfs diff=lfs merge=lfs -text
+tests/storage-compat/collection.snapshot.gz filter=lfs diff=lfs merge=lfs -text
+tests/low-ram/storage.tar.xz filter=lfs diff=lfs merge=lfs -text
+tests/storage-compat/full-snapshot.snapshot.gz filter=lfs diff=lfs merge=lfs -text

.github/ISSUE_TEMPLATE/bug_report.md

@@ -20,6 +20,9 @@ assignees: ''
 3.
 4.
 
+<!--- Please make sure to include the data which could be used to reproduce the problem -->
+
+
 ## Expected Behavior
 <!--- Tell us what should happen -->
 

.github/ISSUE_TEMPLATE/flaky_test.md

@@ -0,0 +1,25 @@
+---
+title: Flaky test `{{ env.TEST_NAME }}`
+---
+
+## Last report
+
+### System error
+
+```text
+{{ env.SYSTEM_ERROR }}
+```
+
+### Context
+
+Date: {{ date | date('DD.MM.YYYY HH:mm') }}
+
+[Flaky failure run](https://github.com/{{ env.REPOSITORY }}/actions/runs/{{ env.RUN_ID }}/job/{{ env.JOB_ID }})
+
+[Commit](https://github.com/{{ env.REPOSITORY }}/tree/{{ env.SHA }})
+
+OS: {{ env.OS }}
+
+Branch: {{ env.BRANCH }}
+
+{{ env.PR }}

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,5 +1,6 @@
 ### All Submissions:
 
+* [ ] Contributions should target the `dev` branch. Did you create your branch from `dev`?
 * [ ] Have you followed the guidelines in our Contributing document?
 * [ ] Have you checked to ensure there aren't other open [Pull Requests](../../../pulls) for the same update/change?
 
@@ -8,8 +9,8 @@
 ### New Feature Submissions:
 
 1. [ ] Does your submission pass tests?
-2. [ ] Have you lint your code locally using ``cargo fmt`` command prior to submission?
-3. [ ] Have you checked your code using ```cargo clippy``` command?
+2. [ ] Have you formatted your code locally using `cargo +nightly fmt --all` command prior to submission?
+3. [ ] Have you checked your code using `cargo clippy --all --all-features` command?
 
 ### Changes to Core Features:
 

.github/dependabot.yml

@@ -4,18 +4,18 @@ updates:
  target-branch: "dev"
  directory: /
  schedule:
- interval: daily
+ interval: weekly
  open-pull-requests-limit: 10
  labels: [dependencies]
  - package-ecosystem: "github-actions"
  target-branch: "dev"
  directory: "/"
  schedule:
- interval: "weekly"
+ interval: weekly
  open-pull-requests-limit: 10
  - package-ecosystem: docker
  target-branch: "dev"
  directory: "/"
  schedule:
- interval: "weekly"
+ interval: weekly
  open-pull-requests-limit: 10

.github/workflows/codespell.yml

@@ -0,0 +1,22 @@
+---
+name: Codespell
+
+on:
+ push:
+ branches: [dev,master]
+ pull_request:
+ branches: [dev,master]
+
+permissions:
+ contents: read
+
+jobs:
+ codespell:
+ name: Check for spelling errors
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+ - name: Codespell
+ uses: codespell-project/actions-codespell@v2

.github/workflows/dev-docker-image-build.yml

@@ -0,0 +1,42 @@
+name: Push dev branch container to ghcr
+
+on:
+ workflow_dispatch:
+ push:
+ branches:
+ - dev
+
+jobs:
+ dev-build-and-push:
+ runs-on: [self-hosted, linux, x64]
+ steps:
+ - uses: actions/checkout@v4
+ - name: Build the Docker image
+ run: |
+ buildx_containers=$(docker container ls -qf "name=buildx_buildkit")
+ buildx_volumes=$(docker volume ls -qf "name=buildx_buildkit")
+
+ if [ -n "$buildx_containers" ]; then
+ echo "Buildx containers to delete: " "$buildx_containers"
+ docker container rm -f "$buildx_containers"
+ fi
+
+ if [ -n "$buildx_volumes" ]; then
+ echo "Buildx volumes to delete: " "$buildx_volumes"
+ docker volume rm "$buildx_volumes"
+ fi
+
+ # Create build container
+ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+ docker buildx create --use
+
+ # Authenticate on registries
+ docker login https://ghcr.io -u qdrant --password ${{ secrets.GITHUB_TOKEN }}
+
+ # Build regular image for Github Registry
+ GITHUB_TAG="-t ghcr.io/qdrant/qdrant:dev-${{ github.sha }} -t ghcr.io/qdrant/qdrant:dev"
+
+ # Pull, retag and push to GitHub packages
+ docker buildx build --platform='linux/amd64,linux/arm64' --build-arg GIT_COMMIT_ID=${{ github.sha }} $GITHUB_TAG --push --label "org.opencontainers.image.revision"=${{ github.sha }} .
+
+

.github/workflows/dev-docker-image-prune.yml

@@ -0,0 +1,25 @@
+name: Delete old container images
+
+on:
+ workflow_dispatch:
+ schedule:
+ - cron: "0 0 * * *" # every day at midnight
+
+
+jobs:
+ dev-clean-ghcr:
+ name: Delete old unused dev container images
+ runs-on: ubuntu-latest
+ steps:
+ - name: Delete dev branch containers older than a month
+ uses: snok/container-retention-policy@v2
+ with:
+ image-names: qdrant
+ cut-off: A month ago UTC
+ account-type: org
+ org-name: qdrant
+ keep-at-least: 10
+ filter-tags: dev-*
+ token-type: github-token
+ token: ${{ secrets.GITHUB_TOKEN }}
+ dry-run: true

.github/workflows/docker-image.yml

@@ -4,42 +4,52 @@ on:
  workflow_dispatch:
  push:
  # Pattern matched against refs/tags
- tags: 
+ tags:
  - '*' # Push events to every tag not containing /
 
 jobs:
 
  build:
-
- runs-on: ubuntu-latest
-
+ # Run build on our self-hosted runner, we had trouble with shared runners
+ runs-on: [self-hosted, linux, x64]
  steps:
- - uses: actions/checkout@v3
- - uses: Swatinem/rust-cache@v1
+ - uses: actions/checkout@v4
+ - uses: Swatinem/rust-cache@v2
  - name: Get current tag
  id: vars
  run: echo ::set-output name=tag::${GITHUB_REF#refs/*/}
  - name: Build the Docker image
  env:
  RELEASE_VERSION: ${{ steps.vars.outputs.tag }}
  run: |
+ # Create build container
  docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
  docker buildx create --use
-
+
+ # Authenticate on registries
  docker login --username generall --password ${{ secrets.DOCKERHUB_TOKEN }}
-
+ docker login https://docker.pkg.github.com -u qdrant --password ${{ secrets.GITHUB_TOKEN }}
+
+ # Build regular image for Docker Hub
  DOCKERHUB_TAG="qdrant/qdrant:${{ github.ref_name }}"
- TAGS="-t ${DOCKERHUB_TAG}"
-
  DOCKERHUB_TAG_LATEST="qdrant/qdrant:latest"
- TAGS="${TAGS} -t ${DOCKERHUB_TAG_LATEST}"
-
+ TAGS="-t ${DOCKERHUB_TAG} -t ${DOCKERHUB_TAG_LATEST}"
  GITHUB_TAG="docker.pkg.github.com/qdrant/qdrant/qdrant:${{ github.ref_name }}"
- 
- docker buildx build --platform='linux/amd64,linux/arm64' $TAGS --push .
-
+
+ # Pull, retag and push to GitHub packages
+ docker buildx build --platform='linux/amd64,linux/arm64' --build-arg GIT_COMMIT_ID=${{ github.sha }} $TAGS --push --label "org.opencontainers.image.version"=$RELEASE_VERSION .
  docker pull $DOCKERHUB_TAG
-
- docker login https://docker.pkg.github.com -u qdrant --password ${{ secrets.GITHUB_TOKEN }}
  docker tag $DOCKERHUB_TAG $GITHUB_TAG
  docker push $GITHUB_TAG
+
+ # Build unprivileged image for Docker Hub
+ DOCKERHUB_TAG_UNPRIVILEGED="qdrant/qdrant:${{ github.ref_name }}-unprivileged"
+ DOCKERHUB_TAG_LATEST_UNPRIVILEGED="qdrant/qdrant:latest-unprivileged"
+ TAGS_UNPRIVILEGED="-t ${DOCKERHUB_TAG_UNPRIVILEGED} -t ${DOCKERHUB_TAG_LATEST_UNPRIVILEGED}"
+ GITHUB_TAG_UNPRIVILEGED="docker.pkg.github.com/qdrant/qdrant/qdrant:${{ github.ref_name }}-unprivileged"
+
+ # Pull, retag and push to GitHub packages
+ docker buildx build --build-arg='USER_ID=1000' --platform='linux/amd64,linux/arm64' $TAGS_UNPRIVILEGED --push --label "org.opencontainers.image.version"=$RELEASE_VERSION .
+ docker pull $DOCKERHUB_TAG_UNPRIVILEGED
+ docker tag $DOCKERHUB_TAG_UNPRIVILEGED $GITHUB_TAG_UNPRIVILEGED
+ docker push $GITHUB_TAG_UNPRIVILEGED