[Strict Mode] Max collection size in distributed setup #14620
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration tests | |
| on: | |
| push: | |
| branches: [ master, dev ] | |
| pull_request: | |
| branches: [ '**' ] | |
| env: | |
| CARGO_TERM_COLOR: always | |
| jobs: | |
| test: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Install minimal stable | |
| uses: dtolnay/rust-toolchain@stable | |
| - uses: actions/checkout@v4 | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v3 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: Swatinem/rust-cache@v2 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get install clang | |
| curl -sSL https://install.python-poetry.org | python3 - | |
| poetry -C tests check --lock | |
| poetry -C tests install --no-root | |
| - name: Build | |
| run: cargo build --features "service_debug data-consistency-check" | |
| - name: Run integration tests | |
| run: poetry -C tests run ./tests/integration-tests.sh | |
| shell: bash | |
| test-consensus: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Install minimal stable | |
| uses: dtolnay/rust-toolchain@stable | |
| - uses: actions/checkout@v4 | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v3 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Install dependencies | |
| run: sudo apt-get install clang | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| run: | | |
| curl -sSL https://install.python-poetry.org | python3 - | |
| poetry -C tests check --lock | |
| poetry -C tests install --no-root | |
| - name: Build | |
| run: cargo build --features "service_debug data-consistency-check" | |
| - name: Run integration tests - 1 peer | |
| run: poetry -C tests run ./tests/integration-tests.sh distributed | |
| shell: bash | |
| - name: Run integration tests - multiple peers - pytest | |
| run: poetry -C tests run pytest tests/consensus_tests | |
| timeout-minutes: 60 | |
| - name: upload logs in case of failure | |
| uses: actions/upload-artifact@v4 | |
| if: failure() || cancelled() | |
| with: | |
| name: consensus-test-logs | |
| retention-days: 90 # default max value | |
| path: consensus_test_logs | |
| test-consensus-compose: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| run: | | |
| curl -sSL https://install.python-poetry.org | python3 - | |
| poetry -C tests check --lock | |
| poetry -C tests install --no-root | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Docker build | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| tags: qdrant_consensus | |
| cache-from: | | |
| type=gha,scope=${{ github.ref }} | |
| type=gha,scope=${{ github.base_ref }} | |
| cache-to: type=gha,mode=max,scope=${{ github.ref }} | |
| load: true | |
| build-args: | | |
| PROFILE=ci | |
| - name: Run integration tests - multiple peers - compose | |
| run: poetry run ./test_restart.sh | |
| working-directory: ./tests/consensus_tests | |
| test-tls-compose: | |
| name: "Basic TLS/HTTPS tests" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Docker build | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| tags: qdrant_tls | |
| cache-from: | | |
| type=gha,scope=${{ github.ref }} | |
| type=gha,scope=${{ github.base_ref }} | |
| cache-to: type=gha,mode=max,scope=${{ github.ref }} | |
| load: true | |
| build-args: | | |
| PROFILE=ci | |
| - name: Run basic TLS/HTTPS test | |
| run: ./test_tls.sh | |
| working-directory: ./tests/tls | |
| - name: Run snapshot shard transfer TLS/HTTPS test | |
| run: ./test_tls_snapshot_shard_transfer.sh | |
| working-directory: ./tests/tls | |
| test-consistency: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Install minimal stable | |
| uses: dtolnay/rust-toolchain@stable | |
| - uses: actions/checkout@v4 | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Install dependencies | |
| run: sudo apt-get install -y jq | |
| - name: Docker build | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: tools/schema2openapi | |
| tags: schema2openapi | |
| cache-from: | | |
| type=gha,scope=${{ github.ref }}-schema2openapi | |
| type=gha,scope=${{ github.base_ref }}-schema2openapi | |
| cache-to: type=gha,mode=max,scope=${{ github.ref }}-schema2openapi | |
| load: true | |
| build-args: | | |
| PROFILE=ci | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v3 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: gRPC file consistency check | |
| run: ./tests/grpc_consistency_check.sh | |
| - name: OpenAPI file consistency check | |
| run: ./tests/openapi_consistency_check.sh | |
| test-low-resources: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Install dependencies | |
| run: sudo apt-get install clang git-lfs jq | |
| - name: Setup git-lfs | |
| run: git lfs install | |
| - name: Install minimal stable | |
| uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - uses: actions/checkout@v4 | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v3 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Docker build | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| tags: qdrant-recovery | |
| cache-from: | | |
| type=gha,scope=${{ github.ref }} | |
| type=gha,scope=${{ github.base_ref }} | |
| cache-to: type=gha,mode=max,scope=${{ github.ref }} | |
| load: true | |
| build-args: | | |
| PROFILE=ci | |
| - name: Run low RAM test | |
| working-directory: ./tests/low-ram | |
| shell: bash | |
| run: ./low-ram.sh | |
| - name: Run low Disk test - search | |
| working-directory: ./tests/low-disk | |
| shell: bash | |
| run: ./low-disk.sh | |
| - name: Run low Disk test - indexing | |
| working-directory: ./tests/low-disk | |
| shell: bash | |
| run: ./low-disk.sh indexing | |
| test-snapshot-operations-s3-minio: | |
| runs-on: ubuntu-latest | |
| # Setup minio server | |
| services: | |
| minio: | |
| image: satantime/minio-server | |
| ports: | |
| - 9000:9000 | |
| env: | |
| MINIO_ACCESS_KEY: "minioadmin" | |
| MINIO_SECRET_KEY: "minioadmin" | |
| steps: | |
| - name: Setup test bucket | |
| env: | |
| AWS_ACCESS_KEY_ID: "minioadmin" | |
| AWS_SECRET_ACCESS_KEY: "minioadmin" | |
| AWS_EC2_METADATA_DISABLED: "true" | |
| run: echo $(aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test-bucket) | |
| - name: Install dependencies | |
| run: sudo apt-get install clang jq | |
| - name: Setup git-lfs | |
| run: git lfs install | |
| - name: Install minimal stable | |
| uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - uses: actions/checkout@v4 | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v3 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Docker build | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| tags: qdrant-snapshots | |
| cache-from: | | |
| type=gha,scope=${{ github.ref }} | |
| type=gha,scope=${{ github.base_ref }} | |
| cache-to: type=gha,mode=max,scope=${{ github.ref }} | |
| load: true | |
| build-args: | | |
| PROFILE=ci | |
| - name: Install dependencies | |
| run: sudo snap install yq | |
| - name: Run test snapshot operations on different storage filesystems | |
| working-directory: ./tests/snapshots | |
| shell: bash | |
| run: ./snapshots-recovery.sh local | |
| - name: Run test snapshot operations on different storage filesystems | |
| working-directory: ./tests/snapshots | |
| shell: bash | |
| run: ./snapshots-recovery.sh s3 | |
| test-shard-snapshot-api-s3-minio: | |
| runs-on: ubuntu-latest | |
| # Setup minio server | |
| services: | |
| minio: | |
| image: satantime/minio-server | |
| ports: | |
| - 9000:9000 | |
| env: | |
| MINIO_ACCESS_KEY: "minioadmin" | |
| MINIO_SECRET_KEY: "minioadmin" | |
| steps: | |
| - name: Setup test bucket | |
| env: | |
| AWS_ACCESS_KEY_ID: "minioadmin" | |
| AWS_SECRET_ACCESS_KEY: "minioadmin" | |
| AWS_EC2_METADATA_DISABLED: "true" | |
| run: echo $(aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test-bucket) | |
| - name: Install minimal stable | |
| uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - uses: actions/checkout@v4 | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v3 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install dependencies | |
| run: sudo apt-get install clang jq | |
| - name: Build | |
| run: cargo build --bin qdrant | |
| - name: Run Shard Snapshot API Tests (local) | |
| shell: bash | |
| run: | | |
| cargo run & | |
| trap 'kill $(jobs -p) &>/dev/null || :' EXIT | |
| sleep 10 | |
| ./tests/shard-snapshot-api.sh test-all | |
| - name: Run Shard Snapshot API Tests (s3) | |
| shell: bash | |
| run: | | |
| export QDRANT__STORAGE__SNAPSHOTS_CONFIG__SNAPSHOTS_STORAGE=s3 | |
| export QDRANT__STORAGE__SNAPSHOTS_CONFIG__S3_CONFIG__BUCKET=test-bucket | |
| export QDRANT__STORAGE__SNAPSHOTS_CONFIG__S3_CONFIG__REGION=us-east-1 | |
| export QDRANT__STORAGE__SNAPSHOTS_CONFIG__S3_CONFIG__ACCESS_KEY=minioadmin | |
| export QDRANT__STORAGE__SNAPSHOTS_CONFIG__S3_CONFIG__SECRET_KEY=minioadmin | |
| export QDRANT__STORAGE__SNAPSHOTS_CONFIG__S3_CONFIG__ENDPOINT_URL=http://127.0.0.1:9000 | |
| cargo run & | |
| trap 'kill $(jobs -p) &>/dev/null || :' EXIT | |
| sleep 10 | |
| ./tests/shard-snapshot-api.sh test-all |