qdrant/.github/workflows/integration-tests.yml

name: Integration tests

on:
  push:
    branches: [ master, dev ]
  pull_request:
    branches: [ '**' ]

env:
  CARGO_TERM_COLOR: always

jobs:
  test:

    runs-on: ubuntu-latest

    steps:
      - name: Install minimal stable
        uses: dtolnay/rust-toolchain@stable 
      - uses: actions/checkout@v4
      - name: Install Protoc
        uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - uses: Swatinem/rust-cache@v2
      - name: Install dependencies
        run: sudo apt-get install clang
      - name: Build
        run: cargo build --features service_debug
      - name: Run integration tests
        run: ./tests/integration-tests.sh
        shell: bash
      - name: Run API Key tests
        run: ./tests/integration-tests-api-key.sh
        shell: bash

  test-consensus:

    runs-on: ubuntu-latest

    steps:
      - name: Install minimal stable
        uses: dtolnay/rust-toolchain@stable
      - uses: actions/checkout@v4
      - name: Install Protoc
        uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - uses: Swatinem/rust-cache@v2
      - name: Install dependencies
        run: sudo apt-get install clang
      - uses: actions/setup-python@v5
        with:
          python-version: '3.10'
      - name: Install pytest
        run: pip install pytest requests
      - name: Build
        run: cargo build --features service_debug
      - name: Run integration tests - 1 peer
        run: ./tests/integration-tests.sh distributed
        shell: bash
      - name: Run integration tests - multiple peers - pytest
        run: pytest ./tests/consensus_tests
        timeout-minutes: 60
      - name: upload logs in case of failure
        uses: actions/upload-artifact@v4
        if: failure() || cancelled()
        with:
          name: consensus-test-logs
          retention-days: 90 # default max value
          path: consensus_test_logs

  test-consensus-compose:

    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.10'
      - name: Install requests python lib
        run: pip install requests
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Docker build
        uses: docker/build-push-action@v5
        with:
          context: .
          tags: qdrant_consensus
          cache-from: |
            type=gha,scope=${{ github.ref }}
            type=gha,scope=${{ github.base_ref }}
          cache-to: type=gha,mode=max,scope=${{ github.ref }}
          load: true
          build-args: |
            PROFILE=ci
      - name: Run integration tests - multiple peers - compose
        run: ./test_restart.sh
        working-directory: ./tests/consensus_tests

  test-tls-compose:

    name: "Basic TLS/HTTPS tests"

    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Docker build
        uses: docker/build-push-action@v5
        with:
          context: .
          tags: qdrant_tls
          cache-from: |
            type=gha,scope=${{ github.ref }}
            type=gha,scope=${{ github.base_ref }}
          cache-to: type=gha,mode=max,scope=${{ github.ref }}
          load: true
          build-args: |
            PROFILE=ci
      - name: Run basic TLS/HTTPS test
        run: ./test_tls.sh
        working-directory: ./tests/tls
      - name: Run snapshot shard transfer TLS/HTTPS test
        run: ./test_tls_snapshot_shard_transfer.sh
        working-directory: ./tests/tls


  test-consistency:

    runs-on: ubuntu-latest

    steps:
      - name: Install minimal stable
        uses: dtolnay/rust-toolchain@stable 
      - uses: actions/checkout@v4
      - uses: Swatinem/rust-cache@v2
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Install dependencies
        run: sudo apt-get install -y jq
      - name: Docker build
        uses: docker/build-push-action@v5
        with:
          context: tools/schema2openapi
          tags: schema2openapi
          cache-from: |
            type=gha,scope=${{ github.ref }}-schema2openapi
            type=gha,scope=${{ github.base_ref }}-schema2openapi
          cache-to: type=gha,mode=max,scope=${{ github.ref }}-schema2openapi
          load: true
          build-args: |
            PROFILE=ci
      - name: Install Protoc
        uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: gRPC file consistency check
        run: ./tests/grpc_consistency_check.sh
      - name: OpenAPI file consistency check
        run: ./tests/openapi_consistency_check.sh


  test-low-ram:
    runs-on: ubuntu-latest

    steps:
      - name: Install dependencies
        run: sudo apt-get install clang git-lfs jq
      - name: Setup git-lfs
        run: git lfs install
      - name: Install minimal stable
        uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
      - uses: actions/checkout@v4
      - name: Install Protoc
        uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Docker build
        uses: docker/build-push-action@v5
        with:
          context: .
          tags: qdrant-recovery
          cache-from: |
            type=gha,scope=${{ github.ref }}
            type=gha,scope=${{ github.base_ref }}
          cache-to: type=gha,mode=max,scope=${{ github.ref }}
          load: true
          build-args: |
            PROFILE=ci
      - name: Run low RAM test
        working-directory: ./tests/low-ram
        shell: bash
        run: ./low-ram.sh

  test-snapshot-operations:
    runs-on: ubuntu-latest

    steps:
      - name: Install dependencies
        run: sudo apt-get install clang jq
      - name: Setup git-lfs
        run: git lfs install
      - name: Install minimal stable
        uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
      - uses: actions/checkout@v4
      - name: Install Protoc
        uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Docker build
        uses: docker/build-push-action@v5
        with:
          context: .
          tags: qdrant-snapshots
          cache-from: |
            type=gha,scope=${{ github.ref }}
            type=gha,scope=${{ github.base_ref }}
          cache-to: type=gha,mode=max,scope=${{ github.ref }}
          load: true
          build-args: |
            PROFILE=ci
      - name: Run test snapshot operations on different storage filesystems
        working-directory: ./tests/snapshots
        shell: bash
        run: ./snapshots-recovery.sh

  test-shard-snapshot-api:
    runs-on: ubuntu-latest

    steps:
      - name: Install minimal stable
        uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
      - uses: actions/checkout@v4
      - name: Install Protoc
        uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Install dependencies
        run: sudo apt-get install clang jq
      - name: Build
        run: cargo build --bin qdrant
      - name: Run Shard Snapshot API Tests
        shell: bash
        run: |
          cargo run &
          trap 'kill $(jobs -p) &>/dev/null || :' EXIT
          sleep 10
          ./tests/shard-snapshot-api.sh test-all