qdrant/.github/workflows/nightly-model-testing.yml

name: Nightly Model Testing

on:
  schedule:
    - cron: '0 2 * * *'  # At 02:00
  workflow_dispatch:
    inputs:
      op_num:
        description: 'Number of ops (leave empty to use the default OP_NUM)'
        required: false
        default: ''
      seed:
        description: 'RNG seed (leave empty for a random seed)'
        required: false
        default: ''

permissions:
  contents: read
  issues: write  # open/update an issue when the nightly fails

env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1
  OP_NUM: 200000
  SHARD_COUNT: 2
  RESTART_PROBABILITY: 0.001
  ID_POOL: 1000

jobs:
  model-testing:
    runs-on: ubuntu-latest

    steps:
      - name: Free disk space
        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
        with:
          tool-cache: true
          android: true
          dotnet: true
          haskell: true
          large-packages: true
          docker-images: true
          swap-storage: true
      - name: Install minimal stable
        uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          # Force dev on the nightly schedule; otherwise let checkout use its default
          # ref (e.g. the PR merge ref). github.ref_name would be "<pr>/merge" here,
          # which checkout wrongly resolves as a branch and fails to fetch.
          ref: ${{ github.event_name == 'schedule' && 'dev' || '' }}
      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
        with:
          # Save only on the nightly run (scheduled events run in the default-branch
          # context, so its cache is restorable by all branches). dispatch/PR runs
          # just restore, avoiding feature-branch caches churning the 10 GB budget.
          save-if: ${{ github.event_name == 'schedule' }}
      - name: Install Protoc
        uses: ./.github/actions/setup-protoc
      - name: Build model testing binary
        run: |
          RUSTFLAGS="--cfg tokio_unstable" \
            cargo build --bin model_testing --features service_debug --profile perf
        shell: bash
        timeout-minutes: 60
      # Resolve the per-run parameters. workflow_dispatch inputs override the env
      # constants when set; otherwise we keep OP_NUM and pick a fresh random seed so
      # each night explores a different op sequence. The seed is logged in each run's
      # banner (model_testing: seed=...), so a failing night stays reproducible.
      # Inputs are read via env (not ${{ }} interpolation) to avoid script injection.
      - name: Resolve run parameters
        env:
          INPUT_OP_NUM: ${{ inputs.op_num }}
          INPUT_SEED: ${{ inputs.seed }}
        run: |
          if [ -n "$INPUT_OP_NUM" ]; then
            echo "OP_NUM=$INPUT_OP_NUM" >> "$GITHUB_ENV"
          fi
          if [ -n "$INPUT_SEED" ]; then
            echo "SEED=$INPUT_SEED" >> "$GITHUB_ENV"
          else
            echo "SEED=$(od -An -N8 -tu8 /dev/urandom | tr -d ' ')" >> "$GITHUB_ENV"
          fi
        shell: bash
      # Both runs reuse the single binary built above and share the same seed.
      - name: Run model testing (async scorer)
        run: |
          target/perf/model_testing \
            --seed "$SEED" \
            --op-num "$OP_NUM" \
            --shard-count "$SHARD_COUNT" \
            --restart-probability "$RESTART_PROBABILITY" \
            --id-pool "$ID_POOL" \
            --on-disk \
            --async-scorer
        shell: bash
        timeout-minutes: 300
      - name: Run model testing (no io_uring)
        # Run even if the async-scorer pass failed, so both signals are collected.
        if: ${{ !cancelled() }}
        run: |
          target/perf/model_testing \
            --seed "$SEED" \
            --op-num "$OP_NUM" \
            --shard-count "$SHARD_COUNT" \
            --restart-probability "$RESTART_PROBABILITY" \
            --id-pool "$ID_POOL" \
            --on-disk
        shell: bash
        timeout-minutes: 300
      # Only alert for unattended nightly failures; dispatch/PR runs are watched live.
      - name: Open issue on failure
        if: ${{ failure() && github.event_name == 'schedule' }}
        continue-on-error: true
        uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SEED: ${{ env.SEED }}
          REPOSITORY: ${{ github.repository }}
          RUN_ID: ${{ github.run_id }}
          SHA: ${{ github.sha }}
        with:
          filename: .github/ISSUE_TEMPLATE/model_testing_failure.md
          update_existing: true