qdrant/.github/workflows/nightly-model-testing.yml

name: Nightly Model Testing

on:
  schedule:
    - cron: '0 2 * * *'  # At 02:00
  workflow_dispatch:
    inputs:
      op_num:
        description: 'Number of ops (leave empty to use the default OP_NUM)'
        required: false
        default: ''
      seed:
        description: 'RNG seed (leave empty for a random seed)'
        required: false
        default: ''

permissions:
  contents: read
  issues: write  # open/update an issue when the nightly fails

env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1
  RUST_LOG: info
  OP_NUM: 500000
  SHARD_COUNT: 2
  RESTART_PROBABILITY: 0.001
  ID_POOL: 1000

jobs:
  model-testing:
    runs-on: ubuntu-latest

    steps:
      - name: Free disk space
        uses: jlumbroso/free-disk-space@ceedf095f4ec1a097402bc6bd80831f2e1a6fde6 # v2.0.0
        with:
          tool-cache: true
          android: true
          dotnet: true
          haskell: true
          large-packages: true
          docker-images: true
          swap-storage: true
      - name: Install minimal stable
        uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          # Force dev on the nightly schedule; otherwise let checkout use its default
          # ref (e.g. the PR merge ref). github.ref_name would be "<pr>/merge" here,
          # which checkout wrongly resolves as a branch and fails to fetch.
          ref: ${{ github.event_name == 'schedule' && 'dev' || '' }}
      - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
        with:
          # Save only on the nightly run (scheduled events run in the default-branch
          # context, so its cache is restorable by all branches). dispatch/PR runs
          # just restore, avoiding feature-branch caches churning the 10 GB budget.
          save-if: ${{ github.event_name == 'schedule' }}
      - name: Install Protoc
        uses: ./.github/actions/setup-protoc
      - name: Build model testing binary
        run: |
          RUSTFLAGS="--cfg tokio_unstable" \
            cargo build --bin model_testing --features service_debug --profile perf
        shell: bash
        timeout-minutes: 60
      # Resolve the per-run parameters. workflow_dispatch inputs override the env
      # constants when set; otherwise we keep OP_NUM and pick a fresh random seed so
      # each night explores a different op sequence. The seed is logged in each run's
      # banner (model_testing: seed=...), so a failing night stays reproducible.
      # Inputs are read via env (not ${{ }} interpolation) to avoid script injection.
      - name: Resolve run parameters
        env:
          INPUT_OP_NUM: ${{ inputs.op_num }}
          INPUT_SEED: ${{ inputs.seed }}
        run: |
          # The commit actually under test. `github.sha` is the default branch (master) on a
          # scheduled run, but the checkout above forces `dev`, so reporting `github.sha` points
          # the failure issue at code that was never run.
          echo "TESTED_SHA=$(git rev-parse HEAD)" >> "$GITHUB_ENV"
          if [ -n "$INPUT_OP_NUM" ]; then
            echo "OP_NUM=$INPUT_OP_NUM" >> "$GITHUB_ENV"
          fi
          if [ -n "$INPUT_SEED" ]; then
            echo "SEED=$INPUT_SEED" >> "$GITHUB_ENV"
          else
            echo "SEED=$(od -An -N8 -tu8 /dev/urandom | tr -d ' ')" >> "$GITHUB_ENV"
          fi
        shell: bash
      # Both runs reuse the single binary built above and share the same seed.
      - name: Run model testing (async scorer)
        id: async_scorer
        run: |
          target/perf/model_testing \
            --seed "$SEED" \
            --op-num "$OP_NUM" \
            --shard-count "$SHARD_COUNT" \
            --restart-probability "$RESTART_PROBABILITY" \
            --id-pool "$ID_POOL" \
            --on-disk \
            --async-scorer
        shell: bash
        timeout-minutes: 300
      - name: Run model testing (no io_uring)
        id: no_io_uring
        # Run even if the async-scorer pass failed, so both signals are collected.
        if: ${{ !cancelled() }}
        run: |
          target/perf/model_testing \
            --seed "$SEED" \
            --op-num "$OP_NUM" \
            --shard-count "$SHARD_COUNT" \
            --restart-probability "$RESTART_PROBABILITY" \
            --id-pool "$ID_POOL" \
            --on-disk
        shell: bash
        timeout-minutes: 300
      # Which of the two passes broke is the first thing triage needs: one failing while the
      # other survives the same seed points at the io_uring read path (or rules it out), and
      # every failure so far has split that way. Rendered here rather than in the issue template
      # so the template stays a plain fill-in-the-blanks form.
      - name: Describe which passes failed
        if: ${{ failure() && github.event_name == 'schedule' }}
        env:
          ASYNC: ${{ steps.async_scorer.conclusion }}
          PLAIN: ${{ steps.no_io_uring.conclusion }}
        run: |
          if [ "$ASYNC" = failure ] && [ "$PLAIN" = failure ]; then
            summary="both passes"
          elif [ "$ASYNC" = failure ]; then
            summary="async scorer only (no io_uring passed the same seed)"
          elif [ "$PLAIN" = failure ]; then
            summary="no io_uring only (async scorer passed the same seed)"
          else
            summary="neither pass, the job broke around them (async=$ASYNC, no-io_uring=$PLAIN)"
          fi
          echo "FAILED_PASSES=$summary" >> "$GITHUB_ENV"
        shell: bash
      # Only alert for unattended nightly failures; dispatch/PR runs are watched live.
      - name: Open issue on failure
        if: ${{ failure() && github.event_name == 'schedule' }}
        continue-on-error: true
        uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SEED: ${{ env.SEED }}
          REPOSITORY: ${{ github.repository }}
          RUN_ID: ${{ github.run_id }}
          # Falls back to github.sha if we failed before resolving the checked-out commit.
          SHA: ${{ env.TESTED_SHA || github.sha }}
        with:
          filename: .github/ISSUE_TEMPLATE/model_testing_failure.md
          update_existing: true