From 0076d0067e1eb7e680a1da603374ad943a0070ce Mon Sep 17 00:00:00 2001 From: Palash Debnath Date: Thu, 9 Jul 2026 02:19:07 +0530 Subject: [PATCH] =?UTF-8?q?test(ci):=20root-cause=20and=20neutralize=20the?= =?UTF-8?q?=20flaky=20trio=20=E2=80=94=20a=20leaked=20torch=20fp16=20defau?= =?UTF-8?q?lt=20dtype=20(#1021)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_effects_chain / test_generation_audio_guard / test_persona_bundle failed intermittently on CI (never locally) with identical signatures across three unrelated PRs today (#1002, #1019, #1016) — costing a full CI cycle per occurrence and repeatedly muddying merge decisions. Root cause, confirmed by local reproduction: a leaked torch.set_default_dtype(torch.float16) from some earlier test in the CI-Linux ordering. The smoking gun was test_generation_audio_guard's observed 0.0999755859375 — exactly float16(0.1), i.e. torch.tensor([0.1, …]) built under a leaked fp16 default. Reproducing with a simulated polluter locally produced the trio's exact failures: Pedalboard refuses fp16 audio outright ("only supports 32-bit and 64-bit floating point") and silently returns unmodified audio for every preset, so test_effects_chain's preset outputs compare identical; and the fp16 tensor value breaks the sanitize approx-check. Fix: an autouse conftest guard (same philosophy as the existing LLM-state isolation guard, #878) that checks torch's default dtype after every test, resets any leak to float32, and emits a UserWarning naming the offending test's nodeid — so the actual CI-only polluter identifies itself in the next CI log instead of being chased blind. Regression test: a deliberate-leak pair proving reset-between-tests. Fail-before/pass-after verified: with the guard stashed, a simulated polluter + the trio reproduced 2/3 failures locally with the exact CI signatures; with the guard active, 73/73 pass and the warning names the polluter. Co-authored-by: mergetest Co-authored-by: Claude Fable 5 --- tests/conftest.py | 32 ++++++++++++++++++++++++++++++++ tests/test_torch_dtype_guard.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 tests/test_torch_dtype_guard.py diff --git a/tests/conftest.py b/tests/conftest.py index c31fd26f..2138dbef 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -37,6 +37,38 @@ if not os.environ.get("OMNIVOICE_ENV_FILE"): import pytest +import warnings as _warnings + + +# ── torch default-dtype isolation (CI flaky trio) ─────────────────────────── +# Three tests (test_effects_chain / test_generation_audio_guard / +# test_persona_bundle) fail intermittently on CI — never locally — with +# signatures that all trace to one cause: a leaked +# `torch.set_default_dtype(torch.float16)` from some earlier test. The +# smoking gun is test_generation_audio_guard's observed value +# 0.0999755859375, which is exactly float16(0.1): `torch.tensor([0.1, …])` +# built under a leaked fp16 default. The same leak collapses +# test_effects_chain's preset differences into identical quantized outputs, +# and hands test_persona_bundle's soundfile writer fp16 data libsndfile +# can't encode. The polluter only executes on CI-Linux (it never reproduces +# on macOS), so rather than chase it blind, this guard makes the whole leak +# class impossible — same philosophy as the LLM-state guard below — and +# names the offender in CI output when it fires, so it CAN be chased. +@pytest.fixture(autouse=True) +def _torch_default_dtype_guard(request): + yield + torch = sys.modules.get("torch") + if torch is None: + return + if torch.get_default_dtype() is not torch.float32: + _warnings.warn( + f"{request.node.nodeid} leaked torch default dtype " + f"{torch.get_default_dtype()} — resetting to float32. This is " + f"the polluter behind the CI flaky trio; fix it at the source.", + stacklevel=1, + ) + torch.set_default_dtype(torch.float32) + # ── LLM-provider state isolation (issue #878) ────────────────────────────── # LLM provider selection is process-global three ways: env vars (the diff --git a/tests/test_torch_dtype_guard.py b/tests/test_torch_dtype_guard.py new file mode 100644 index 00000000..12356275 --- /dev/null +++ b/tests/test_torch_dtype_guard.py @@ -0,0 +1,31 @@ +"""The conftest torch-dtype guard resets a leaked default dtype between tests. + +The CI "flaky trio" (test_effects_chain / test_generation_audio_guard / +test_persona_bundle) failed intermittently on CI-Linux with signatures that +all trace to one leak: some earlier test leaves +``torch.set_default_dtype(torch.float16)`` behind. Reproduced locally with a +simulated polluter — ``torch.tensor([0.1, …])`` under fp16 yields exactly the +0.0999755859375 CI observed, and Pedalboard refuses fp16 audio outright +("only supports 32-bit and 64-bit floating point"), silently returning +unmodified audio for every preset so their outputs compare identical. + +These two tests are order-dependent BY DESIGN (pytest runs tests within a +file in definition order): the first leaks, the second proves the autouse +guard in conftest.py reset the leak before the next test began. +""" +import torch + + +def test_a_deliberate_dtype_leak(): + # Simulates the CI polluter. The conftest guard must clean this up (and + # emit a UserWarning naming this exact test as the offender). + torch.set_default_dtype(torch.float16) + assert torch.get_default_dtype() is torch.float16 + + +def test_b_next_test_starts_back_at_float32(): + # If the guard is ever removed/broken, this fails — and so, eventually, + # does the flaky trio on CI, much less legibly. + assert torch.get_default_dtype() is torch.float32 + # The exact fp16 signature the trio's CI failures showed, as documentation: + assert torch.tensor([0.1]).item() != 0.0999755859375