Palash DebnathandClaude Fable 5
94ba362ef2
* feat(triage): crash-class recurrence report — the reliability metric scripts/crash_class_report.py measures the "backend died / never came up" class (the project's #1 lifetime failure, ~1 in 5 of all issues) filtered to reports from the current version — the definition of done for the reliability cycle. Buckets by the bug reporter's Build-status stamp (#1547): current / outdated / unknown (pre-deflection builds), so deflection-miss noise never pollutes the number the work is judged on. tests/scripts/test_crash_class_report.py pins the title→sub-class mapping against the real historical title shapes and locks the stamp literals to frontend/src/utils/bugReport.js so a reworded marker fails in CI instead of silently zeroing the metric. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(triage): --version is authoritative; loud fetch-cap warning Bot-review harvest on #1549: with --version, the Environment Version line now decides the bucket (extracted to pure classify_build + tests) — a report stamped "current at filing time" during another version's window no longer counts toward this version's recurrence. Hitting the 500-issue fetch cap now warns loudly instead of silently understating. The stamp lockstep test asserts the full Build-status prefix. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
85 lines
3.8 KiB
Python
85 lines
3.8 KiB
Python
"""Bucketing contract for scripts/crash_class_report.py — the reliability
|
|
cycle's recurrence metric.
|
|
|
|
Pins two things:
|
|
- title → sub-class mapping matches the auto-reporter's real title shapes
|
|
(seeded from error messages, so titles are machine-stable), and
|
|
- body → build bucket mapping matches the exact `**Build status:**` lines
|
|
the bug reporter stamps (frontend/src/utils/bugReport.js) — a drift in
|
|
either side silently zeroes the metric.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
SCRIPT_PATH = Path(__file__).resolve().parents[2] / "scripts" / "crash_class_report.py"
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def report():
|
|
spec = importlib.util.spec_from_file_location("crash_class_report", SCRIPT_PATH)
|
|
mod = importlib.util.module_from_spec(spec)
|
|
sys.modules["crash_class_report"] = mod
|
|
spec.loader.exec_module(mod)
|
|
return mod
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("title", "expected"),
|
|
[
|
|
# The real title shapes from the historical corpus.
|
|
("[Bug] Can't reach the local VoiceStudio backend. It has not answered", "unreachable"),
|
|
("[Bug] Can't reach the local OmniVoice backend — it may still be starti", "unreachable"),
|
|
("[Bug] It was answering 6 minutes ago and then stopped responding", "unreachable"),
|
|
("[Crash] Backend died (exit code 1)", "backend-died"),
|
|
("[Crash] Backend ended uncleanly (previous run)", "backend-died"),
|
|
("[Backend] Backend failed to start", "never-started"),
|
|
# Non-class titles must not count toward the metric.
|
|
("[Bug] Synthesize produced silence in Design mode", None),
|
|
("[Feature] Add a voice changer", None),
|
|
],
|
|
)
|
|
def test_title_bucketing(report, title, expected):
|
|
sub = next((k for k, rx in report.CRASH_CLASS.items() if rx.search(title)), None)
|
|
assert sub == expected
|
|
|
|
|
|
def test_build_status_lines_match_the_reporter_stamps(report):
|
|
# These literals must stay in lockstep with frontend/src/utils/bugReport.js
|
|
# captureContext() — the stamps are the metric's only version signal.
|
|
assert report.OUTDATED.search("**Build status:** OUTDATED — `v0.5.0` was already out when this was filed")
|
|
assert report.CURRENT.search("**Build status:** current at filing time (latest `v0.5.0`)")
|
|
assert not report.OUTDATED.search("**Build status:** current at filing time (latest `v0.5.0`)")
|
|
assert report.VERSION_LINE.search("**Version:** `0.5.1-3`").group(1) == "0.5.1-3"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("body", "version", "expected"),
|
|
[
|
|
# Stamp-based (no target version).
|
|
("**Build status:** OUTDATED — `v0.5.0` was already out when this was filed", None, "outdated"),
|
|
("**Build status:** current at filing time (latest `v0.5.0`)", None, "current"),
|
|
("no stamp at all", None, "unknown"),
|
|
# With --version the Version line is authoritative: a report stamped
|
|
# "current" during ANOTHER version's window must not count here.
|
|
("**Version:** `0.4.2`\n**Build status:** current at filing time (latest `v0.4.2`)", "0.5.0", "outdated"),
|
|
("**Version:** `0.5.0`\n**Build status:** current at filing time (latest `v0.5.0`)", "0.5.0", "current"),
|
|
("**Version:** `0.5.1-3`", "0.5.1", "current"), # preview stamps match by base
|
|
("no version line", "0.5.0", "unknown"),
|
|
],
|
|
)
|
|
def test_classify_build(report, body, version, expected):
|
|
assert report.classify_build(body, version) == expected
|
|
|
|
|
|
def test_reporter_source_still_emits_the_stamps(report):
|
|
# Fail here (not in production silence) if bugReport.js rewords the marker.
|
|
src = (SCRIPT_PATH.parents[1] / "frontend/src/utils/bugReport.js").read_text(encoding="utf-8")
|
|
assert "**Build status:** OUTDATED" in src
|
|
assert "**Build status:** current at filing time" in src
|