[DTP-Worker] Sprint 7 · Générateur Audit 4Big qualité (95+/100 sur 100% deliverables) (QA · roadmap L69)
Audit de méta-niveau + gate : note la qualité 4Big de 100% des livrables gated et bloque (FAIL) si un module < 95/100 (CLAUDE.md #5). Couverture PROUVÉE par recoupement bijectif registre ↔ working-directory du CI (moins l'auditeur · SoD ISA 315). 5 critères déterministes (DOC/CONTRAT/TESTS/CLI/HANDOFF) renormalisés par archétype. Anti-invention (#6) : chaque note est recalculée depuis des faits du dépôt, jamais saisie ; un invariant recompute chaque note. Résultat : PASS · 17/17 modules à 100/100. Régression 442 tests verts (+34). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Code DTP Worker
@@ -0,0 +1,349 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests du générateur d'audit 4Big qualité (QA · Sprint 7).
|
||||
|
||||
Stdlib pur (`unittest`) → aucune installation pip requise sur le runner Gitea
|
||||
(CLAUDE.md #2). Couvre : les 5 critères (positif + injection négative sur arbre
|
||||
de module synthétique en tmpdir), la renormalisation par archétype, la preuve de
|
||||
couverture 100% vs CI, les invariants du générateur, le déterminisme, le schéma.
|
||||
|
||||
Axe central : une note ne peut pas être « fabriquée pour faire 95 » — elle est
|
||||
RECALCULÉE depuis des faits du dépôt (#6). Un module dont un critère applicable
|
||||
échoue tombe sous 95 → FAIL global (l'audit est un GATE).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
_HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
_MOD = os.path.normpath(os.path.join(_HERE, ".."))
|
||||
sys.path.insert(0, _MOD)
|
||||
|
||||
from q4lib import builder, criteria, registry, scoring # noqa: E402
|
||||
import audit_4big_gen as gen # noqa: E402
|
||||
|
||||
|
||||
def _spec():
|
||||
with open(os.path.join(_MOD, "quality_spec.json"), encoding="utf-8") as fh:
|
||||
return json.load(fh)
|
||||
|
||||
|
||||
def _make_module(root: str, *, doc=True, doc_bytes=800, schema=True,
|
||||
tests=12, cli=True, handoff=True) -> str:
|
||||
"""Construit un arbre de module synthétique et renvoie son chemin."""
|
||||
mod = os.path.join(root, "mod")
|
||||
os.makedirs(mod, exist_ok=True)
|
||||
if doc:
|
||||
with open(os.path.join(mod, "README.md"), "w", encoding="utf-8") as fh:
|
||||
fh.write("# Module\n" + ("x" * doc_bytes))
|
||||
if schema:
|
||||
with open(os.path.join(mod, "out.schema.json"), "w", encoding="utf-8") as fh:
|
||||
json.dump({"type": "object"}, fh)
|
||||
tdir = os.path.join(mod, "tests")
|
||||
os.makedirs(tdir, exist_ok=True)
|
||||
body = "".join(f" def test_case_{i}(self):\n pass\n"
|
||||
for i in range(tests))
|
||||
with open(os.path.join(tdir, "test_mod.py"), "w", encoding="utf-8") as fh:
|
||||
fh.write("import unittest\nclass T(unittest.TestCase):\n" +
|
||||
(body or " pass\n"))
|
||||
if cli:
|
||||
with open(os.path.join(mod, "mod_gen.py"), "w", encoding="utf-8") as fh:
|
||||
fh.write('import argparse\n'
|
||||
'ap = argparse.ArgumentParser()\n'
|
||||
'sub = ap.add_subparsers()\n'
|
||||
'sub.add_parser("build")\n'
|
||||
'if __name__ == "__main__":\n pass\n')
|
||||
if handoff:
|
||||
odir = os.path.join(mod, "out")
|
||||
os.makedirs(odir, exist_ok=True)
|
||||
with open(os.path.join(odir, "MANIFEST.json"), "w", encoding="utf-8") as fh:
|
||||
json.dump({"a": 1}, fh)
|
||||
with open(os.path.join(odir, "artifact.json"), "w", encoding="utf-8") as fh:
|
||||
json.dump({"b": 2}, fh)
|
||||
return mod
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 1. Critères — positifs et injections négatives (arbre synthétique). #
|
||||
# --------------------------------------------------------------------------- #
|
||||
class CriteriaTest(unittest.TestCase):
|
||||
def test_all_criteria_pass_on_complete_module(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
mod = _make_module(d)
|
||||
self.assertTrue(criteria.crit_doc(mod, 400)["passed"])
|
||||
self.assertTrue(criteria.crit_contrat(mod)["passed"])
|
||||
self.assertTrue(criteria.crit_tests(mod, 8)["passed"])
|
||||
self.assertTrue(criteria.crit_cli(mod)["passed"])
|
||||
self.assertTrue(criteria.crit_handoff(mod)["passed"])
|
||||
|
||||
def test_doc_fails_when_too_small(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
mod = _make_module(d, doc_bytes=10)
|
||||
self.assertFalse(criteria.crit_doc(mod, 400)["passed"])
|
||||
|
||||
def test_doc_fails_when_absent(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
mod = _make_module(d, doc=False)
|
||||
self.assertFalse(criteria.crit_doc(mod, 400)["passed"])
|
||||
|
||||
def test_contrat_fails_without_schema(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
mod = _make_module(d, schema=False)
|
||||
self.assertFalse(criteria.crit_contrat(mod)["passed"])
|
||||
|
||||
def test_tests_counts_methods_and_thresholds(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
mod = _make_module(d, tests=5)
|
||||
res = criteria.crit_tests(mod, 8)
|
||||
self.assertFalse(res["passed"])
|
||||
self.assertIn("5 méthodes", res["evidence"])
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
mod = _make_module(d, tests=9)
|
||||
self.assertTrue(criteria.crit_tests(mod, 8)["passed"])
|
||||
|
||||
def test_cli_fails_without_entrypoint(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
mod = _make_module(d, cli=False)
|
||||
self.assertFalse(criteria.crit_cli(mod)["passed"])
|
||||
|
||||
def test_cli_ignores_tests_dir_scripts(self):
|
||||
# Un script argparse dans tests/ ne compte pas comme entrypoint racine.
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
mod = _make_module(d, cli=False)
|
||||
self.assertFalse(criteria.crit_cli(mod)["passed"])
|
||||
|
||||
def test_handoff_fails_without_manifest(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
mod = _make_module(d, handoff=False)
|
||||
self.assertFalse(criteria.crit_handoff(mod)["passed"])
|
||||
|
||||
def test_handoff_fails_when_manifest_only(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
mod = _make_module(d)
|
||||
os.remove(os.path.join(mod, "out", "artifact.json"))
|
||||
self.assertFalse(criteria.crit_handoff(mod)["passed"])
|
||||
|
||||
def test_handoff_fails_on_broken_json(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
mod = _make_module(d)
|
||||
with open(os.path.join(mod, "out", "artifact.json"), "w") as fh:
|
||||
fh.write("{ not json")
|
||||
self.assertFalse(criteria.crit_handoff(mod)["passed"])
|
||||
|
||||
def test_handoff_fails_on_empty_artifact(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
mod = _make_module(d)
|
||||
with open(os.path.join(mod, "out", "artifact.json"), "w") as fh:
|
||||
json.dump({}, fh)
|
||||
self.assertFalse(criteria.crit_handoff(mod)["passed"])
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 2. Scoring — renormalisation par archétype + verdict. #
|
||||
# --------------------------------------------------------------------------- #
|
||||
class ScoringTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.spec = _spec()
|
||||
self._orig_root = scoring.DELIVERABLES_ROOT
|
||||
|
||||
def tearDown(self):
|
||||
scoring.DELIVERABLES_ROOT = self._orig_root
|
||||
|
||||
def _score(self, archetype, **kw):
|
||||
d = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: __import__("shutil").rmtree(d, ignore_errors=True))
|
||||
_make_module(d, **kw)
|
||||
scoring.DELIVERABLES_ROOT = d
|
||||
return scoring.score_module(self.spec, {
|
||||
"id": "x", "path": "mod", "sprint": "S9", "archetype": archetype})
|
||||
|
||||
def test_generator_full_is_100(self):
|
||||
r = self._score("generator")
|
||||
self.assertEqual(r["score"], 100)
|
||||
self.assertEqual(r["verdict"], "PASS")
|
||||
self.assertEqual(r["applicable_weight"], 100)
|
||||
|
||||
def test_generator_missing_handoff_fails(self):
|
||||
r = self._score("generator", handoff=False)
|
||||
self.assertEqual(r["score"], 80) # 80/100
|
||||
self.assertEqual(r["verdict"], "FAIL")
|
||||
|
||||
def test_contract_excludes_cli_and_handoff(self):
|
||||
r = self._score("contract", cli=False, handoff=False)
|
||||
self.assertEqual(r["applicable_weight"], 65) # DOC20+CONTRAT20+TESTS25
|
||||
self.assertEqual(r["score"], 100)
|
||||
self.assertEqual({c["criterion"] for c in r["checks"]},
|
||||
{"DOC", "CONTRAT", "TESTS"})
|
||||
|
||||
def test_parser_excludes_contrat(self):
|
||||
r = self._score("parser", schema=False, handoff=False)
|
||||
self.assertEqual({c["criterion"] for c in r["checks"]},
|
||||
{"DOC", "TESTS", "CLI"})
|
||||
self.assertEqual(r["score"], 100)
|
||||
|
||||
def test_data_room_missing_cli_drops_below_threshold(self):
|
||||
r = self._score("data_room", cli=False, handoff=False)
|
||||
# applicable DOC20+CONTRAT20+TESTS25+CLI15 = 80 ; earned 65 → 81
|
||||
self.assertEqual(r["applicable_weight"], 80)
|
||||
self.assertEqual(r["score"], 81)
|
||||
self.assertEqual(r["verdict"], "FAIL")
|
||||
|
||||
def test_missing_module_dir_raises(self):
|
||||
scoring.DELIVERABLES_ROOT = "/does/not/exist"
|
||||
with self.assertRaises(FileNotFoundError):
|
||||
scoring.score_module(self.spec, {
|
||||
"id": "x", "path": "nope", "sprint": "S9",
|
||||
"archetype": "generator"})
|
||||
|
||||
def test_round_half_up_is_deterministic(self):
|
||||
self.assertEqual(scoring._round_half_up(80.5), 81)
|
||||
self.assertEqual(scoring._round_half_up(81.25), 81)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 3. Couverture 100% vs CI (preuve, pas déclaration). #
|
||||
# --------------------------------------------------------------------------- #
|
||||
class CoverageTest(unittest.TestCase):
|
||||
def test_real_coverage_is_bijective(self):
|
||||
cov = registry.coverage_report(_spec())
|
||||
self.assertTrue(cov["ok"], cov)
|
||||
self.assertEqual(cov["missing_in_registry"], [])
|
||||
self.assertEqual(cov["missing_in_ci"], [])
|
||||
self.assertEqual(cov["not_in_gate"], [])
|
||||
self.assertEqual(cov["ci_modules_count"], cov["registry_modules_count"])
|
||||
|
||||
def test_auditor_excluded_from_scope(self):
|
||||
cov = registry.coverage_report(_spec())
|
||||
self.assertEqual(cov["self_module_excluded"], "qa/audit_4big")
|
||||
self.assertNotIn("qa/audit_4big",
|
||||
{m["path"] for m in _spec()["modules"]})
|
||||
|
||||
def test_extra_registry_module_flags_missing_in_ci(self):
|
||||
spec = _spec()
|
||||
spec["modules"].append({"id": "ghost", "path": "ghost/dir",
|
||||
"sprint": "S9", "archetype": "generator",
|
||||
"source": "test"})
|
||||
cov = registry.coverage_report(spec)
|
||||
self.assertFalse(cov["ok"])
|
||||
self.assertIn("ghost/dir", cov["missing_in_ci"])
|
||||
|
||||
def test_dropped_registry_module_flags_missing_in_registry(self):
|
||||
spec = _spec()
|
||||
spec["modules"] = spec["modules"][:-1] # retire un module gated
|
||||
cov = registry.coverage_report(spec)
|
||||
self.assertFalse(cov["ok"])
|
||||
self.assertTrue(cov["missing_in_registry"])
|
||||
|
||||
def test_ci_parsing_finds_gate_needs(self):
|
||||
ci = registry.parse_ci()
|
||||
# le job d'agrégat `gate` liste ses dépendances (needs) — un job de
|
||||
# test réel doit y figurer, preuve que le parsing du needs fonctionne.
|
||||
self.assertIn("seo-tests", ci["gate_needs"])
|
||||
self.assertGreaterEqual(len(ci["gate_needs"]), 17)
|
||||
self.assertGreaterEqual(len(ci["job_to_path"]), 17)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 4. Build réel + invariants + schéma + déterminisme. #
|
||||
# --------------------------------------------------------------------------- #
|
||||
class BuildTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.spec = _spec()
|
||||
self.report = builder.build(self.spec)
|
||||
|
||||
def test_real_build_passes_gate(self):
|
||||
self.assertEqual(self.report["verdict"], "PASS")
|
||||
self.assertEqual(self.report["totals"]["fail"], 0)
|
||||
self.assertEqual(self.report["totals"]["modules"], 17)
|
||||
self.assertGreaterEqual(self.report["totals"]["min_score"], 95)
|
||||
|
||||
def test_invariants_clean_on_real_report(self):
|
||||
errs = gen.check_invariants(self.report, self.spec)
|
||||
self.assertEqual(errs, [], errs)
|
||||
|
||||
def test_schema_validates(self):
|
||||
with open(os.path.join(_MOD, "quality.schema.json"), encoding="utf-8") as fh:
|
||||
schema = json.load(fh)
|
||||
from q4lib.deps import validate
|
||||
self.assertEqual(validate(self.report, schema), [])
|
||||
|
||||
def test_determinism(self):
|
||||
again = builder.build(_spec())
|
||||
self.assertEqual(json.dumps(self.report, sort_keys=True),
|
||||
json.dumps(again, sort_keys=True))
|
||||
|
||||
def test_pass_score_is_95(self):
|
||||
self.assertEqual(self.report["pass_score"], 95)
|
||||
self.assertEqual(self.spec["thresholds"]["pass_score"], 95)
|
||||
|
||||
def test_every_module_path_exists(self):
|
||||
from q4lib.deps import DELIVERABLES_ROOT
|
||||
for m in self.report["modules"]:
|
||||
self.assertTrue(
|
||||
os.path.isdir(os.path.join(DELIVERABLES_ROOT, m["path"])),
|
||||
m["path"])
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 5. Invariants — détection d'une note fabriquée / d'un module en échec. #
|
||||
# --------------------------------------------------------------------------- #
|
||||
class InvariantGuardTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.spec = _spec()
|
||||
self.report = builder.build(self.spec)
|
||||
|
||||
def test_fabricated_score_is_caught(self):
|
||||
bad = copy.deepcopy(self.report)
|
||||
# On force une note sans toucher aux checks → note non recomputable.
|
||||
bad["modules"][0]["score"] = 60
|
||||
errs = gen.check_invariants(bad, self.spec)
|
||||
self.assertTrue(any("INV6" in e or "INV7" in e for e in errs), errs)
|
||||
|
||||
def test_failing_module_forces_global_fail(self):
|
||||
bad = copy.deepcopy(self.report)
|
||||
m = bad["modules"][0]
|
||||
# On retire un check passant → earned baisse, note recomputée < 95.
|
||||
for c in m["checks"]:
|
||||
if c["passed"]:
|
||||
c["passed"] = False
|
||||
break
|
||||
m["earned_weight"] = sum(c["weight"] for c in m["checks"] if c["passed"])
|
||||
m["score"] = scoring._round_half_up(
|
||||
100.0 * m["earned_weight"] / m["applicable_weight"])
|
||||
m["verdict"] = "PASS" if m["score"] >= 95 else "FAIL"
|
||||
bad["totals"]["fail"] = sum(1 for x in bad["modules"]
|
||||
if x["verdict"] != "PASS")
|
||||
bad["totals"]["pass"] = 17 - bad["totals"]["fail"]
|
||||
bad["verdict"] = "FAIL"
|
||||
errs = gen.check_invariants(bad, self.spec)
|
||||
# Le rapport est cohérent en interne mais porte un FAIL → INV7 le signale.
|
||||
self.assertTrue(any("< seuil 4Big" in e for e in errs), errs)
|
||||
|
||||
def test_self_module_in_scope_is_caught(self):
|
||||
bad = copy.deepcopy(self.report)
|
||||
bad["modules"].append(copy.deepcopy(bad["modules"][0]))
|
||||
bad["modules"][-1]["path"] = self.spec["self_module"]
|
||||
errs = gen.check_invariants(bad, self.spec)
|
||||
self.assertTrue(any("INV4" in e for e in errs), errs)
|
||||
|
||||
def test_weights_sum_invariant(self):
|
||||
bad_spec = copy.deepcopy(self.spec)
|
||||
bad_spec["criteria"][0]["weight"] = 999
|
||||
errs = gen.check_invariants(self.report, bad_spec)
|
||||
self.assertTrue(any("INV2" in e for e in errs), errs)
|
||||
|
||||
def test_cli_build_and_validate_exit_zero(self):
|
||||
self.assertEqual(gen.main(["validate"]), 0)
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
self.assertEqual(gen.main(["build", "-o", d]), 0)
|
||||
self.assertTrue(os.path.isfile(os.path.join(d, "quality_report.json")))
|
||||
self.assertTrue(os.path.isfile(os.path.join(d, "MANIFEST.json")))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
Reference in New Issue
Block a user