63 lines
2.6 KiB
Python
63 lines
2.6 KiB
Python
"""Сравнение сегментации: правила vs обученный сегментатор (predict_release).
|
|
|
|
Метрика — пары (тип, автор) на уровне релиза против меток датасета, только по
|
|
страницам, которых сегментатор не видел при обучении (is_test_release).
|
|
Запуск: .venv/Scripts/python tests/e2e_segmenter.py
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
from src.html_extract import read_input # noqa: E402
|
|
from src.predict import DubPredictor # noqa: E402
|
|
from src.release import predict_release # noqa: E402
|
|
from src.segmenter import is_test_release # noqa: E402
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def run(predictor, rows, mode: str, test_only: bool) -> dict:
|
|
by_rel = defaultdict(list)
|
|
for r in rows:
|
|
by_rel[r["release_id"]].append(r)
|
|
per_src = defaultdict(list)
|
|
for page in sorted((ROOT / "data" / "pages").glob("*.txt")):
|
|
rid = page.stem
|
|
if is_test_release(rid) != test_only or rid not in by_rel:
|
|
continue
|
|
gold = {(r["translation_type"], r["studio"] or None) for r in by_rel[rid]}
|
|
res = predict_release(predictor, text=read_input(page), segmenter=mode)
|
|
pred = {(t["translation_type"], t["studio"]) for t in res["tracks"]}
|
|
tp = len(gold & pred)
|
|
p = tp / len(pred) if pred else 0.0
|
|
r = tp / len(gold) if gold else 0.0
|
|
per_src[by_rel[rid][0]["source"]].append((p, r))
|
|
out = {}
|
|
for s, v in per_src.items():
|
|
out[s] = (len(v), sum(x[0] for x in v) / len(v), sum(x[1] for x in v) / len(v))
|
|
return out
|
|
|
|
|
|
def main() -> None:
|
|
rows = json.loads((ROOT / "data" / "processed" / "dataset.json").read_text(encoding="utf-8"))
|
|
predictor = DubPredictor()
|
|
if predictor.segmenter is None:
|
|
sys.exit("нет models/segmenter: python -m src.segmenter --train")
|
|
for test_only, name in ((True, "тестовые страницы (не виделись сегментатору)"),
|
|
(False, "train-страницы (для справки, оптимистично)")):
|
|
print(f"== {name} ==")
|
|
for mode in ("rules", "model"):
|
|
res = run(predictor, rows, mode, test_only)
|
|
print(f" {mode:5s}: " + "; ".join(
|
|
f"{s} n={n} P={p:.3f} R={r:.3f}" for s, (n, p, r) in res.items()))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|