Docker-сервис, выбор основы и сравнение моделей, сборщик датасета
docker / build (push) Failing after 39s
docker / build (push) Failing after 39s
- Docker: Dockerfile (CPU), compose с FlareSolverr, настройка через DUB_* переменные, токен доступа к /predict, tools/pack_models.py, workflow Gitea Actions, docs/docker.md - Модель грузится офлайн (config.json рядом с весами); общий код вынесен в src/spans.py, сервису не нужны sklearn/matplotlib/pandas - Обучение: профили tiny2/base/large/modern, запуски в models/runs, реестр и честное сравнение на одной тестовой выборке (src/runs.py), пункт меню «Модели» - Датасет: язык и аудио-параметры дорожек, 18+ по строкам «Аудио», ручные метки по ссылкам, переиндексация сохранённого HTML, параллельная сборка, сбалансированная выборка - Тексты страниц (data/pages) больше не хранятся в git Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
DenozordecandClaude Sonnet 5.5 committed
1 parent
d155b8584b
commit
efc9a4f56a
411 files changed
+87495
-27363
No files matched your search
+314
@@ -0,0 +1,314 @@
|
||||
"""Метаданные релиза со страницы трекера (фильм / сериал / мультфильм / аниме).
|
||||
|
||||
extract(site, rec, section_name="") -> dict # одна строка data/meta.csv
|
||||
|
||||
Вход — запись сборщика (`data/raw/<трекер>/<id>.json` или старый harvest):
|
||||
rutracker / nnm: поля `title` и `post`; kinozal: `h1`, `release_tab`, `tech_tab`.
|
||||
Всё берётся из сохранённого текста, поэтому парсер можно улучшать и пересобирать
|
||||
таблицу без повторных заходов на трекер.
|
||||
|
||||
Дорожки (`audio_tracks`, JSON): язык, кодек, число каналов, битрейт, тип
|
||||
(DUB/MVO/DVO/AVO/VO/ORIG), студия, диапазон серий, 18+.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
# Поднимайте при правке извлечения: кэш строк meta.csv сбросится и пересоберётся один раз.
|
||||
META_VERSION = 2
|
||||
|
||||
FIELDS = [
|
||||
"source", "release_id", "title", "ru_title", "orig_title", "year", "kind", "is_series",
|
||||
"season", "episodes", "episodes_total", "section",
|
||||
"quality", "resolution", "video_codec", "hdr", "bit_depth", "fps", "container",
|
||||
"size_gb", "duration_min", "duration_per_episode",
|
||||
"countries", "genres", "directors", "cast", "age_limit", "mpaa", "release_group", "source_media",
|
||||
"audio_langs", "orig_audio", "n_audio", "audio_tracks",
|
||||
"sub_langs", "sub_forced", "sub_format",
|
||||
]
|
||||
|
||||
LANGS = [("русск", "RUS"), ("russian", "RUS"), ("рус", "RUS"), ("rus", "RUS"),
|
||||
("англий", "ENG"), ("english", "ENG"), ("eng", "ENG"),
|
||||
("украин", "UKR"), ("ukrainian", "UKR"), ("ukr", "UKR"),
|
||||
("япон", "JPN"), ("japanese", "JPN"), ("jpn", "JPN"),
|
||||
("корей", "KOR"), ("korean", "KOR"), ("француз", "FRA"), ("french", "FRA"),
|
||||
("немец", "DEU"), ("german", "DEU"), ("испан", "SPA"), ("spanish", "SPA"),
|
||||
("итальян", "ITA"), ("italian", "ITA"), ("китайск", "CHI"), ("chinese", "CHI"),
|
||||
("польск", "POL"), ("polish", "POL"), ("белорус", "BEL"), ("чешск", "CZE")]
|
||||
|
||||
CODEC_AUDIO = re.compile(
|
||||
r"(?i)E-?AC-?3(?:\s*JOC)?|AC-?3|DTS-HD\s*MA|DTS-HD|DTS:X|DTS|TrueHD|FLAC|AAC|Opus|MP3|L?PCM")
|
||||
TAG = re.compile(r"(?i)\b(DUB|MVO|DVO|AVO|VO|Original)\b")
|
||||
TAG_WORDS = [("дублир", "DUB"), ("дубляж", "DUB"), ("многоголос", "MVO"), ("двухголос", "DVO"),
|
||||
("одноголос", "AVO"), ("оригинал", "ORIG")]
|
||||
RE_BITRATE = re.compile(r"(?i)(\d+(?:[.,]\d+)?)\s*(?:k|к)(?:bps|бит)")
|
||||
RE_CHANNELS = re.compile(r"(?i)(\d+(?:\.\d)?)\s*(?:ch\b|канал)")
|
||||
RE_LAYOUT = re.compile(r"(\d)/(\d)(?:\.\d)?")
|
||||
RE_EPISODES_IN_TRACK = re.compile(r"(?i)сери[яию]\s*([\d\s,\-–]+)")
|
||||
|
||||
KEYS = {
|
||||
"year": ("Год выпуска", "Год"),
|
||||
"country": ("Страна", "Производство", "Выпущено"),
|
||||
"genre": ("Жанр",),
|
||||
"director": ("Режиссер", "Режиссёр"),
|
||||
"cast": ("В ролях", "Актеры", "Актёры"),
|
||||
"duration": ("Продолжительность",),
|
||||
"quality": ("Качество видео", "Тип релиза", "Качество"),
|
||||
"container": ("Контейнер",),
|
||||
"size": ("Размер",),
|
||||
"age": ("Возраст",),
|
||||
"mpaa": ("Рейтинг MPAA",),
|
||||
"group": ("Релиз от", "Релиз группы", "Релизер"),
|
||||
"source_media": ("Источник", "Исходник"),
|
||||
"orig_audio": ("Оригинальная аудиодорожка",),
|
||||
"voice_langs": ("Язык озвучки",),
|
||||
"subs": ("Субтитры",),
|
||||
"sub_format": ("Формат субтитров",),
|
||||
"video": ("Видео",),
|
||||
}
|
||||
|
||||
|
||||
def lang_of(word: str) -> str | None:
|
||||
w = (word or "").strip().lower()
|
||||
for stem, code in LANGS:
|
||||
if w.startswith(stem):
|
||||
return code
|
||||
return None
|
||||
|
||||
|
||||
def langs_in(text: str) -> list[str]:
|
||||
out: list[str] = []
|
||||
for part in re.split(r"[,;/+|]|\bи\b", text or ""):
|
||||
code = lang_of(re.sub(r"[^A-Za-zА-Яа-яЁё]+", " ", part).strip().split(" ")[0] if part.strip() else "")
|
||||
if code and code not in out:
|
||||
out.append(code)
|
||||
return out
|
||||
|
||||
|
||||
def field(text: str, key: str) -> str:
|
||||
for label in KEYS[key]:
|
||||
m = re.search(rf"(?im)(?:^|\s){re.escape(label)}\s*:[ \t]*([^\n]+)", text)
|
||||
if m and m.group(1).strip():
|
||||
return m.group(1).strip()
|
||||
return ""
|
||||
|
||||
|
||||
def _num_range(s: str) -> str:
|
||||
return re.sub(r"\s+", "", s).replace("–", "-")
|
||||
|
||||
|
||||
def parse_title(title: str) -> dict:
|
||||
t = re.sub(r"\s+", " ", title or "").strip()
|
||||
parts = [p.strip() for p in t.split(" / ")]
|
||||
ru = parts[0] if parts else ""
|
||||
orig = ""
|
||||
if len(parts) > 1 and not re.match(r"^(?:19|20)\d\d\b", parts[1]):
|
||||
orig = re.split(r"\s*[\(\[]", parts[1])[0].strip()
|
||||
year = re.search(r"\b(19\d\d|20[0-2]\d)\b", t)
|
||||
season = re.search(r"(?i)сезон[ы]?\s*(\d+(?:\s*[-–]\s*\d+)?)", t) or re.search(r"\bS(\d{1,2})\b", t)
|
||||
ep = re.search(r"(?i)сери[ияй]\s*(\d+(?:\s*[-–]\s*\d+)?)(?:\s*из\s*(\d+))?", t)
|
||||
return {"ru_title": ru, "orig_title": orig, "year": year.group(1) if year else "",
|
||||
"season": _num_range(season.group(1)) if season else "",
|
||||
"episodes": _num_range(ep.group(1)) if ep else "",
|
||||
"episodes_total": ep.group(2) if ep and ep.group(2) else ""}
|
||||
|
||||
|
||||
def parse_video(line: str, extra: str = "") -> dict:
|
||||
s = f"{line} {extra}"
|
||||
codec = ""
|
||||
if re.search(r"(?i)HEVC|H\.?265|x265", s):
|
||||
codec = "HEVC"
|
||||
elif re.search(r"(?i)AVC|H\.?264|x264", s):
|
||||
codec = "AVC"
|
||||
elif re.search(r"(?i)MPEG-?2", s):
|
||||
codec = "MPEG2"
|
||||
elif re.search(r"(?i)VC-?1", s):
|
||||
codec = "VC1"
|
||||
elif re.search(r"(?i)\bAV1\b", s):
|
||||
codec = "AV1"
|
||||
elif re.search(r"(?i)xvid|divx", s):
|
||||
codec = "XviD/DivX"
|
||||
res = ""
|
||||
m = re.search(r"(\d{3,4})\s*[x×]\s*(\d{3,4})", s)
|
||||
if m:
|
||||
w = int(m.group(1))
|
||||
res = "2160p" if w >= 3500 else "1080p" if w >= 1800 else "720p" if w >= 1200 else "480p"
|
||||
else:
|
||||
m = re.search(r"(?i)\b(2160|1440|1080|720|576|480)\s*p\b", s)
|
||||
res = f"{m.group(1)}p" if m else ("2160p" if re.search(r"(?i)\b4K\b|UHD", s) else "")
|
||||
hdr = []
|
||||
for pat, name in ((r"HDR10\+", "HDR10+"), (r"HDR10(?!\+)", "HDR10"), (r"Dolby\s*Vision|\bDV\b", "DV"),
|
||||
(r"(?<![A-Za-z0-9])HDR(?!10)", "HDR")):
|
||||
if re.search(pat, s):
|
||||
hdr.append(name)
|
||||
bd = re.search(r"(?i)(\d{1,2})\s*-?\s*bit", s)
|
||||
fps = re.search(r"(?i)(\d{2}(?:\.\d+)?)\s*(?:fps|кадр)", s)
|
||||
return {"video_codec": codec, "resolution": res, "hdr": "+".join(dict.fromkeys(hdr)),
|
||||
"bit_depth": bd.group(1) if bd else "", "fps": fps.group(1) if fps else ""}
|
||||
|
||||
|
||||
_HOMOGLYPH = str.maketrans("АСЕ", "ACE") # «АС3» кириллицей на rutracker
|
||||
|
||||
|
||||
def _track(body: str, lang: str | None = None) -> dict:
|
||||
codec = CODEC_AUDIO.search(body.translate(_HOMOGLYPH))
|
||||
ch = RE_CHANNELS.search(body)
|
||||
channels = ""
|
||||
if ch:
|
||||
channels = ch.group(1).split(".")[0] if "." in ch.group(1) and ch.group(1).endswith(".0") else ch.group(1)
|
||||
else:
|
||||
lay = RE_LAYOUT.search(body)
|
||||
if lay:
|
||||
channels = str(int(lay.group(1)) + int(lay.group(2)) + (1 if re.search(r"LFE", body) else 0))
|
||||
br = RE_BITRATE.search(body)
|
||||
note = re.split(r"\s\|\s|\s-\s|\|", body, maxsplit=1)
|
||||
note = note[1] if len(note) > 1 else ""
|
||||
tag_m = TAG.search(note) or TAG.search(body)
|
||||
tag = tag_m.group(1).upper() if tag_m else ""
|
||||
if not tag:
|
||||
low = note.lower()
|
||||
tag = next((v for k, v in TAG_WORDS if k in low), "")
|
||||
tag = "ORIG" if tag == "ORIGINAL" else tag
|
||||
if lang is None:
|
||||
lead = re.match(r"\s*([A-Za-z]{2,3})\s*:", body) # «RUS: AC3, …»
|
||||
lang = lang_of(lead.group(1)) if lead else None
|
||||
if lang is None:
|
||||
for w in re.findall(r"[A-Za-zА-Яа-яЁё]+", note):
|
||||
lang = lang_of(w)
|
||||
if lang:
|
||||
break
|
||||
if lang is None and tag in ("DUB", "MVO", "DVO", "AVO", "VO"):
|
||||
lang = "RUS"
|
||||
studio = ""
|
||||
m = re.search(r"\(([^()]*)\)", note)
|
||||
if m:
|
||||
studio = TAG.sub("", m.group(1)).strip(" ,;")
|
||||
else: # rutracker: «… 192 kbps |HDrezka| 18+»
|
||||
piped = re.search(r"\|\s*([^|]+?)\s*\|", body)
|
||||
if piped and not re.fullmatch(r"(?i)original|оригинал|18\s*\+", piped.group(1)):
|
||||
studio = TAG.sub("", piped.group(1)).strip(" ,;")
|
||||
try:
|
||||
from src.episodes import format_ranges, parse_episodes
|
||||
episodes = format_ranges(parse_episodes(note)["ranges"])
|
||||
except ImportError: # без репозитория рядом
|
||||
eps = RE_EPISODES_IN_TRACK.search(note)
|
||||
episodes = _num_range(eps.group(1)).strip(",") if eps else ""
|
||||
return {"lang": lang or "", "codec": re.sub(r"\s+", " ", codec.group(0)).upper() if codec else "",
|
||||
"channels": channels, "bitrate_kbps": int(float(br.group(1).replace(",", "."))) if br else "",
|
||||
"tag": tag, "studio": studio, "episodes": episodes,
|
||||
"is_18plus": "18+" in body}
|
||||
|
||||
|
||||
def parse_audio(text: str) -> list[dict]:
|
||||
tracks: list[dict] = []
|
||||
# «Аудио: …», «Аудио 2: …» и rutracker-вид «Аудио RUS: AC3, 2 ch … |Студия| 18+»
|
||||
for m in re.finditer(r"(?im)^\s*Аудио(?:\s*0*(\d*)\s*:[ \t]*|[^\S\n]+(?=[A-Za-z]{2,3}\s*:))([^\n]+)", text):
|
||||
body = m.group(2).strip()
|
||||
groups = re.findall(r"\(([^()]*(?:ch|канал)[^()]*)\)", body, flags=re.I)
|
||||
if groups and not m.group(1): # kinozal: «Русский (AC3, 2 ch, …), (…), английский (…)»
|
||||
lang = None
|
||||
for g in re.finditer(r"(?:([A-Za-zА-Яа-яЁё]+)\s*)?\(([^()]*(?:ch|канал)[^()]*)\)", body, flags=re.I):
|
||||
lang = lang_of(g.group(1)) or lang
|
||||
tracks.append(_track(g.group(2), lang))
|
||||
elif body:
|
||||
tracks.append(_track(body))
|
||||
return tracks
|
||||
|
||||
|
||||
RE_TRANSLATION = re.compile(r"(?im)^\s*(?:Перевод|Озвучка|Озвучивание)\s*\d*\s*:[^\S\n]*([^\n]+)")
|
||||
|
||||
|
||||
def translation_tags(text: str) -> dict[str, str]:
|
||||
"""{студия: DUB/MVO/DVO/AVO} из строк «Перевод: Профессиональный (многоголосый) Студия»."""
|
||||
out: dict[str, str] = {}
|
||||
for m in RE_TRANSLATION.finditer(text):
|
||||
body = m.group(1)
|
||||
low = body.lower()
|
||||
tag = next((v for k, v in TAG_WORDS if k in low), "")
|
||||
studio = re.sub(r"\[.*?\]|\(.*?\)|18\s*\+", " ", body).strip(" ,;|-")
|
||||
studio = studio.split("|")[-1].strip()
|
||||
studio = re.sub(r"(?i)^(?:(?:профессиональн|любительск|авторск|дублирован|многоголос|двухголос|"
|
||||
r"одноголос|закадров)\w*[\s,]*)+", "", studio).strip()
|
||||
if tag and studio:
|
||||
out.setdefault(_studio_canon(studio).casefold(), tag)
|
||||
return out
|
||||
|
||||
|
||||
def kind_of(section: str, title: str, genres: str, is_series: bool) -> str:
|
||||
blob = f"{section} {genres} {title}".lower()
|
||||
if re.search(r"аниме|anime|онгоинг", blob):
|
||||
return "anime"
|
||||
if re.search(r"мульт|анимац|animation", blob):
|
||||
return "cartoon"
|
||||
return "series" if is_series else "film"
|
||||
|
||||
|
||||
def _studio_canon(s: str) -> str:
|
||||
try:
|
||||
from src.studio_norm import canonical
|
||||
return canonical(s) or s
|
||||
except Exception: # noqa: BLE001
|
||||
return s
|
||||
|
||||
|
||||
def extract(site: str, rec: dict, section_name: str = "", page_text: str = "") -> dict:
|
||||
"""page_text — текст полной страницы (если сохранён HTML): добавляет размер раздачи."""
|
||||
if site == "kinozal":
|
||||
title = rec.get("h1") or rec.get("page_title") or ""
|
||||
# жанр, страна, режиссёр — в теле страницы; вкладки идут первыми, чтобы
|
||||
# их поля (качество, аудио) имели приоритет над списками «подобных раздач»
|
||||
body = (rec.get("page_text") or "")[:8000]
|
||||
text = "\n".join(x for x in (title, rec.get("release_tab"), rec.get("tech_tab"), body) if x)
|
||||
else:
|
||||
title = rec.get("title") or rec.get("page_title") or ""
|
||||
text = f"{title}\n{rec.get('post') or ''}"
|
||||
tp = parse_title(title)
|
||||
quality = field(text, "quality")
|
||||
video_line = field(text, "video")
|
||||
vid = parse_video(video_line, f"{title} {quality}")
|
||||
tracks = parse_audio(text)
|
||||
for t in tracks:
|
||||
if t["studio"]:
|
||||
t["studio"] = _studio_canon(t["studio"])
|
||||
by_studio = translation_tags(text) # тип озвучки из строк «Перевод»
|
||||
for t in tracks:
|
||||
if not t["tag"] and t["studio"]:
|
||||
t["tag"] = by_studio.get(t["studio"].casefold(), "")
|
||||
orig_audio = langs_in(field(text, "orig_audio"))
|
||||
audio_langs = list(dict.fromkeys(
|
||||
[t["lang"] for t in tracks if t["lang"]] + langs_in(field(text, "voice_langs")) + orig_audio))
|
||||
subs = field(text, "subs")
|
||||
size = re.search(r"(?i)([\d.,]+)\s*(ТБ|ГБ|GB|МБ|MB)", field(text, "size"))
|
||||
if not size and page_text:
|
||||
size = re.search(r"(?i)(?:Размер|Size)\s*[:\s]\s*([\d.,]+)\s*(ТБ|ГБ|GB|МБ|MB)", page_text)
|
||||
size_gb = ""
|
||||
if size:
|
||||
v = float(size.group(1).replace(",", "."))
|
||||
size_gb = round(v * {"ТБ": 1024, "ГБ": 1, "GB": 1, "МБ": 1 / 1024, "MB": 1 / 1024}[size.group(2).upper()], 2)
|
||||
dur_line = field(text, "duration")
|
||||
dur = re.search(r"(\d+):(\d\d):(\d\d)", dur_line)
|
||||
genres = field(text, "genre")
|
||||
is_series = bool(tp["season"] or tp["episodes"] or re.search(r"(?i)сериал", section_name)
|
||||
or re.search(r"(?i)сериал", title))
|
||||
age = re.search(r"(\d+)\s*\+", field(text, "age"))
|
||||
cast = [c.strip() for c in re.split(r"[,;]", field(text, "cast")) if c.strip()][:8]
|
||||
return {
|
||||
"source": site, "release_id": str(rec.get("id", "")), "title": re.sub(r"\s+", " ", title).strip(),
|
||||
"ru_title": tp["ru_title"], "orig_title": tp["orig_title"],
|
||||
"year": tp["year"] or (re.search(r"\d{4}", field(text, "year")) or [""])[0],
|
||||
"kind": kind_of(section_name, title, genres, is_series), "is_series": is_series,
|
||||
"season": tp["season"], "episodes": tp["episodes"], "episodes_total": tp["episodes_total"],
|
||||
"section": section_name or rec.get("section_ref", ""),
|
||||
"quality": quality, **vid, "container": field(text, "container"),
|
||||
"size_gb": size_gb, "duration_min": int(dur.group(1)) * 60 + int(dur.group(2)) if dur else "",
|
||||
"duration_per_episode": bool(dur and re.search(r"(?i)сери", dur_line)),
|
||||
"countries": field(text, "country"), "genres": genres, "directors": field(text, "director"),
|
||||
"cast": "; ".join(cast), "age_limit": age.group(1) if age else "", "mpaa": field(text, "mpaa"),
|
||||
"release_group": field(text, "group"), "source_media": field(text, "source_media"),
|
||||
"audio_langs": "+".join(audio_langs), "orig_audio": "+".join(orig_audio), "n_audio": len(tracks),
|
||||
"audio_tracks": json.dumps(tracks, ensure_ascii=False) if tracks else "",
|
||||
"sub_langs": "+".join(langs_in(subs)), "sub_forced": bool(re.search(r"(?i)forced|форсир", subs)),
|
||||
"sub_format": field(text, "sub_format"),
|
||||
}
|
||||
Reference in new issue
Block a user