feat: add session-level publish and comment flow

This commit is contained in:
theshy
2026-04-14 16:00:17 +08:00
parent 862db502b0
commit d5d9693581
42 changed files with 2478 additions and 181 deletions

View File

@ -31,7 +31,8 @@ def main() -> None:
worker_parser.add_argument("--interval", type=int, default=5)
create_task_parser = sub.add_parser("create-task")
create_task_parser.add_argument("source_path")
create_task_parser.add_argument("source")
create_task_parser.add_argument("--source-type", choices=["local_file", "bilibili_url"], default="local_file")
serve_parser = sub.add_parser("serve")
serve_parser.add_argument("--host", default="127.0.0.1")
@ -95,10 +96,11 @@ def main() -> None:
state = ensure_initialized()
settings = dict(state["settings"]["ingest"])
settings.update(state["settings"]["paths"])
task = state["ingest_service"].create_task_from_file(
Path(args.source_path),
settings,
)
if args.source_type == "bilibili_url":
settings["provider"] = "bilibili_url"
task = state["ingest_service"].create_task_from_url(args.source, settings)
else:
task = state["ingest_service"].create_task_from_file(Path(args.source), settings)
print(json.dumps(task.to_dict(), ensure_ascii=False, indent=2))
return

View File

@ -137,13 +137,23 @@ class ControlPlanePostDispatcher:
return result, HTTPStatus.CREATED
def handle_create_task(self, payload: object) -> tuple[object, HTTPStatus]:
source_path = payload.get("source_path") if isinstance(payload, dict) else None
if not source_path:
return {"error": "missing source_path"}, HTTPStatus.BAD_REQUEST
if not isinstance(payload, dict):
return {"error": "invalid payload"}, HTTPStatus.BAD_REQUEST
source_type = str(payload.get("source_type") or "local_file")
try:
settings = dict(self.state["settings"]["ingest"])
settings.update(self.state["settings"]["paths"])
task = self.state["ingest_service"].create_task_from_file(Path(source_path), settings)
if source_type == "bilibili_url":
source_url = payload.get("source_url")
if not source_url:
return {"error": "missing source_url"}, HTTPStatus.BAD_REQUEST
settings["provider"] = "bilibili_url"
task = self.state["ingest_service"].create_task_from_url(str(source_url), settings)
else:
source_path = payload.get("source_path")
if not source_path:
return {"error": "missing source_path"}, HTTPStatus.BAD_REQUEST
task = self.state["ingest_service"].create_task_from_file(Path(str(source_path)), settings)
except Exception as exc:
status = HTTPStatus.CONFLICT if exc.__class__.__name__ == "ModuleError" else HTTPStatus.INTERNAL_SERVER_ERROR
body = exc.to_dict() if hasattr(exc, "to_dict") else {"error": str(exc)}

View File

@ -155,6 +155,10 @@ class SessionDeliveryService:
if session_key is None and source_title is None:
return {"error": {"code": "SESSION_KEY_OR_SOURCE_TITLE_REQUIRED", "message": "session_key or source_title required"}}
source_contexts = self.repo.list_task_contexts_by_source_title(source_title) if source_title else []
if session_key is None and source_contexts:
session_key = source_contexts[0].session_key
now = utc_now_iso()
self.repo.upsert_session_binding(
SessionBinding(
@ -170,8 +174,8 @@ class SessionDeliveryService:
)
contexts = self.repo.list_task_contexts_by_session_key(session_key) if session_key else []
if not contexts and source_title:
contexts = self.repo.list_task_contexts_by_source_title(source_title)
if not contexts and source_contexts:
contexts = source_contexts
updated_tasks: list[dict[str, object]] = []
for context in contexts:

View File

@ -1,5 +1,7 @@
from __future__ import annotations
from datetime import datetime
from biliup_next.app.retry_meta import retry_meta_for_step
@ -90,6 +92,14 @@ def next_runnable_step(task, steps: dict[str, object], state: dict[str, object])
if task.status == "split_done":
step = steps.get("publish")
if step and step.status in {"pending", "failed_retryable"}:
session_publish = _session_publish_gate(task.id, state)
if session_publish is not None:
if session_publish["session_published"]:
return "publish", None
if not session_publish["is_anchor"]:
return None, None
if not session_publish["all_split_ready"]:
return None, None
return "publish", None
if task.status in {"published", "collection_synced"}:
if state["settings"]["comment"].get("enabled", True):
@ -113,6 +123,49 @@ def next_runnable_step(task, steps: dict[str, object], state: dict[str, object])
return None, None
def _session_publish_gate(task_id: str, state: dict[str, object]) -> dict[str, object] | None:
repo = state.get("repo")
if repo is None or not hasattr(repo, "get_task_context"):
return None
context = repo.get_task_context(task_id)
if context is None or not context.session_key or context.session_key.startswith("task:"):
return None
contexts = list(repo.list_task_contexts_by_session_key(context.session_key))
if len(contexts) <= 1:
return None
ordered = sorted(
contexts,
key=lambda item: (
_parse_dt(item.segment_started_at),
item.source_title or item.task_id,
),
)
anchor_id = ordered[0].task_id
sibling_tasks = [repo.get_task(item.task_id) for item in ordered]
session_published = any(
sibling is not None and sibling.status in {"published", "commented", "collection_synced"}
for sibling in sibling_tasks
)
all_split_ready = all(
sibling is not None and sibling.status in {"split_done", "published", "commented", "collection_synced"}
for sibling in sibling_tasks
)
return {
"is_anchor": task_id == anchor_id,
"session_published": session_published,
"all_split_ready": all_split_ready,
}
def _parse_dt(value: str | None) -> datetime:
if not value:
return datetime.max
try:
return datetime.fromisoformat(value)
except ValueError:
return datetime.max
def execute_step(state: dict[str, object], task_id: str, step_name: str) -> dict[str, object]:
if step_name == "transcribe":
artifact = state["transcribe_service"].run(task_id, settings_for(state, "transcribe"))

View File

@ -36,7 +36,15 @@ def resolve_failure(task, repo, state: dict[str, object], exc) -> dict[str, obje
next_status = "failed_retryable" if exc.retryable else "failed_manual"
next_retry_delay_seconds: int | None = None
if exc.retryable and step_name == "publish":
schedule = publish_retry_schedule_seconds(settings_for(state, "publish"))
publish_settings = settings_for(state, "publish")
if exc.code == "PUBLISH_RATE_LIMITED":
custom_schedule = publish_settings.get("rate_limit_retry_schedule_minutes")
if isinstance(custom_schedule, list) and custom_schedule:
schedule = [max(0, int(minutes)) * 60 for minutes in custom_schedule]
else:
schedule = publish_retry_schedule_seconds(publish_settings)
else:
schedule = publish_retry_schedule_seconds(publish_settings)
if next_retry_count > len(schedule):
next_status = "failed_manual"
else:

View File

@ -86,4 +86,13 @@ def process_task(task_id: str, *, reset_step: str | None = None, include_stage_s
failure = resolve_failure(task, repo, state, exc)
processed.append(failure["payload"])
record_task_action(repo, task_id, failure["step_name"], "error", failure["summary"], failure["payload"])
except Exception as exc:
unexpected = ModuleError(
code="UNHANDLED_EXCEPTION",
message=f"unexpected error: {exc}",
retryable=False,
)
failure = resolve_failure(task, repo, state, unexpected)
processed.append(failure["payload"])
record_task_action(repo, task_id, failure["step_name"], "error", failure["summary"], failure["payload"])
return {"processed": processed}

View File

@ -160,9 +160,11 @@ class SettingsService:
("paths", "cookies_file"),
("paths", "upload_config_file"),
("ingest", "ffprobe_bin"),
("ingest", "yt_dlp_cmd"),
("transcribe", "ffmpeg_bin"),
("split", "ffmpeg_bin"),
("song_detect", "codex_cmd"),
("song_detect", "qwen_cmd"),
("publish", "biliup_path"),
("publish", "cookie_file"),
):

View File

@ -1,27 +1,158 @@
from __future__ import annotations
import subprocess
from datetime import datetime
from pathlib import Path
import shlex
from biliup_next.core.errors import ModuleError
class BiliupCliAdapter:
def run(self, cmd: list[str], *, label: str) -> subprocess.CompletedProcess[str]:
def run(
self,
cmd: list[str],
*,
label: str,
timeout_seconds: int | None = None,
log_path: Path | None = None,
) -> subprocess.CompletedProcess[str]:
try:
return subprocess.run(cmd, capture_output=True, text=True, check=False)
result = subprocess.run(
cmd,
capture_output=True,
text=True,
check=False,
timeout=timeout_seconds,
)
self._append_log(
log_path,
label=label,
cmd=cmd,
returncode=result.returncode,
stdout=result.stdout,
stderr=result.stderr,
timeout_seconds=timeout_seconds,
)
return result
except FileNotFoundError as exc:
raise ModuleError(
code="BILIUP_NOT_FOUND",
message=f"找不到 biliup 命令: {cmd[0]} ({label})",
retryable=False,
) from exc
except subprocess.TimeoutExpired as exc:
stdout = self._stringify_output(exc.stdout)
stderr = self._stringify_output(exc.stderr)
self._append_log(
log_path,
label=label,
cmd=cmd,
returncode=None,
stdout=stdout,
stderr=stderr,
timeout_seconds=timeout_seconds,
)
raise ModuleError(
code="BILIUP_TIMEOUT",
message=f"biliup 命令超时: {label}",
retryable=True,
details={
"label": label,
"timeout_seconds": timeout_seconds,
"stdout": stdout[-2000:],
"stderr": stderr[-2000:],
},
) from exc
def run_optional(self, cmd: list[str]) -> None:
def run_optional(
self,
cmd: list[str],
*,
label: str,
timeout_seconds: int | None = None,
log_path: Path | None = None,
) -> None:
try:
subprocess.run(cmd, capture_output=True, text=True, check=False)
result = subprocess.run(
cmd,
capture_output=True,
text=True,
check=False,
timeout=timeout_seconds,
)
self._append_log(
log_path,
label=label,
cmd=cmd,
returncode=result.returncode,
stdout=result.stdout,
stderr=result.stderr,
timeout_seconds=timeout_seconds,
)
except FileNotFoundError as exc:
raise ModuleError(
code="BILIUP_NOT_FOUND",
message=f"找不到 biliup 命令: {cmd[0]}",
retryable=False,
) from exc
except subprocess.TimeoutExpired as exc:
stdout = self._stringify_output(exc.stdout)
stderr = self._stringify_output(exc.stderr)
self._append_log(
log_path,
label=label,
cmd=cmd,
returncode=None,
stdout=stdout,
stderr=stderr,
timeout_seconds=timeout_seconds,
)
raise ModuleError(
code="BILIUP_TIMEOUT",
message=f"biliup 命令超时: {label}",
retryable=True,
details={
"label": label,
"timeout_seconds": timeout_seconds,
"stdout": stdout[-2000:],
"stderr": stderr[-2000:],
},
) from exc
@staticmethod
def _append_log(
log_path: Path | None,
*,
label: str,
cmd: list[str],
returncode: int | None,
stdout: str,
stderr: str,
timeout_seconds: int | None = None,
) -> None:
if log_path is None:
return
timestamp = datetime.now().isoformat(timespec="seconds")
log_path.parent.mkdir(parents=True, exist_ok=True)
lines = [
f"[{timestamp}] {label}",
f"cmd: {shlex.join(cmd)}",
]
if timeout_seconds is not None:
lines.append(f"timeout_seconds: {timeout_seconds}")
lines.append(f"exit: {returncode if returncode is not None else 'timeout'}")
if stdout:
lines.extend(["stdout:", stdout.rstrip()])
if stderr:
lines.extend(["stderr:", stderr.rstrip()])
lines.append("")
log_path.write_text(log_path.read_text(encoding="utf-8") + "\n".join(lines), encoding="utf-8") if log_path.exists() else log_path.write_text("\n".join(lines), encoding="utf-8")
@staticmethod
def _stringify_output(value: str | bytes | None) -> str:
if value is None:
return ""
if isinstance(value, bytes):
return value.decode("utf-8", errors="replace")
return value

View File

@ -0,0 +1,36 @@
from __future__ import annotations
import subprocess
from pathlib import Path
from biliup_next.core.errors import ModuleError
class QwenCliAdapter:
def run_song_detect(
self,
*,
qwen_cmd: str,
work_dir: Path,
prompt: str,
) -> subprocess.CompletedProcess[str]:
cmd = [
qwen_cmd,
"--yolo",
"-p",
prompt,
]
try:
return subprocess.run(
cmd,
cwd=str(work_dir),
capture_output=True,
text=True,
check=False,
)
except FileNotFoundError as exc:
raise ModuleError(
code="QWEN_NOT_FOUND",
message=f"找不到 qwen 命令: {qwen_cmd}",
retryable=False,
) from exc

View File

@ -0,0 +1,76 @@
from __future__ import annotations
import json
import subprocess
from typing import Any
from biliup_next.core.errors import ModuleError
class YtDlpAdapter:
def probe(self, *, yt_dlp_cmd: str, source_url: str) -> dict[str, Any]:
cmd = [
yt_dlp_cmd,
"--dump-single-json",
"--no-warnings",
"--no-playlist",
source_url,
]
result = self._run(cmd)
if result.returncode != 0:
raise ModuleError(
code="YT_DLP_PROBE_FAILED",
message="yt-dlp 获取视频信息失败",
retryable=True,
details={"stdout": result.stdout[-2000:], "stderr": result.stderr[-2000:]},
)
try:
payload = json.loads(result.stdout)
except json.JSONDecodeError as exc:
raise ModuleError(
code="YT_DLP_PROBE_INVALID_JSON",
message="yt-dlp 返回了非法 JSON",
retryable=True,
details={"stdout": result.stdout[-2000:]},
) from exc
if not isinstance(payload, dict):
raise ModuleError(
code="YT_DLP_PROBE_INVALID_JSON",
message="yt-dlp 返回结果不是对象",
retryable=True,
)
return payload
def download(
self,
*,
yt_dlp_cmd: str,
source_url: str,
output_template: str,
format_selector: str | None = None,
) -> subprocess.CompletedProcess[str]:
cmd = [
yt_dlp_cmd,
"--no-warnings",
"--no-playlist",
]
if format_selector:
cmd.extend(["-f", format_selector])
cmd.extend([
"--merge-output-format",
"mp4",
"-o",
output_template,
source_url,
])
return self._run(cmd)
def _run(self, cmd: list[str]) -> subprocess.CompletedProcess[str]:
try:
return subprocess.run(cmd, capture_output=True, text=True, check=False)
except FileNotFoundError as exc:
raise ModuleError(
code="YT_DLP_NOT_FOUND",
message=f"找不到 yt-dlp 命令: {cmd[0]}",
retryable=False,
) from exc

View File

@ -40,13 +40,25 @@ class RuntimeDoctor:
for group, name in (
("ingest", "ffprobe_bin"),
("transcribe", "ffmpeg_bin"),
("song_detect", "codex_cmd"),
):
value = settings[group][name]
found = shutil.which(value) if "/" not in value else str((self.root_dir / value).resolve())
ok = bool(found) and (Path(found).exists() if "/" in str(found) else True)
checks.append({"name": f"{group}.{name}", "ok": ok, "detail": str(found or value)})
if str(settings.get("ingest", {}).get("provider", "local_file")) == "bilibili_url":
value = str(settings["ingest"].get("yt_dlp_cmd", "yt-dlp"))
found = shutil.which(value) if "/" not in value else str((self.root_dir / value).resolve())
ok = bool(found) and (Path(found).exists() if "/" in str(found) else True)
checks.append({"name": "ingest.yt_dlp_cmd", "ok": ok, "detail": str(found or value)})
song_detect_provider = str(settings.get("song_detect", {}).get("provider", "codex"))
song_detect_cmd_name = "qwen_cmd" if song_detect_provider == "qwen_cli" else "codex_cmd"
song_detect_cmd = str(settings["song_detect"].get(song_detect_cmd_name, ""))
found = shutil.which(song_detect_cmd) if "/" not in song_detect_cmd else str((self.root_dir / song_detect_cmd).resolve())
ok = bool(found) and (Path(found).exists() if "/" in str(found) else True)
checks.append({"name": f"song_detect.{song_detect_cmd_name}", "ok": ok, "detail": str(found or song_detect_cmd)})
publish_biliup_path = Path(str(settings["publish"]["biliup_path"])).resolve()
checks.append(
{

View File

@ -2,6 +2,7 @@ from __future__ import annotations
import json
import time
from datetime import datetime
from pathlib import Path
from typing import Any
@ -39,7 +40,7 @@ class BilibiliTopCommentProvider:
)
timeline_content = songs_path.read_text(encoding="utf-8").strip()
split_content = self._build_split_comment_content(songs_json_path, songs_path)
split_content, split_reason = self._build_split_comment(task, settings)
if not timeline_content and not split_content:
self._touch_comment_flags(session_dir, split_done=True, full_done=True)
return {"status": "skipped", "reason": "comment_content_empty"}
@ -62,7 +63,9 @@ class BilibiliTopCommentProvider:
if settings.get("post_split_comment", True) and not split_done:
split_bvid = bvid_path.read_text(encoding="utf-8").strip()
if split_content:
if split_reason is not None:
split_result = {"status": "skipped", "reason": split_reason}
elif split_content:
split_result = self._post_and_top_comment(session, csrf, split_bvid, split_content, "split")
else:
split_result = {"status": "skipped", "reason": "split_comment_empty"}
@ -74,8 +77,11 @@ class BilibiliTopCommentProvider:
if settings.get("post_full_video_timeline_comment", True) and not full_done:
full_bvid = resolve_full_video_bvid(task.title, session_dir, settings)
if full_bvid and timeline_content:
full_result = self._post_and_top_comment(session, csrf, full_bvid, timeline_content, "full")
full_content, full_reason = self._build_full_comment_content(task, settings)
if full_reason is not None:
full_result = {"status": "skipped", "reason": full_reason}
elif full_bvid and full_content:
full_result = self._post_and_top_comment(session, csrf, full_bvid, full_content, "full")
else:
reason = "full_video_bvid_not_found" if not full_bvid else "timeline_comment_empty"
full_result = {"status": "skipped", "reason": reason}
@ -104,13 +110,18 @@ class BilibiliTopCommentProvider:
error_message=f"获取{target}视频信息失败",
)
aid = int(view["aid"])
add_res = self.bilibili_api.add_reply(
session,
csrf=csrf,
aid=aid,
content=content,
error_message=f"发布{target}评论失败",
)
try:
add_res = self.bilibili_api.add_reply(
session,
csrf=csrf,
aid=aid,
content=content,
error_message=f"发布{target}评论失败",
)
except ModuleError as exc:
if self._is_comment_closed_error(exc):
return {"status": "skipped", "reason": "comment_disabled", "bvid": bvid, "aid": aid}
raise
rpid = int(add_res["rpid"])
time.sleep(3)
self.bilibili_api.top_reply(
@ -151,6 +162,80 @@ class BilibiliTopCommentProvider:
return "\n".join(lines)
return ""
def _build_split_comment(self, task: Task, settings: dict[str, Any]) -> tuple[str, str | None]:
repo = settings.get("__repo")
session_dir_root = Path(str(settings["session_dir"]))
if repo is None or not hasattr(repo, "get_task_context") or not hasattr(repo, "list_task_contexts_by_session_key"):
session_dir = session_dir_root / task.title
return self._build_split_comment_content(session_dir / "songs.json", session_dir / "songs.txt"), None
context = repo.get_task_context(task.id)
if context is None or not context.session_key or context.session_key.startswith("task:"):
session_dir = session_dir_root / task.title
return self._build_split_comment_content(session_dir / "songs.json", session_dir / "songs.txt"), None
ordered_contexts = self._ordered_session_contexts(repo, context.session_key)
if not ordered_contexts:
return "", "split_comment_empty"
anchor_task_id = ordered_contexts[0].task_id
if task.id != anchor_task_id:
return "", "session_split_comment_owned_by_anchor"
blocks: list[str] = []
for index, session_context in enumerate(ordered_contexts, start=1):
task_dir = session_dir_root / session_context.task_id
content = self._build_split_comment_content(task_dir / "songs.json", task_dir / "songs.txt")
if not content:
continue
blocks.append(f"P{index}:\n{content}")
if not blocks:
return "", "split_comment_empty"
return "\n\n".join(blocks), None
def _build_full_comment_content(self, task: Task, settings: dict[str, Any]) -> tuple[str, str | None]:
repo = settings.get("__repo")
if repo is None or not hasattr(repo, "get_task_context") or not hasattr(repo, "list_task_contexts_by_session_key"):
session_dir = Path(str(settings["session_dir"])) / task.title
content = session_dir.joinpath("songs.txt").read_text(encoding="utf-8").strip()
return content, None if content else "timeline_comment_empty"
context = repo.get_task_context(task.id)
if context is None or not context.session_key or context.session_key.startswith("task:"):
session_dir = Path(str(settings["session_dir"])) / task.title
content = session_dir.joinpath("songs.txt").read_text(encoding="utf-8").strip()
return content, None if content else "timeline_comment_empty"
ordered_contexts = self._ordered_session_contexts(repo, context.session_key)
if not ordered_contexts:
return "", "timeline_comment_empty"
anchor_task_id = ordered_contexts[0].task_id
if task.id != anchor_task_id:
return "", "session_full_comment_owned_by_anchor"
blocks: list[str] = []
for index, session_context in enumerate(ordered_contexts, start=1):
task_dir = Path(str(settings["session_dir"])) / session_context.task_id
songs_path = task_dir / "songs.txt"
if not songs_path.exists():
continue
content = songs_path.read_text(encoding="utf-8").strip()
if not content:
continue
blocks.append(f"P{index}:\n{content}")
if not blocks:
return "", "timeline_comment_empty"
return "\n\n".join(blocks), None
def _ordered_session_contexts(self, repo, session_key: str) -> list[object]: # type: ignore[no-untyped-def]
contexts = list(repo.list_task_contexts_by_session_key(session_key))
return sorted(
contexts,
key=lambda item: (
self._parse_started_at(item.segment_started_at),
item.source_title or item.task_id,
),
)
@staticmethod
def _touch_comment_flags(session_dir: Path, *, split_done: bool, full_done: bool) -> None:
if split_done:
@ -159,3 +244,19 @@ class BilibiliTopCommentProvider:
(session_dir / "comment_full_done.flag").touch()
if split_done and full_done:
(session_dir / "comment_done.flag").touch()
@staticmethod
def _is_comment_closed_error(exc: ModuleError) -> bool:
message = exc.message or ""
details = exc.details or {}
combined = f"{message}\n{details}".lower()
return "评论功能已关闭" in message or "comment disabled" in combined or "reply not allowed" in combined
@staticmethod
def _parse_started_at(value: str | None) -> datetime:
if not value:
return datetime.max
try:
return datetime.fromisoformat(value)
except ValueError:
return datetime.max

View File

@ -15,9 +15,11 @@ class CommentService:
if task is None:
raise RuntimeError(f"task not found: {task_id}")
provider = self.registry.get("comment_provider", str(settings.get("provider", "bilibili_top_comment")))
provider_settings = dict(settings)
provider_settings["__repo"] = self.repo
started_at = utc_now_iso()
self.repo.update_step_status(task_id, "comment", "running", started_at=started_at)
result = provider.comment(task, settings)
result = provider.comment(task, provider_settings)
finished_at = utc_now_iso()
self.repo.update_step_status(task_id, "comment", "succeeded", finished_at=finished_at)
self.repo.update_task_status(task_id, "commented", finished_at)

View File

@ -0,0 +1,128 @@
from __future__ import annotations
import re
from pathlib import Path
from typing import Any
from biliup_next.core.errors import ModuleError
from biliup_next.core.providers import ProviderManifest
from biliup_next.infra.adapters.yt_dlp import YtDlpAdapter
FORBIDDEN_PATH_CHARS = re.compile(r'[<>:"/\\|?*\x00-\x1f]')
URL_PREFIXES = (
"https://www.bilibili.com/video/",
"http://www.bilibili.com/video/",
"https://b23.tv/",
"http://b23.tv/",
)
class BilibiliUrlIngestProvider:
def __init__(self, yt_dlp: YtDlpAdapter | None = None) -> None:
self.yt_dlp = yt_dlp or YtDlpAdapter()
manifest = ProviderManifest(
id="bilibili_url",
name="Bilibili URL Ingest",
version="0.1.0",
provider_type="ingest_provider",
entrypoint="biliup_next.modules.ingest.providers.bilibili_url:BilibiliUrlIngestProvider",
capabilities=["ingest"],
enabled_by_default=True,
)
def validate_source(self, source_url: str, settings: dict[str, Any]) -> None:
if not isinstance(source_url, str) or not source_url.strip():
raise ModuleError(
code="SOURCE_URL_MISSING",
message="缺少 source_url",
retryable=False,
)
source_url = source_url.strip()
if not source_url.startswith(URL_PREFIXES):
raise ModuleError(
code="SOURCE_URL_NOT_SUPPORTED",
message=f"当前仅支持 B 站视频链接: {source_url}",
retryable=False,
)
def resolve_source(self, source_url: str, settings: dict[str, Any]) -> dict[str, Any]:
yt_dlp_cmd = str(settings.get("yt_dlp_cmd", "yt-dlp"))
info = self.yt_dlp.probe(yt_dlp_cmd=yt_dlp_cmd, source_url=source_url)
video_id = str(info.get("id") or "").strip()
title = str(info.get("title") or video_id or "bilibili-video").strip()
if not video_id:
raise ModuleError(
code="YT_DLP_METADATA_MISSING_ID",
message="yt-dlp 未返回视频 ID",
retryable=True,
)
return {
"task_id": self._safe_name(f"{title} [{video_id}]"),
"title": title,
"video_id": video_id,
"streamer": self._clean_text(info.get("uploader") or info.get("channel")),
"segment_duration_seconds": self._coerce_float(info.get("duration")),
"source_url": source_url,
}
def download_source(
self,
source_url: str,
task_dir: Path,
settings: dict[str, Any],
*,
task_id: str,
) -> Path:
yt_dlp_cmd = str(settings.get("yt_dlp_cmd", "yt-dlp"))
format_selector = str(settings.get("yt_dlp_format", "")).strip() or None
task_dir.mkdir(parents=True, exist_ok=True)
output_template = str((task_dir / f"{task_id}.%(ext)s").resolve())
result = self.yt_dlp.download(
yt_dlp_cmd=yt_dlp_cmd,
source_url=source_url,
output_template=output_template,
format_selector=format_selector,
)
if result.returncode != 0:
raise ModuleError(
code="YT_DLP_DOWNLOAD_FAILED",
message="yt-dlp 下载视频失败",
retryable=True,
details={"stdout": result.stdout[-2000:], "stderr": result.stderr[-2000:]},
)
candidates = [
path
for path in sorted(task_dir.iterdir())
if path.is_file() and path.stem == task_id and path.suffix.lower() in {".mp4", ".mkv", ".flv", ".mov"}
]
if not candidates:
raise ModuleError(
code="YT_DLP_OUTPUT_MISSING",
message=f"下载完成但未找到目标视频文件: {task_dir}",
retryable=True,
details={"stdout": result.stdout[-2000:], "stderr": result.stderr[-2000:]},
)
return candidates[0].resolve()
@staticmethod
def _safe_name(value: str) -> str:
cleaned = FORBIDDEN_PATH_CHARS.sub("_", value).strip().rstrip(".")
cleaned = re.sub(r"\s+", " ", cleaned)
return cleaned[:180] or "bilibili-video"
@staticmethod
def _clean_text(value: object) -> str | None:
if value is None:
return None
text = str(value).strip()
return text or None
@staticmethod
def _coerce_float(value: object) -> float | None:
if value in {None, ""}:
return None
try:
return float(value)
except (TypeError, ValueError):
return None

View File

@ -31,8 +31,12 @@ class IngestService:
settings: dict[str, object],
*,
context_payload: dict[str, object] | None = None,
provider_id: str | None = None,
source_type: str = "local_file",
title_override: str | None = None,
source_ref: str | None = None,
) -> Task:
provider_id = str(settings.get("provider", "local_file"))
provider_id = provider_id or str(settings.get("provider", "local_file"))
provider = self.registry.get("ingest_provider", provider_id)
provider.validate_source(source_path, settings)
source_path = source_path.resolve()
@ -59,9 +63,9 @@ class IngestService:
context_payload = context_payload or {}
task = Task(
id=task_id,
source_type="local_file",
source_type=source_type,
source_path=str(source_path),
title=source_path.stem,
title=title_override or source_path.stem,
status="created",
created_at=now,
updated_at=now,
@ -86,7 +90,7 @@ class IngestService:
task_id=task_id,
artifact_type="source_video",
path=str(source_path),
metadata_json=json.dumps({"provider": provider_id}),
metadata_json=json.dumps({"provider": provider_id, "source_ref": source_ref}),
created_at=now,
)
)
@ -103,6 +107,41 @@ class IngestService:
(source_path.parent / "full_video_bvid.txt").write_text(full_video_bvid, encoding="utf-8")
return task
def create_task_from_url(self, source_url: str, settings: dict[str, object]) -> Task:
provider_id = str(settings.get("provider", "bilibili_url"))
provider = self.registry.get("ingest_provider", provider_id)
provider.validate_source(source_url, settings)
session_dir = Path(str(settings["session_dir"])).resolve()
session_dir.mkdir(parents=True, exist_ok=True)
resolved = provider.resolve_source(source_url, settings)
task_id = str(resolved["task_id"])
if self.repo.get_task(task_id):
raise ModuleError(
code="TASK_ALREADY_EXISTS",
message=f"任务已存在: {task_id}",
retryable=False,
)
task_dir = session_dir / task_id
downloaded_path = provider.download_source(source_url, task_dir, settings, task_id=task_id)
context_payload = {
"source_title": resolved.get("title"),
"streamer": resolved.get("streamer"),
"segment_duration_seconds": resolved.get("segment_duration_seconds"),
"reference_timestamp": time.time(),
}
return self.create_task_from_file(
downloaded_path,
settings,
context_payload=context_payload,
provider_id=provider_id,
source_type="bilibili_url",
title_override=str(resolved.get("title") or task_id),
source_ref=source_url,
)
def scan_stage(self, settings: dict[str, object]) -> dict[str, object]:
stage_dir = Path(str(settings["stage_dir"])).resolve()
backup_dir = Path(str(settings["backup_dir"])).resolve()
@ -315,8 +354,7 @@ class IngestService:
streamer=streamer,
room_id=room_id,
segment_started_at=segment_started_at,
segment_duration_seconds=segment_duration,
fallback_task_id=task.id,
source_title=source_title,
gap_minutes=session_gap_minutes,
)
if full_video_bvid is None:
@ -413,8 +451,7 @@ class IngestService:
streamer: str | None,
room_id: str | None,
segment_started_at: str | None,
segment_duration_seconds: float | None,
fallback_task_id: str,
source_title: str,
gap_minutes: int,
) -> tuple[str | None, str | None]:
if not streamer or not segment_started_at:
@ -424,27 +461,35 @@ class IngestService:
except ValueError:
return None, None
tolerance = timedelta(minutes=max(gap_minutes, 0))
tolerance = timedelta(minutes=max(gap_minutes, 180))
matched_contexts: list[TaskContext] = []
for context in self.repo.find_recent_task_contexts(streamer):
if room_id and context.room_id and room_id != context.room_id:
continue
candidate_end = self._context_end_time(context)
if candidate_end is None:
candidate_start = self._context_start_time(context)
if candidate_start is None:
continue
if segment_start >= candidate_end and segment_start - candidate_end <= tolerance:
return context.session_key, context.full_video_bvid
date_tag = segment_start.astimezone(SHANGHAI_TZ).strftime("%Y%m%dT%H%M")
return f"{streamer}:{date_tag}", None
if abs(segment_start - candidate_start) <= tolerance:
matched_contexts.append(context)
if matched_contexts:
anchor = min(
matched_contexts,
key=lambda context: (
self._context_start_time(context) or segment_start,
context.source_title or context.session_key,
),
)
return anchor.session_key, anchor.full_video_bvid
return source_title, None
@staticmethod
def _context_end_time(context: TaskContext) -> datetime | None:
if not context.segment_started_at or context.segment_duration_seconds is None:
def _context_start_time(context: TaskContext) -> datetime | None:
if not context.segment_started_at:
return None
try:
started_at = datetime.fromisoformat(context.segment_started_at)
return datetime.fromisoformat(context.segment_started_at)
except ValueError:
return None
return started_at + timedelta(seconds=float(context.segment_duration_seconds))
def _find_full_video_bvid_by_session_key(self, session_key: str) -> str | None:
for context in self.repo.list_task_contexts_by_session_key(session_key):

View File

@ -31,6 +31,8 @@ class BiliupCliPublishProvider:
work_dir = Path(str(settings["session_dir"])) / task.title
bvid_file = work_dir / "bvid.txt"
upload_done = work_dir / "upload_done.flag"
publish_log = work_dir / "publish.log"
publish_progress = work_dir / "publish_progress.json"
config = self._load_upload_config(Path(str(settings["upload_config_file"])))
video_files = [artifact.path for artifact in clip_videos]
@ -45,8 +47,8 @@ class BiliupCliPublishProvider:
streamer = parsed.get("streamer", task.title)
date = parsed.get("date", "")
songs_txt = work_dir / "songs.txt"
songs_json = work_dir / "songs.json"
songs_txt = Path(str(settings.get("publish_songs_txt_path", work_dir / "songs.txt")))
songs_json = Path(str(settings.get("publish_songs_json_path", work_dir / "songs.json")))
songs_list = songs_txt.read_text(encoding="utf-8").strip() if songs_txt.exists() else ""
song_count = 0
if songs_json.exists():
@ -75,13 +77,20 @@ class BiliupCliPublishProvider:
biliup_path = str(settings["biliup_path"])
cookie_file = str(settings["cookie_file"])
retry_count = max(1, int(settings.get("retry_count", 5)))
command_timeout_seconds = max(1, int(settings.get("command_timeout_seconds", 1800)))
self.adapter.run_optional([biliup_path, "-u", cookie_file, "renew"])
self.adapter.run_optional(
[biliup_path, "-u", cookie_file, "renew"],
label="刷新 biliup 登录态",
timeout_seconds=command_timeout_seconds,
log_path=publish_log,
)
first_batch = video_files[:5]
remaining_batches = [video_files[i:i + 5] for i in range(5, len(video_files), 5)]
existing_bvid = bvid_file.read_text(encoding="utf-8").strip() if bvid_file.exists() else ""
progress = self._load_publish_progress(publish_progress)
if upload_done.exists() and existing_bvid.startswith("BV"):
return PublishRecord(
id=None,
@ -93,21 +102,35 @@ class BiliupCliPublishProvider:
published_at=utc_now_iso(),
)
bvid = existing_bvid if existing_bvid.startswith("BV") else self._upload_first_batch(
biliup_path=biliup_path,
cookie_file=cookie_file,
first_batch=first_batch,
title=title,
tid=tid,
tags=tags,
description=description,
dynamic=dynamic,
upload_settings=upload_settings,
retry_count=retry_count,
)
bvid_file.write_text(bvid, encoding="utf-8")
if existing_bvid.startswith("BV") and not publish_progress.exists() and remaining_batches:
progress = {"bvid": existing_bvid, "completed_append_batches": []}
self._write_publish_progress(publish_progress, progress)
if existing_bvid.startswith("BV") and publish_progress.exists():
bvid = existing_bvid
else:
bvid = self._upload_first_batch(
biliup_path=biliup_path,
cookie_file=cookie_file,
first_batch=first_batch,
title=title,
tid=tid,
tags=tags,
description=description,
dynamic=dynamic,
upload_settings=upload_settings,
retry_count=retry_count,
command_timeout_seconds=command_timeout_seconds,
publish_log=publish_log,
)
bvid_file.write_text(bvid, encoding="utf-8")
progress = {"bvid": bvid, "completed_append_batches": []}
self._write_publish_progress(publish_progress, progress)
completed_append_batches = set(self._progress_batches(progress))
for batch_index, batch in enumerate(remaining_batches, start=2):
if batch_index in completed_append_batches:
continue
self._append_batch(
biliup_path=biliup_path,
cookie_file=cookie_file,
@ -115,9 +138,16 @@ class BiliupCliPublishProvider:
batch=batch,
batch_index=batch_index,
retry_count=retry_count,
command_timeout_seconds=command_timeout_seconds,
publish_log=publish_log,
)
completed_append_batches.add(batch_index)
progress = {"bvid": bvid, "completed_append_batches": sorted(completed_append_batches)}
self._write_publish_progress(publish_progress, progress)
upload_done.touch()
if publish_progress.exists():
publish_progress.unlink()
return PublishRecord(
id=None,
task_id=task.id,
@ -141,6 +171,8 @@ class BiliupCliPublishProvider:
dynamic: str,
upload_settings: dict[str, Any],
retry_count: int,
command_timeout_seconds: int,
publish_log: Path,
) -> str:
upload_cmd = [
biliup_path,
@ -168,20 +200,20 @@ class BiliupCliPublishProvider:
upload_cmd.extend(["--cover", cover])
for attempt in range(1, retry_count + 1):
result = self.adapter.run(upload_cmd, label=f"首批上传[{attempt}/{retry_count}]")
result = self.adapter.run(
upload_cmd,
label=f"首批上传[{attempt}/{retry_count}]",
timeout_seconds=command_timeout_seconds,
log_path=publish_log,
)
if result.returncode == 0:
match = re.search(r'"bvid":"(BV[A-Za-z0-9]+)"', result.stdout) or re.search(r"(BV[A-Za-z0-9]+)", result.stdout)
match = self._extract_bvid(result.stdout)
if match:
return match.group(1)
return match
if attempt < retry_count:
time.sleep(self._wait_seconds(attempt - 1))
continue
raise ModuleError(
code="PUBLISH_UPLOAD_FAILED",
message="首批上传失败",
retryable=True,
details={"stdout": result.stdout[-2000:], "stderr": result.stderr[-2000:]},
)
raise self._classify_publish_error(result, default_code="PUBLISH_UPLOAD_FAILED", default_message="首批上传失败")
raise AssertionError("unreachable")
def _append_batch(
@ -193,21 +225,27 @@ class BiliupCliPublishProvider:
batch: list[str],
batch_index: int,
retry_count: int,
command_timeout_seconds: int,
publish_log: Path,
) -> None:
time.sleep(45)
append_cmd = [biliup_path, "-u", cookie_file, "append", "--vid", bvid, *batch]
for attempt in range(1, retry_count + 1):
result = self.adapter.run(append_cmd, label=f"追加第{batch_index}批[{attempt}/{retry_count}]")
result = self.adapter.run(
append_cmd,
label=f"追加第{batch_index}批[{attempt}/{retry_count}]",
timeout_seconds=command_timeout_seconds,
log_path=publish_log,
)
if result.returncode == 0:
return
if attempt < retry_count:
time.sleep(self._wait_seconds(attempt - 1))
continue
raise ModuleError(
code="PUBLISH_APPEND_FAILED",
message=f"追加第 {batch_index} 批失败",
retryable=True,
details={"stdout": result.stdout[-2000:], "stderr": result.stderr[-2000:]},
raise self._classify_publish_error(
result,
default_code="PUBLISH_APPEND_FAILED",
default_message=f"追加第 {batch_index} 批失败",
)
@staticmethod
@ -245,3 +283,74 @@ class BiliupCliPublishProvider:
if not quotes:
return {"text": "", "author": ""}
return random.choice(quotes)
@staticmethod
def _classify_publish_error(
result,
*,
default_code: str,
default_message: str,
) -> ModuleError:
stdout = result.stdout[-2000:]
stderr = result.stderr[-2000:]
combined = f"{result.stdout}\n{result.stderr}"
details = {"stdout": stdout, "stderr": stderr}
if "code: 601" in combined or "上传视频过快" in combined:
return ModuleError(
code="PUBLISH_RATE_LIMITED",
message="B站上传限流请稍后重试",
retryable=True,
details=details,
)
if "code: 21021" in combined or "转载来源不能为空" in combined:
return ModuleError(
code="PUBLISH_SOURCE_REQUIRED",
message="转载稿件缺少来源说明",
retryable=False,
details=details,
)
return ModuleError(
code=default_code,
message=default_message,
retryable=True,
details=details,
)
@staticmethod
def _extract_bvid(output: str) -> str | None:
patterns = (
r'"bvid":"(BV[A-Za-z0-9]+)"',
r'String\("?(BV[A-Za-z0-9]+)"?\)',
r"\b(BV[A-Za-z0-9]+)\b",
)
for pattern in patterns:
match = re.search(pattern, output)
if match:
return match.group(1)
return None
@staticmethod
def _load_publish_progress(path: Path) -> dict[str, Any]:
if not path.exists():
return {}
try:
return json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError:
return {}
@staticmethod
def _write_publish_progress(path: Path, progress: dict[str, Any]) -> None:
path.write_text(json.dumps(progress, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
@staticmethod
def _progress_batches(progress: dict[str, Any]) -> list[int]:
raw = progress.get("completed_append_batches")
if not isinstance(raw, list):
return []
batches: list[int] = []
for item in raw:
try:
batches.append(int(item))
except (TypeError, ValueError):
continue
return batches

View File

@ -1,9 +1,11 @@
from __future__ import annotations
import json
from datetime import datetime
from pathlib import Path
from typing import Any
from biliup_next.core.models import Artifact, PublishRecord, utc_now_iso
from biliup_next.core.models import Artifact, PublishRecord, TaskContext, utc_now_iso
from biliup_next.core.registry import Registry
from biliup_next.infra.task_repository import TaskRepository
@ -17,22 +19,75 @@ class PublishService:
task = self.repo.get_task(task_id)
if task is None:
raise RuntimeError(f"task not found: {task_id}")
artifacts = self.repo.list_artifacts(task_id)
clip_videos = [a for a in artifacts if a.artifact_type == "clip_video"]
provider = self.registry.get("publish_provider", str(settings.get("provider", "biliup_cli")))
started_at = utc_now_iso()
self.repo.update_step_status(task_id, "publish", "running", started_at=started_at)
record = provider.publish(task, clip_videos, settings)
session_contexts = self._session_contexts(task_id)
if len(session_contexts) <= 1:
clip_videos = self._clip_videos_for_task(task_id)
record = provider.publish(task, clip_videos, settings)
self._persist_publish_success(task_id, task.title, record, settings)
return record
anchor_context = session_contexts[0]
shared_bvid = self._shared_session_bvid(session_contexts, settings)
if task_id != anchor_context.task_id and shared_bvid:
record = PublishRecord(
id=None,
task_id=task_id,
platform="bilibili",
aid=None,
bvid=shared_bvid,
title=task.title,
published_at=utc_now_iso(),
)
self._persist_publish_success(task_id, task.title, record, settings)
return record
clip_videos = self._session_clip_videos(session_contexts)
anchor_task = self.repo.get_task(anchor_context.task_id)
if anchor_task is None:
raise RuntimeError(f"anchor task not found: {anchor_context.task_id}")
session_settings = dict(settings)
session_settings.update(self._session_publish_metadata(anchor_task, session_contexts, settings))
record = provider.publish(anchor_task, clip_videos, session_settings)
for context in session_contexts:
session_task = self.repo.get_task(context.task_id)
if session_task is None:
continue
session_record = PublishRecord(
id=None,
task_id=context.task_id,
platform=record.platform,
aid=record.aid,
bvid=record.bvid,
title=record.title,
published_at=record.published_at,
)
self._persist_publish_success(context.task_id, session_task.title, session_record, settings)
return PublishRecord(
id=None,
task_id=task_id,
platform=record.platform,
aid=record.aid,
bvid=record.bvid,
title=record.title,
published_at=record.published_at,
)
def _persist_publish_success(self, task_id: str, task_title: str, record: PublishRecord, settings: dict[str, object]) -> None:
self.repo.add_publish_record(record)
if record.bvid:
session_dir = Path(str(settings.get("session_dir", "session"))) / task.title
bvid_path = str((session_dir / "bvid.txt").resolve())
session_dir = Path(str(settings.get("session_dir", "session"))) / task_title
session_dir.mkdir(parents=True, exist_ok=True)
bvid_path_obj = session_dir / "bvid.txt"
bvid_path_obj.write_text(record.bvid, encoding="utf-8")
self.repo.add_artifact(
Artifact(
id=None,
task_id=task_id,
artifact_type="publish_bvid",
path=bvid_path,
path=str(bvid_path_obj.resolve()),
metadata_json=json.dumps({}),
created_at=utc_now_iso(),
)
@ -40,4 +95,95 @@ class PublishService:
finished_at = utc_now_iso()
self.repo.update_step_status(task_id, "publish", "succeeded", finished_at=finished_at)
self.repo.update_task_status(task_id, "published", finished_at)
return record
def _session_contexts(self, task_id: str) -> list[TaskContext]:
context = self.repo.get_task_context(task_id)
if context is None or not context.session_key or context.session_key.startswith("task:"):
return [context] if context is not None else []
contexts = list(self.repo.list_task_contexts_by_session_key(context.session_key))
return sorted(
contexts,
key=lambda item: (
self._parse_started_at(item.segment_started_at),
item.source_title or item.task_id,
),
)
def _clip_videos_for_task(self, task_id: str) -> list[Artifact]:
artifacts = self.repo.list_artifacts(task_id)
return [artifact for artifact in artifacts if artifact.artifact_type == "clip_video"]
def _session_clip_videos(self, contexts: list[TaskContext]) -> list[Artifact]:
aggregated: list[Artifact] = []
for context in contexts:
aggregated.extend(self._clip_videos_for_task(context.task_id))
return aggregated
def _shared_session_bvid(self, contexts: list[TaskContext], settings: dict[str, object]) -> str | None:
session_dir_root = Path(str(settings.get("session_dir", "session")))
for context in contexts:
task = self.repo.get_task(context.task_id)
if task is None:
continue
bvid_path = session_dir_root / task.title / "bvid.txt"
if bvid_path.exists():
bvid = bvid_path.read_text(encoding="utf-8").strip()
if bvid.startswith("BV"):
return bvid
return None
def _session_publish_metadata(
self,
anchor_task,
contexts: list[TaskContext],
settings: dict[str, object],
) -> dict[str, Any]: # type: ignore[no-untyped-def]
session_dir_root = Path(str(settings.get("session_dir", "session")))
anchor_work_dir = (session_dir_root / anchor_task.title).resolve()
anchor_work_dir.mkdir(parents=True, exist_ok=True)
aggregate_txt_lines: list[str] = []
aggregate_songs: list[dict[str, object]] = []
for index, context in enumerate(contexts, start=1):
task = self.repo.get_task(context.task_id)
if task is None:
continue
task_work_dir = (session_dir_root / task.title).resolve()
songs_txt = task_work_dir / "songs.txt"
songs_json = task_work_dir / "songs.json"
if songs_txt.exists():
lines = [line.strip() for line in songs_txt.read_text(encoding="utf-8").splitlines() if line.strip()]
if lines:
aggregate_txt_lines.append(f"P{index}:")
aggregate_txt_lines.extend(lines)
aggregate_txt_lines.append("")
if songs_json.exists():
try:
songs = json.loads(songs_json.read_text(encoding="utf-8")).get("songs", [])
except json.JSONDecodeError:
songs = []
if isinstance(songs, list):
aggregate_songs.extend(song for song in songs if isinstance(song, dict))
aggregate_txt_path = anchor_work_dir / "session_split_songs.txt"
aggregate_json_path = anchor_work_dir / "session_split_songs.json"
aggregate_txt_path.write_text("\n".join(aggregate_txt_lines).strip() + "\n", encoding="utf-8")
aggregate_json_path.write_text(
json.dumps({"songs": aggregate_songs}, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
return {
"publish_songs_txt_path": str(aggregate_txt_path),
"publish_songs_json_path": str(aggregate_json_path),
}
@staticmethod
def _parse_started_at(value: str | None) -> datetime:
if not value:
return datetime.max
try:
return datetime.fromisoformat(value)
except ValueError:
return datetime.max

View File

@ -8,51 +8,7 @@ from biliup_next.core.errors import ModuleError
from biliup_next.core.models import Artifact, Task, utc_now_iso
from biliup_next.core.providers import ProviderManifest
from biliup_next.infra.adapters.codex_cli import CodexCliAdapter
SONG_SCHEMA = {
"type": "object",
"properties": {
"songs": {
"type": "array",
"items": {
"type": "object",
"properties": {
"start": {"type": "string"},
"end": {"type": "string"},
"title": {"type": "string"},
"artist": {"type": "string"},
"confidence": {"type": "number"},
"evidence": {"type": "string"},
},
"required": ["start", "end", "title", "artist", "confidence", "evidence"],
"additionalProperties": False,
},
}
},
"required": ["songs"],
"additionalProperties": False,
}
TASK_PROMPT = """你是音乐片段识别助手。当前目录下有一个字幕文件。
任务:
1. 结合字幕内容并允许联网搜索进行纠错(识别同音字、唱错等)。
2. 识别出直播中唱过的所有歌曲,给出精确的开始和结束时间。歌曲开始时间规则:
- 歌曲开始时间应使用“上一句字幕的结束时间”作为 start_time。
- 这样可以尽量保留歌曲可能存在的前奏。
3. 同一首歌间隔 ≤160s 合并,>160s 分开。若连续识别出相同歌曲,且中间只有短暂对白、空白、转场或无歌词段,应合并为同一首歌.
4. 忽略纯聊天片段。
5. 无法确认的歌曲丢弃,宁缺毋滥:你的输出将直接面向最终用户。
6. 忽略短片段:如果一段演唱持续时间总和少于 15 秒,视为随口哼唱,请直接忽略,不计入列表。
7. 仔细分析每一句歌词,识别出相关歌曲后, 使用该歌曲歌词上下文对比字幕上下文,确定歌曲起始与停止时间
8.歌曲标注规则:
- 可以在歌曲名称后使用括号 () 添加补充说明。
- 常见标注示例:
- (片段):歌曲演唱时间较短,例如 < 60 秒
- (清唱):无伴奏演唱
- (副歌):只演唱副歌部分
- 标注应简洁,仅在确有必要时使用。
9. 通过歌曲起始和结束时间自检, 一般歌曲长度在5分钟以内, 1分钟以上, 可疑片段重新联网搜索检查.
最后请严格按照 Schema 生成 JSON 数据。"""
from biliup_next.modules.song_detect.providers.common import TASK_PROMPT, ensure_song_outputs, write_song_schema
class CodexSongDetector:
@ -71,10 +27,9 @@ class CodexSongDetector:
def detect(self, task: Task, subtitle_srt: Artifact, settings: dict[str, Any]) -> tuple[Artifact, Artifact]:
work_dir = Path(subtitle_srt.path).resolve().parent
schema_path = work_dir / "song_schema.json"
write_song_schema(work_dir)
songs_json_path = work_dir / "songs.json"
songs_txt_path = work_dir / "songs.txt"
schema_path.write_text(json.dumps(SONG_SCHEMA, ensure_ascii=False, indent=2), encoding="utf-8")
codex_cmd = str(settings.get("codex_cmd", "codex"))
result = self.adapter.run_song_detect(
@ -91,16 +46,13 @@ class CodexSongDetector:
details={"stdout": result.stdout[-2000:], "stderr": result.stderr[-2000:]},
)
if songs_json_path.exists() and not songs_txt_path.exists():
self._generate_txt_fallback(songs_json_path, songs_txt_path)
if not songs_json_path.exists() or not songs_txt_path.exists():
raise ModuleError(
code="SONG_DETECT_OUTPUT_MISSING",
message=f"未生成 songs.json/songs.txt: {work_dir}",
retryable=True,
details={"stdout": result.stdout[-2000:], "stderr": result.stderr[-2000:]},
)
ensure_song_outputs(
songs_json_path=songs_json_path,
songs_txt_path=songs_txt_path,
stdout=result.stdout,
stderr=result.stderr,
provider_name="codex",
)
return (
Artifact(
@ -120,19 +72,3 @@ class CodexSongDetector:
created_at=utc_now_iso(),
),
)
def _generate_txt_fallback(self, songs_json_path: Path, songs_txt_path: Path) -> None:
try:
data = json.loads(songs_json_path.read_text(encoding="utf-8"))
songs = data.get("songs", [])
with songs_txt_path.open("w", encoding="utf-8") as file_handle:
for song in songs:
start_time = str(song["start"]).split(",")[0].split(".")[0]
file_handle.write(f"{start_time} {song['title']}{song['artist']}\n")
except Exception as exc: # noqa: BLE001
raise ModuleError(
code="SONGS_TXT_GENERATE_FAILED",
message=f"生成 songs.txt 失败: {songs_txt_path}",
retryable=False,
details={"error": str(exc)},
) from exc

View File

@ -0,0 +1,122 @@
from __future__ import annotations
import json
from pathlib import Path
from biliup_next.core.errors import ModuleError
SONG_SCHEMA = {
"type": "object",
"properties": {
"songs": {
"type": "array",
"items": {
"type": "object",
"properties": {
"start": {"type": "string"},
"end": {"type": "string"},
"title": {"type": "string"},
"artist": {"type": "string"},
"confidence": {"type": "number"},
"evidence": {"type": "string"},
},
"required": ["start", "end", "title", "artist", "confidence", "evidence"],
"additionalProperties": False,
},
}
},
"required": ["songs"],
"additionalProperties": False,
}
TASK_PROMPT = """你是音乐片段识别助手。当前目录下有一个字幕文件。
任务:
1. 结合字幕内容并允许联网搜索进行纠错(识别同音字、唱错等)。
2. 识别出直播中唱过的所有歌曲,给出精确的开始和结束时间。歌曲开始时间规则:
- 歌曲开始时间应使用“上一句字幕的结束时间”作为 start_time。
- 这样可以尽量保留歌曲可能存在的前奏。
3. 同一首歌间隔 ≤160s 合并,>160s 分开。若连续识别出相同歌曲,且中间只有短暂对白、空白、转场或无歌词段,应合并为同一首歌.
4. 忽略纯聊天片段。
5. 无法确认的歌曲丢弃,宁缺毋滥:你的输出将直接面向最终用户。
6. 忽略短片段:如果一段演唱持续时间总和少于 15 秒,视为随口哼唱,请直接忽略,不计入列表。
7. 仔细分析每一句歌词,识别出相关歌曲后, 使用该歌曲歌词上下文对比字幕上下文,确定歌曲起始与停止时间
8.歌曲标注规则:
- 可以在歌曲名称后使用括号 () 添加补充说明。
- 常见标注示例:
- (片段):歌曲演唱时间较短,例如 < 60 秒
- (清唱):无伴奏演唱
- (副歌):只演唱副歌部分
- 标注应简洁,仅在确有必要时使用。
9. 通过歌曲起始和结束时间自检, 一般歌曲长度在5分钟以内, 1分钟以上, 可疑片段重新联网搜索检查.
最后请严格按照 Schema 生成 JSON 数据。"""
QWEN_TASK_PROMPT = """你是音乐片段识别助手。当前目录下有一个字幕文件 `subtitle.srt` 和一个 JSON schema 文件 `song_schema.json`。
任务:
1. 结合字幕内容并允许联网搜索进行纠错(识别同音字、唱错等)。
2. 识别出直播中唱过的所有歌曲,给出精确的开始和结束时间。歌曲开始时间规则:
- 歌曲开始时间应使用“上一句字幕的结束时间”作为 start_time。
- 这样可以尽量保留歌曲可能存在的前奏。
3. 同一首歌间隔 ≤160s 合并,>160s 分开。若连续识别出相同歌曲,且中间只有短暂对白、空白、转场或无歌词段,应合并为同一首歌。
4. 忽略纯聊天片段。
5. 无法确认的歌曲丢弃,宁缺毋滥:你的输出将直接面向最终用户。
6. 忽略短片段:如果一段演唱持续时间总和少于 15 秒,视为随口哼唱,请直接忽略,不计入列表。
7. 仔细分析每一句歌词,识别出相关歌曲后,使用该歌曲歌词上下文对比字幕上下文,确定歌曲起始与停止时间。
8. 歌曲名称后可以按需补充 `(片段)`、`(清唱)`、`(副歌)` 等简短标注。
9. 通过歌曲起始和结束时间自检,一般歌曲长度在 5 分钟以内、1 分钟以上,可疑片段重新联网搜索检查。
输出要求:
1. 读取 `song_schema.json`,生成严格符合 schema 的 JSON。
2. 把 JSON 保存到当前目录的 `songs.json`。
3. 再生成一个 `songs.txt`,每行格式为 `HH:MM:SS 歌曲名 — 歌手`,其中时间取每首歌的开始时间,忽略毫秒。
4. 不要修改其他文件。
5. 完成后只输出简短结果说明。
"""
def write_song_schema(work_dir: Path) -> Path:
schema_path = work_dir / "song_schema.json"
schema_path.write_text(json.dumps(SONG_SCHEMA, ensure_ascii=False, indent=2), encoding="utf-8")
return schema_path
def ensure_song_outputs(
*,
songs_json_path: Path,
songs_txt_path: Path,
stdout: str,
stderr: str,
provider_name: str,
) -> None:
if songs_json_path.exists() and not songs_txt_path.exists():
generate_txt_fallback(songs_json_path, songs_txt_path)
if songs_json_path.exists() and songs_txt_path.exists():
return
raise ModuleError(
code="SONG_DETECT_OUTPUT_MISSING",
message=f"未生成 songs.json/songs.txt: {songs_json_path.parent}",
retryable=True,
details={
"provider": provider_name,
"stdout": stdout[-2000:],
"stderr": stderr[-2000:],
},
)
def generate_txt_fallback(songs_json_path: Path, songs_txt_path: Path) -> None:
try:
data = json.loads(songs_json_path.read_text(encoding="utf-8"))
songs = data.get("songs", [])
with songs_txt_path.open("w", encoding="utf-8") as file_handle:
for song in songs:
start_time = str(song["start"]).split(",")[0].split(".")[0]
file_handle.write(f"{start_time} {song['title']}{song['artist']}\n")
except Exception as exc: # noqa: BLE001
raise ModuleError(
code="SONGS_TXT_GENERATE_FAILED",
message=f"生成 songs.txt 失败: {songs_txt_path}",
retryable=False,
details={"error": str(exc)},
) from exc

View File

@ -0,0 +1,78 @@
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
from biliup_next.core.errors import ModuleError
from biliup_next.core.models import Artifact, Task, utc_now_iso
from biliup_next.core.providers import ProviderManifest
from biliup_next.infra.adapters.qwen_cli import QwenCliAdapter
from biliup_next.modules.song_detect.providers.common import (
QWEN_TASK_PROMPT,
ensure_song_outputs,
write_song_schema,
)
class QwenCliSongDetector:
def __init__(self, adapter: QwenCliAdapter | None = None) -> None:
self.adapter = adapter or QwenCliAdapter()
manifest = ProviderManifest(
id="qwen_cli",
name="Qwen CLI Song Detector",
version="0.1.0",
provider_type="song_detector",
entrypoint="biliup_next.modules.song_detect.providers.qwen_cli:QwenCliSongDetector",
capabilities=["song_detect"],
enabled_by_default=True,
)
def detect(self, task: Task, subtitle_srt: Artifact, settings: dict[str, Any]) -> tuple[Artifact, Artifact]:
work_dir = Path(subtitle_srt.path).resolve().parent
write_song_schema(work_dir)
songs_json_path = work_dir / "songs.json"
songs_txt_path = work_dir / "songs.txt"
qwen_cmd = str(settings.get("qwen_cmd", "qwen"))
result = self.adapter.run_song_detect(
qwen_cmd=qwen_cmd,
work_dir=work_dir,
prompt=QWEN_TASK_PROMPT,
)
if result.returncode != 0:
raise ModuleError(
code="SONG_DETECT_FAILED",
message="qwen -p 执行失败",
retryable=True,
details={"stdout": result.stdout[-2000:], "stderr": result.stderr[-2000:]},
)
ensure_song_outputs(
songs_json_path=songs_json_path,
songs_txt_path=songs_txt_path,
stdout=result.stdout,
stderr=result.stderr,
provider_name="qwen_cli",
)
return (
Artifact(
id=None,
task_id=task.id,
artifact_type="songs_json",
path=str(songs_json_path.resolve()),
metadata_json=json.dumps({"provider": "qwen_cli"}),
created_at=utc_now_iso(),
),
Artifact(
id=None,
task_id=task.id,
artifact_type="songs_txt",
path=str(songs_txt_path.resolve()),
metadata_json=json.dumps({"provider": "qwen_cli"}),
created_at=utc_now_iso(),
),
)

View File

@ -0,0 +1,9 @@
{
"id": "bilibili_url",
"name": "Bilibili URL Ingest",
"version": "0.1.0",
"provider_type": "ingest_provider",
"entrypoint": "biliup_next.modules.ingest.providers.bilibili_url:BilibiliUrlIngestProvider",
"capabilities": ["ingest"],
"enabled_by_default": true
}

View File

@ -0,0 +1,9 @@
{
"id": "qwen_cli",
"name": "Qwen CLI Song Detector",
"version": "0.1.0",
"provider_type": "song_detector",
"entrypoint": "biliup_next.modules.song_detect.providers.qwen_cli:QwenCliSongDetector",
"capabilities": ["song_detect"],
"enabled_by_default": true
}