feat: professionalize control plane and standalone delivery
This commit is contained in:
@ -18,7 +18,6 @@ src/biliup_next/core/models.py
|
||||
src/biliup_next/core/providers.py
|
||||
src/biliup_next/core/registry.py
|
||||
src/biliup_next/infra/db.py
|
||||
src/biliup_next/infra/legacy_paths.py
|
||||
src/biliup_next/infra/log_reader.py
|
||||
src/biliup_next/infra/plugin_loader.py
|
||||
src/biliup_next/infra/runtime_doctor.py
|
||||
@ -26,17 +25,18 @@ src/biliup_next/infra/stage_importer.py
|
||||
src/biliup_next/infra/systemd_runtime.py
|
||||
src/biliup_next/infra/task_repository.py
|
||||
src/biliup_next/infra/task_reset.py
|
||||
src/biliup_next/infra/adapters/bilibili_collection_legacy.py
|
||||
src/biliup_next/infra/adapters/bilibili_top_comment_legacy.py
|
||||
src/biliup_next/infra/adapters/biliup_publish_legacy.py
|
||||
src/biliup_next/infra/adapters/codex_legacy.py
|
||||
src/biliup_next/infra/adapters/ffmpeg_split_legacy.py
|
||||
src/biliup_next/infra/adapters/groq_legacy.py
|
||||
src/biliup_next/infra/adapters/full_video_locator.py
|
||||
src/biliup_next/modules/collection/service.py
|
||||
src/biliup_next/modules/collection/providers/bilibili_collection.py
|
||||
src/biliup_next/modules/comment/service.py
|
||||
src/biliup_next/modules/comment/providers/bilibili_top_comment.py
|
||||
src/biliup_next/modules/ingest/service.py
|
||||
src/biliup_next/modules/ingest/providers/local_file.py
|
||||
src/biliup_next/modules/publish/service.py
|
||||
src/biliup_next/modules/publish/providers/biliup_cli.py
|
||||
src/biliup_next/modules/song_detect/service.py
|
||||
src/biliup_next/modules/song_detect/providers/codex.py
|
||||
src/biliup_next/modules/split/service.py
|
||||
src/biliup_next/modules/transcribe/service.py
|
||||
src/biliup_next/modules/split/providers/ffmpeg_copy.py
|
||||
src/biliup_next/modules/transcribe/service.py
|
||||
src/biliup_next/modules/transcribe/providers/groq.py
|
||||
|
||||
@ -8,13 +8,21 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qs, unquote, urlparse
|
||||
|
||||
from biliup_next.app.task_actions import bind_full_video_action
|
||||
from biliup_next.app.task_actions import merge_session_action
|
||||
from biliup_next.app.task_actions import receive_full_video_webhook
|
||||
from biliup_next.app.task_actions import rebind_session_full_video_action
|
||||
from biliup_next.app.task_actions import reset_to_step_action
|
||||
from biliup_next.app.task_actions import retry_step_action
|
||||
from biliup_next.app.task_actions import run_task_action
|
||||
from biliup_next.app.bootstrap import ensure_initialized
|
||||
from biliup_next.app.bootstrap import reset_initialized_state
|
||||
from biliup_next.app.control_plane_get_dispatcher import ControlPlaneGetDispatcher
|
||||
from biliup_next.app.dashboard import render_dashboard_html
|
||||
from biliup_next.app.control_plane_post_dispatcher import ControlPlanePostDispatcher
|
||||
from biliup_next.app.retry_meta import retry_meta_for_step
|
||||
from biliup_next.app.scheduler import build_scheduler_preview
|
||||
from biliup_next.app.serializers import ControlPlaneSerializer
|
||||
from biliup_next.app.worker import run_once
|
||||
from biliup_next.core.config import SettingsService
|
||||
from biliup_next.core.models import ActionRecord, utc_now_iso
|
||||
@ -28,61 +36,32 @@ from biliup_next.infra.systemd_runtime import SystemdRuntime
|
||||
class ApiHandler(BaseHTTPRequestHandler):
|
||||
server_version = "biliup-next/0.1"
|
||||
|
||||
def _task_payload(self, task_id: str, state: dict[str, object]) -> dict[str, object] | None:
|
||||
task = state["repo"].get_task(task_id)
|
||||
if task is None:
|
||||
return None
|
||||
payload = task.to_dict()
|
||||
retry_state = self._task_retry_state(task_id, state)
|
||||
if retry_state:
|
||||
payload["retry_state"] = retry_state
|
||||
payload["delivery_state"] = self._task_delivery_state(task_id, state)
|
||||
return payload
|
||||
@staticmethod
|
||||
def _attention_state(task_payload: dict[str, object]) -> str:
|
||||
if task_payload.get("status") == "failed_manual":
|
||||
return "manual_now"
|
||||
retry_state = task_payload.get("retry_state")
|
||||
if isinstance(retry_state, dict) and retry_state.get("retry_due"):
|
||||
return "retry_now"
|
||||
if task_payload.get("status") == "failed_retryable" and isinstance(retry_state, dict) and retry_state.get("next_retry_at"):
|
||||
return "waiting_retry"
|
||||
if task_payload.get("status") == "running":
|
||||
return "running"
|
||||
return "stable"
|
||||
|
||||
@staticmethod
|
||||
def _delivery_state_label(task_payload: dict[str, object]) -> str:
|
||||
delivery_state = task_payload.get("delivery_state")
|
||||
if not isinstance(delivery_state, dict):
|
||||
return "stable"
|
||||
if delivery_state.get("split_comment") == "pending" or delivery_state.get("full_video_timeline_comment") == "pending":
|
||||
return "pending_comment"
|
||||
if delivery_state.get("source_video_present") is False or delivery_state.get("split_videos_present") is False:
|
||||
return "cleanup_removed"
|
||||
return "stable"
|
||||
|
||||
def _step_payload(self, step, state: dict[str, object]) -> dict[str, object]: # type: ignore[no-untyped-def]
|
||||
payload = step.to_dict()
|
||||
retry_meta = retry_meta_for_step(step, state["settings"])
|
||||
if retry_meta:
|
||||
payload.update(retry_meta)
|
||||
return payload
|
||||
|
||||
def _task_retry_state(self, task_id: str, state: dict[str, object]) -> dict[str, object] | None:
|
||||
for step in state["repo"].list_steps(task_id):
|
||||
retry_meta = retry_meta_for_step(step, state["settings"])
|
||||
if retry_meta:
|
||||
return {"step_name": step.step_name, **retry_meta}
|
||||
return None
|
||||
|
||||
def _task_delivery_state(self, task_id: str, state: dict[str, object]) -> dict[str, object]:
|
||||
task = state["repo"].get_task(task_id)
|
||||
if task is None:
|
||||
return {}
|
||||
session_dir = Path(str(state["settings"]["paths"]["session_dir"])) / task.title
|
||||
source_path = Path(task.source_path)
|
||||
split_dir = session_dir / "split_video"
|
||||
legacy_comment_done = (session_dir / "comment_done.flag").exists()
|
||||
|
||||
def comment_status(flag_name: str, *, enabled: bool) -> str:
|
||||
if not enabled:
|
||||
return "disabled"
|
||||
if flag_name == "comment_full_done.flag" and legacy_comment_done and not (session_dir / flag_name).exists():
|
||||
return "legacy_untracked"
|
||||
return "done" if (session_dir / flag_name).exists() else "pending"
|
||||
|
||||
return {
|
||||
"split_comment": comment_status("comment_split_done.flag", enabled=state["settings"]["comment"].get("post_split_comment", True)),
|
||||
"full_video_timeline_comment": comment_status(
|
||||
"comment_full_done.flag",
|
||||
enabled=state["settings"]["comment"].get("post_full_video_timeline_comment", True),
|
||||
),
|
||||
"full_video_bvid_resolved": (session_dir / "full_video_bvid.txt").exists(),
|
||||
"source_video_present": source_path.exists(),
|
||||
"split_videos_present": split_dir.exists(),
|
||||
"cleanup_enabled": {
|
||||
"delete_source_video_after_collection_synced": state["settings"].get("cleanup", {}).get("delete_source_video_after_collection_synced", False),
|
||||
"delete_split_videos_after_collection_synced": state["settings"].get("cleanup", {}).get("delete_split_videos_after_collection_synced", False),
|
||||
},
|
||||
}
|
||||
return ControlPlaneSerializer(state).step_payload(step)
|
||||
|
||||
def _serve_asset(self, asset_name: str) -> None:
|
||||
root = ensure_initialized()["root"]
|
||||
@ -116,10 +95,22 @@ class ApiHandler(BaseHTTPRequestHandler):
|
||||
dist = self._frontend_dist_dir()
|
||||
if not (dist / "index.html").exists():
|
||||
return False
|
||||
if parsed_path in {"/ui", "/ui/"}:
|
||||
if parsed_path in {"/", "/ui", "/ui/"}:
|
||||
self._html((dist / "index.html").read_text(encoding="utf-8"))
|
||||
return True
|
||||
|
||||
if parsed_path.startswith("/assets/"):
|
||||
relative = parsed_path.removeprefix("/")
|
||||
asset_path = dist / relative
|
||||
if asset_path.exists() and asset_path.is_file():
|
||||
body = asset_path.read_bytes()
|
||||
self.send_response(HTTPStatus.OK)
|
||||
self.send_header("Content-Type", self._guess_content_type(asset_path))
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
return True
|
||||
|
||||
if not parsed_path.startswith("/ui/"):
|
||||
return False
|
||||
|
||||
@ -143,13 +134,16 @@ class ApiHandler(BaseHTTPRequestHandler):
|
||||
|
||||
def do_GET(self) -> None: # noqa: N802
|
||||
parsed = urlparse(self.path)
|
||||
if parsed.path.startswith("/ui") and self._serve_frontend_dist(parsed.path):
|
||||
if (parsed.path == "/" or parsed.path.startswith("/ui") or parsed.path.startswith("/assets/")) and self._serve_frontend_dist(parsed.path):
|
||||
return
|
||||
if not self._check_auth(parsed.path):
|
||||
return
|
||||
if parsed.path.startswith("/assets/"):
|
||||
self._serve_asset(parsed.path.removeprefix("/assets/"))
|
||||
return
|
||||
if parsed.path == "/classic":
|
||||
self._html(render_dashboard_html())
|
||||
return
|
||||
if parsed.path == "/":
|
||||
self._html(render_dashboard_html())
|
||||
return
|
||||
@ -158,16 +152,23 @@ class ApiHandler(BaseHTTPRequestHandler):
|
||||
self._json({"ok": True})
|
||||
return
|
||||
|
||||
state = ensure_initialized()
|
||||
get_dispatcher = ControlPlaneGetDispatcher(
|
||||
state,
|
||||
attention_state_fn=self._attention_state,
|
||||
delivery_state_label_fn=self._delivery_state_label,
|
||||
build_scheduler_preview_fn=build_scheduler_preview,
|
||||
settings_service_factory=SettingsService,
|
||||
)
|
||||
|
||||
if parsed.path == "/settings":
|
||||
state = ensure_initialized()
|
||||
service = SettingsService(state["root"])
|
||||
self._json(service.load_redacted().settings)
|
||||
body, status = get_dispatcher.handle_settings()
|
||||
self._json(body, status=status)
|
||||
return
|
||||
|
||||
if parsed.path == "/settings/schema":
|
||||
state = ensure_initialized()
|
||||
service = SettingsService(state["root"])
|
||||
self._json(service.load().schema)
|
||||
body, status = get_dispatcher.handle_settings_schema()
|
||||
self._json(body, status=status)
|
||||
return
|
||||
|
||||
if parsed.path == "/doctor":
|
||||
@ -180,8 +181,8 @@ class ApiHandler(BaseHTTPRequestHandler):
|
||||
return
|
||||
|
||||
if parsed.path == "/scheduler/preview":
|
||||
state = ensure_initialized()
|
||||
self._json(build_scheduler_preview(state, include_stage_scan=False, limit=200))
|
||||
body, status = get_dispatcher.handle_scheduler_preview()
|
||||
self._json(body, status=status)
|
||||
return
|
||||
|
||||
if parsed.path == "/logs":
|
||||
@ -196,146 +197,78 @@ class ApiHandler(BaseHTTPRequestHandler):
|
||||
return
|
||||
|
||||
if parsed.path == "/history":
|
||||
state = ensure_initialized()
|
||||
query = parse_qs(parsed.query)
|
||||
limit = int(query.get("limit", ["100"])[0])
|
||||
task_id = query.get("task_id", [None])[0]
|
||||
action_name = query.get("action_name", [None])[0]
|
||||
status = query.get("status", [None])[0]
|
||||
items = [
|
||||
item.to_dict()
|
||||
for item in state["repo"].list_action_records(
|
||||
task_id=task_id,
|
||||
limit=limit,
|
||||
action_name=action_name,
|
||||
status=status,
|
||||
)
|
||||
]
|
||||
self._json({"items": items})
|
||||
body, http_status = get_dispatcher.handle_history(
|
||||
limit=limit,
|
||||
task_id=task_id,
|
||||
action_name=action_name,
|
||||
status=status,
|
||||
)
|
||||
self._json(body, status=http_status)
|
||||
return
|
||||
|
||||
if parsed.path == "/modules":
|
||||
state = ensure_initialized()
|
||||
self._json({"items": state["registry"].list_manifests(), "discovered_manifests": state["manifests"]})
|
||||
body, status = get_dispatcher.handle_modules()
|
||||
self._json(body, status=status)
|
||||
return
|
||||
|
||||
if parsed.path == "/tasks":
|
||||
state = ensure_initialized()
|
||||
query = parse_qs(parsed.query)
|
||||
limit = int(query.get("limit", ["100"])[0])
|
||||
tasks = [self._task_payload(task.id, state) for task in state["repo"].list_tasks(limit=limit)]
|
||||
self._json({"items": tasks})
|
||||
offset = int(query.get("offset", ["0"])[0])
|
||||
status = query.get("status", [None])[0]
|
||||
search = query.get("search", [None])[0]
|
||||
sort = query.get("sort", ["updated_desc"])[0]
|
||||
attention = query.get("attention", [None])[0]
|
||||
delivery = query.get("delivery", [None])[0]
|
||||
body, http_status = get_dispatcher.handle_tasks(
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
status=status,
|
||||
search=search,
|
||||
sort=sort,
|
||||
attention=attention,
|
||||
delivery=delivery,
|
||||
)
|
||||
self._json(body, status=http_status)
|
||||
return
|
||||
|
||||
if parsed.path.startswith("/tasks/"):
|
||||
state = ensure_initialized()
|
||||
if parsed.path.startswith("/sessions/"):
|
||||
parts = [unquote(p) for p in parsed.path.split("/") if p]
|
||||
if len(parts) == 2:
|
||||
task = self._task_payload(parts[1], state)
|
||||
if task is None:
|
||||
self._json({"error": "task not found"}, status=HTTPStatus.NOT_FOUND)
|
||||
return
|
||||
self._json(task)
|
||||
body, status = get_dispatcher.handle_session(parts[1])
|
||||
self._json(body, status=status)
|
||||
return
|
||||
|
||||
if parsed.path.startswith("/tasks/"):
|
||||
parts = [unquote(p) for p in parsed.path.split("/") if p]
|
||||
if len(parts) == 2:
|
||||
body, status = get_dispatcher.handle_task(parts[1])
|
||||
self._json(body, status=status)
|
||||
return
|
||||
if len(parts) == 3 and parts[2] == "steps":
|
||||
steps = [self._step_payload(step, state) for step in state["repo"].list_steps(parts[1])]
|
||||
self._json({"items": steps})
|
||||
body, status = get_dispatcher.handle_task_steps(parts[1])
|
||||
self._json(body, status=status)
|
||||
return
|
||||
if len(parts) == 3 and parts[2] == "context":
|
||||
body, status = get_dispatcher.handle_task_context(parts[1])
|
||||
self._json(body, status=status)
|
||||
return
|
||||
if len(parts) == 3 and parts[2] == "artifacts":
|
||||
artifacts = [artifact.to_dict() for artifact in state["repo"].list_artifacts(parts[1])]
|
||||
self._json({"items": artifacts})
|
||||
body, status = get_dispatcher.handle_task_artifacts(parts[1])
|
||||
self._json(body, status=status)
|
||||
return
|
||||
if len(parts) == 3 and parts[2] == "history":
|
||||
actions = [item.to_dict() for item in state["repo"].list_action_records(parts[1], limit=100)]
|
||||
self._json({"items": actions})
|
||||
body, status = get_dispatcher.handle_task_history(parts[1])
|
||||
self._json(body, status=status)
|
||||
return
|
||||
if len(parts) == 3 and parts[2] == "timeline":
|
||||
task = state["repo"].get_task(parts[1])
|
||||
if task is None:
|
||||
self._json({"error": "task not found"}, status=HTTPStatus.NOT_FOUND)
|
||||
return
|
||||
steps = state["repo"].list_steps(parts[1])
|
||||
artifacts = state["repo"].list_artifacts(parts[1])
|
||||
actions = state["repo"].list_action_records(parts[1], limit=200)
|
||||
items: list[dict[str, object]] = []
|
||||
if task.created_at:
|
||||
items.append({
|
||||
"kind": "task",
|
||||
"time": task.created_at,
|
||||
"title": "Task Created",
|
||||
"summary": task.title,
|
||||
"status": task.status,
|
||||
})
|
||||
if task.updated_at and task.updated_at != task.created_at:
|
||||
items.append({
|
||||
"kind": "task",
|
||||
"time": task.updated_at,
|
||||
"title": "Task Updated",
|
||||
"summary": task.status,
|
||||
"status": task.status,
|
||||
})
|
||||
for step in steps:
|
||||
if step.started_at:
|
||||
items.append({
|
||||
"kind": "step",
|
||||
"time": step.started_at,
|
||||
"title": f"{step.step_name} started",
|
||||
"summary": step.status,
|
||||
"status": step.status,
|
||||
})
|
||||
if step.finished_at:
|
||||
retry_meta = retry_meta_for_step(step, state["settings"])
|
||||
retry_note = ""
|
||||
if retry_meta and retry_meta.get("next_retry_at"):
|
||||
retry_note = f" | next retry: {retry_meta['next_retry_at']}"
|
||||
items.append({
|
||||
"kind": "step",
|
||||
"time": step.finished_at,
|
||||
"title": f"{step.step_name} finished",
|
||||
"summary": f"{step.error_message or step.status}{retry_note}",
|
||||
"status": step.status,
|
||||
"retry_state": retry_meta,
|
||||
})
|
||||
for artifact in artifacts:
|
||||
if artifact.created_at:
|
||||
items.append({
|
||||
"kind": "artifact",
|
||||
"time": artifact.created_at,
|
||||
"title": artifact.artifact_type,
|
||||
"summary": artifact.path,
|
||||
"status": "created",
|
||||
})
|
||||
for action in actions:
|
||||
summary = action.summary
|
||||
try:
|
||||
details = json.loads(action.details_json or "{}")
|
||||
except json.JSONDecodeError:
|
||||
details = {}
|
||||
if action.action_name == "comment" and isinstance(details, dict):
|
||||
split_status = details.get("split", {}).get("status")
|
||||
full_status = details.get("full", {}).get("status")
|
||||
fragments = []
|
||||
if split_status:
|
||||
fragments.append(f"split={split_status}")
|
||||
if full_status:
|
||||
fragments.append(f"full={full_status}")
|
||||
if fragments:
|
||||
summary = f"{summary} | {' '.join(fragments)}"
|
||||
if action.action_name in {"collection_a", "collection_b"} and isinstance(details, dict):
|
||||
cleanup = details.get("result", {}).get("cleanup") or details.get("cleanup")
|
||||
if isinstance(cleanup, dict):
|
||||
removed = cleanup.get("removed") or []
|
||||
if removed:
|
||||
summary = f"{summary} | cleanup removed={len(removed)}"
|
||||
items.append({
|
||||
"kind": "action",
|
||||
"time": action.created_at,
|
||||
"title": action.action_name,
|
||||
"summary": summary,
|
||||
"status": action.status,
|
||||
})
|
||||
items.sort(key=lambda item: str(item["time"]), reverse=True)
|
||||
self._json({"items": items})
|
||||
body, status = get_dispatcher.handle_task_timeline(parts[1])
|
||||
self._json(body, status=status)
|
||||
return
|
||||
|
||||
self._json({"error": "not found"}, status=HTTPStatus.NOT_FOUND)
|
||||
@ -353,74 +286,86 @@ class ApiHandler(BaseHTTPRequestHandler):
|
||||
service = SettingsService(root)
|
||||
service.save_staged_from_redacted(payload)
|
||||
service.promote_staged()
|
||||
reset_initialized_state()
|
||||
ensure_initialized()
|
||||
self._json({"ok": True})
|
||||
|
||||
def do_POST(self) -> None: # noqa: N802
|
||||
parsed = urlparse(self.path)
|
||||
if not self._check_auth(parsed.path):
|
||||
return
|
||||
state = ensure_initialized()
|
||||
dispatcher = ControlPlanePostDispatcher(
|
||||
state,
|
||||
bind_full_video_action=bind_full_video_action,
|
||||
merge_session_action=merge_session_action,
|
||||
receive_full_video_webhook=receive_full_video_webhook,
|
||||
rebind_session_full_video_action=rebind_session_full_video_action,
|
||||
reset_to_step_action=reset_to_step_action,
|
||||
retry_step_action=retry_step_action,
|
||||
run_task_action=run_task_action,
|
||||
run_once=run_once,
|
||||
stage_importer_factory=StageImporter,
|
||||
systemd_runtime_factory=SystemdRuntime,
|
||||
)
|
||||
if parsed.path == "/webhooks/full-video-uploaded":
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
payload = json.loads(self.rfile.read(length) or b"{}")
|
||||
body, status = dispatcher.handle_webhook_full_video(payload)
|
||||
self._json(body, status=status)
|
||||
return
|
||||
if parsed.path != "/tasks":
|
||||
if parsed.path.startswith("/sessions/"):
|
||||
parts = [unquote(p) for p in parsed.path.split("/") if p]
|
||||
if len(parts) == 3 and parts[0] == "sessions" and parts[2] == "merge":
|
||||
session_key = parts[1]
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
payload = json.loads(self.rfile.read(length) or b"{}")
|
||||
body, status = dispatcher.handle_session_merge(session_key, payload)
|
||||
self._json(body, status=status)
|
||||
return
|
||||
if len(parts) == 3 and parts[0] == "sessions" and parts[2] == "rebind":
|
||||
session_key = parts[1]
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
payload = json.loads(self.rfile.read(length) or b"{}")
|
||||
body, status = dispatcher.handle_session_rebind(session_key, payload)
|
||||
self._json(body, status=status)
|
||||
return
|
||||
if parsed.path.startswith("/tasks/"):
|
||||
parts = [unquote(p) for p in parsed.path.split("/") if p]
|
||||
if len(parts) == 3 and parts[0] == "tasks" and parts[2] == "bind-full-video":
|
||||
task_id = parts[1]
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
payload = json.loads(self.rfile.read(length) or b"{}")
|
||||
body, status = dispatcher.handle_bind_full_video(task_id, payload)
|
||||
self._json(body, status=status)
|
||||
return
|
||||
if len(parts) == 4 and parts[0] == "tasks" and parts[2] == "actions":
|
||||
task_id = parts[1]
|
||||
action = parts[3]
|
||||
if action == "run":
|
||||
result = run_task_action(task_id)
|
||||
self._json(result, status=HTTPStatus.ACCEPTED)
|
||||
return
|
||||
if action == "retry-step":
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
payload = json.loads(self.rfile.read(length) or b"{}")
|
||||
step_name = payload.get("step_name")
|
||||
if not step_name:
|
||||
self._json({"error": "missing step_name"}, status=HTTPStatus.BAD_REQUEST)
|
||||
return
|
||||
result = retry_step_action(task_id, step_name)
|
||||
self._json(result, status=HTTPStatus.ACCEPTED)
|
||||
return
|
||||
if action == "reset-to-step":
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
payload = json.loads(self.rfile.read(length) or b"{}")
|
||||
step_name = payload.get("step_name")
|
||||
if not step_name:
|
||||
self._json({"error": "missing step_name"}, status=HTTPStatus.BAD_REQUEST)
|
||||
return
|
||||
result = reset_to_step_action(task_id, step_name)
|
||||
self._json(result, status=HTTPStatus.ACCEPTED)
|
||||
if action in {"run", "retry-step", "reset-to-step"}:
|
||||
payload = {}
|
||||
if action != "run":
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
payload = json.loads(self.rfile.read(length) or b"{}")
|
||||
body, status = dispatcher.handle_task_action(task_id, action, payload)
|
||||
self._json(body, status=status)
|
||||
return
|
||||
if parsed.path == "/worker/run-once":
|
||||
payload = run_once()
|
||||
self._record_action(None, "worker_run_once", "ok", "worker run once invoked", payload)
|
||||
self._json(payload, status=HTTPStatus.ACCEPTED)
|
||||
body, status = dispatcher.handle_worker_run_once()
|
||||
self._json(body, status=status)
|
||||
return
|
||||
if parsed.path.startswith("/runtime/services/"):
|
||||
parts = [unquote(p) for p in parsed.path.split("/") if p]
|
||||
if len(parts) == 4 and parts[0] == "runtime" and parts[1] == "services":
|
||||
try:
|
||||
payload = SystemdRuntime().act(parts[2], parts[3])
|
||||
except ValueError as exc:
|
||||
self._json({"error": str(exc)}, status=HTTPStatus.BAD_REQUEST)
|
||||
return
|
||||
self._record_action(None, "service_action", "ok" if payload.get("command_ok") else "error", f"{parts[3]} {parts[2]}", payload)
|
||||
self._json(payload, status=HTTPStatus.ACCEPTED)
|
||||
body, status = dispatcher.handle_runtime_service_action(parts[2], parts[3])
|
||||
self._json(body, status=status)
|
||||
return
|
||||
if parsed.path == "/stage/import":
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
payload = json.loads(self.rfile.read(length) or b"{}")
|
||||
source_path = payload.get("source_path")
|
||||
if not source_path:
|
||||
self._json({"error": "missing source_path"}, status=HTTPStatus.BAD_REQUEST)
|
||||
return
|
||||
state = ensure_initialized()
|
||||
stage_dir = Path(state["settings"]["paths"]["stage_dir"])
|
||||
try:
|
||||
result = StageImporter().import_file(Path(source_path), stage_dir)
|
||||
except Exception as exc:
|
||||
self._json({"error": str(exc)}, status=HTTPStatus.BAD_REQUEST)
|
||||
return
|
||||
self._record_action(None, "stage_import", "ok", "imported file into stage", result)
|
||||
self._json(result, status=HTTPStatus.CREATED)
|
||||
body, status = dispatcher.handle_stage_import(payload)
|
||||
self._json(body, status=status)
|
||||
return
|
||||
if parsed.path == "/stage/upload":
|
||||
content_type = self.headers.get("Content-Type", "")
|
||||
@ -437,44 +382,19 @@ class ApiHandler(BaseHTTPRequestHandler):
|
||||
},
|
||||
)
|
||||
file_item = form["file"] if "file" in form else None
|
||||
if file_item is None or not getattr(file_item, "filename", None):
|
||||
self._json({"error": "missing file"}, status=HTTPStatus.BAD_REQUEST)
|
||||
return
|
||||
state = ensure_initialized()
|
||||
stage_dir = Path(state["settings"]["paths"]["stage_dir"])
|
||||
try:
|
||||
result = StageImporter().import_upload(file_item.filename, file_item.file, stage_dir)
|
||||
except Exception as exc:
|
||||
self._json({"error": str(exc)}, status=HTTPStatus.BAD_REQUEST)
|
||||
return
|
||||
self._record_action(None, "stage_upload", "ok", "uploaded file into stage", result)
|
||||
self._json(result, status=HTTPStatus.CREATED)
|
||||
body, status = dispatcher.handle_stage_upload(file_item)
|
||||
self._json(body, status=status)
|
||||
return
|
||||
if parsed.path == "/scheduler/run-once":
|
||||
result = run_once()
|
||||
self._record_action(None, "scheduler_run_once", "ok", "scheduler run once completed", result.get("scheduler", {}))
|
||||
self._json(result, status=HTTPStatus.ACCEPTED)
|
||||
body, status = dispatcher.handle_scheduler_run_once()
|
||||
self._json(body, status=status)
|
||||
return
|
||||
self._json({"error": "not found"}, status=HTTPStatus.NOT_FOUND)
|
||||
return
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
payload = json.loads(self.rfile.read(length) or b"{}")
|
||||
source_path = payload.get("source_path")
|
||||
if not source_path:
|
||||
self._json({"error": "missing source_path"}, status=HTTPStatus.BAD_REQUEST)
|
||||
return
|
||||
state = ensure_initialized()
|
||||
try:
|
||||
task = state["ingest_service"].create_task_from_file(
|
||||
Path(source_path),
|
||||
state["settings"]["ingest"],
|
||||
)
|
||||
except Exception as exc: # keep API small for now
|
||||
status = HTTPStatus.CONFLICT if exc.__class__.__name__ == "ModuleError" else HTTPStatus.INTERNAL_SERVER_ERROR
|
||||
payload = exc.to_dict() if hasattr(exc, "to_dict") else {"error": str(exc)}
|
||||
self._json(payload, status=status)
|
||||
return
|
||||
self._json(task.to_dict(), status=HTTPStatus.CREATED)
|
||||
body, status = dispatcher.handle_create_task(payload)
|
||||
self._json(body, status=status)
|
||||
|
||||
def log_message(self, format: str, *args) -> None: # noqa: A003
|
||||
return
|
||||
@ -510,7 +430,7 @@ class ApiHandler(BaseHTTPRequestHandler):
|
||||
)
|
||||
|
||||
def _check_auth(self, path: str) -> bool:
|
||||
if path in {"/", "/health", "/ui", "/ui/"} or path.startswith("/assets/") or path.startswith("/ui/assets/"):
|
||||
if path in {"/", "/health", "/ui", "/ui/", "/classic"} or path.startswith("/assets/") or path.startswith("/ui/assets/"):
|
||||
return True
|
||||
state = ensure_initialized()
|
||||
expected = str(state["settings"]["runtime"].get("control_token", "")).strip()
|
||||
|
||||
@ -1,11 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from dataclasses import asdict
|
||||
from pathlib import Path
|
||||
from threading import RLock
|
||||
|
||||
from biliup_next.core.config import SettingsService
|
||||
from biliup_next.core.registry import Registry
|
||||
from biliup_next.infra.comment_flag_migration import CommentFlagMigrationService
|
||||
from biliup_next.infra.db import Database
|
||||
from biliup_next.infra.plugin_loader import PluginLoader
|
||||
from biliup_next.infra.task_repository import TaskRepository
|
||||
@ -22,56 +22,67 @@ def project_root() -> Path:
|
||||
return Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
_APP_STATE: dict[str, object] | None = None
|
||||
_APP_STATE_LOCK = RLock()
|
||||
|
||||
|
||||
def reset_initialized_state() -> None:
|
||||
global _APP_STATE
|
||||
with _APP_STATE_LOCK:
|
||||
_APP_STATE = None
|
||||
|
||||
|
||||
def ensure_initialized() -> dict[str, object]:
|
||||
root = project_root()
|
||||
settings_service = SettingsService(root)
|
||||
bundle = settings_service.load()
|
||||
db_path = (root / bundle.settings["runtime"]["database_path"]).resolve()
|
||||
db = Database(db_path)
|
||||
db.initialize()
|
||||
repo = TaskRepository(db)
|
||||
registry = Registry()
|
||||
plugin_loader = PluginLoader(root)
|
||||
manifests = plugin_loader.load_manifests()
|
||||
for manifest in manifests:
|
||||
if not manifest.enabled_by_default:
|
||||
continue
|
||||
provider = plugin_loader.instantiate_provider(manifest)
|
||||
provider_manifest = getattr(provider, "manifest", None)
|
||||
if provider_manifest is None:
|
||||
raise RuntimeError(f"provider missing manifest: {manifest.entrypoint}")
|
||||
if provider_manifest.id != manifest.id or provider_manifest.provider_type != manifest.provider_type:
|
||||
raise RuntimeError(f"provider manifest mismatch: {manifest.entrypoint}")
|
||||
registry.register(
|
||||
manifest.provider_type,
|
||||
manifest.id,
|
||||
provider,
|
||||
provider_manifest,
|
||||
)
|
||||
session_dir = (root / bundle.settings["paths"]["session_dir"]).resolve()
|
||||
imported = repo.bootstrap_from_legacy_sessions(session_dir)
|
||||
comment_flag_migration = CommentFlagMigrationService().migrate(session_dir)
|
||||
ingest_service = IngestService(registry, repo)
|
||||
transcribe_service = TranscribeService(registry, repo)
|
||||
song_detect_service = SongDetectService(registry, repo)
|
||||
split_service = SplitService(registry, repo)
|
||||
publish_service = PublishService(registry, repo)
|
||||
comment_service = CommentService(registry, repo)
|
||||
collection_service = CollectionService(registry, repo)
|
||||
return {
|
||||
"root": root,
|
||||
"settings": bundle.settings,
|
||||
"db": db,
|
||||
"repo": repo,
|
||||
"registry": registry,
|
||||
"manifests": [asdict(m) for m in manifests],
|
||||
"ingest_service": ingest_service,
|
||||
"transcribe_service": transcribe_service,
|
||||
"song_detect_service": song_detect_service,
|
||||
"split_service": split_service,
|
||||
"publish_service": publish_service,
|
||||
"comment_service": comment_service,
|
||||
"collection_service": collection_service,
|
||||
"imported": imported,
|
||||
"comment_flag_migration": comment_flag_migration,
|
||||
}
|
||||
global _APP_STATE
|
||||
with _APP_STATE_LOCK:
|
||||
if _APP_STATE is not None:
|
||||
return _APP_STATE
|
||||
|
||||
root = project_root()
|
||||
settings_service = SettingsService(root)
|
||||
bundle = settings_service.load()
|
||||
db_path = (root / bundle.settings["runtime"]["database_path"]).resolve()
|
||||
db = Database(db_path)
|
||||
db.initialize()
|
||||
repo = TaskRepository(db)
|
||||
registry = Registry()
|
||||
plugin_loader = PluginLoader(root)
|
||||
manifests = plugin_loader.load_manifests()
|
||||
for manifest in manifests:
|
||||
if not manifest.enabled_by_default:
|
||||
continue
|
||||
provider = plugin_loader.instantiate_provider(manifest)
|
||||
provider_manifest = getattr(provider, "manifest", None)
|
||||
if provider_manifest is None:
|
||||
raise RuntimeError(f"provider missing manifest: {manifest.entrypoint}")
|
||||
if provider_manifest.id != manifest.id or provider_manifest.provider_type != manifest.provider_type:
|
||||
raise RuntimeError(f"provider manifest mismatch: {manifest.entrypoint}")
|
||||
registry.register(
|
||||
manifest.provider_type,
|
||||
manifest.id,
|
||||
provider,
|
||||
provider_manifest,
|
||||
)
|
||||
ingest_service = IngestService(registry, repo)
|
||||
transcribe_service = TranscribeService(registry, repo)
|
||||
song_detect_service = SongDetectService(registry, repo)
|
||||
split_service = SplitService(registry, repo)
|
||||
publish_service = PublishService(registry, repo)
|
||||
comment_service = CommentService(registry, repo)
|
||||
collection_service = CollectionService(registry, repo)
|
||||
_APP_STATE = {
|
||||
"root": root,
|
||||
"settings": bundle.settings,
|
||||
"db": db,
|
||||
"repo": repo,
|
||||
"registry": registry,
|
||||
"manifests": [asdict(m) for m in manifests],
|
||||
"ingest_service": ingest_service,
|
||||
"transcribe_service": transcribe_service,
|
||||
"song_detect_service": song_detect_service,
|
||||
"split_service": split_service,
|
||||
"publish_service": publish_service,
|
||||
"comment_service": comment_service,
|
||||
"collection_service": collection_service,
|
||||
}
|
||||
return _APP_STATE
|
||||
|
||||
@ -40,8 +40,8 @@ def main() -> None:
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "init":
|
||||
state = ensure_initialized()
|
||||
print(json.dumps({"ok": True, "imported": state["imported"]}, ensure_ascii=False, indent=2))
|
||||
ensure_initialized()
|
||||
print(json.dumps({"ok": True}, ensure_ascii=False, indent=2))
|
||||
return
|
||||
|
||||
if args.command == "doctor":
|
||||
@ -93,9 +93,11 @@ def main() -> None:
|
||||
|
||||
if args.command == "create-task":
|
||||
state = ensure_initialized()
|
||||
settings = dict(state["settings"]["ingest"])
|
||||
settings.update(state["settings"]["paths"])
|
||||
task = state["ingest_service"].create_task_from_file(
|
||||
Path(args.source_path),
|
||||
state["settings"]["ingest"],
|
||||
settings,
|
||||
)
|
||||
print(json.dumps(task.to_dict(), ensure_ascii=False, indent=2))
|
||||
return
|
||||
|
||||
123
src/biliup_next/app/control_plane_get_dispatcher.py
Normal file
123
src/biliup_next/app/control_plane_get_dispatcher.py
Normal file
@ -0,0 +1,123 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from http import HTTPStatus
|
||||
|
||||
from biliup_next.app.serializers import ControlPlaneSerializer
|
||||
|
||||
|
||||
class ControlPlaneGetDispatcher:
|
||||
def __init__(
|
||||
self,
|
||||
state: dict[str, object],
|
||||
*,
|
||||
attention_state_fn,
|
||||
delivery_state_label_fn,
|
||||
build_scheduler_preview_fn,
|
||||
settings_service_factory,
|
||||
) -> None: # type: ignore[no-untyped-def]
|
||||
self.state = state
|
||||
self.repo = state["repo"]
|
||||
self.serializer = ControlPlaneSerializer(state)
|
||||
self.attention_state_fn = attention_state_fn
|
||||
self.delivery_state_label_fn = delivery_state_label_fn
|
||||
self.build_scheduler_preview_fn = build_scheduler_preview_fn
|
||||
self.settings_service_factory = settings_service_factory
|
||||
|
||||
def handle_settings(self) -> tuple[object, HTTPStatus]:
|
||||
service = self.settings_service_factory(self.state["root"])
|
||||
return service.load_redacted().settings, HTTPStatus.OK
|
||||
|
||||
def handle_settings_schema(self) -> tuple[object, HTTPStatus]:
|
||||
service = self.settings_service_factory(self.state["root"])
|
||||
return service.load().schema, HTTPStatus.OK
|
||||
|
||||
def handle_scheduler_preview(self) -> tuple[object, HTTPStatus]:
|
||||
return self.build_scheduler_preview_fn(self.state, include_stage_scan=False, limit=200), HTTPStatus.OK
|
||||
|
||||
def handle_history(self, *, limit: int, task_id: str | None, action_name: str | None, status: str | None) -> tuple[object, HTTPStatus]:
|
||||
items = [
|
||||
item.to_dict()
|
||||
for item in self.repo.list_action_records(
|
||||
task_id=task_id,
|
||||
limit=limit,
|
||||
action_name=action_name,
|
||||
status=status,
|
||||
)
|
||||
]
|
||||
return {"items": items}, HTTPStatus.OK
|
||||
|
||||
def handle_modules(self) -> tuple[object, HTTPStatus]:
|
||||
return {"items": self.state["registry"].list_manifests(), "discovered_manifests": self.state["manifests"]}, HTTPStatus.OK
|
||||
|
||||
def handle_tasks(
|
||||
self,
|
||||
*,
|
||||
limit: int,
|
||||
offset: int,
|
||||
status: str | None,
|
||||
search: str | None,
|
||||
sort: str,
|
||||
attention: str | None,
|
||||
delivery: str | None,
|
||||
) -> tuple[object, HTTPStatus]:
|
||||
if attention or delivery:
|
||||
task_items, _ = self.repo.query_tasks(
|
||||
limit=5000,
|
||||
offset=0,
|
||||
status=status,
|
||||
search=search,
|
||||
sort=sort,
|
||||
)
|
||||
all_tasks = self.serializer.task_payloads_from_tasks(task_items)
|
||||
filtered_tasks: list[dict[str, object]] = []
|
||||
for item in all_tasks:
|
||||
if attention and self.attention_state_fn(item) != attention:
|
||||
continue
|
||||
if delivery and self.delivery_state_label_fn(item) != delivery:
|
||||
continue
|
||||
filtered_tasks.append(item)
|
||||
total = len(filtered_tasks)
|
||||
tasks = filtered_tasks[offset:offset + limit]
|
||||
else:
|
||||
task_items, total = self.repo.query_tasks(
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
status=status,
|
||||
search=search,
|
||||
sort=sort,
|
||||
)
|
||||
tasks = self.serializer.task_payloads_from_tasks(task_items)
|
||||
return {"items": tasks, "total": total, "limit": limit, "offset": offset}, HTTPStatus.OK
|
||||
|
||||
def handle_session(self, session_key: str) -> tuple[object, HTTPStatus]:
|
||||
payload = self.serializer.session_payload(session_key)
|
||||
if payload is None:
|
||||
return {"error": "session not found"}, HTTPStatus.NOT_FOUND
|
||||
return payload, HTTPStatus.OK
|
||||
|
||||
def handle_task(self, task_id: str) -> tuple[object, HTTPStatus]:
|
||||
payload = self.serializer.task_payload(task_id)
|
||||
if payload is None:
|
||||
return {"error": "task not found"}, HTTPStatus.NOT_FOUND
|
||||
return payload, HTTPStatus.OK
|
||||
|
||||
def handle_task_steps(self, task_id: str) -> tuple[object, HTTPStatus]:
|
||||
return {"items": [self.serializer.step_payload(step) for step in self.repo.list_steps(task_id)]}, HTTPStatus.OK
|
||||
|
||||
def handle_task_context(self, task_id: str) -> tuple[object, HTTPStatus]:
|
||||
payload = self.serializer.task_context_payload(task_id)
|
||||
if payload is None:
|
||||
return {"error": "task context not found"}, HTTPStatus.NOT_FOUND
|
||||
return payload, HTTPStatus.OK
|
||||
|
||||
def handle_task_artifacts(self, task_id: str) -> tuple[object, HTTPStatus]:
|
||||
return {"items": [artifact.to_dict() for artifact in self.repo.list_artifacts(task_id)]}, HTTPStatus.OK
|
||||
|
||||
def handle_task_history(self, task_id: str) -> tuple[object, HTTPStatus]:
|
||||
return {"items": [item.to_dict() for item in self.repo.list_action_records(task_id, limit=100)]}, HTTPStatus.OK
|
||||
|
||||
def handle_task_timeline(self, task_id: str) -> tuple[object, HTTPStatus]:
|
||||
payload = self.serializer.timeline_payload(task_id)
|
||||
if payload is None:
|
||||
return {"error": "task not found"}, HTTPStatus.NOT_FOUND
|
||||
return payload, HTTPStatus.OK
|
||||
164
src/biliup_next/app/control_plane_post_dispatcher.py
Normal file
164
src/biliup_next/app/control_plane_post_dispatcher.py
Normal file
@ -0,0 +1,164 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from http import HTTPStatus
|
||||
from pathlib import Path
|
||||
|
||||
from biliup_next.core.models import ActionRecord, utc_now_iso
|
||||
from biliup_next.infra.storage_guard import mb_to_bytes
|
||||
|
||||
|
||||
class ControlPlanePostDispatcher:
|
||||
def __init__(
|
||||
self,
|
||||
state: dict[str, object],
|
||||
*,
|
||||
bind_full_video_action,
|
||||
merge_session_action,
|
||||
receive_full_video_webhook,
|
||||
rebind_session_full_video_action,
|
||||
reset_to_step_action,
|
||||
retry_step_action,
|
||||
run_task_action,
|
||||
run_once,
|
||||
stage_importer_factory,
|
||||
systemd_runtime_factory,
|
||||
) -> None: # type: ignore[no-untyped-def]
|
||||
self.state = state
|
||||
self.repo = state["repo"]
|
||||
self.bind_full_video_action = bind_full_video_action
|
||||
self.merge_session_action = merge_session_action
|
||||
self.receive_full_video_webhook = receive_full_video_webhook
|
||||
self.rebind_session_full_video_action = rebind_session_full_video_action
|
||||
self.reset_to_step_action = reset_to_step_action
|
||||
self.retry_step_action = retry_step_action
|
||||
self.run_task_action = run_task_action
|
||||
self.run_once = run_once
|
||||
self.stage_importer_factory = stage_importer_factory
|
||||
self.systemd_runtime_factory = systemd_runtime_factory
|
||||
|
||||
def handle_webhook_full_video(self, payload: object) -> tuple[object, HTTPStatus]:
|
||||
if not isinstance(payload, dict):
|
||||
return {"error": "invalid payload"}, HTTPStatus.BAD_REQUEST
|
||||
result = self.receive_full_video_webhook(payload)
|
||||
if "error" in result:
|
||||
return result, HTTPStatus.BAD_REQUEST
|
||||
return result, HTTPStatus.ACCEPTED
|
||||
|
||||
def handle_session_merge(self, session_key: str, payload: object) -> tuple[object, HTTPStatus]:
|
||||
if not isinstance(payload, dict) or not isinstance(payload.get("task_ids"), list):
|
||||
return {"error": "missing task_ids"}, HTTPStatus.BAD_REQUEST
|
||||
result = self.merge_session_action(session_key, [str(item) for item in payload["task_ids"]])
|
||||
if "error" in result:
|
||||
return result, HTTPStatus.BAD_REQUEST
|
||||
return result, HTTPStatus.ACCEPTED
|
||||
|
||||
def handle_session_rebind(self, session_key: str, payload: object) -> tuple[object, HTTPStatus]:
|
||||
full_video_bvid = str((payload or {}).get("full_video_bvid", "")).strip() if isinstance(payload, dict) else ""
|
||||
if not full_video_bvid:
|
||||
return {"error": "missing full_video_bvid"}, HTTPStatus.BAD_REQUEST
|
||||
result = self.rebind_session_full_video_action(session_key, full_video_bvid)
|
||||
if "error" in result:
|
||||
status = HTTPStatus.NOT_FOUND if result["error"].get("code") == "SESSION_NOT_FOUND" else HTTPStatus.BAD_REQUEST
|
||||
return result, status
|
||||
return result, HTTPStatus.ACCEPTED
|
||||
|
||||
def handle_bind_full_video(self, task_id: str, payload: object) -> tuple[object, HTTPStatus]:
|
||||
full_video_bvid = str((payload or {}).get("full_video_bvid", "")).strip() if isinstance(payload, dict) else ""
|
||||
if not full_video_bvid:
|
||||
return {"error": "missing full_video_bvid"}, HTTPStatus.BAD_REQUEST
|
||||
result = self.bind_full_video_action(task_id, full_video_bvid)
|
||||
if "error" in result:
|
||||
status = HTTPStatus.NOT_FOUND if result["error"].get("code") == "TASK_NOT_FOUND" else HTTPStatus.BAD_REQUEST
|
||||
return result, status
|
||||
return result, HTTPStatus.ACCEPTED
|
||||
|
||||
def handle_task_action(self, task_id: str, action: str, payload: object) -> tuple[object, HTTPStatus]:
|
||||
if action == "run":
|
||||
return self.run_task_action(task_id), HTTPStatus.ACCEPTED
|
||||
if action == "retry-step":
|
||||
step_name = payload.get("step_name") if isinstance(payload, dict) else None
|
||||
if not step_name:
|
||||
return {"error": "missing step_name"}, HTTPStatus.BAD_REQUEST
|
||||
return self.retry_step_action(task_id, step_name), HTTPStatus.ACCEPTED
|
||||
if action == "reset-to-step":
|
||||
step_name = payload.get("step_name") if isinstance(payload, dict) else None
|
||||
if not step_name:
|
||||
return {"error": "missing step_name"}, HTTPStatus.BAD_REQUEST
|
||||
return self.reset_to_step_action(task_id, step_name), HTTPStatus.ACCEPTED
|
||||
return {"error": "not found"}, HTTPStatus.NOT_FOUND
|
||||
|
||||
def handle_worker_run_once(self) -> tuple[object, HTTPStatus]:
|
||||
payload = self.run_once()
|
||||
self._record_action(None, "worker_run_once", "ok", "worker run once invoked", payload)
|
||||
return payload, HTTPStatus.ACCEPTED
|
||||
|
||||
def handle_scheduler_run_once(self) -> tuple[object, HTTPStatus]:
|
||||
payload = self.run_once()
|
||||
self._record_action(None, "scheduler_run_once", "ok", "scheduler run once completed", payload.get("scheduler", {}))
|
||||
return payload, HTTPStatus.ACCEPTED
|
||||
|
||||
def handle_runtime_service_action(self, service_name: str, action: str) -> tuple[object, HTTPStatus]:
|
||||
try:
|
||||
payload = self.systemd_runtime_factory().act(service_name, action)
|
||||
except ValueError as exc:
|
||||
return {"error": str(exc)}, HTTPStatus.BAD_REQUEST
|
||||
self._record_action(None, "service_action", "ok" if payload.get("command_ok") else "error", f"{action} {service_name}", payload)
|
||||
return payload, HTTPStatus.ACCEPTED
|
||||
|
||||
def handle_stage_import(self, payload: object) -> tuple[object, HTTPStatus]:
|
||||
source_path = payload.get("source_path") if isinstance(payload, dict) else None
|
||||
if not source_path:
|
||||
return {"error": "missing source_path"}, HTTPStatus.BAD_REQUEST
|
||||
stage_dir = Path(self.state["settings"]["paths"]["stage_dir"])
|
||||
min_free_bytes = mb_to_bytes(self.state["settings"]["ingest"].get("stage_min_free_space_mb", 0))
|
||||
try:
|
||||
result = self.stage_importer_factory().import_file(Path(source_path), stage_dir, min_free_bytes=min_free_bytes)
|
||||
except Exception as exc:
|
||||
return {"error": str(exc)}, HTTPStatus.BAD_REQUEST
|
||||
self._record_action(None, "stage_import", "ok", "imported file into stage", result)
|
||||
return result, HTTPStatus.CREATED
|
||||
|
||||
def handle_stage_upload(self, file_item) -> tuple[object, HTTPStatus]: # type: ignore[no-untyped-def]
|
||||
if file_item is None or not getattr(file_item, "filename", None):
|
||||
return {"error": "missing file"}, HTTPStatus.BAD_REQUEST
|
||||
stage_dir = Path(self.state["settings"]["paths"]["stage_dir"])
|
||||
min_free_bytes = mb_to_bytes(self.state["settings"]["ingest"].get("stage_min_free_space_mb", 0))
|
||||
try:
|
||||
result = self.stage_importer_factory().import_upload(
|
||||
file_item.filename,
|
||||
file_item.file,
|
||||
stage_dir,
|
||||
min_free_bytes=min_free_bytes,
|
||||
)
|
||||
except Exception as exc:
|
||||
return {"error": str(exc)}, HTTPStatus.BAD_REQUEST
|
||||
self._record_action(None, "stage_upload", "ok", "uploaded file into stage", result)
|
||||
return result, HTTPStatus.CREATED
|
||||
|
||||
def handle_create_task(self, payload: object) -> tuple[object, HTTPStatus]:
|
||||
source_path = payload.get("source_path") if isinstance(payload, dict) else None
|
||||
if not source_path:
|
||||
return {"error": "missing source_path"}, HTTPStatus.BAD_REQUEST
|
||||
try:
|
||||
settings = dict(self.state["settings"]["ingest"])
|
||||
settings.update(self.state["settings"]["paths"])
|
||||
task = self.state["ingest_service"].create_task_from_file(Path(source_path), settings)
|
||||
except Exception as exc:
|
||||
status = HTTPStatus.CONFLICT if exc.__class__.__name__ == "ModuleError" else HTTPStatus.INTERNAL_SERVER_ERROR
|
||||
body = exc.to_dict() if hasattr(exc, "to_dict") else {"error": str(exc)}
|
||||
return body, status
|
||||
return task.to_dict(), HTTPStatus.CREATED
|
||||
|
||||
def _record_action(self, task_id: str | None, action_name: str, status: str, summary: str, details: dict[str, object]) -> None:
|
||||
self.repo.add_action_record(
|
||||
ActionRecord(
|
||||
id=None,
|
||||
task_id=task_id,
|
||||
action_name=action_name,
|
||||
status=status,
|
||||
summary=summary,
|
||||
details_json=json.dumps(details, ensure_ascii=False),
|
||||
created_at=utc_now_iso(),
|
||||
)
|
||||
)
|
||||
@ -215,7 +215,6 @@ def render_dashboard_html() -> str:
|
||||
</select>
|
||||
<select id="taskDeliveryFilter">
|
||||
<option value="">全部交付状态</option>
|
||||
<option value="legacy_untracked">主视频评论未追踪</option>
|
||||
<option value="pending_comment">评论待完成</option>
|
||||
<option value="cleanup_removed">已清理视频</option>
|
||||
</select>
|
||||
@ -249,6 +248,17 @@ def render_dashboard_html() -> str:
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="panel">
|
||||
<div class="panel-head">
|
||||
<h3>Session Workspace</h3>
|
||||
<div class="button-row">
|
||||
<button id="refreshSessionBtn" class="secondary compact">刷新 Session</button>
|
||||
</div>
|
||||
</div>
|
||||
<div id="sessionWorkspaceState" class="task-workspace-state show">当前任务如果已绑定 session_key,这里会显示同场片段和完整版绑定信息。</div>
|
||||
<div id="sessionPanel" class="summary-card session-panel"></div>
|
||||
</section>
|
||||
|
||||
<div class="panel-grid two-up">
|
||||
<section class="panel">
|
||||
<div class="panel-head"><h3>Steps</h3></div>
|
||||
|
||||
@ -2,6 +2,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
STEP_SETTINGS_GROUP = {
|
||||
"publish": "publish",
|
||||
"comment": "comment",
|
||||
}
|
||||
|
||||
|
||||
def parse_iso(value: str | None) -> datetime | None:
|
||||
if not value:
|
||||
@ -12,7 +17,14 @@ def parse_iso(value: str | None) -> datetime | None:
|
||||
return None
|
||||
|
||||
|
||||
def publish_retry_schedule_seconds(settings: dict[str, object]) -> list[int]:
|
||||
def retry_schedule_seconds(
|
||||
settings: dict[str, object],
|
||||
*,
|
||||
count_key: str,
|
||||
backoff_key: str,
|
||||
default_count: int,
|
||||
default_backoff: int,
|
||||
) -> list[int]:
|
||||
raw_schedule = settings.get("retry_schedule_minutes")
|
||||
if isinstance(raw_schedule, list):
|
||||
schedule: list[int] = []
|
||||
@ -21,25 +33,57 @@ def publish_retry_schedule_seconds(settings: dict[str, object]) -> list[int]:
|
||||
schedule.append(item * 60)
|
||||
if schedule:
|
||||
return schedule
|
||||
retry_count = settings.get("retry_count", 5)
|
||||
retry_count = retry_count if isinstance(retry_count, int) and not isinstance(retry_count, bool) else 5
|
||||
|
||||
retry_count = settings.get(count_key, default_count)
|
||||
retry_count = retry_count if isinstance(retry_count, int) and not isinstance(retry_count, bool) else default_count
|
||||
retry_count = max(retry_count, 0)
|
||||
retry_backoff = settings.get("retry_backoff_seconds", 300)
|
||||
retry_backoff = retry_backoff if isinstance(retry_backoff, int) and not isinstance(retry_backoff, bool) else 300
|
||||
|
||||
retry_backoff = settings.get(backoff_key, default_backoff)
|
||||
retry_backoff = retry_backoff if isinstance(retry_backoff, int) and not isinstance(retry_backoff, bool) else default_backoff
|
||||
retry_backoff = max(retry_backoff, 0)
|
||||
return [retry_backoff] * retry_count
|
||||
|
||||
|
||||
def publish_retry_schedule_seconds(settings: dict[str, object]) -> list[int]:
|
||||
return retry_schedule_seconds(
|
||||
settings,
|
||||
count_key="retry_count",
|
||||
backoff_key="retry_backoff_seconds",
|
||||
default_count=5,
|
||||
default_backoff=300,
|
||||
)
|
||||
|
||||
|
||||
def comment_retry_schedule_seconds(settings: dict[str, object]) -> list[int]:
|
||||
return retry_schedule_seconds(
|
||||
settings,
|
||||
count_key="max_retries",
|
||||
backoff_key="base_delay_seconds",
|
||||
default_count=5,
|
||||
default_backoff=180,
|
||||
)
|
||||
|
||||
|
||||
def retry_meta_for_step(step, settings_by_group: dict[str, object]) -> dict[str, object] | None: # type: ignore[no-untyped-def]
|
||||
if getattr(step, "status", None) != "failed_retryable" or getattr(step, "retry_count", 0) <= 0:
|
||||
return None
|
||||
if getattr(step, "step_name", None) != "publish":
|
||||
|
||||
step_name = getattr(step, "step_name", None)
|
||||
settings_group = STEP_SETTINGS_GROUP.get(step_name)
|
||||
if settings_group is None:
|
||||
return None
|
||||
|
||||
group_settings = settings_by_group.get(settings_group, {})
|
||||
if not isinstance(group_settings, dict):
|
||||
group_settings = {}
|
||||
|
||||
if step_name == "publish":
|
||||
schedule = publish_retry_schedule_seconds(group_settings)
|
||||
elif step_name == "comment":
|
||||
schedule = comment_retry_schedule_seconds(group_settings)
|
||||
else:
|
||||
return None
|
||||
|
||||
publish_settings = settings_by_group.get("publish", {})
|
||||
if not isinstance(publish_settings, dict):
|
||||
publish_settings = {}
|
||||
schedule = publish_retry_schedule_seconds(publish_settings)
|
||||
attempt_index = step.retry_count - 1
|
||||
if attempt_index >= len(schedule):
|
||||
return {
|
||||
|
||||
254
src/biliup_next/app/serializers.py
Normal file
254
src/biliup_next/app/serializers.py
Normal file
@ -0,0 +1,254 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from biliup_next.app.retry_meta import retry_meta_for_step
|
||||
|
||||
|
||||
class ControlPlaneSerializer:
|
||||
def __init__(self, state: dict[str, object]):
|
||||
self.state = state
|
||||
|
||||
@staticmethod
|
||||
def video_url(bvid: object) -> str | None:
|
||||
if isinstance(bvid, str) and bvid.startswith("BV"):
|
||||
return f"https://www.bilibili.com/video/{bvid}"
|
||||
return None
|
||||
|
||||
def task_related_maps(
|
||||
self,
|
||||
tasks,
|
||||
) -> tuple[dict[str, object], dict[str, list[object]]]: # type: ignore[no-untyped-def]
|
||||
task_ids = [task.id for task in tasks]
|
||||
contexts_by_task_id = self.state["repo"].list_task_contexts_for_task_ids(task_ids)
|
||||
steps_by_task_id = self.state["repo"].list_steps_for_task_ids(task_ids)
|
||||
return contexts_by_task_id, steps_by_task_id
|
||||
|
||||
def task_payload(self, task_id: str) -> dict[str, object] | None:
|
||||
task = self.state["repo"].get_task(task_id)
|
||||
if task is None:
|
||||
return None
|
||||
return self.task_payload_from_task(task)
|
||||
|
||||
def task_payloads_from_tasks(self, tasks) -> list[dict[str, object]]: # type: ignore[no-untyped-def]
|
||||
contexts_by_task_id, steps_by_task_id = self.task_related_maps(tasks)
|
||||
return [
|
||||
self.task_payload_from_task(
|
||||
task,
|
||||
context=contexts_by_task_id.get(task.id),
|
||||
steps=steps_by_task_id.get(task.id, []),
|
||||
)
|
||||
for task in tasks
|
||||
]
|
||||
|
||||
def task_payload_from_task(
|
||||
self,
|
||||
task,
|
||||
*,
|
||||
context=None, # type: ignore[no-untyped-def]
|
||||
steps=None, # type: ignore[no-untyped-def]
|
||||
) -> dict[str, object]:
|
||||
payload = task.to_dict()
|
||||
session_context = self.task_context_payload(task.id, task=task, context=context)
|
||||
if session_context:
|
||||
payload["session_context"] = session_context
|
||||
retry_state = self.task_retry_state(task.id, steps=steps)
|
||||
if retry_state:
|
||||
payload["retry_state"] = retry_state
|
||||
payload["delivery_state"] = self.task_delivery_state(task.id, task=task)
|
||||
return payload
|
||||
|
||||
def step_payload(self, step) -> dict[str, object]: # type: ignore[no-untyped-def]
|
||||
payload = step.to_dict()
|
||||
retry_meta = retry_meta_for_step(step, self.state["settings"])
|
||||
if retry_meta:
|
||||
payload.update(retry_meta)
|
||||
return payload
|
||||
|
||||
def task_retry_state(self, task_id: str, *, steps=None) -> dict[str, object] | None: # type: ignore[no-untyped-def]
|
||||
step_items = steps if steps is not None else self.state["repo"].list_steps(task_id)
|
||||
for step in step_items:
|
||||
retry_meta = retry_meta_for_step(step, self.state["settings"])
|
||||
if retry_meta:
|
||||
return {"step_name": step.step_name, **retry_meta}
|
||||
return None
|
||||
|
||||
def task_delivery_state(self, task_id: str, *, task=None) -> dict[str, object]: # type: ignore[no-untyped-def]
|
||||
task = task or self.state["repo"].get_task(task_id)
|
||||
if task is None:
|
||||
return {}
|
||||
session_dir = Path(str(self.state["settings"]["paths"]["session_dir"])) / task.title
|
||||
source_path = Path(task.source_path)
|
||||
split_dir = session_dir / "split_video"
|
||||
|
||||
def comment_status(flag_name: str, *, enabled: bool) -> str:
|
||||
if not enabled:
|
||||
return "disabled"
|
||||
return "done" if (session_dir / flag_name).exists() else "pending"
|
||||
|
||||
return {
|
||||
"split_comment": comment_status("comment_split_done.flag", enabled=self.state["settings"]["comment"].get("post_split_comment", True)),
|
||||
"full_video_timeline_comment": comment_status(
|
||||
"comment_full_done.flag",
|
||||
enabled=self.state["settings"]["comment"].get("post_full_video_timeline_comment", True),
|
||||
),
|
||||
"full_video_bvid_resolved": (session_dir / "full_video_bvid.txt").exists(),
|
||||
"source_video_present": source_path.exists(),
|
||||
"split_videos_present": split_dir.exists(),
|
||||
"cleanup_enabled": {
|
||||
"delete_source_video_after_collection_synced": self.state["settings"].get("cleanup", {}).get("delete_source_video_after_collection_synced", False),
|
||||
"delete_split_videos_after_collection_synced": self.state["settings"].get("cleanup", {}).get("delete_split_videos_after_collection_synced", False),
|
||||
},
|
||||
}
|
||||
|
||||
def task_context_payload(self, task_id: str, *, task=None, context=None) -> dict[str, object] | None: # type: ignore[no-untyped-def]
|
||||
task = task or self.state["repo"].get_task(task_id)
|
||||
if task is None:
|
||||
return None
|
||||
context = context or self.state["repo"].get_task_context(task_id)
|
||||
if context is None:
|
||||
payload = {
|
||||
"task_id": task.id,
|
||||
"session_key": None,
|
||||
"streamer": None,
|
||||
"room_id": None,
|
||||
"source_title": task.title,
|
||||
"segment_started_at": None,
|
||||
"segment_duration_seconds": None,
|
||||
"full_video_bvid": None,
|
||||
"created_at": task.created_at,
|
||||
"updated_at": task.updated_at,
|
||||
"context_source": "fallback",
|
||||
}
|
||||
else:
|
||||
payload = context.to_dict()
|
||||
payload["context_source"] = "task_context"
|
||||
payload["split_bvid"] = self.read_task_text_artifact(task_id, "bvid.txt", task=task)
|
||||
full_video_bvid = self.read_task_text_artifact(task_id, "full_video_bvid.txt", task=task)
|
||||
if full_video_bvid:
|
||||
payload["full_video_bvid"] = full_video_bvid
|
||||
payload["video_links"] = {
|
||||
"split_video_url": self.video_url(payload.get("split_bvid")),
|
||||
"full_video_url": self.video_url(payload.get("full_video_bvid")),
|
||||
}
|
||||
return payload
|
||||
|
||||
def session_payload(self, session_key: str) -> dict[str, object] | None:
|
||||
contexts = self.state["repo"].list_task_contexts_by_session_key(session_key)
|
||||
if not contexts:
|
||||
return None
|
||||
tasks = []
|
||||
full_video_bvid = None
|
||||
for context in contexts:
|
||||
task = self.state["repo"].get_task(context.task_id)
|
||||
if task is None:
|
||||
continue
|
||||
tasks.append(task)
|
||||
if not full_video_bvid and context.full_video_bvid:
|
||||
full_video_bvid = context.full_video_bvid
|
||||
return {
|
||||
"session_key": session_key,
|
||||
"task_count": len(tasks),
|
||||
"full_video_bvid": full_video_bvid,
|
||||
"full_video_url": self.video_url(full_video_bvid),
|
||||
"tasks": self.task_payloads_from_tasks(tasks),
|
||||
}
|
||||
|
||||
def timeline_payload(self, task_id: str) -> dict[str, object] | None:
|
||||
task = self.state["repo"].get_task(task_id)
|
||||
if task is None:
|
||||
return None
|
||||
steps = self.state["repo"].list_steps(task_id)
|
||||
artifacts = self.state["repo"].list_artifacts(task_id)
|
||||
actions = self.state["repo"].list_action_records(task_id, limit=200)
|
||||
items: list[dict[str, object]] = []
|
||||
if task.created_at:
|
||||
items.append({
|
||||
"kind": "task",
|
||||
"time": task.created_at,
|
||||
"title": "Task Created",
|
||||
"summary": task.title,
|
||||
"status": task.status,
|
||||
})
|
||||
if task.updated_at and task.updated_at != task.created_at:
|
||||
items.append({
|
||||
"kind": "task",
|
||||
"time": task.updated_at,
|
||||
"title": "Task Updated",
|
||||
"summary": task.status,
|
||||
"status": task.status,
|
||||
})
|
||||
for step in steps:
|
||||
if step.started_at:
|
||||
items.append({
|
||||
"kind": "step",
|
||||
"time": step.started_at,
|
||||
"title": f"{step.step_name} started",
|
||||
"summary": step.status,
|
||||
"status": step.status,
|
||||
})
|
||||
if step.finished_at:
|
||||
retry_meta = retry_meta_for_step(step, self.state["settings"])
|
||||
retry_note = ""
|
||||
if retry_meta and retry_meta.get("next_retry_at"):
|
||||
retry_note = f" | next retry: {retry_meta['next_retry_at']}"
|
||||
items.append({
|
||||
"kind": "step",
|
||||
"time": step.finished_at,
|
||||
"title": f"{step.step_name} finished",
|
||||
"summary": f"{step.error_message or step.status}{retry_note}",
|
||||
"status": step.status,
|
||||
"retry_state": retry_meta,
|
||||
})
|
||||
for artifact in artifacts:
|
||||
if artifact.created_at:
|
||||
items.append({
|
||||
"kind": "artifact",
|
||||
"time": artifact.created_at,
|
||||
"title": artifact.artifact_type,
|
||||
"summary": artifact.path,
|
||||
"status": "created",
|
||||
})
|
||||
for action in actions:
|
||||
summary = action.summary
|
||||
try:
|
||||
details = json.loads(action.details_json or "{}")
|
||||
except json.JSONDecodeError:
|
||||
details = {}
|
||||
if action.action_name == "comment" and isinstance(details, dict):
|
||||
split_status = details.get("split", {}).get("status")
|
||||
full_status = details.get("full", {}).get("status")
|
||||
fragments = []
|
||||
if split_status:
|
||||
fragments.append(f"split={split_status}")
|
||||
if full_status:
|
||||
fragments.append(f"full={full_status}")
|
||||
if fragments:
|
||||
summary = f"{summary} | {' '.join(fragments)}"
|
||||
if action.action_name in {"collection_a", "collection_b"} and isinstance(details, dict):
|
||||
cleanup = details.get("result", {}).get("cleanup") or details.get("cleanup")
|
||||
if isinstance(cleanup, dict):
|
||||
removed = cleanup.get("removed") or []
|
||||
if removed:
|
||||
summary = f"{summary} | cleanup removed={len(removed)}"
|
||||
items.append({
|
||||
"kind": "action",
|
||||
"time": action.created_at,
|
||||
"title": action.action_name,
|
||||
"summary": summary,
|
||||
"status": action.status,
|
||||
})
|
||||
items.sort(key=lambda item: str(item["time"]), reverse=True)
|
||||
return {"items": items}
|
||||
|
||||
def read_task_text_artifact(self, task_id: str, filename: str, *, task=None) -> str | None: # type: ignore[no-untyped-def]
|
||||
task = task or self.state["repo"].get_task(task_id)
|
||||
if task is None:
|
||||
return None
|
||||
session_dir = Path(str(self.state["settings"]["paths"]["session_dir"])) / task.title
|
||||
path = session_dir / filename
|
||||
if not path.exists():
|
||||
return None
|
||||
value = path.read_text(encoding="utf-8").strip()
|
||||
return value or None
|
||||
254
src/biliup_next/app/session_delivery_service.py
Normal file
254
src/biliup_next/app/session_delivery_service.py
Normal file
@ -0,0 +1,254 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from biliup_next.core.models import ActionRecord, SessionBinding, TaskContext, utc_now_iso
|
||||
|
||||
|
||||
class SessionDeliveryService:
|
||||
def __init__(self, state: dict[str, object]):
|
||||
self.state = state
|
||||
self.repo = state["repo"]
|
||||
self.settings = state["settings"]
|
||||
|
||||
def bind_task_full_video(self, task_id: str, full_video_bvid: str) -> dict[str, object]:
|
||||
task = self.repo.get_task(task_id)
|
||||
if task is None:
|
||||
return {"error": {"code": "TASK_NOT_FOUND", "message": f"task not found: {task_id}"}}
|
||||
|
||||
bvid = self._normalize_bvid(full_video_bvid)
|
||||
if bvid is None:
|
||||
return {"error": {"code": "INVALID_BVID", "message": f"invalid bvid: {full_video_bvid}"}}
|
||||
|
||||
now = utc_now_iso()
|
||||
context = self.repo.get_task_context(task_id)
|
||||
if context is None:
|
||||
context = TaskContext(
|
||||
id=None,
|
||||
task_id=task.id,
|
||||
session_key=f"task:{task.id}",
|
||||
streamer=None,
|
||||
room_id=None,
|
||||
source_title=task.title,
|
||||
segment_started_at=None,
|
||||
segment_duration_seconds=None,
|
||||
full_video_bvid=bvid,
|
||||
created_at=task.created_at,
|
||||
updated_at=now,
|
||||
)
|
||||
full_video_bvid_path = self._persist_task_full_video_bvid(task, context, bvid, now=now)
|
||||
return {
|
||||
"task_id": task.id,
|
||||
"session_key": context.session_key,
|
||||
"full_video_bvid": bvid,
|
||||
"path": str(full_video_bvid_path),
|
||||
}
|
||||
|
||||
def rebind_session_full_video(self, session_key: str, full_video_bvid: str) -> dict[str, object]:
|
||||
bvid = self._normalize_bvid(full_video_bvid)
|
||||
if bvid is None:
|
||||
return {"error": {"code": "INVALID_BVID", "message": f"invalid bvid: {full_video_bvid}"}}
|
||||
|
||||
contexts = self.repo.list_task_contexts_by_session_key(session_key)
|
||||
if not contexts:
|
||||
return {"error": {"code": "SESSION_NOT_FOUND", "message": f"session not found: {session_key}"}}
|
||||
|
||||
now = utc_now_iso()
|
||||
self.repo.update_session_full_video_bvid(session_key, bvid, now)
|
||||
|
||||
updated_tasks: list[dict[str, object]] = []
|
||||
for context in contexts:
|
||||
task = self.repo.get_task(context.task_id)
|
||||
if task is None:
|
||||
continue
|
||||
full_video_bvid_path = self._persist_task_full_video_bvid(task, context, bvid, now=now)
|
||||
updated_tasks.append({"task_id": task.id, "path": str(full_video_bvid_path)})
|
||||
|
||||
return {
|
||||
"session_key": session_key,
|
||||
"full_video_bvid": bvid,
|
||||
"updated_count": len(updated_tasks),
|
||||
"tasks": updated_tasks,
|
||||
}
|
||||
|
||||
def merge_session(self, session_key: str, task_ids: list[str]) -> dict[str, object]:
|
||||
normalized_task_ids: list[str] = []
|
||||
for raw in task_ids:
|
||||
task_id = str(raw).strip()
|
||||
if task_id and task_id not in normalized_task_ids:
|
||||
normalized_task_ids.append(task_id)
|
||||
if not normalized_task_ids:
|
||||
return {"error": {"code": "TASK_IDS_EMPTY", "message": "task_ids is empty"}}
|
||||
|
||||
now = utc_now_iso()
|
||||
inherited_bvid = None
|
||||
existing_contexts = self.repo.list_task_contexts_by_session_key(session_key)
|
||||
for context in existing_contexts:
|
||||
if context.full_video_bvid:
|
||||
inherited_bvid = context.full_video_bvid
|
||||
break
|
||||
|
||||
merged_tasks: list[dict[str, object]] = []
|
||||
missing_tasks: list[str] = []
|
||||
|
||||
for task_id in normalized_task_ids:
|
||||
task = self.repo.get_task(task_id)
|
||||
if task is None:
|
||||
missing_tasks.append(task_id)
|
||||
continue
|
||||
|
||||
context = self.repo.get_task_context(task_id)
|
||||
if context is None:
|
||||
context = TaskContext(
|
||||
id=None,
|
||||
task_id=task.id,
|
||||
session_key=session_key,
|
||||
streamer=None,
|
||||
room_id=None,
|
||||
source_title=task.title,
|
||||
segment_started_at=None,
|
||||
segment_duration_seconds=None,
|
||||
full_video_bvid=inherited_bvid,
|
||||
created_at=task.created_at,
|
||||
updated_at=now,
|
||||
)
|
||||
else:
|
||||
context.session_key = session_key
|
||||
context.updated_at = now
|
||||
if inherited_bvid and not context.full_video_bvid:
|
||||
context.full_video_bvid = inherited_bvid
|
||||
self.repo.upsert_task_context(context)
|
||||
|
||||
if context.full_video_bvid:
|
||||
full_video_bvid_path = self._persist_task_full_video_bvid(task, context, context.full_video_bvid, now=now)
|
||||
else:
|
||||
full_video_bvid_path = None
|
||||
|
||||
payload = {
|
||||
"task_id": task.id,
|
||||
"session_key": session_key,
|
||||
"full_video_bvid": context.full_video_bvid,
|
||||
}
|
||||
if full_video_bvid_path is not None:
|
||||
payload["path"] = str(full_video_bvid_path)
|
||||
merged_tasks.append(payload)
|
||||
|
||||
return {
|
||||
"session_key": session_key,
|
||||
"merged_count": len(merged_tasks),
|
||||
"tasks": merged_tasks,
|
||||
"missing_task_ids": missing_tasks,
|
||||
}
|
||||
|
||||
def receive_full_video_webhook(self, payload: dict[str, object]) -> dict[str, object]:
|
||||
raw_bvid = str(payload.get("full_video_bvid") or payload.get("bvid") or "").strip()
|
||||
bvid = self._normalize_bvid(raw_bvid)
|
||||
if bvid is None:
|
||||
return {"error": {"code": "INVALID_BVID", "message": f"invalid bvid: {raw_bvid}"}}
|
||||
|
||||
session_key = str(payload.get("session_key") or "").strip() or None
|
||||
source_title = str(payload.get("source_title") or "").strip() or None
|
||||
streamer = str(payload.get("streamer") or "").strip() or None
|
||||
room_id = str(payload.get("room_id") or "").strip() or None
|
||||
if session_key is None and source_title is None:
|
||||
return {"error": {"code": "SESSION_KEY_OR_SOURCE_TITLE_REQUIRED", "message": "session_key or source_title required"}}
|
||||
|
||||
now = utc_now_iso()
|
||||
self.repo.upsert_session_binding(
|
||||
SessionBinding(
|
||||
id=None,
|
||||
session_key=session_key,
|
||||
source_title=source_title,
|
||||
streamer=streamer,
|
||||
room_id=room_id,
|
||||
full_video_bvid=bvid,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
|
||||
contexts = self.repo.list_task_contexts_by_session_key(session_key) if session_key else []
|
||||
if not contexts and source_title:
|
||||
contexts = self.repo.list_task_contexts_by_source_title(source_title)
|
||||
|
||||
updated_tasks: list[dict[str, object]] = []
|
||||
for context in contexts:
|
||||
task = self.repo.get_task(context.task_id)
|
||||
if task is None:
|
||||
continue
|
||||
if session_key and (context.session_key.startswith("task:") or context.session_key != session_key):
|
||||
context.session_key = session_key
|
||||
full_video_bvid_path = self._persist_task_full_video_bvid(task, context, bvid, now=now)
|
||||
updated_tasks.append({"task_id": task.id, "path": str(full_video_bvid_path)})
|
||||
|
||||
self.repo.add_action_record(
|
||||
ActionRecord(
|
||||
id=None,
|
||||
task_id=None,
|
||||
action_name="webhook_full_video_uploaded",
|
||||
status="ok",
|
||||
summary=f"full video webhook received: {bvid}",
|
||||
details_json=json.dumps(
|
||||
{
|
||||
"session_key": session_key,
|
||||
"source_title": source_title,
|
||||
"streamer": streamer,
|
||||
"room_id": room_id,
|
||||
"updated_count": len(updated_tasks),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
created_at=now,
|
||||
)
|
||||
)
|
||||
return {
|
||||
"ok": True,
|
||||
"session_key": session_key,
|
||||
"source_title": source_title,
|
||||
"full_video_bvid": bvid,
|
||||
"updated_count": len(updated_tasks),
|
||||
"tasks": updated_tasks,
|
||||
}
|
||||
|
||||
def _normalize_bvid(self, full_video_bvid: str) -> str | None:
|
||||
bvid = full_video_bvid.strip()
|
||||
if not re.fullmatch(r"BV[0-9A-Za-z]+", bvid):
|
||||
return None
|
||||
return bvid
|
||||
|
||||
def _full_video_bvid_path(self, task_title: str) -> Path:
|
||||
session_dir = Path(str(self.settings["paths"]["session_dir"])) / task_title
|
||||
session_dir.mkdir(parents=True, exist_ok=True)
|
||||
return session_dir / "full_video_bvid.txt"
|
||||
|
||||
def _upsert_session_binding_for_context(self, context: TaskContext, full_video_bvid: str, now: str) -> None:
|
||||
self.repo.upsert_session_binding(
|
||||
SessionBinding(
|
||||
id=None,
|
||||
session_key=context.session_key,
|
||||
source_title=context.source_title,
|
||||
streamer=context.streamer,
|
||||
room_id=context.room_id,
|
||||
full_video_bvid=full_video_bvid,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
|
||||
def _persist_task_full_video_bvid(
|
||||
self,
|
||||
task,
|
||||
context: TaskContext,
|
||||
full_video_bvid: str,
|
||||
*,
|
||||
now: str,
|
||||
) -> Path: # type: ignore[no-untyped-def]
|
||||
context.full_video_bvid = full_video_bvid
|
||||
context.updated_at = now
|
||||
self.repo.upsert_task_context(context)
|
||||
self._upsert_session_binding_for_context(context, full_video_bvid, now)
|
||||
path = self._full_video_bvid_path(task.title)
|
||||
path.write_text(full_video_bvid, encoding="utf-8")
|
||||
return path
|
||||
@ -9,13 +9,14 @@ import {
|
||||
setTaskPageSize,
|
||||
state,
|
||||
} from "./state.js";
|
||||
import { showBanner, syncSettingsEditorFromState } from "./utils.js";
|
||||
import { showBanner, syncSettingsEditorFromState, withButtonBusy } from "./utils.js";
|
||||
import { renderSettingsForm } from "./views/settings.js";
|
||||
import { renderTasks } from "./views/tasks.js";
|
||||
|
||||
export function bindActions({
|
||||
loadOverview,
|
||||
loadTaskDetail,
|
||||
refreshSelectedTaskOnly,
|
||||
refreshLog,
|
||||
handleSettingsFieldChange,
|
||||
}) {
|
||||
@ -170,29 +171,33 @@ export function bindActions({
|
||||
|
||||
document.getElementById("runTaskBtn").onclick = async () => {
|
||||
if (!state.selectedTaskId) return showBanner("当前没有选中的任务", "warn");
|
||||
try {
|
||||
const result = await fetchJson(`/tasks/${state.selectedTaskId}/actions/run`, { method: "POST" });
|
||||
await loadOverview();
|
||||
showBanner(`任务已推进,processed=${result.processed.length}`, "ok");
|
||||
} catch (err) {
|
||||
showBanner(`任务执行失败: ${err}`, "err");
|
||||
}
|
||||
await withButtonBusy(document.getElementById("runTaskBtn"), "执行中…", async () => {
|
||||
try {
|
||||
const result = await fetchJson(`/tasks/${state.selectedTaskId}/actions/run`, { method: "POST" });
|
||||
await refreshSelectedTaskOnly(state.selectedTaskId);
|
||||
showBanner(`任务已推进,processed=${result.processed.length}`, "ok");
|
||||
} catch (err) {
|
||||
showBanner(`任务执行失败: ${err}`, "err");
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
document.getElementById("retryStepBtn").onclick = async () => {
|
||||
if (!state.selectedTaskId) return showBanner("当前没有选中的任务", "warn");
|
||||
if (!state.selectedStepName) return showBanner("请先在 Steps 区域选中一个 step", "warn");
|
||||
try {
|
||||
const result = await fetchJson(`/tasks/${state.selectedTaskId}/actions/retry-step`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ step_name: state.selectedStepName }),
|
||||
});
|
||||
await loadOverview();
|
||||
showBanner(`已重试 step=${state.selectedStepName},processed=${result.processed.length}`, "ok");
|
||||
} catch (err) {
|
||||
showBanner(`重试失败: ${err}`, "err");
|
||||
}
|
||||
await withButtonBusy(document.getElementById("retryStepBtn"), "重试中…", async () => {
|
||||
try {
|
||||
const result = await fetchJson(`/tasks/${state.selectedTaskId}/actions/retry-step`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ step_name: state.selectedStepName }),
|
||||
});
|
||||
await refreshSelectedTaskOnly(state.selectedTaskId);
|
||||
showBanner(`已重试 step=${state.selectedStepName},processed=${result.processed.length}`, "ok");
|
||||
} catch (err) {
|
||||
showBanner(`重试失败: ${err}`, "err");
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
document.getElementById("resetStepBtn").onclick = async () => {
|
||||
@ -200,16 +205,18 @@ export function bindActions({
|
||||
if (!state.selectedStepName) return showBanner("请先在 Steps 区域选中一个 step", "warn");
|
||||
const ok = window.confirm(`确认重置到 step=${state.selectedStepName} 并清理其后的产物吗?`);
|
||||
if (!ok) return;
|
||||
try {
|
||||
const result = await fetchJson(`/tasks/${state.selectedTaskId}/actions/reset-to-step`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ step_name: state.selectedStepName }),
|
||||
});
|
||||
await loadOverview();
|
||||
showBanner(`已重置并重跑 step=${state.selectedStepName},processed=${result.run.processed.length}`, "ok");
|
||||
} catch (err) {
|
||||
showBanner(`重置失败: ${err}`, "err");
|
||||
}
|
||||
await withButtonBusy(document.getElementById("resetStepBtn"), "重置中…", async () => {
|
||||
try {
|
||||
const result = await fetchJson(`/tasks/${state.selectedTaskId}/actions/reset-to-step`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ step_name: state.selectedStepName }),
|
||||
});
|
||||
await refreshSelectedTaskOnly(state.selectedTaskId);
|
||||
showBanner(`已重置并重跑 step=${state.selectedStepName},processed=${result.run.processed.length}`, "ok");
|
||||
} catch (err) {
|
||||
showBanner(`重置失败: ${err}`, "err");
|
||||
}
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
@ -40,13 +40,22 @@ export async function loadOverviewPayload() {
|
||||
return { health, doctor, tasks, modules, settings, settingsSchema, services, logs, history, scheduler };
|
||||
}
|
||||
|
||||
export async function loadTasksPayload(limit = 100) {
|
||||
return fetchJson(`/tasks?limit=${limit}`);
|
||||
}
|
||||
|
||||
export async function loadTaskPayload(taskId) {
|
||||
const [task, steps, artifacts, history, timeline] = await Promise.all([
|
||||
const [task, steps, artifacts, history, timeline, context] = await Promise.all([
|
||||
fetchJson(`/tasks/${taskId}`),
|
||||
fetchJson(`/tasks/${taskId}/steps`),
|
||||
fetchJson(`/tasks/${taskId}/artifacts`),
|
||||
fetchJson(`/tasks/${taskId}/history`),
|
||||
fetchJson(`/tasks/${taskId}/timeline`),
|
||||
fetchJson(`/tasks/${taskId}/context`).catch(() => null),
|
||||
]);
|
||||
return { task, steps, artifacts, history, timeline };
|
||||
return { task, steps, artifacts, history, timeline, context };
|
||||
}
|
||||
|
||||
export async function loadSessionPayload(sessionKey) {
|
||||
return fetchJson(`/sessions/${encodeURIComponent(sessionKey)}`);
|
||||
}
|
||||
|
||||
70
src/biliup_next/app/static/app/components/session-panel.js
Normal file
70
src/biliup_next/app/static/app/components/session-panel.js
Normal file
@ -0,0 +1,70 @@
|
||||
import { escapeHtml, taskDisplayStatus } from "../utils.js";
|
||||
|
||||
export function renderSessionPanel(session, actions = {}) {
|
||||
const wrap = document.getElementById("sessionPanel");
|
||||
const stateEl = document.getElementById("sessionWorkspaceState");
|
||||
if (!wrap || !stateEl) return;
|
||||
if (!session) {
|
||||
stateEl.className = "task-workspace-state show";
|
||||
stateEl.textContent = "当前任务如果已绑定 session_key,这里会显示同场片段和完整版绑定信息。";
|
||||
wrap.innerHTML = "";
|
||||
return;
|
||||
}
|
||||
|
||||
stateEl.className = "task-workspace-state";
|
||||
const tasks = session.tasks || [];
|
||||
wrap.innerHTML = `
|
||||
<div class="session-hero">
|
||||
<div>
|
||||
<div class="summary-title">Session Key</div>
|
||||
<div class="session-key">${escapeHtml(session.session_key || "-")}</div>
|
||||
</div>
|
||||
<div class="session-meta-strip">
|
||||
<span class="pill">${escapeHtml(`tasks ${session.task_count || tasks.length || 0}`)}</span>
|
||||
<span class="pill">${escapeHtml(`full BV ${session.full_video_bvid || "-"}`)}</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="session-actions-grid">
|
||||
<div class="bind-form">
|
||||
<div class="summary-title">Session Rebind</div>
|
||||
<input id="sessionRebindInput" value="${escapeHtml(session.full_video_bvid || "")}" placeholder="BV1..." />
|
||||
<div class="button-row">
|
||||
<button id="sessionRebindBtn" class="secondary compact">整个 Session 重绑 BV</button>
|
||||
${session.full_video_url ? `<a class="detail-link session-link-btn" href="${escapeHtml(session.full_video_url)}" target="_blank" rel="noreferrer">打开完整版</a>` : ""}
|
||||
</div>
|
||||
</div>
|
||||
<div class="bind-form">
|
||||
<div class="summary-title">Merge Tasks</div>
|
||||
<input id="sessionMergeInput" placeholder="输入 task id,用逗号分隔" />
|
||||
<div class="button-row">
|
||||
<button id="sessionMergeBtn" class="secondary compact">合并到当前 Session</button>
|
||||
</div>
|
||||
<div class="muted-note">适用于同一场直播断流后产生的多个片段。</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="summary-title" style="margin-top:14px;">Session Tasks</div>
|
||||
<div class="stack-list">
|
||||
${tasks.map((task) => `
|
||||
<div class="row-card session-task-card" data-session-task-id="${escapeHtml(task.id)}">
|
||||
<div class="step-card-title">
|
||||
<strong>${escapeHtml(task.title)}</strong>
|
||||
<span class="pill">${escapeHtml(taskDisplayStatus(task))}</span>
|
||||
</div>
|
||||
<div class="muted-note">${escapeHtml(task.session_context?.split_bvid || "-")} · ${escapeHtml(task.session_context?.full_video_bvid || "-")}</div>
|
||||
</div>
|
||||
`).join("")}
|
||||
</div>
|
||||
`;
|
||||
|
||||
const rebindBtn = document.getElementById("sessionRebindBtn");
|
||||
if (rebindBtn) {
|
||||
rebindBtn.onclick = () => actions.onRebind?.(session.session_key, document.getElementById("sessionRebindInput")?.value || "");
|
||||
}
|
||||
const mergeBtn = document.getElementById("sessionMergeBtn");
|
||||
if (mergeBtn) {
|
||||
mergeBtn.onclick = () => actions.onMerge?.(session.session_key, document.getElementById("sessionMergeInput")?.value || "");
|
||||
}
|
||||
wrap.querySelectorAll("[data-session-task-id]").forEach((node) => {
|
||||
node.onclick = () => actions.onSelectTask?.(node.dataset.sessionTaskId);
|
||||
});
|
||||
}
|
||||
@ -1,22 +1,41 @@
|
||||
import { escapeHtml, statusClass } from "../utils.js";
|
||||
|
||||
function displayTaskStatus(task) {
|
||||
if (task.status === "failed_manual") return "需人工处理";
|
||||
if (task.status === "failed_retryable" && task.retry_state?.step_name === "comment") return "等待B站可见";
|
||||
if (task.status === "failed_retryable") return "等待自动重试";
|
||||
return {
|
||||
created: "已接收",
|
||||
transcribed: "已转录",
|
||||
songs_detected: "已识歌",
|
||||
split_done: "已切片",
|
||||
published: "已上传",
|
||||
collection_synced: "已完成",
|
||||
running: "处理中",
|
||||
}[task.status] || task.status || "-";
|
||||
}
|
||||
|
||||
export function renderTaskHero(task, steps) {
|
||||
const wrap = document.getElementById("taskHero");
|
||||
const succeeded = steps.items.filter((step) => step.status === "succeeded").length;
|
||||
const running = steps.items.filter((step) => step.status === "running").length;
|
||||
const failed = steps.items.filter((step) => step.status.startsWith("failed")).length;
|
||||
const delivery = task.delivery_state || {};
|
||||
const sessionContext = task.session_context || {};
|
||||
wrap.className = "task-hero";
|
||||
wrap.innerHTML = `
|
||||
<div class="task-hero-title">${escapeHtml(task.title)}</div>
|
||||
<div class="task-hero-subtitle">${escapeHtml(task.id)} · ${escapeHtml(task.source_path)}</div>
|
||||
<div class="hero-meta-grid">
|
||||
<div class="mini-stat"><div class="mini-stat-label">Task Status</div><div class="mini-stat-value"><span class="pill ${statusClass(task.status)}">${escapeHtml(task.status)}</span></div></div>
|
||||
<div class="mini-stat"><div class="mini-stat-label">Task Status</div><div class="mini-stat-value"><span class="pill ${statusClass(task.status)}">${escapeHtml(displayTaskStatus(task))}</span></div></div>
|
||||
<div class="mini-stat"><div class="mini-stat-label">Succeeded Steps</div><div class="mini-stat-value">${succeeded}/${steps.items.length}</div></div>
|
||||
<div class="mini-stat"><div class="mini-stat-label">Running / Failed</div><div class="mini-stat-value">${running} / ${failed}</div></div>
|
||||
</div>
|
||||
<div class="task-hero-delivery muted-note">
|
||||
split comment=${escapeHtml(delivery.split_comment || "-")} · full timeline=${escapeHtml(delivery.full_video_timeline_comment || "-")} · source=${delivery.source_video_present ? "present" : "removed"} · split videos=${delivery.split_videos_present ? "present" : "removed"}
|
||||
</div>
|
||||
<div class="task-hero-delivery muted-note">
|
||||
session=${escapeHtml(sessionContext.session_key || "-")} · split_bv=${escapeHtml(sessionContext.split_bvid || "-")} · full_bv=${escapeHtml(sessionContext.full_video_bvid || "-")}
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { fetchJson, loadOverviewPayload, loadTaskPayload } from "./api.js";
|
||||
import { fetchJson, loadOverviewPayload, loadSessionPayload, loadTaskPayload, loadTasksPayload } from "./api.js";
|
||||
import { bindActions } from "./actions.js";
|
||||
import { currentRoute, initRouter, navigate } from "./router.js";
|
||||
import {
|
||||
@ -11,11 +11,12 @@ import {
|
||||
setSelectedLog,
|
||||
setSelectedStep,
|
||||
setSelectedTask,
|
||||
setCurrentSession,
|
||||
setTaskDetailStatus,
|
||||
setTaskListLoading,
|
||||
state,
|
||||
} from "./state.js";
|
||||
import { settingsFieldKey, showBanner } from "./utils.js";
|
||||
import { settingsFieldKey, showBanner, withButtonBusy } from "./utils.js";
|
||||
import {
|
||||
renderDoctor,
|
||||
renderModules,
|
||||
@ -27,6 +28,7 @@ import {
|
||||
import { renderLogContent, renderLogsList } from "./views/logs.js";
|
||||
import { renderSettingsForm } from "./views/settings.js";
|
||||
import { renderTaskDetail, renderTasks, renderTaskWorkspaceState } from "./views/tasks.js";
|
||||
import { renderSessionPanel } from "./components/session-panel.js";
|
||||
|
||||
async function refreshLog() {
|
||||
const name = state.selectedLogName;
|
||||
@ -56,7 +58,41 @@ async function loadTaskDetail(taskId) {
|
||||
renderTaskDetail(payload, async (stepName) => {
|
||||
setSelectedStep(stepName);
|
||||
await loadTaskDetail(taskId);
|
||||
}, {
|
||||
onBindFullVideo: async (currentTaskId, fullVideoBvid) => {
|
||||
const button = document.getElementById("bindFullVideoBtn");
|
||||
const bvid = String(fullVideoBvid || "").trim();
|
||||
if (!/^BV[0-9A-Za-z]+$/.test(bvid)) {
|
||||
showBanner("请输入合法的 BV 号", "warn");
|
||||
return;
|
||||
}
|
||||
await withButtonBusy(button, "绑定中…", async () => {
|
||||
try {
|
||||
await fetchJson(`/tasks/${currentTaskId}/bind-full-video`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ full_video_bvid: bvid }),
|
||||
});
|
||||
await refreshSelectedTaskOnly(currentTaskId);
|
||||
showBanner(`已绑定完整版 BV: ${bvid}`, "ok");
|
||||
} catch (err) {
|
||||
showBanner(`绑定完整版失败: ${err}`, "err");
|
||||
}
|
||||
});
|
||||
},
|
||||
onOpenSession: async (sessionKey) => {
|
||||
if (!sessionKey) {
|
||||
showBanner("当前任务没有可用的 session_key", "warn");
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await loadSessionDetail(sessionKey);
|
||||
} catch (err) {
|
||||
showBanner(`读取 Session 失败: ${err}`, "err");
|
||||
}
|
||||
},
|
||||
});
|
||||
await loadSessionDetail(payload.task.session_context?.session_key || payload.context?.session_key || null);
|
||||
setTaskDetailStatus("ready");
|
||||
renderTaskWorkspaceState("ready");
|
||||
} catch (err) {
|
||||
@ -67,6 +103,79 @@ async function loadTaskDetail(taskId) {
|
||||
}
|
||||
}
|
||||
|
||||
async function loadSessionDetail(sessionKey) {
|
||||
if (!sessionKey) {
|
||||
setCurrentSession(null);
|
||||
renderSessionPanel(null);
|
||||
return;
|
||||
}
|
||||
const session = await loadSessionPayload(sessionKey);
|
||||
setCurrentSession(session);
|
||||
renderSessionPanel(session, {
|
||||
onSelectTask: async (taskId) => {
|
||||
if (!taskId) return;
|
||||
taskSelectHandler(taskId);
|
||||
},
|
||||
onRebind: async (currentSessionKey, fullVideoBvid) => {
|
||||
const button = document.getElementById("sessionRebindBtn");
|
||||
const bvid = String(fullVideoBvid || "").trim();
|
||||
if (!/^BV[0-9A-Za-z]+$/.test(bvid)) {
|
||||
showBanner("请输入合法的 BV 号", "warn");
|
||||
return;
|
||||
}
|
||||
await withButtonBusy(button, "重绑中…", async () => {
|
||||
try {
|
||||
await fetchJson(`/sessions/${encodeURIComponent(currentSessionKey)}/rebind`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ full_video_bvid: bvid }),
|
||||
});
|
||||
await refreshSelectedTaskOnly();
|
||||
showBanner(`Session 已重绑完整版 BV: ${bvid}`, "ok");
|
||||
} catch (err) {
|
||||
showBanner(`Session 重绑失败: ${err}`, "err");
|
||||
}
|
||||
});
|
||||
},
|
||||
onMerge: async (currentSessionKey, rawTaskIds) => {
|
||||
const button = document.getElementById("sessionMergeBtn");
|
||||
const taskIds = String(rawTaskIds || "")
|
||||
.split(",")
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean);
|
||||
if (!taskIds.length) {
|
||||
showBanner("请先输入至少一个 task id", "warn");
|
||||
return;
|
||||
}
|
||||
await withButtonBusy(button, "合并中…", async () => {
|
||||
try {
|
||||
await fetchJson(`/sessions/${encodeURIComponent(currentSessionKey)}/merge`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ task_ids: taskIds }),
|
||||
});
|
||||
await refreshSelectedTaskOnly();
|
||||
showBanner(`已合并 ${taskIds.length} 个任务到当前 Session`, "ok");
|
||||
} catch (err) {
|
||||
showBanner(`Session 合并失败: ${err}`, "err");
|
||||
}
|
||||
});
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function refreshTaskListOnly() {
|
||||
const payload = await loadTasksPayload(100);
|
||||
state.currentTasks = payload.items || [];
|
||||
renderTasks(taskSelectHandler, taskRowActionHandler);
|
||||
}
|
||||
|
||||
async function refreshSelectedTaskOnly(taskId = state.selectedTaskId) {
|
||||
if (!taskId) return;
|
||||
await refreshTaskListOnly();
|
||||
await loadTaskDetail(taskId);
|
||||
}
|
||||
|
||||
function taskSelectHandler(taskId) {
|
||||
setSelectedTask(taskId);
|
||||
setSelectedStep(null);
|
||||
@ -79,7 +188,7 @@ async function taskRowActionHandler(action, taskId) {
|
||||
if (action !== "run") return;
|
||||
try {
|
||||
const result = await fetchJson(`/tasks/${taskId}/actions/run`, { method: "POST" });
|
||||
await loadOverview();
|
||||
await refreshSelectedTaskOnly(taskId);
|
||||
showBanner(`任务已推进: ${taskId} / processed=${result.processed.length}`, "ok");
|
||||
} catch (err) {
|
||||
showBanner(`任务执行失败: ${err}`, "err");
|
||||
@ -201,6 +310,7 @@ async function handleRouteChange(route) {
|
||||
bindActions({
|
||||
loadOverview,
|
||||
loadTaskDetail,
|
||||
refreshSelectedTaskOnly,
|
||||
refreshLog,
|
||||
handleSettingsFieldChange,
|
||||
});
|
||||
|
||||
@ -13,6 +13,7 @@ export const state = {
|
||||
taskListLoading: true,
|
||||
taskDetailStatus: "idle",
|
||||
taskDetailError: "",
|
||||
currentSession: null,
|
||||
currentLogs: [],
|
||||
selectedLogName: null,
|
||||
logListLoading: true,
|
||||
@ -74,6 +75,10 @@ export function setTaskDetailStatus(status, error = "") {
|
||||
state.taskDetailError = error;
|
||||
}
|
||||
|
||||
export function setCurrentSession(session) {
|
||||
state.currentSession = session;
|
||||
}
|
||||
|
||||
export function setLogs(logs) {
|
||||
state.currentLogs = logs;
|
||||
}
|
||||
|
||||
@ -1,9 +1,11 @@
|
||||
import { state } from "./state.js";
|
||||
|
||||
let bannerTimer = null;
|
||||
|
||||
export function statusClass(status) {
|
||||
if (["collection_synced", "published", "commented", "succeeded", "active"].includes(status)) return "good";
|
||||
if (["done", "resolved", "present"].includes(status)) return "good";
|
||||
if (["legacy_untracked", "pending", "unresolved"].includes(status)) return "warn";
|
||||
if (["pending", "unresolved"].includes(status)) return "warn";
|
||||
if (["removed", "disabled"].includes(status)) return "";
|
||||
if (["failed_manual", "failed_retryable", "inactive"].includes(status)) return "hot";
|
||||
if (["running", "activating", "songs_detected", "split_done", "transcribed", "created", "pending"].includes(status)) return "warn";
|
||||
@ -14,6 +16,11 @@ export function showBanner(message, kind) {
|
||||
const el = document.getElementById("banner");
|
||||
el.textContent = message;
|
||||
el.className = `banner show ${kind}`;
|
||||
if (bannerTimer) window.clearTimeout(bannerTimer);
|
||||
bannerTimer = window.setTimeout(() => {
|
||||
el.className = "banner";
|
||||
el.textContent = "";
|
||||
}, kind === "err" ? 6000 : 3200);
|
||||
}
|
||||
|
||||
export function escapeHtml(text) {
|
||||
@ -59,3 +66,92 @@ export function compareFieldEntries(a, b) {
|
||||
export function settingsFieldKey(group, field) {
|
||||
return `${group}.${field}`;
|
||||
}
|
||||
|
||||
export function taskDisplayStatus(task) {
|
||||
if (!task) return "-";
|
||||
if (task.status === "failed_manual") return "需人工处理";
|
||||
if (task.status === "failed_retryable" && task.retry_state?.step_name === "comment") return "等待B站可见";
|
||||
if (task.status === "failed_retryable") return "等待自动重试";
|
||||
return {
|
||||
created: "已接收",
|
||||
transcribed: "已转录",
|
||||
songs_detected: "已识歌",
|
||||
split_done: "已切片",
|
||||
published: "已上传",
|
||||
commented: "评论完成",
|
||||
collection_synced: "已完成",
|
||||
running: "处理中",
|
||||
}[task.status] || task.status || "-";
|
||||
}
|
||||
|
||||
export function taskPrimaryActionLabel(task) {
|
||||
if (!task) return "执行";
|
||||
if (task.status === "failed_manual") return "人工重跑";
|
||||
if (task.retry_state?.retry_due) return "立即重试";
|
||||
if (task.status === "failed_retryable") return "继续等待";
|
||||
if (task.status === "collection_synced") return "查看结果";
|
||||
return "执行";
|
||||
}
|
||||
|
||||
export function taskCurrentStep(task, steps = []) {
|
||||
const running = steps.find((step) => step.status === "running");
|
||||
if (running) return stepLabel(running.step_name);
|
||||
if (task?.retry_state?.step_name) return `${stepLabel(task.retry_state.step_name)}: ${taskDisplayStatus(task)}`;
|
||||
const pending = steps.find((step) => step.status === "pending");
|
||||
if (pending) return stepLabel(pending.step_name);
|
||||
return {
|
||||
created: "转录字幕",
|
||||
transcribed: "识别歌曲",
|
||||
songs_detected: "切分分P",
|
||||
split_done: "上传分P",
|
||||
published: "评论与合集",
|
||||
commented: "同步合集",
|
||||
collection_synced: "链路完成",
|
||||
}[task?.status] || "-";
|
||||
}
|
||||
|
||||
export function stepLabel(stepName) {
|
||||
return {
|
||||
ingest: "接收视频",
|
||||
transcribe: "转录字幕",
|
||||
song_detect: "识别歌曲",
|
||||
split: "切分分P",
|
||||
publish: "上传分P",
|
||||
comment: "发布评论",
|
||||
collection_a: "加入完整版合集",
|
||||
collection_b: "加入分P合集",
|
||||
}[stepName] || stepName || "-";
|
||||
}
|
||||
|
||||
export function actionAdvice(task) {
|
||||
if (!task) return "";
|
||||
if (task.status === "failed_retryable" && task.retry_state?.step_name === "comment") {
|
||||
return "B站通常需要一段时间完成转码和审核,系统会自动重试评论。";
|
||||
}
|
||||
if (task.status === "failed_retryable") {
|
||||
return "当前错误可自动恢复,等到重试时间或手工触发即可。";
|
||||
}
|
||||
if (task.status === "failed_manual") {
|
||||
return "这个任务需要人工判断,先看错误信息,再决定是重试当前步骤还是绑定完整版 BV。";
|
||||
}
|
||||
if (task.status === "collection_synced") {
|
||||
return "链路已完成,可以直接打开分P链接检查结果。";
|
||||
}
|
||||
return "系统会继续推进后续步骤,必要时可在这里手工干预。";
|
||||
}
|
||||
|
||||
export async function withButtonBusy(button, loadingText, fn) {
|
||||
if (!button) return fn();
|
||||
const originalHtml = button.innerHTML;
|
||||
const originalDisabled = button.disabled;
|
||||
button.disabled = true;
|
||||
button.classList.add("is-busy");
|
||||
if (loadingText) button.textContent = loadingText;
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
button.disabled = originalDisabled;
|
||||
button.classList.remove("is-busy");
|
||||
button.innerHTML = originalHtml;
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,5 +1,14 @@
|
||||
import { state, setTaskPage } from "../state.js";
|
||||
import { escapeHtml, formatDate, formatDuration, statusClass } from "../utils.js";
|
||||
import {
|
||||
actionAdvice,
|
||||
escapeHtml,
|
||||
formatDate,
|
||||
formatDuration,
|
||||
statusClass,
|
||||
taskCurrentStep,
|
||||
taskDisplayStatus,
|
||||
taskPrimaryActionLabel,
|
||||
} from "../utils.js";
|
||||
import { renderArtifactList } from "../components/artifact-list.js";
|
||||
import { renderHistoryList } from "../components/history-list.js";
|
||||
import { renderRetryPanel } from "../components/retry-banner.js";
|
||||
@ -8,13 +17,13 @@ import { renderTaskHero } from "../components/task-hero.js";
|
||||
import { renderTimelineList } from "../components/timeline-list.js";
|
||||
|
||||
const STATUS_LABELS = {
|
||||
created: "待转录",
|
||||
transcribed: "待识歌",
|
||||
songs_detected: "待切歌",
|
||||
split_done: "待上传",
|
||||
published: "待收尾",
|
||||
created: "已接收",
|
||||
transcribed: "已转录",
|
||||
songs_detected: "已识歌",
|
||||
split_done: "已切片",
|
||||
published: "已上传",
|
||||
collection_synced: "已完成",
|
||||
failed_retryable: "待重试",
|
||||
failed_retryable: "等待重试",
|
||||
failed_manual: "待人工",
|
||||
running: "处理中",
|
||||
};
|
||||
@ -22,15 +31,17 @@ const STATUS_LABELS = {
|
||||
const DELIVERY_LABELS = {
|
||||
done: "已发送",
|
||||
pending: "待处理",
|
||||
legacy_untracked: "历史未追踪",
|
||||
resolved: "已定位",
|
||||
unresolved: "未定位",
|
||||
present: "保留",
|
||||
removed: "已清理",
|
||||
};
|
||||
|
||||
function displayStatus(status) {
|
||||
return STATUS_LABELS[status] || status || "-";
|
||||
function displayTaskStatus(task) {
|
||||
if (task.status === "failed_manual") return "需人工处理";
|
||||
if (task.status === "failed_retryable" && task.retry_state?.step_name === "comment") return "等待B站可见";
|
||||
if (task.status === "failed_retryable") return "等待自动重试";
|
||||
return taskDisplayStatus(task);
|
||||
}
|
||||
|
||||
function displayDelivery(status) {
|
||||
@ -162,7 +173,6 @@ export function filteredTasks() {
|
||||
if (search && !haystack.includes(search)) return false;
|
||||
if (status && task.status !== status) return false;
|
||||
const deliveryState = task.delivery_state || {};
|
||||
if (delivery === "legacy_untracked" && deliveryState.full_video_timeline_comment !== "legacy_untracked") return false;
|
||||
if (delivery === "pending_comment" && deliveryState.split_comment !== "pending" && deliveryState.full_video_timeline_comment !== "pending") return false;
|
||||
if (delivery === "cleanup_removed" && deliveryState.source_video_present !== false && deliveryState.split_videos_present !== false) return false;
|
||||
if (attention && attentionState(task) !== attention) return false;
|
||||
@ -304,9 +314,9 @@ export function renderTasks(onSelect, onRowAction = null) {
|
||||
row.innerHTML = `
|
||||
<td>
|
||||
<div class="task-cell-title">${escapeHtml(item.title)}</div>
|
||||
<div class="task-cell-subtitle">${escapeHtml(item.id)}</div>
|
||||
<div class="task-cell-subtitle">${escapeHtml(taskCurrentStep(item))}</div>
|
||||
</td>
|
||||
<td><span class="pill ${statusClass(item.status)}">${escapeHtml(displayStatus(item.status))}</span></td>
|
||||
<td><span class="pill ${statusClass(item.status)}">${escapeHtml(displayTaskStatus(item))}</span></td>
|
||||
<td><span class="pill ${attentionClass(attention)}">${escapeHtml(displayAttention(attention))}</span></td>
|
||||
<td><span class="pill ${statusClass(delivery.split_comment || "")}">${escapeHtml(displayDelivery(delivery.split_comment || "-"))}</span></td>
|
||||
<td><span class="pill ${statusClass(delivery.full_video_timeline_comment || "")}">${escapeHtml(displayDelivery(delivery.full_video_timeline_comment || "-"))}</span></td>
|
||||
@ -321,7 +331,7 @@ export function renderTasks(onSelect, onRowAction = null) {
|
||||
</td>
|
||||
<td class="task-table-actions">
|
||||
<button class="secondary compact inline-action-btn" data-task-action="open">打开</button>
|
||||
<button class="compact inline-action-btn" data-task-action="run">${attention === "manual_now" || attention === "retry_now" ? "重跑" : "执行"}</button>
|
||||
<button class="compact inline-action-btn" data-task-action="run">${escapeHtml(taskPrimaryActionLabel(item))}</button>
|
||||
</td>
|
||||
`;
|
||||
row.onclick = () => onSelect(item.id);
|
||||
@ -346,7 +356,7 @@ export function renderTasks(onSelect, onRowAction = null) {
|
||||
wrap.appendChild(table);
|
||||
}
|
||||
|
||||
export function renderTaskDetail(payload, onStepSelect) {
|
||||
export function renderTaskDetail(payload, onStepSelect, actions = {}) {
|
||||
const { task, steps, artifacts, history, timeline } = payload;
|
||||
renderTaskHero(task, steps);
|
||||
renderRetryPanel(task);
|
||||
@ -355,7 +365,8 @@ export function renderTaskDetail(payload, onStepSelect) {
|
||||
detail.innerHTML = "";
|
||||
[
|
||||
["Task ID", task.id],
|
||||
["Status", task.status],
|
||||
["Status", displayTaskStatus(task)],
|
||||
["Current Step", taskCurrentStep(task, steps.items)],
|
||||
["Created", formatDate(task.created_at)],
|
||||
["Updated", formatDate(task.updated_at)],
|
||||
["Source", task.source_path],
|
||||
@ -385,10 +396,40 @@ export function renderTaskDetail(payload, onStepSelect) {
|
||||
}
|
||||
}
|
||||
const delivery = task.delivery_state || {};
|
||||
const sessionContext = task.session_context || {};
|
||||
const splitVideoUrl = sessionContext.video_links?.split_video_url;
|
||||
const fullVideoUrl = sessionContext.video_links?.full_video_url;
|
||||
const summaryEl = document.getElementById("taskSummary");
|
||||
summaryEl.innerHTML = `
|
||||
<div class="summary-title">Recent Result</div>
|
||||
<div class="summary-text">${escapeHtml(summaryText)}</div>
|
||||
<div class="summary-title" style="margin-top:14px;">Recommended Next Step</div>
|
||||
<div class="summary-text">${escapeHtml(actionAdvice(task))}</div>
|
||||
<div class="summary-title" style="margin-top:14px;">Delivery Links</div>
|
||||
<div class="delivery-grid">
|
||||
${renderDeliveryState("Split BV", sessionContext.split_bvid || "-", "")}
|
||||
${renderDeliveryState("Full BV", sessionContext.full_video_bvid || "-", "")}
|
||||
${renderLinkState("Split Video", splitVideoUrl)}
|
||||
${renderLinkState("Full Video", fullVideoUrl)}
|
||||
</div>
|
||||
<div class="summary-title" style="margin-top:14px;">Session Context</div>
|
||||
<div class="delivery-grid">
|
||||
${renderDeliveryState("Session Key", sessionContext.session_key || "-", "")}
|
||||
${renderDeliveryState("Streamer", sessionContext.streamer || "-", "")}
|
||||
${renderDeliveryState("Room ID", sessionContext.room_id || "-", "")}
|
||||
${renderDeliveryState("Context Source", sessionContext.context_source || "-", "")}
|
||||
${renderDeliveryState("Segment Start", sessionContext.segment_started_at ? formatDate(sessionContext.segment_started_at) : "-", "")}
|
||||
${renderDeliveryState("Segment Duration", sessionContext.segment_duration_seconds != null ? formatDuration(sessionContext.segment_duration_seconds) : "-", "")}
|
||||
</div>
|
||||
<div class="summary-title" style="margin-top:14px;">Bind Full Video BV</div>
|
||||
<div class="bind-form">
|
||||
<input id="bindFullVideoInput" value="${escapeHtml(sessionContext.full_video_bvid || "")}" placeholder="BV1..." />
|
||||
<div class="button-row">
|
||||
<button id="bindFullVideoBtn" class="secondary compact">绑定完整版 BV</button>
|
||||
${sessionContext.session_key ? `<button id="openSessionBtn" class="secondary compact">查看 Session</button>` : ""}
|
||||
</div>
|
||||
<div class="muted-note">用于修复评论 / 合集查不到完整版视频的问题。</div>
|
||||
</div>
|
||||
<div class="summary-title" style="margin-top:14px;">Delivery State</div>
|
||||
<div class="delivery-grid">
|
||||
${renderDeliveryState("Split Comment", delivery.split_comment || "-")}
|
||||
@ -403,6 +444,14 @@ export function renderTaskDetail(payload, onStepSelect) {
|
||||
)}
|
||||
</div>
|
||||
`;
|
||||
const bindBtn = document.getElementById("bindFullVideoBtn");
|
||||
if (bindBtn) {
|
||||
bindBtn.onclick = () => actions.onBindFullVideo?.(task.id, document.getElementById("bindFullVideoInput")?.value || "");
|
||||
}
|
||||
const openSessionBtn = document.getElementById("openSessionBtn");
|
||||
if (openSessionBtn) {
|
||||
openSessionBtn.onclick = () => actions.onOpenSession?.(sessionContext.session_key);
|
||||
}
|
||||
|
||||
renderStepList(steps, onStepSelect);
|
||||
renderArtifactList(artifacts);
|
||||
@ -420,8 +469,21 @@ function renderDeliveryState(label, value, forcedClass = null) {
|
||||
`;
|
||||
}
|
||||
|
||||
function renderLinkState(label, url) {
|
||||
return `
|
||||
<div class="delivery-card">
|
||||
<div class="delivery-label">${escapeHtml(label)}</div>
|
||||
<div class="delivery-value">
|
||||
${url ? `<a class="detail-link" href="${escapeHtml(url)}" target="_blank" rel="noreferrer">打开</a>` : `<span class="muted-note">-</span>`}
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
export function renderTaskWorkspaceState(mode, message = "") {
|
||||
const stateEl = document.getElementById("taskWorkspaceState");
|
||||
const sessionStateEl = document.getElementById("sessionWorkspaceState");
|
||||
const sessionPanel = document.getElementById("sessionPanel");
|
||||
const hero = document.getElementById("taskHero");
|
||||
const retry = document.getElementById("taskRetryPanel");
|
||||
const detail = document.getElementById("taskDetail");
|
||||
@ -459,4 +521,11 @@ export function renderTaskWorkspaceState(mode, message = "") {
|
||||
artifactList.innerHTML = "";
|
||||
historyList.innerHTML = "";
|
||||
timelineList.innerHTML = "";
|
||||
if (sessionStateEl) {
|
||||
sessionStateEl.className = "task-workspace-state show";
|
||||
sessionStateEl.textContent = mode === "error"
|
||||
? "Session 区域暂不可用。"
|
||||
: "当前任务如果已绑定 session_key,这里会显示同场片段和完整版绑定信息。";
|
||||
}
|
||||
if (sessionPanel) sessionPanel.innerHTML = "";
|
||||
}
|
||||
|
||||
@ -134,6 +134,11 @@ button.compact {
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
button.is-busy {
|
||||
opacity: 0.72;
|
||||
cursor: wait;
|
||||
}
|
||||
|
||||
.content {
|
||||
display: grid;
|
||||
gap: 16px;
|
||||
@ -258,6 +263,79 @@ button.compact {
|
||||
line-height: 1.6;
|
||||
}
|
||||
|
||||
.task-cell-subtitle {
|
||||
margin-top: 4px;
|
||||
color: var(--muted);
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.bind-form {
|
||||
display: grid;
|
||||
gap: 10px;
|
||||
margin-top: 10px;
|
||||
}
|
||||
|
||||
.bind-form input {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.detail-link {
|
||||
color: var(--accent-2);
|
||||
text-decoration: none;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.detail-link:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
|
||||
.session-panel {
|
||||
display: grid;
|
||||
gap: 16px;
|
||||
}
|
||||
|
||||
.session-hero {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
gap: 12px;
|
||||
align-items: flex-start;
|
||||
}
|
||||
|
||||
.session-key {
|
||||
margin-top: 6px;
|
||||
font-size: 20px;
|
||||
font-weight: 700;
|
||||
letter-spacing: -0.02em;
|
||||
}
|
||||
|
||||
.session-meta-strip,
|
||||
.session-actions-grid {
|
||||
display: grid;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.session-actions-grid {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
}
|
||||
|
||||
.session-task-card {
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.session-task-card:hover {
|
||||
border-color: var(--line-strong);
|
||||
}
|
||||
|
||||
.session-link-btn {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 12px;
|
||||
padding: 8px 12px;
|
||||
background: rgba(255,255,255,0.78);
|
||||
}
|
||||
|
||||
.delivery-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
|
||||
@ -1,29 +1,98 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from biliup_next.app.bootstrap import ensure_initialized
|
||||
from biliup_next.app.task_control_service import TaskControlService
|
||||
from biliup_next.app.session_delivery_service import SessionDeliveryService
|
||||
from biliup_next.app.task_audit import record_task_action
|
||||
from biliup_next.app.task_runner import process_task
|
||||
from biliup_next.infra.task_reset import TaskResetService
|
||||
|
||||
|
||||
def run_task_action(task_id: str) -> dict[str, object]:
|
||||
result = process_task(task_id)
|
||||
state = ensure_initialized()
|
||||
result = TaskControlService(state).run_task(task_id)
|
||||
record_task_action(state["repo"], task_id, "task_run", "ok", "task run invoked", result)
|
||||
return result
|
||||
|
||||
|
||||
def retry_step_action(task_id: str, step_name: str) -> dict[str, object]:
|
||||
result = process_task(task_id, reset_step=step_name)
|
||||
state = ensure_initialized()
|
||||
result = TaskControlService(state).retry_step(task_id, step_name)
|
||||
record_task_action(state["repo"], task_id, "retry_step", "ok", f"retry step invoked: {step_name}", result)
|
||||
return result
|
||||
|
||||
|
||||
def reset_to_step_action(task_id: str, step_name: str) -> dict[str, object]:
|
||||
state = ensure_initialized()
|
||||
reset_result = TaskResetService(state["repo"]).reset_to_step(task_id, step_name)
|
||||
process_result = process_task(task_id)
|
||||
payload = {"reset": reset_result, "run": process_result}
|
||||
payload = TaskControlService(state).reset_to_step(task_id, step_name)
|
||||
record_task_action(state["repo"], task_id, "reset_to_step", "ok", f"reset to step invoked: {step_name}", payload)
|
||||
return payload
|
||||
|
||||
|
||||
def bind_full_video_action(task_id: str, full_video_bvid: str) -> dict[str, object]:
|
||||
state = ensure_initialized()
|
||||
payload = SessionDeliveryService(state).bind_task_full_video(task_id, full_video_bvid)
|
||||
if "error" in payload:
|
||||
return payload
|
||||
record_task_action(
|
||||
state["repo"],
|
||||
task_id,
|
||||
"bind_full_video",
|
||||
"ok",
|
||||
f"full video bvid bound: {payload['full_video_bvid']}",
|
||||
payload,
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
def rebind_session_full_video_action(session_key: str, full_video_bvid: str) -> dict[str, object]:
|
||||
state = ensure_initialized()
|
||||
payload = SessionDeliveryService(state).rebind_session_full_video(session_key, full_video_bvid)
|
||||
if "error" in payload:
|
||||
return payload
|
||||
|
||||
for item in payload["tasks"]:
|
||||
record_task_action(
|
||||
state["repo"],
|
||||
item["task_id"],
|
||||
"rebind_session_full_video",
|
||||
"ok",
|
||||
f"session full video bvid rebound: {payload['full_video_bvid']}",
|
||||
{
|
||||
"session_key": session_key,
|
||||
"full_video_bvid": payload["full_video_bvid"],
|
||||
"path": item["path"],
|
||||
},
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
def merge_session_action(session_key: str, task_ids: list[str]) -> dict[str, object]:
|
||||
state = ensure_initialized()
|
||||
payload = SessionDeliveryService(state).merge_session(session_key, task_ids)
|
||||
if "error" in payload:
|
||||
return payload
|
||||
for item in payload["tasks"]:
|
||||
record_task_action(state["repo"], item["task_id"], "merge_session", "ok", f"task merged into session: {session_key}", item)
|
||||
return payload
|
||||
|
||||
|
||||
def receive_full_video_webhook(payload: dict[str, object]) -> dict[str, object]:
|
||||
state = ensure_initialized()
|
||||
result = SessionDeliveryService(state).receive_full_video_webhook(payload)
|
||||
if "error" in result:
|
||||
return result
|
||||
|
||||
for item in result["tasks"]:
|
||||
record_task_action(
|
||||
state["repo"],
|
||||
item["task_id"],
|
||||
"webhook_full_video_uploaded",
|
||||
"ok",
|
||||
f"full video bvid received via webhook: {result['full_video_bvid']}",
|
||||
{
|
||||
"session_key": result["session_key"],
|
||||
"source_title": result["source_title"],
|
||||
"full_video_bvid": result["full_video_bvid"],
|
||||
"path": item["path"],
|
||||
},
|
||||
)
|
||||
return result
|
||||
|
||||
25
src/biliup_next/app/task_control_service.py
Normal file
25
src/biliup_next/app/task_control_service.py
Normal file
@ -0,0 +1,25 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from biliup_next.app.task_runner import process_task
|
||||
from biliup_next.infra.task_reset import TaskResetService
|
||||
|
||||
|
||||
class TaskControlService:
|
||||
def __init__(self, state: dict[str, object]):
|
||||
self.state = state
|
||||
|
||||
def run_task(self, task_id: str) -> dict[str, object]:
|
||||
return process_task(task_id)
|
||||
|
||||
def retry_step(self, task_id: str, step_name: str) -> dict[str, object]:
|
||||
return process_task(task_id, reset_step=step_name)
|
||||
|
||||
def reset_to_step(self, task_id: str, step_name: str) -> dict[str, object]:
|
||||
reset_result = TaskResetService(
|
||||
self.state["repo"],
|
||||
Path(str(self.state["settings"]["paths"]["session_dir"])),
|
||||
).reset_to_step(task_id, step_name)
|
||||
process_result = process_task(task_id)
|
||||
return {"reset": reset_result, "run": process_result}
|
||||
@ -22,6 +22,12 @@ def settings_for(state: dict[str, object], group: str) -> dict[str, object]:
|
||||
|
||||
|
||||
def infer_error_step_name(task, steps: dict[str, object]) -> str: # type: ignore[no-untyped-def]
|
||||
running = next((step for step in steps.values() if step.status == "running"), None)
|
||||
if running is not None:
|
||||
return running.step_name
|
||||
failed = next((step for step in steps.values() if step.status == "failed_retryable"), None)
|
||||
if failed is not None:
|
||||
return failed.step_name
|
||||
if task.status in {"created", "failed_retryable"} and steps.get("transcribe") and steps["transcribe"].status in {"pending", "failed_retryable", "running"}:
|
||||
return "transcribe"
|
||||
if task.status == "transcribed":
|
||||
@ -57,6 +63,9 @@ def retry_wait_payload(task_id: str, step, state: dict[str, object]) -> dict[str
|
||||
|
||||
|
||||
def next_runnable_step(task, steps: dict[str, object], state: dict[str, object]) -> tuple[str | None, dict[str, object] | None]: # type: ignore[no-untyped-def]
|
||||
if any(step.status == "running" for step in steps.values()):
|
||||
return None, None
|
||||
|
||||
if task.status == "failed_retryable":
|
||||
failed = next((step for step in steps.values() if step.status == "failed_retryable"), None)
|
||||
if failed is None:
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from biliup_next.app.retry_meta import comment_retry_schedule_seconds
|
||||
from biliup_next.app.retry_meta import publish_retry_schedule_seconds
|
||||
from biliup_next.app.task_engine import infer_error_step_name, settings_for as task_engine_settings_for
|
||||
from biliup_next.core.models import utc_now_iso
|
||||
@ -40,6 +41,12 @@ def resolve_failure(task, repo, state: dict[str, object], exc) -> dict[str, obje
|
||||
next_status = "failed_manual"
|
||||
else:
|
||||
next_retry_delay_seconds = schedule[next_retry_count - 1]
|
||||
if exc.retryable and step_name == "comment":
|
||||
schedule = comment_retry_schedule_seconds(settings_for(state, "comment"))
|
||||
if next_retry_count > len(schedule):
|
||||
next_status = "failed_manual"
|
||||
else:
|
||||
next_retry_delay_seconds = schedule[next_retry_count - 1]
|
||||
failed_at = utc_now_iso()
|
||||
repo.update_step_status(
|
||||
task.id,
|
||||
|
||||
@ -10,6 +10,7 @@ from biliup_next.app.task_policies import apply_disabled_step_fallbacks
|
||||
from biliup_next.app.task_policies import resolve_failure
|
||||
from biliup_next.core.errors import ModuleError
|
||||
from biliup_next.core.models import utc_now_iso
|
||||
from biliup_next.infra.task_reset import STATUS_BEFORE_STEP
|
||||
|
||||
|
||||
def process_task(task_id: str, *, reset_step: str | None = None, include_stage_scan: bool = False) -> dict[str, object]:
|
||||
@ -41,7 +42,8 @@ def process_task(task_id: str, *, reset_step: str | None = None, include_stage_s
|
||||
started_at=None,
|
||||
finished_at=None,
|
||||
)
|
||||
repo.update_task_status(task_id, task.status, utc_now_iso())
|
||||
target_status = STATUS_BEFORE_STEP.get(reset_step, "created")
|
||||
repo.update_task_status(task_id, target_status, utc_now_iso())
|
||||
processed.append({"task_id": task_id, "step": reset_step, "reset": True})
|
||||
record_task_action(repo, task_id, "retry_step", "ok", f"step reset to pending: {reset_step}", {"step_name": reset_step})
|
||||
|
||||
@ -60,6 +62,19 @@ def process_task(task_id: str, *, reset_step: str | None = None, include_stage_s
|
||||
if step_name is None:
|
||||
break
|
||||
|
||||
claimed_at = utc_now_iso()
|
||||
if not repo.claim_step_running(task.id, step_name, started_at=claimed_at):
|
||||
processed.append(
|
||||
{
|
||||
"task_id": task.id,
|
||||
"step": step_name,
|
||||
"skipped": True,
|
||||
"reason": "step_already_claimed",
|
||||
}
|
||||
)
|
||||
return {"processed": processed}
|
||||
repo.update_task_status(task.id, "running", claimed_at)
|
||||
|
||||
payload = execute_step(state, task.id, step_name)
|
||||
if current_task.status == "failed_retryable":
|
||||
payload["retry"] = True
|
||||
|
||||
@ -25,12 +25,13 @@ class SettingsService:
|
||||
self.schema_path = self.config_dir / "settings.schema.json"
|
||||
self.settings_path = self.config_dir / "settings.json"
|
||||
self.staged_path = self.config_dir / "settings.staged.json"
|
||||
self.standalone_example_path = self.config_dir / "settings.standalone.example.json"
|
||||
|
||||
def load(self) -> SettingsBundle:
|
||||
self.ensure_local_settings()
|
||||
schema = self._read_json(self.schema_path)
|
||||
settings = self._read_json(self.settings_path)
|
||||
settings = self._apply_schema_defaults(settings, schema)
|
||||
settings = self._apply_legacy_env_overrides(settings, schema)
|
||||
settings = self._normalize_paths(settings)
|
||||
self.validate(settings, schema)
|
||||
return SettingsBundle(schema=schema, settings=settings)
|
||||
@ -49,6 +50,7 @@ class SettingsService:
|
||||
self._validate_field(group_name, field_name, group_value[field_name], field_schema)
|
||||
|
||||
def save_staged(self, settings: dict[str, Any]) -> None:
|
||||
self.ensure_local_settings()
|
||||
schema = self._read_json(self.schema_path)
|
||||
settings = self._apply_schema_defaults(settings, schema)
|
||||
self.validate(settings, schema)
|
||||
@ -68,12 +70,23 @@ class SettingsService:
|
||||
self._write_json(self.staged_path, merged)
|
||||
|
||||
def promote_staged(self) -> None:
|
||||
self.ensure_local_settings()
|
||||
staged = self._read_json(self.staged_path)
|
||||
schema = self._read_json(self.schema_path)
|
||||
staged = self._apply_schema_defaults(staged, schema)
|
||||
self.validate(staged, schema)
|
||||
self._write_json(self.settings_path, staged)
|
||||
|
||||
def ensure_local_settings(self) -> None:
|
||||
if not self.settings_path.exists():
|
||||
if not self.standalone_example_path.exists():
|
||||
raise ConfigError(f"配置文件不存在: {self.settings_path}")
|
||||
example_settings = self._read_json(self.standalone_example_path)
|
||||
self._write_json(self.settings_path, example_settings)
|
||||
if not self.staged_path.exists():
|
||||
settings = self._read_json(self.settings_path)
|
||||
self._write_json(self.staged_path, settings)
|
||||
|
||||
def _validate_field(self, group: str, name: str, value: Any, field_schema: dict[str, Any]) -> None:
|
||||
expected = field_schema.get("type")
|
||||
if expected == "string" and not isinstance(value, str):
|
||||
@ -130,38 +143,6 @@ class SettingsService:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
f.write("\n")
|
||||
|
||||
def _apply_legacy_env_overrides(self, settings: dict[str, Any], schema: dict[str, Any]) -> dict[str, Any]:
|
||||
env_path = self.root_dir.parent / ".env"
|
||||
if not env_path.exists():
|
||||
return settings
|
||||
env_map: dict[str, str] = {}
|
||||
with env_path.open("r", encoding="utf-8") as f:
|
||||
for raw_line in f:
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
env_map[key.strip()] = value.strip()
|
||||
|
||||
overrides = {
|
||||
("transcribe", "groq_api_key"): env_map.get("GROQ_API_KEY"),
|
||||
("song_detect", "codex_cmd"): self._resolve_legacy_path(env_map.get("CODEX_CMD")),
|
||||
("transcribe", "ffmpeg_bin"): self._resolve_legacy_path(env_map.get("FFMPEG_BIN")),
|
||||
("split", "ffmpeg_bin"): self._resolve_legacy_path(env_map.get("FFMPEG_BIN")),
|
||||
("ingest", "ffprobe_bin"): self._resolve_legacy_path(env_map.get("FFPROBE_BIN")),
|
||||
("publish", "biliup_path"): self._resolve_legacy_path(env_map.get("BILIUP_PATH")),
|
||||
("publish", "cookie_file"): self._resolve_legacy_path(env_map.get("BILIUP_COOKIE_FILE")),
|
||||
("paths", "cookies_file"): self._resolve_legacy_path(env_map.get("BILIUP_COOKIE_FILE")),
|
||||
}
|
||||
merged = json.loads(json.dumps(settings))
|
||||
defaults = schema.get("groups", {})
|
||||
for (group, field), value in overrides.items():
|
||||
default_value = defaults.get(group, {}).get(field, {}).get("default")
|
||||
current_value = merged.get(group, {}).get(field)
|
||||
if value and (current_value in ("", None) or current_value == default_value):
|
||||
merged[group][field] = value
|
||||
return merged
|
||||
|
||||
def _resolve_legacy_path(self, value: str | None) -> str | None:
|
||||
if not value:
|
||||
return value
|
||||
|
||||
@ -78,3 +78,36 @@ class ActionRecord:
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class TaskContext:
|
||||
id: int | None
|
||||
task_id: str
|
||||
session_key: str
|
||||
streamer: str | None
|
||||
room_id: str | None
|
||||
source_title: str | None
|
||||
segment_started_at: str | None
|
||||
segment_duration_seconds: float | None
|
||||
full_video_bvid: str | None
|
||||
created_at: str
|
||||
updated_at: str
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class SessionBinding:
|
||||
id: int | None
|
||||
session_key: str | None
|
||||
source_title: str | None
|
||||
streamer: str | None
|
||||
room_id: str | None
|
||||
full_video_bvid: str
|
||||
created_at: str
|
||||
updated_at: str
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
113
src/biliup_next/infra/adapters/bilibili_api.py
Normal file
113
src/biliup_next/infra/adapters/bilibili_api.py
Normal file
@ -0,0 +1,113 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from biliup_next.core.errors import ModuleError
|
||||
|
||||
|
||||
class BilibiliApiAdapter:
|
||||
def load_cookies(self, path: Path) -> dict[str, str]:
|
||||
with path.open("r", encoding="utf-8") as file_handle:
|
||||
data = json.load(file_handle)
|
||||
if "cookie_info" in data:
|
||||
return {c["name"]: c["value"] for c in data.get("cookie_info", {}).get("cookies", [])}
|
||||
return data
|
||||
|
||||
def build_session(
|
||||
self,
|
||||
*,
|
||||
cookies: dict[str, str],
|
||||
referer: str,
|
||||
origin: str | None = None,
|
||||
) -> requests.Session:
|
||||
session = requests.Session()
|
||||
session.cookies.update(cookies)
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0",
|
||||
"Referer": referer,
|
||||
}
|
||||
if origin:
|
||||
headers["Origin"] = origin
|
||||
session.headers.update(headers)
|
||||
return session
|
||||
|
||||
def get_video_view(self, session: requests.Session, bvid: str, *, error_code: str, error_message: str) -> dict[str, Any]:
|
||||
result = session.get("https://api.bilibili.com/x/web-interface/view", params={"bvid": bvid}, timeout=15).json()
|
||||
if result.get("code") != 0:
|
||||
raise ModuleError(
|
||||
code=error_code,
|
||||
message=f"{error_message}: {result.get('message')}",
|
||||
retryable=True,
|
||||
)
|
||||
return dict(result["data"])
|
||||
|
||||
def add_reply(self, session: requests.Session, *, csrf: str, aid: int, content: str, error_message: str) -> dict[str, Any]:
|
||||
result = session.post(
|
||||
"https://api.bilibili.com/x/v2/reply/add",
|
||||
data={"type": 1, "oid": aid, "message": content, "plat": 1, "csrf": csrf},
|
||||
timeout=15,
|
||||
).json()
|
||||
if result.get("code") != 0:
|
||||
raise ModuleError(
|
||||
code="COMMENT_POST_FAILED",
|
||||
message=f"{error_message}: {result.get('message')}",
|
||||
retryable=True,
|
||||
)
|
||||
return dict(result["data"])
|
||||
|
||||
def top_reply(self, session: requests.Session, *, csrf: str, aid: int, rpid: int, error_message: str) -> None:
|
||||
result = session.post(
|
||||
"https://api.bilibili.com/x/v2/reply/top",
|
||||
data={"type": 1, "oid": aid, "rpid": rpid, "action": 1, "csrf": csrf},
|
||||
timeout=15,
|
||||
).json()
|
||||
if result.get("code") != 0:
|
||||
raise ModuleError(
|
||||
code="COMMENT_TOP_FAILED",
|
||||
message=f"{error_message}: {result.get('message')}",
|
||||
retryable=True,
|
||||
)
|
||||
|
||||
def list_seasons(self, session: requests.Session) -> dict[str, Any]:
|
||||
result = session.get("https://member.bilibili.com/x2/creative/web/seasons", params={"pn": 1, "ps": 50}, timeout=15).json()
|
||||
return dict(result)
|
||||
|
||||
def add_section_episodes(
|
||||
self,
|
||||
session: requests.Session,
|
||||
*,
|
||||
csrf: str,
|
||||
section_id: int,
|
||||
episodes: list[dict[str, object]],
|
||||
) -> dict[str, Any]:
|
||||
return dict(
|
||||
session.post(
|
||||
"https://member.bilibili.com/x2/creative/web/season/section/episodes/add",
|
||||
params={"csrf": csrf},
|
||||
json={"sectionId": section_id, "episodes": episodes},
|
||||
timeout=20,
|
||||
).json()
|
||||
)
|
||||
|
||||
def get_section_detail(self, session: requests.Session, *, section_id: int) -> dict[str, Any]:
|
||||
return dict(
|
||||
session.get(
|
||||
"https://member.bilibili.com/x2/creative/web/season/section",
|
||||
params={"id": section_id},
|
||||
timeout=20,
|
||||
).json()
|
||||
)
|
||||
|
||||
def edit_section(self, session: requests.Session, *, csrf: str, payload: dict[str, object]) -> dict[str, Any]:
|
||||
return dict(
|
||||
session.post(
|
||||
"https://member.bilibili.com/x2/creative/web/season/section/edit",
|
||||
params={"csrf": csrf},
|
||||
json=payload,
|
||||
timeout=20,
|
||||
).json()
|
||||
)
|
||||
27
src/biliup_next/infra/adapters/biliup_cli.py
Normal file
27
src/biliup_next/infra/adapters/biliup_cli.py
Normal file
@ -0,0 +1,27 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
|
||||
from biliup_next.core.errors import ModuleError
|
||||
|
||||
|
||||
class BiliupCliAdapter:
|
||||
def run(self, cmd: list[str], *, label: str) -> subprocess.CompletedProcess[str]:
|
||||
try:
|
||||
return subprocess.run(cmd, capture_output=True, text=True, check=False)
|
||||
except FileNotFoundError as exc:
|
||||
raise ModuleError(
|
||||
code="BILIUP_NOT_FOUND",
|
||||
message=f"找不到 biliup 命令: {cmd[0]} ({label})",
|
||||
retryable=False,
|
||||
) from exc
|
||||
|
||||
def run_optional(self, cmd: list[str]) -> None:
|
||||
try:
|
||||
subprocess.run(cmd, capture_output=True, text=True, check=False)
|
||||
except FileNotFoundError as exc:
|
||||
raise ModuleError(
|
||||
code="BILIUP_NOT_FOUND",
|
||||
message=f"找不到 biliup 命令: {cmd[0]}",
|
||||
retryable=False,
|
||||
) from exc
|
||||
@ -1,176 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from biliup_next.core.errors import ModuleError
|
||||
from biliup_next.core.models import PublishRecord, Task, utc_now_iso
|
||||
from biliup_next.core.providers import ProviderManifest
|
||||
from biliup_next.infra.legacy_paths import legacy_project_root
|
||||
|
||||
|
||||
class LegacyBiliupPublishProvider:
|
||||
manifest = ProviderManifest(
|
||||
id="biliup_cli",
|
||||
name="Legacy biliup CLI Publish Provider",
|
||||
version="0.1.0",
|
||||
provider_type="publish_provider",
|
||||
entrypoint="biliup_next.infra.adapters.biliup_publish_legacy:LegacyBiliupPublishProvider",
|
||||
capabilities=["publish"],
|
||||
enabled_by_default=True,
|
||||
)
|
||||
|
||||
def __init__(self, next_root: Path):
|
||||
self.next_root = next_root
|
||||
self.legacy_root = legacy_project_root(next_root)
|
||||
|
||||
def publish(self, task: Task, clip_videos: list, settings: dict[str, Any]) -> PublishRecord:
|
||||
work_dir = Path(str(settings.get("session_dir", str(self.legacy_root / "session")))) / task.title
|
||||
bvid_file = work_dir / "bvid.txt"
|
||||
upload_done = work_dir / "upload_done.flag"
|
||||
config = self._load_upload_config(Path(str(settings.get("upload_config_file", str(self.legacy_root / "upload_config.json")))))
|
||||
if bvid_file.exists():
|
||||
bvid = bvid_file.read_text(encoding="utf-8").strip()
|
||||
return PublishRecord(
|
||||
id=None,
|
||||
task_id=task.id,
|
||||
platform="bilibili",
|
||||
aid=None,
|
||||
bvid=bvid,
|
||||
title=task.title,
|
||||
published_at=utc_now_iso(),
|
||||
)
|
||||
|
||||
video_files = [artifact.path for artifact in clip_videos]
|
||||
if not video_files:
|
||||
raise ModuleError(
|
||||
code="PUBLISH_NO_CLIPS",
|
||||
message=f"没有可上传的切片: {task.id}",
|
||||
retryable=False,
|
||||
)
|
||||
|
||||
parsed = self._parse_filename(task.title, config)
|
||||
streamer = parsed.get("streamer", task.title)
|
||||
date = parsed.get("date", "")
|
||||
|
||||
songs_txt = work_dir / "songs.txt"
|
||||
songs_list = songs_txt.read_text(encoding="utf-8").strip() if songs_txt.exists() else ""
|
||||
songs_json = work_dir / "songs.json"
|
||||
song_count = 0
|
||||
if songs_json.exists():
|
||||
song_count = len(json.loads(songs_json.read_text(encoding="utf-8")).get("songs", []))
|
||||
|
||||
quote = self._get_random_quote(config)
|
||||
template_vars = {
|
||||
"streamer": streamer,
|
||||
"date": date,
|
||||
"song_count": song_count,
|
||||
"songs_list": songs_list,
|
||||
"daily_quote": quote.get("text", ""),
|
||||
"quote_author": quote.get("author", ""),
|
||||
}
|
||||
template = config.get("template", {})
|
||||
title = template.get("title", "{streamer}_{date}").format(**template_vars)
|
||||
description = template.get("description", "{songs_list}").format(**template_vars)
|
||||
dynamic = template.get("dynamic", "").format(**template_vars)
|
||||
tags = template.get("tag", "翻唱,唱歌,音乐").format(**template_vars)
|
||||
streamer_cfg = config.get("streamers", {})
|
||||
if streamer in streamer_cfg:
|
||||
tags = streamer_cfg[streamer].get("tags", tags)
|
||||
|
||||
upload_settings = config.get("upload_settings", {})
|
||||
tid = upload_settings.get("tid", 31)
|
||||
biliup_path = str(settings.get("biliup_path", str(self.legacy_root / "biliup")))
|
||||
cookie_file = str(settings.get("cookie_file", str(self.legacy_root / "cookies.json")))
|
||||
|
||||
subprocess.run([biliup_path, "-u", cookie_file, "renew"], capture_output=True, text=True)
|
||||
|
||||
first_batch = video_files[:5]
|
||||
remaining_batches = [video_files[i:i + 5] for i in range(5, len(video_files), 5)]
|
||||
upload_cmd = [
|
||||
biliup_path, "-u", cookie_file, "upload",
|
||||
*first_batch,
|
||||
"--title", title,
|
||||
"--tid", str(tid),
|
||||
"--tag", tags,
|
||||
"--copyright", str(upload_settings.get("copyright", 2)),
|
||||
"--source", upload_settings.get("source", "直播回放"),
|
||||
"--desc", description,
|
||||
]
|
||||
if dynamic:
|
||||
upload_cmd.extend(["--dynamic", dynamic])
|
||||
|
||||
bvid = self._run_upload(upload_cmd, "首批上传")
|
||||
bvid_file.write_text(bvid, encoding="utf-8")
|
||||
|
||||
for idx, batch in enumerate(remaining_batches, 2):
|
||||
append_cmd = [biliup_path, "-u", cookie_file, "append", "--vid", bvid, *batch]
|
||||
self._run_append(append_cmd, f"追加第 {idx} 批")
|
||||
|
||||
upload_done.touch()
|
||||
return PublishRecord(
|
||||
id=None,
|
||||
task_id=task.id,
|
||||
platform="bilibili",
|
||||
aid=None,
|
||||
bvid=bvid,
|
||||
title=title,
|
||||
published_at=utc_now_iso(),
|
||||
)
|
||||
|
||||
def _run_upload(self, cmd: list[str], label: str) -> str:
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode == 0:
|
||||
match = re.search(r'"bvid":"(BV[A-Za-z0-9]+)"', result.stdout) or re.search(r'(BV[A-Za-z0-9]+)', result.stdout)
|
||||
if match:
|
||||
return match.group(1)
|
||||
raise ModuleError(
|
||||
code="PUBLISH_UPLOAD_FAILED",
|
||||
message=f"{label}失败",
|
||||
retryable=True,
|
||||
details={"stdout": result.stdout[-2000:], "stderr": result.stderr[-2000:]},
|
||||
)
|
||||
|
||||
def _run_append(self, cmd: list[str], label: str) -> None:
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode == 0:
|
||||
return
|
||||
raise ModuleError(
|
||||
code="PUBLISH_APPEND_FAILED",
|
||||
message=f"{label}失败",
|
||||
retryable=True,
|
||||
details={"stdout": result.stdout[-2000:], "stderr": result.stderr[-2000:]},
|
||||
)
|
||||
|
||||
def _load_upload_config(self, path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
def _parse_filename(self, filename: str, config: dict[str, Any] | None = None) -> dict[str, str]:
|
||||
config = config or {}
|
||||
patterns = config.get("filename_patterns", {}).get("patterns", [])
|
||||
for pattern_config in patterns:
|
||||
regex = pattern_config.get("regex")
|
||||
if not regex:
|
||||
continue
|
||||
match = re.match(regex, filename)
|
||||
if match:
|
||||
data = match.groupdict()
|
||||
date_format = pattern_config.get("date_format", "{date}")
|
||||
try:
|
||||
data["date"] = date_format.format(**data)
|
||||
except KeyError:
|
||||
pass
|
||||
return data
|
||||
return {"streamer": filename, "date": ""}
|
||||
|
||||
def _get_random_quote(self, config: dict[str, Any]) -> dict[str, str]:
|
||||
quotes = config.get("quotes", [])
|
||||
if not quotes:
|
||||
return {"text": "", "author": ""}
|
||||
return random.choice(quotes)
|
||||
44
src/biliup_next/infra/adapters/codex_cli.py
Normal file
44
src/biliup_next/infra/adapters/codex_cli.py
Normal file
@ -0,0 +1,44 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from biliup_next.core.errors import ModuleError
|
||||
|
||||
|
||||
class CodexCliAdapter:
|
||||
def run_song_detect(
|
||||
self,
|
||||
*,
|
||||
codex_cmd: str,
|
||||
work_dir: Path,
|
||||
prompt: str,
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
cmd = [
|
||||
codex_cmd,
|
||||
"exec",
|
||||
prompt.replace("\n", " "),
|
||||
"--full-auto",
|
||||
"--sandbox",
|
||||
"workspace-write",
|
||||
"--output-schema",
|
||||
"./song_schema.json",
|
||||
"-o",
|
||||
"songs.json",
|
||||
"--skip-git-repo-check",
|
||||
"--json",
|
||||
]
|
||||
try:
|
||||
return subprocess.run(
|
||||
cmd,
|
||||
cwd=str(work_dir),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
except FileNotFoundError as exc:
|
||||
raise ModuleError(
|
||||
code="CODEX_NOT_FOUND",
|
||||
message=f"找不到 codex 命令: {codex_cmd}",
|
||||
retryable=False,
|
||||
) from exc
|
||||
@ -1,79 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from biliup_next.core.errors import ModuleError
|
||||
from biliup_next.core.models import Artifact, Task, utc_now_iso
|
||||
from biliup_next.core.providers import ProviderManifest
|
||||
from biliup_next.infra.legacy_paths import legacy_project_root
|
||||
|
||||
|
||||
class LegacyGroqTranscribeProvider:
|
||||
manifest = ProviderManifest(
|
||||
id="groq",
|
||||
name="Legacy Groq Transcribe Provider",
|
||||
version="0.1.0",
|
||||
provider_type="transcribe_provider",
|
||||
entrypoint="biliup_next.infra.adapters.groq_legacy:LegacyGroqTranscribeProvider",
|
||||
capabilities=["transcribe"],
|
||||
enabled_by_default=True,
|
||||
)
|
||||
|
||||
def __init__(self, next_root: Path):
|
||||
self.next_root = next_root
|
||||
self.legacy_root = legacy_project_root(next_root)
|
||||
self.python_bin = self._resolve_python_bin()
|
||||
|
||||
def transcribe(self, task: Task, source_video: Artifact, settings: dict[str, Any]) -> Artifact:
|
||||
session_dir = Path(str(settings.get("session_dir", str(self.legacy_root / "session"))))
|
||||
work_dir = (session_dir / task.title).resolve()
|
||||
cmd = [
|
||||
self.python_bin,
|
||||
"video2srt.py",
|
||||
source_video.path,
|
||||
str(work_dir),
|
||||
]
|
||||
env = {
|
||||
**os.environ,
|
||||
"GROQ_API_KEY": str(settings.get("groq_api_key", "")),
|
||||
"FFMPEG_BIN": str(settings.get("ffmpeg_bin", "ffmpeg")),
|
||||
}
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
cwd=str(self.legacy_root),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise ModuleError(
|
||||
code="TRANSCRIBE_FAILED",
|
||||
message="legacy video2srt.py 执行失败",
|
||||
retryable=True,
|
||||
details={"stderr": result.stderr[-2000:], "stdout": result.stdout[-2000:]},
|
||||
)
|
||||
srt_path = work_dir / f"{task.title}.srt"
|
||||
if not srt_path.exists():
|
||||
raise ModuleError(
|
||||
code="TRANSCRIBE_OUTPUT_MISSING",
|
||||
message=f"未找到字幕文件: {srt_path}",
|
||||
retryable=False,
|
||||
)
|
||||
return Artifact(
|
||||
id=None,
|
||||
task_id=task.id,
|
||||
artifact_type="subtitle_srt",
|
||||
path=str(srt_path),
|
||||
metadata_json=json.dumps({"provider": "groq_legacy"}),
|
||||
created_at=utc_now_iso(),
|
||||
)
|
||||
|
||||
def _resolve_python_bin(self) -> str:
|
||||
venv_python = self.legacy_root / ".venv" / "bin" / "python"
|
||||
if venv_python.exists():
|
||||
return str(venv_python)
|
||||
return "python"
|
||||
@ -1,27 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class CommentFlagMigrationService:
|
||||
def migrate(self, session_dir: Path) -> dict[str, int]:
|
||||
migrated_split_flags = 0
|
||||
legacy_untracked_full = 0
|
||||
if not session_dir.exists():
|
||||
return {"migrated_split_flags": 0, "legacy_untracked_full": 0}
|
||||
|
||||
for folder in sorted(p for p in session_dir.iterdir() if p.is_dir()):
|
||||
comment_done = folder / "comment_done.flag"
|
||||
split_done = folder / "comment_split_done.flag"
|
||||
full_done = folder / "comment_full_done.flag"
|
||||
if not comment_done.exists():
|
||||
continue
|
||||
if not split_done.exists():
|
||||
split_done.touch()
|
||||
migrated_split_flags += 1
|
||||
if not full_done.exists():
|
||||
legacy_untracked_full += 1
|
||||
return {
|
||||
"migrated_split_flags": migrated_split_flags,
|
||||
"legacy_untracked_full": legacy_untracked_full,
|
||||
}
|
||||
@ -59,6 +59,37 @@ CREATE TABLE IF NOT EXISTS action_records (
|
||||
created_at TEXT NOT NULL,
|
||||
FOREIGN KEY(task_id) REFERENCES tasks(id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS task_contexts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
task_id TEXT NOT NULL UNIQUE,
|
||||
session_key TEXT NOT NULL,
|
||||
streamer TEXT,
|
||||
room_id TEXT,
|
||||
source_title TEXT,
|
||||
segment_started_at TEXT,
|
||||
segment_duration_seconds REAL,
|
||||
full_video_bvid TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
FOREIGN KEY(task_id) REFERENCES tasks(id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS session_bindings (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
session_key TEXT UNIQUE,
|
||||
source_title TEXT,
|
||||
streamer TEXT,
|
||||
room_id TEXT,
|
||||
full_video_bvid TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_task_contexts_session_key ON task_contexts(session_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_task_contexts_streamer_started_at ON task_contexts(streamer, segment_started_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_session_bindings_source_title ON session_bindings(source_title);
|
||||
CREATE INDEX IF NOT EXISTS idx_session_bindings_streamer_room_id ON session_bindings(streamer, room_id);
|
||||
"""
|
||||
|
||||
|
||||
@ -70,6 +101,10 @@ class Database:
|
||||
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
conn.execute("PRAGMA busy_timeout = 5000")
|
||||
conn.execute("PRAGMA journal_mode = WAL")
|
||||
conn.execute("PRAGMA synchronous = NORMAL")
|
||||
return conn
|
||||
|
||||
def initialize(self) -> None:
|
||||
|
||||
@ -1,7 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def legacy_project_root(next_root: Path) -> Path:
|
||||
return next_root.parent
|
||||
@ -2,18 +2,27 @@ from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
ALLOWED_LOG_FILES = {
|
||||
"monitor.log": Path("/home/theshy/biliup/logs/system/monitor.log"),
|
||||
"monitorSrt.log": Path("/home/theshy/biliup/logs/system/monitorSrt.log"),
|
||||
"monitorSongs.log": Path("/home/theshy/biliup/logs/system/monitorSongs.log"),
|
||||
"upload.log": Path("/home/theshy/biliup/logs/system/upload.log"),
|
||||
"session_top_comment.py.log": Path("/home/theshy/biliup/logs/system/session_top_comment.py.log"),
|
||||
"add_to_collection.py.log": Path("/home/theshy/biliup/logs/system/add_to_collection.py.log"),
|
||||
}
|
||||
|
||||
|
||||
class LogReader:
|
||||
def __init__(self, root_dir: Path | None = None):
|
||||
self.root_dir = (root_dir or Path(__file__).resolve().parents[3]).resolve()
|
||||
self.log_dirs = [
|
||||
self.root_dir / "logs",
|
||||
self.root_dir / "runtime" / "logs",
|
||||
self.root_dir / "data" / "workspace" / "logs",
|
||||
]
|
||||
|
||||
def _allowed_log_files(self) -> dict[str, Path]:
|
||||
items: dict[str, Path] = {}
|
||||
for log_dir in self.log_dirs:
|
||||
if not log_dir.exists():
|
||||
continue
|
||||
for path in sorted(p for p in log_dir.rglob("*.log") if p.is_file()):
|
||||
items.setdefault(path.name, path.resolve())
|
||||
return items
|
||||
|
||||
def list_logs(self) -> dict[str, object]:
|
||||
allowed_log_files = self._allowed_log_files()
|
||||
return {
|
||||
"items": [
|
||||
{
|
||||
@ -21,14 +30,15 @@ class LogReader:
|
||||
"path": str(path),
|
||||
"exists": path.exists(),
|
||||
}
|
||||
for name, path in sorted(ALLOWED_LOG_FILES.items())
|
||||
for name, path in sorted(allowed_log_files.items())
|
||||
]
|
||||
}
|
||||
|
||||
def tail(self, name: str, lines: int = 200, contains: str | None = None) -> dict[str, object]:
|
||||
if name not in ALLOWED_LOG_FILES:
|
||||
allowed_log_files = self._allowed_log_files()
|
||||
if name not in allowed_log_files:
|
||||
raise ValueError(f"unsupported log: {name}")
|
||||
path = ALLOWED_LOG_FILES[name]
|
||||
path = allowed_log_files[name]
|
||||
if not path.exists():
|
||||
return {"name": name, "path": str(path), "exists": False, "content": ""}
|
||||
content = path.read_text(encoding="utf-8", errors="replace").splitlines()
|
||||
|
||||
@ -8,7 +8,7 @@ from biliup_next.core.config import SettingsService
|
||||
|
||||
class RuntimeDoctor:
|
||||
def __init__(self, root_dir: Path):
|
||||
self.root_dir = root_dir
|
||||
self.root_dir = root_dir.resolve()
|
||||
self.settings_service = SettingsService(root_dir)
|
||||
|
||||
def run(self) -> dict[str, object]:
|
||||
@ -28,27 +28,47 @@ class RuntimeDoctor:
|
||||
("paths", "cookies_file"),
|
||||
("paths", "upload_config_file"),
|
||||
):
|
||||
path = (self.root_dir / settings[group][name]).resolve()
|
||||
detail = str(path)
|
||||
if path.exists() and not str(path).startswith(str(self.root_dir)):
|
||||
detail = f"{path} (external)"
|
||||
checks.append({"name": f"{group}.{name}", "ok": path.exists(), "detail": detail})
|
||||
path = Path(str(settings[group][name])).resolve()
|
||||
checks.append(
|
||||
{
|
||||
"name": f"{group}.{name}",
|
||||
"ok": path.exists() and self._is_internal_path(path),
|
||||
"detail": self._internal_path_detail(path),
|
||||
}
|
||||
)
|
||||
|
||||
for group, name in (
|
||||
("ingest", "ffprobe_bin"),
|
||||
("transcribe", "ffmpeg_bin"),
|
||||
("song_detect", "codex_cmd"),
|
||||
("publish", "biliup_path"),
|
||||
):
|
||||
value = settings[group][name]
|
||||
found = shutil.which(value) if "/" not in value else str((self.root_dir / value).resolve())
|
||||
ok = bool(found) and (Path(found).exists() if "/" in str(found) else True)
|
||||
detail = str(found or value)
|
||||
if ok and "/" in detail and not detail.startswith(str(self.root_dir)):
|
||||
detail = f"{detail} (external)"
|
||||
checks.append({"name": f"{group}.{name}", "ok": ok, "detail": detail})
|
||||
checks.append({"name": f"{group}.{name}", "ok": ok, "detail": str(found or value)})
|
||||
|
||||
publish_biliup_path = Path(str(settings["publish"]["biliup_path"])).resolve()
|
||||
checks.append(
|
||||
{
|
||||
"name": "publish.biliup_path",
|
||||
"ok": publish_biliup_path.exists() and self._is_internal_path(publish_biliup_path),
|
||||
"detail": self._internal_path_detail(publish_biliup_path),
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"ok": all(item["ok"] for item in checks),
|
||||
"checks": checks,
|
||||
}
|
||||
|
||||
def _is_internal_path(self, path: Path) -> bool:
|
||||
try:
|
||||
path.relative_to(self.root_dir)
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
def _internal_path_detail(self, path: Path) -> str:
|
||||
if self._is_internal_path(path):
|
||||
return str(path)
|
||||
return f"{path} (must live under {self.root_dir})"
|
||||
|
||||
@ -5,7 +5,6 @@ import subprocess
|
||||
ALLOWED_SERVICES = {
|
||||
"biliup-next-worker.service",
|
||||
"biliup-next-api.service",
|
||||
"biliup-python.service",
|
||||
}
|
||||
ALLOWED_ACTIONS = {"start", "stop", "restart"}
|
||||
|
||||
|
||||
@ -1,26 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from biliup_next.core.models import ActionRecord, Artifact, PublishRecord, Task, TaskStep
|
||||
from biliup_next.core.models import ActionRecord, Artifact, PublishRecord, SessionBinding, Task, TaskContext, TaskStep
|
||||
from biliup_next.infra.db import Database
|
||||
|
||||
|
||||
TASK_STATUS_ORDER = {
|
||||
"created": 0,
|
||||
"transcribed": 1,
|
||||
"songs_detected": 2,
|
||||
"split_done": 3,
|
||||
"published": 4,
|
||||
"commented": 5,
|
||||
"collection_synced": 6,
|
||||
"failed_retryable": 7,
|
||||
"failed_manual": 8,
|
||||
}
|
||||
|
||||
|
||||
class TaskRepository:
|
||||
def __init__(self, db: Database):
|
||||
self.db = db
|
||||
@ -58,6 +41,24 @@ class TaskRepository:
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def _build_task_query(
|
||||
self,
|
||||
*,
|
||||
status: str | None = None,
|
||||
search: str | None = None,
|
||||
) -> tuple[str, list[object]]:
|
||||
conditions: list[str] = []
|
||||
params: list[object] = []
|
||||
if status:
|
||||
conditions.append("status = ?")
|
||||
params.append(status)
|
||||
if search:
|
||||
conditions.append("(id LIKE ? OR title LIKE ?)")
|
||||
needle = f"%{search}%"
|
||||
params.extend([needle, needle])
|
||||
where_clause = f"WHERE {' AND '.join(conditions)}" if conditions else ""
|
||||
return where_clause, params
|
||||
|
||||
def list_tasks(self, limit: int = 100) -> list[Task]:
|
||||
with self.db.connect() as conn:
|
||||
rows = conn.execute(
|
||||
@ -67,6 +68,42 @@ class TaskRepository:
|
||||
).fetchall()
|
||||
return [Task(**dict(row)) for row in rows]
|
||||
|
||||
def query_tasks(
|
||||
self,
|
||||
*,
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
status: str | None = None,
|
||||
search: str | None = None,
|
||||
sort: str = "updated_desc",
|
||||
) -> tuple[list[Task], int]:
|
||||
sort_sql = {
|
||||
"updated_desc": "updated_at DESC",
|
||||
"updated_asc": "updated_at ASC",
|
||||
"title_asc": "title COLLATE NOCASE ASC",
|
||||
"title_desc": "title COLLATE NOCASE DESC",
|
||||
"created_desc": "created_at DESC",
|
||||
"created_asc": "created_at ASC",
|
||||
"status_asc": "status ASC, updated_at DESC",
|
||||
}.get(sort, "updated_at DESC")
|
||||
where_clause, params = self._build_task_query(status=status, search=search)
|
||||
with self.db.connect() as conn:
|
||||
total = conn.execute(
|
||||
f"SELECT COUNT(*) AS count FROM tasks {where_clause}",
|
||||
params,
|
||||
).fetchone()["count"]
|
||||
rows = conn.execute(
|
||||
f"""
|
||||
SELECT id, source_type, source_path, title, status, created_at, updated_at
|
||||
FROM tasks
|
||||
{where_clause}
|
||||
ORDER BY {sort_sql}
|
||||
LIMIT ? OFFSET ?
|
||||
""",
|
||||
[*params, limit, offset],
|
||||
).fetchall()
|
||||
return [Task(**dict(row)) for row in rows], int(total)
|
||||
|
||||
def get_task(self, task_id: str) -> Task | None:
|
||||
with self.db.connect() as conn:
|
||||
row = conn.execute(
|
||||
@ -81,6 +118,7 @@ class TaskRepository:
|
||||
conn.execute("DELETE FROM action_records WHERE task_id = ?", (task_id,))
|
||||
conn.execute("DELETE FROM publish_records WHERE task_id = ?", (task_id,))
|
||||
conn.execute("DELETE FROM artifacts WHERE task_id = ?", (task_id,))
|
||||
conn.execute("DELETE FROM task_contexts WHERE task_id = ?", (task_id,))
|
||||
conn.execute("DELETE FROM task_steps WHERE task_id = ?", (task_id,))
|
||||
conn.execute("DELETE FROM tasks WHERE id = ?", (task_id,))
|
||||
conn.commit()
|
||||
@ -172,6 +210,19 @@ class TaskRepository:
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def claim_step_running(self, task_id: str, step_name: str, *, started_at: str) -> bool:
|
||||
with self.db.connect() as conn:
|
||||
result = conn.execute(
|
||||
"""
|
||||
UPDATE task_steps
|
||||
SET status = ?, started_at = ?, finished_at = NULL, error_code = NULL, error_message = NULL
|
||||
WHERE task_id = ? AND step_name = ? AND status IN (?, ?)
|
||||
""",
|
||||
("running", started_at, task_id, step_name, "pending", "failed_retryable"),
|
||||
)
|
||||
conn.commit()
|
||||
return result.rowcount == 1
|
||||
|
||||
def add_artifact(self, artifact: Artifact) -> None:
|
||||
with self.db.connect() as conn:
|
||||
existing = conn.execute(
|
||||
@ -265,6 +316,250 @@ class TaskRepository:
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def upsert_task_context(self, context: TaskContext) -> None:
|
||||
with self.db.connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO task_contexts (
|
||||
task_id, session_key, streamer, room_id, source_title,
|
||||
segment_started_at, segment_duration_seconds, full_video_bvid,
|
||||
created_at, updated_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(task_id) DO UPDATE SET
|
||||
session_key=excluded.session_key,
|
||||
streamer=excluded.streamer,
|
||||
room_id=excluded.room_id,
|
||||
source_title=excluded.source_title,
|
||||
segment_started_at=excluded.segment_started_at,
|
||||
segment_duration_seconds=excluded.segment_duration_seconds,
|
||||
full_video_bvid=excluded.full_video_bvid,
|
||||
updated_at=excluded.updated_at
|
||||
""",
|
||||
(
|
||||
context.task_id,
|
||||
context.session_key,
|
||||
context.streamer,
|
||||
context.room_id,
|
||||
context.source_title,
|
||||
context.segment_started_at,
|
||||
context.segment_duration_seconds,
|
||||
context.full_video_bvid,
|
||||
context.created_at,
|
||||
context.updated_at,
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def get_task_context(self, task_id: str) -> TaskContext | None:
|
||||
with self.db.connect() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT id, task_id, session_key, streamer, room_id, source_title,
|
||||
segment_started_at, segment_duration_seconds, full_video_bvid,
|
||||
created_at, updated_at
|
||||
FROM task_contexts
|
||||
WHERE task_id = ?
|
||||
""",
|
||||
(task_id,),
|
||||
).fetchone()
|
||||
return TaskContext(**dict(row)) if row else None
|
||||
|
||||
def list_task_contexts_by_session_key(self, session_key: str) -> list[TaskContext]:
|
||||
with self.db.connect() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT id, task_id, session_key, streamer, room_id, source_title,
|
||||
segment_started_at, segment_duration_seconds, full_video_bvid,
|
||||
created_at, updated_at
|
||||
FROM task_contexts
|
||||
WHERE session_key = ?
|
||||
ORDER BY segment_started_at ASC, id ASC
|
||||
""",
|
||||
(session_key,),
|
||||
).fetchall()
|
||||
return [TaskContext(**dict(row)) for row in rows]
|
||||
|
||||
def list_task_contexts_by_source_title(self, source_title: str) -> list[TaskContext]:
|
||||
with self.db.connect() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT id, task_id, session_key, streamer, room_id, source_title,
|
||||
segment_started_at, segment_duration_seconds, full_video_bvid,
|
||||
created_at, updated_at
|
||||
FROM task_contexts
|
||||
WHERE source_title = ?
|
||||
ORDER BY COALESCE(segment_started_at, updated_at) ASC, id ASC
|
||||
""",
|
||||
(source_title,),
|
||||
).fetchall()
|
||||
return [TaskContext(**dict(row)) for row in rows]
|
||||
|
||||
def list_task_contexts_for_task_ids(self, task_ids: list[str]) -> dict[str, TaskContext]:
|
||||
if not task_ids:
|
||||
return {}
|
||||
placeholders = ", ".join("?" for _ in task_ids)
|
||||
with self.db.connect() as conn:
|
||||
rows = conn.execute(
|
||||
f"""
|
||||
SELECT id, task_id, session_key, streamer, room_id, source_title,
|
||||
segment_started_at, segment_duration_seconds, full_video_bvid,
|
||||
created_at, updated_at
|
||||
FROM task_contexts
|
||||
WHERE task_id IN ({placeholders})
|
||||
""",
|
||||
task_ids,
|
||||
).fetchall()
|
||||
return {row["task_id"]: TaskContext(**dict(row)) for row in rows}
|
||||
|
||||
def find_recent_task_contexts(self, streamer: str, limit: int = 20) -> list[TaskContext]:
|
||||
with self.db.connect() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT id, task_id, session_key, streamer, room_id, source_title,
|
||||
segment_started_at, segment_duration_seconds, full_video_bvid,
|
||||
created_at, updated_at
|
||||
FROM task_contexts
|
||||
WHERE streamer = ?
|
||||
ORDER BY COALESCE(segment_started_at, updated_at) DESC, id DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
(streamer, limit),
|
||||
).fetchall()
|
||||
return [TaskContext(**dict(row)) for row in rows]
|
||||
|
||||
def list_steps_for_task_ids(self, task_ids: list[str]) -> dict[str, list[TaskStep]]:
|
||||
if not task_ids:
|
||||
return {}
|
||||
placeholders = ", ".join("?" for _ in task_ids)
|
||||
with self.db.connect() as conn:
|
||||
rows = conn.execute(
|
||||
f"""
|
||||
SELECT id, task_id, step_name, status, error_code, error_message,
|
||||
retry_count, started_at, finished_at
|
||||
FROM task_steps
|
||||
WHERE task_id IN ({placeholders})
|
||||
ORDER BY id ASC
|
||||
""",
|
||||
task_ids,
|
||||
).fetchall()
|
||||
result: dict[str, list[TaskStep]] = {}
|
||||
for row in rows:
|
||||
step = TaskStep(**dict(row))
|
||||
result.setdefault(step.task_id, []).append(step)
|
||||
return result
|
||||
|
||||
def update_session_full_video_bvid(self, session_key: str, full_video_bvid: str, updated_at: str) -> int:
|
||||
with self.db.connect() as conn:
|
||||
result = conn.execute(
|
||||
"""
|
||||
UPDATE task_contexts
|
||||
SET full_video_bvid = ?, updated_at = ?
|
||||
WHERE session_key = ?
|
||||
""",
|
||||
(full_video_bvid, updated_at, session_key),
|
||||
)
|
||||
conn.commit()
|
||||
return result.rowcount
|
||||
|
||||
def upsert_session_binding(self, binding: SessionBinding) -> None:
|
||||
with self.db.connect() as conn:
|
||||
if binding.session_key:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO session_bindings (
|
||||
session_key, source_title, streamer, room_id, full_video_bvid, created_at, updated_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(session_key) DO UPDATE SET
|
||||
source_title=excluded.source_title,
|
||||
streamer=excluded.streamer,
|
||||
room_id=excluded.room_id,
|
||||
full_video_bvid=excluded.full_video_bvid,
|
||||
updated_at=excluded.updated_at
|
||||
""",
|
||||
(
|
||||
binding.session_key,
|
||||
binding.source_title,
|
||||
binding.streamer,
|
||||
binding.room_id,
|
||||
binding.full_video_bvid,
|
||||
binding.created_at,
|
||||
binding.updated_at,
|
||||
),
|
||||
)
|
||||
else:
|
||||
existing = conn.execute(
|
||||
"""
|
||||
SELECT id
|
||||
FROM session_bindings
|
||||
WHERE source_title = ?
|
||||
ORDER BY id DESC
|
||||
LIMIT 1
|
||||
""",
|
||||
(binding.source_title,),
|
||||
).fetchone()
|
||||
if existing:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE session_bindings
|
||||
SET streamer = ?, room_id = ?, full_video_bvid = ?, updated_at = ?
|
||||
WHERE id = ?
|
||||
""",
|
||||
(
|
||||
binding.streamer,
|
||||
binding.room_id,
|
||||
binding.full_video_bvid,
|
||||
binding.updated_at,
|
||||
existing["id"],
|
||||
),
|
||||
)
|
||||
else:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO session_bindings (
|
||||
session_key, source_title, streamer, room_id, full_video_bvid, created_at, updated_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
binding.session_key,
|
||||
binding.source_title,
|
||||
binding.streamer,
|
||||
binding.room_id,
|
||||
binding.full_video_bvid,
|
||||
binding.created_at,
|
||||
binding.updated_at,
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def get_session_binding(self, *, session_key: str | None = None, source_title: str | None = None) -> SessionBinding | None:
|
||||
with self.db.connect() as conn:
|
||||
row = None
|
||||
if session_key:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT id, session_key, source_title, streamer, room_id, full_video_bvid, created_at, updated_at
|
||||
FROM session_bindings
|
||||
WHERE session_key = ?
|
||||
LIMIT 1
|
||||
""",
|
||||
(session_key,),
|
||||
).fetchone()
|
||||
if row is None and source_title:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT id, session_key, source_title, streamer, room_id, full_video_bvid, created_at, updated_at
|
||||
FROM session_bindings
|
||||
WHERE source_title = ?
|
||||
ORDER BY id DESC
|
||||
LIMIT 1
|
||||
""",
|
||||
(source_title,),
|
||||
).fetchone()
|
||||
return SessionBinding(**dict(row)) if row else None
|
||||
|
||||
def list_action_records(
|
||||
self,
|
||||
task_id: str | None = None,
|
||||
@ -297,162 +592,3 @@ class TaskRepository:
|
||||
(*params, limit),
|
||||
).fetchall()
|
||||
return [ActionRecord(**dict(row)) for row in rows]
|
||||
|
||||
def bootstrap_from_legacy_sessions(self, session_dir: Path) -> int:
|
||||
synced = 0
|
||||
if not session_dir.exists():
|
||||
return synced
|
||||
for folder in sorted(p for p in session_dir.iterdir() if p.is_dir()):
|
||||
task_id = folder.name
|
||||
existing_task = self.get_task(task_id)
|
||||
derived_status = "created"
|
||||
if (folder / "transcribe_done.flag").exists():
|
||||
derived_status = "transcribed"
|
||||
if (folder / "songs.json").exists():
|
||||
derived_status = "songs_detected"
|
||||
if (folder / "split_done.flag").exists():
|
||||
derived_status = "split_done"
|
||||
if (folder / "upload_done.flag").exists():
|
||||
derived_status = "published"
|
||||
if (folder / "comment_done.flag").exists():
|
||||
derived_status = "commented"
|
||||
if (folder / "collection_a_done.flag").exists() or (folder / "collection_b_done.flag").exists():
|
||||
derived_status = "collection_synced"
|
||||
effective_status = self._merge_task_status(existing_task.status if existing_task else None, derived_status)
|
||||
created_at = (
|
||||
existing_task.created_at
|
||||
if existing_task and existing_task.created_at
|
||||
else self._folder_time_iso(folder)
|
||||
)
|
||||
updated_at = (
|
||||
existing_task.updated_at
|
||||
if existing_task and existing_task.updated_at
|
||||
else created_at
|
||||
)
|
||||
task = Task(
|
||||
id=task_id,
|
||||
source_type=existing_task.source_type if existing_task else "legacy_session",
|
||||
source_path=existing_task.source_path if existing_task else str(folder),
|
||||
title=folder.name,
|
||||
status=effective_status,
|
||||
created_at=created_at,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
self.upsert_task(task)
|
||||
steps = self._merge_steps(folder, task_id)
|
||||
self.replace_steps(task_id, steps)
|
||||
self._bootstrap_artifacts(folder, task_id)
|
||||
synced += 1
|
||||
return synced
|
||||
|
||||
def _infer_steps(self, folder: Path, task_id: str) -> list[TaskStep]:
|
||||
flags = {
|
||||
"ingest": True,
|
||||
"transcribe": (folder / "transcribe_done.flag").exists(),
|
||||
"song_detect": (folder / "songs.json").exists(),
|
||||
"split": (folder / "split_done.flag").exists(),
|
||||
"publish": (folder / "upload_done.flag").exists(),
|
||||
"comment": (folder / "comment_done.flag").exists(),
|
||||
"collection_a": (folder / "collection_a_done.flag").exists(),
|
||||
"collection_b": (folder / "collection_b_done.flag").exists(),
|
||||
}
|
||||
steps: list[TaskStep] = []
|
||||
for name, done in flags.items():
|
||||
steps.append(
|
||||
TaskStep(
|
||||
id=None,
|
||||
task_id=task_id,
|
||||
step_name=name,
|
||||
status="succeeded" if done else "pending",
|
||||
error_code=None,
|
||||
error_message=None,
|
||||
retry_count=0,
|
||||
started_at=None,
|
||||
finished_at=None,
|
||||
)
|
||||
)
|
||||
return steps
|
||||
|
||||
def _merge_steps(self, folder: Path, task_id: str) -> list[TaskStep]:
|
||||
inferred_steps = {step.step_name: step for step in self._infer_steps(folder, task_id)}
|
||||
current_steps = {step.step_name: step for step in self.list_steps(task_id)}
|
||||
merged: list[TaskStep] = []
|
||||
for step_name, inferred in inferred_steps.items():
|
||||
current = current_steps.get(step_name)
|
||||
if current is None:
|
||||
merged.append(inferred)
|
||||
continue
|
||||
if inferred.status == "succeeded":
|
||||
merged.append(
|
||||
TaskStep(
|
||||
id=None,
|
||||
task_id=task_id,
|
||||
step_name=step_name,
|
||||
status="succeeded",
|
||||
error_code=None,
|
||||
error_message=None,
|
||||
retry_count=current.retry_count,
|
||||
started_at=current.started_at,
|
||||
finished_at=current.finished_at,
|
||||
)
|
||||
)
|
||||
continue
|
||||
if current.status != "pending":
|
||||
merged.append(
|
||||
TaskStep(
|
||||
id=None,
|
||||
task_id=task_id,
|
||||
step_name=step_name,
|
||||
status=current.status,
|
||||
error_code=current.error_code,
|
||||
error_message=current.error_message,
|
||||
retry_count=current.retry_count,
|
||||
started_at=current.started_at,
|
||||
finished_at=current.finished_at,
|
||||
)
|
||||
)
|
||||
continue
|
||||
merged.append(inferred)
|
||||
return merged
|
||||
|
||||
@staticmethod
|
||||
def _merge_task_status(existing_status: str | None, derived_status: str) -> str:
|
||||
if not existing_status:
|
||||
return derived_status
|
||||
existing_rank = TASK_STATUS_ORDER.get(existing_status, -1)
|
||||
derived_rank = TASK_STATUS_ORDER.get(derived_status, -1)
|
||||
return existing_status if existing_rank >= derived_rank else derived_status
|
||||
|
||||
@staticmethod
|
||||
def _folder_time_iso(folder: Path) -> str:
|
||||
return datetime.fromtimestamp(folder.stat().st_mtime, tz=timezone.utc).isoformat()
|
||||
|
||||
def _bootstrap_artifacts(self, folder: Path, task_id: str) -> None:
|
||||
artifacts = []
|
||||
if any(folder.glob("*.srt")):
|
||||
for srt in folder.glob("*.srt"):
|
||||
artifacts.append(("subtitle_srt", srt))
|
||||
for name in ("songs.json", "songs.txt", "bvid.txt"):
|
||||
path = folder / name
|
||||
if path.exists():
|
||||
artifact_type = {
|
||||
"songs.json": "songs_json",
|
||||
"songs.txt": "songs_txt",
|
||||
"bvid.txt": "publish_bvid",
|
||||
}[name]
|
||||
artifacts.append((artifact_type, path))
|
||||
existing = {(a.artifact_type, a.path) for a in self.list_artifacts(task_id)}
|
||||
for artifact_type, path in artifacts:
|
||||
key = (artifact_type, str(path))
|
||||
if key in existing:
|
||||
continue
|
||||
self.add_artifact(
|
||||
Artifact(
|
||||
id=None,
|
||||
task_id=task_id,
|
||||
artifact_type=artifact_type,
|
||||
path=str(path),
|
||||
metadata_json=json.dumps({}),
|
||||
created_at="",
|
||||
)
|
||||
)
|
||||
|
||||
@ -29,8 +29,9 @@ STATUS_BEFORE_STEP = {
|
||||
|
||||
|
||||
class TaskResetService:
|
||||
def __init__(self, repo: TaskRepository):
|
||||
def __init__(self, repo: TaskRepository, session_dir: Path):
|
||||
self.repo = repo
|
||||
self.session_dir = session_dir.resolve()
|
||||
|
||||
def reset_to_step(self, task_id: str, step_name: str) -> dict[str, object]:
|
||||
task = self.repo.get_task(task_id)
|
||||
@ -39,7 +40,7 @@ class TaskResetService:
|
||||
if step_name not in STEP_ORDER:
|
||||
raise RuntimeError(f"unsupported step: {step_name}")
|
||||
|
||||
work_dir = self._resolve_work_dir(task)
|
||||
work_dir = self._resolve_work_dir(task, self.session_dir)
|
||||
self._cleanup_files(work_dir, step_name)
|
||||
self._cleanup_artifacts(task_id, step_name)
|
||||
self._reset_steps(task_id, step_name)
|
||||
@ -48,9 +49,14 @@ class TaskResetService:
|
||||
return {"task_id": task_id, "reset_to": step_name, "work_dir": str(work_dir)}
|
||||
|
||||
@staticmethod
|
||||
def _resolve_work_dir(task) -> Path: # type: ignore[no-untyped-def]
|
||||
source = Path(task.source_path)
|
||||
return source.parent if source.is_file() else source
|
||||
def _resolve_work_dir(task, session_dir: Path) -> Path: # type: ignore[no-untyped-def]
|
||||
source = Path(task.source_path).resolve()
|
||||
work_dir = source.parent if source.is_file() else source
|
||||
try:
|
||||
work_dir.relative_to(session_dir)
|
||||
except ValueError as exc:
|
||||
raise RuntimeError(f"task work_dir outside managed session_dir: {work_dir}") from exc
|
||||
return work_dir
|
||||
|
||||
@staticmethod
|
||||
def _remove_path(path: Path) -> None:
|
||||
|
||||
@ -20,8 +20,13 @@ class WorkspaceCleanupService:
|
||||
skipped: list[str] = []
|
||||
|
||||
if settings.get("delete_source_video_after_collection_synced", False):
|
||||
source_path = Path(task.source_path)
|
||||
if source_path.exists():
|
||||
source_path = Path(task.source_path).resolve()
|
||||
try:
|
||||
source_path.relative_to(session_dir)
|
||||
source_managed = True
|
||||
except ValueError:
|
||||
source_managed = False
|
||||
if source_path.exists() and source_managed:
|
||||
source_path.unlink()
|
||||
self.repo.delete_artifact_by_path(task_id, str(source_path.resolve()))
|
||||
removed.append(str(source_path))
|
||||
|
||||
@ -2,47 +2,42 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from biliup_next.core.errors import ModuleError
|
||||
from biliup_next.core.models import Task
|
||||
from biliup_next.core.providers import ProviderManifest
|
||||
from biliup_next.infra.adapters.bilibili_api import BilibiliApiAdapter
|
||||
from biliup_next.infra.adapters.full_video_locator import resolve_full_video_bvid
|
||||
|
||||
|
||||
class LegacyBilibiliCollectionProvider:
|
||||
class BilibiliCollectionProvider:
|
||||
def __init__(self, bilibili_api: BilibiliApiAdapter | None = None) -> None:
|
||||
self.bilibili_api = bilibili_api or BilibiliApiAdapter()
|
||||
self._section_cache: dict[int, int | None] = {}
|
||||
|
||||
manifest = ProviderManifest(
|
||||
id="bilibili_collection",
|
||||
name="Legacy Bilibili Collection Provider",
|
||||
name="Bilibili Collection Provider",
|
||||
version="0.1.0",
|
||||
provider_type="collection_provider",
|
||||
entrypoint="biliup_next.infra.adapters.bilibili_collection_legacy:LegacyBilibiliCollectionProvider",
|
||||
entrypoint="biliup_next.modules.collection.providers.bilibili_collection:BilibiliCollectionProvider",
|
||||
capabilities=["collection"],
|
||||
enabled_by_default=True,
|
||||
)
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._section_cache: dict[int, int | None] = {}
|
||||
|
||||
def sync(self, task: Task, target: str, settings: dict[str, Any]) -> dict[str, object]:
|
||||
session_dir = Path(str(settings["session_dir"])) / task.title
|
||||
cookies = self._load_cookies(Path(str(settings["cookies_file"])))
|
||||
cookies = self.bilibili_api.load_cookies(Path(str(settings["cookies_file"])))
|
||||
csrf = cookies.get("bili_jct")
|
||||
if not csrf:
|
||||
raise ModuleError(code="COOKIE_CSRF_MISSING", message="Cookie 缺少 bili_jct", retryable=False)
|
||||
session = requests.Session()
|
||||
session.cookies.update(cookies)
|
||||
session.headers.update(
|
||||
{
|
||||
"User-Agent": "Mozilla/5.0",
|
||||
"Referer": "https://member.bilibili.com/platform/upload-manager/distribution",
|
||||
}
|
||||
|
||||
session = self.bilibili_api.build_session(
|
||||
cookies=cookies,
|
||||
referer="https://member.bilibili.com/platform/upload-manager/distribution",
|
||||
)
|
||||
|
||||
if target == "a":
|
||||
@ -69,12 +64,11 @@ class LegacyBilibiliCollectionProvider:
|
||||
raise ModuleError(code="COLLECTION_SECTION_NOT_FOUND", message=f"未找到合集 section: {season_id}", retryable=True)
|
||||
|
||||
info = self._get_video_info(session, bvid)
|
||||
episodes = [info]
|
||||
add_result = self._add_videos_batch(session, csrf, section_id, episodes)
|
||||
add_result = self._add_videos_batch(session, csrf, section_id, [info])
|
||||
if add_result["status"] == "failed":
|
||||
raise ModuleError(
|
||||
code="COLLECTION_ADD_FAILED",
|
||||
message=add_result["message"],
|
||||
message=str(add_result["message"]),
|
||||
retryable=True,
|
||||
details=add_result,
|
||||
)
|
||||
@ -83,21 +77,13 @@ class LegacyBilibiliCollectionProvider:
|
||||
if add_result["status"] == "added":
|
||||
append_key = "append_collection_a_new_to_end" if target == "a" else "append_collection_b_new_to_end"
|
||||
if settings.get(append_key, True):
|
||||
self._move_videos_to_section_end(session, csrf, section_id, [info["aid"]])
|
||||
self._move_videos_to_section_end(session, csrf, section_id, [int(info["aid"])])
|
||||
return {"status": add_result["status"], "target": target, "bvid": bvid, "season_id": season_id}
|
||||
|
||||
@staticmethod
|
||||
def _load_cookies(path: Path) -> dict[str, str]:
|
||||
with path.open("r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
if "cookie_info" in data:
|
||||
return {c["name"]: c["value"] for c in data.get("cookie_info", {}).get("cookies", [])}
|
||||
return data
|
||||
|
||||
def _resolve_section_id(self, session: requests.Session, season_id: int) -> int | None:
|
||||
def _resolve_section_id(self, session, season_id: int) -> int | None: # type: ignore[no-untyped-def]
|
||||
if season_id in self._section_cache:
|
||||
return self._section_cache[season_id]
|
||||
result = session.get("https://member.bilibili.com/x2/creative/web/seasons", params={"pn": 1, "ps": 50}, timeout=15).json()
|
||||
result = self.bilibili_api.list_seasons(session)
|
||||
if result.get("code") != 0:
|
||||
return None
|
||||
for season in result.get("data", {}).get("seasons", []):
|
||||
@ -109,40 +95,31 @@ class LegacyBilibiliCollectionProvider:
|
||||
self._section_cache[season_id] = None
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _get_video_info(session: requests.Session, bvid: str) -> dict[str, object]:
|
||||
result = session.get("https://api.bilibili.com/x/web-interface/view", params={"bvid": bvid}, timeout=15).json()
|
||||
if result.get("code") != 0:
|
||||
raise ModuleError(
|
||||
code="COLLECTION_VIDEO_INFO_FAILED",
|
||||
message=f"获取视频信息失败: {result.get('message')}",
|
||||
retryable=True,
|
||||
)
|
||||
data = result["data"]
|
||||
def _get_video_info(self, session, bvid: str) -> dict[str, object]: # type: ignore[no-untyped-def]
|
||||
data = self.bilibili_api.get_video_view(
|
||||
session,
|
||||
bvid,
|
||||
error_code="COLLECTION_VIDEO_INFO_FAILED",
|
||||
error_message="获取视频信息失败",
|
||||
)
|
||||
return {"aid": data["aid"], "cid": data["cid"], "title": data["title"], "charging_pay": 0}
|
||||
|
||||
@staticmethod
|
||||
def _add_videos_batch(session: requests.Session, csrf: str, section_id: int, episodes: list[dict[str, object]]) -> dict[str, object]:
|
||||
def _add_videos_batch(self, session, csrf: str, section_id: int, episodes: list[dict[str, object]]) -> dict[str, object]: # type: ignore[no-untyped-def]
|
||||
time.sleep(random.uniform(5.0, 10.0))
|
||||
result = session.post(
|
||||
"https://member.bilibili.com/x2/creative/web/season/section/episodes/add",
|
||||
params={"csrf": csrf},
|
||||
json={"sectionId": section_id, "episodes": episodes},
|
||||
timeout=20,
|
||||
).json()
|
||||
result = self.bilibili_api.add_section_episodes(
|
||||
session,
|
||||
csrf=csrf,
|
||||
section_id=section_id,
|
||||
episodes=episodes,
|
||||
)
|
||||
if result.get("code") == 0:
|
||||
return {"status": "added"}
|
||||
if result.get("code") == 20080:
|
||||
return {"status": "already_exists", "message": result.get("message", "")}
|
||||
return {"status": "failed", "message": result.get("message", "unknown error"), "code": result.get("code")}
|
||||
|
||||
@staticmethod
|
||||
def _move_videos_to_section_end(session: requests.Session, csrf: str, section_id: int, added_aids: list[int]) -> bool:
|
||||
detail = session.get(
|
||||
"https://member.bilibili.com/x2/creative/web/season/section",
|
||||
params={"id": section_id},
|
||||
timeout=20,
|
||||
).json()
|
||||
def _move_videos_to_section_end(self, session, csrf: str, section_id: int, added_aids: list[int]) -> bool: # type: ignore[no-untyped-def]
|
||||
detail = self.bilibili_api.get_section_detail(session, section_id=section_id)
|
||||
if detail.get("code") != 0:
|
||||
return False
|
||||
section = detail.get("data", {}).get("section", {})
|
||||
@ -168,12 +145,7 @@ class LegacyBilibiliCollectionProvider:
|
||||
"title": section["title"],
|
||||
"type": section["type"],
|
||||
},
|
||||
"sorts": [{"id": item["id"], "sort": idx + 1} for idx, item in enumerate(ordered)],
|
||||
"sorts": [{"id": item["id"], "sort": index + 1} for index, item in enumerate(ordered)],
|
||||
}
|
||||
result = session.post(
|
||||
"https://member.bilibili.com/x2/creative/web/season/section/edit",
|
||||
params={"csrf": csrf},
|
||||
json=payload,
|
||||
timeout=20,
|
||||
).json()
|
||||
result = self.bilibili_api.edit_section(session, csrf=csrf, payload=payload)
|
||||
return result.get("code") == 0
|
||||
@ -5,21 +5,23 @@ import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from biliup_next.core.errors import ModuleError
|
||||
from biliup_next.core.models import Task
|
||||
from biliup_next.core.providers import ProviderManifest
|
||||
from biliup_next.infra.adapters.bilibili_api import BilibiliApiAdapter
|
||||
from biliup_next.infra.adapters.full_video_locator import resolve_full_video_bvid
|
||||
|
||||
|
||||
class LegacyBilibiliTopCommentProvider:
|
||||
class BilibiliTopCommentProvider:
|
||||
def __init__(self, bilibili_api: BilibiliApiAdapter | None = None) -> None:
|
||||
self.bilibili_api = bilibili_api or BilibiliApiAdapter()
|
||||
|
||||
manifest = ProviderManifest(
|
||||
id="bilibili_top_comment",
|
||||
name="Legacy Bilibili Top Comment Provider",
|
||||
name="Bilibili Top Comment Provider",
|
||||
version="0.1.0",
|
||||
provider_type="comment_provider",
|
||||
entrypoint="biliup_next.infra.adapters.bilibili_top_comment_legacy:LegacyBilibiliTopCommentProvider",
|
||||
entrypoint="biliup_next.modules.comment.providers.bilibili_top_comment:BilibiliTopCommentProvider",
|
||||
capabilities=["comment"],
|
||||
enabled_by_default=True,
|
||||
)
|
||||
@ -42,19 +44,15 @@ class LegacyBilibiliTopCommentProvider:
|
||||
self._touch_comment_flags(session_dir, split_done=True, full_done=True)
|
||||
return {"status": "skipped", "reason": "comment_content_empty"}
|
||||
|
||||
cookies = self._load_cookies(Path(str(settings["cookies_file"])))
|
||||
cookies = self.bilibili_api.load_cookies(Path(str(settings["cookies_file"])))
|
||||
csrf = cookies.get("bili_jct")
|
||||
if not csrf:
|
||||
raise ModuleError(code="COOKIE_CSRF_MISSING", message="Cookie 缺少 bili_jct", retryable=False)
|
||||
|
||||
session = requests.Session()
|
||||
session.cookies.update(cookies)
|
||||
session.headers.update(
|
||||
{
|
||||
"User-Agent": "Mozilla/5.0",
|
||||
"Referer": "https://www.bilibili.com/",
|
||||
"Origin": "https://www.bilibili.com",
|
||||
}
|
||||
session = self.bilibili_api.build_session(
|
||||
cookies=cookies,
|
||||
referer="https://www.bilibili.com/",
|
||||
origin="https://www.bilibili.com",
|
||||
)
|
||||
|
||||
split_result = {"status": "skipped", "reason": "disabled"}
|
||||
@ -79,7 +77,8 @@ class LegacyBilibiliTopCommentProvider:
|
||||
if full_bvid and timeline_content:
|
||||
full_result = self._post_and_top_comment(session, csrf, full_bvid, timeline_content, "full")
|
||||
else:
|
||||
full_result = {"status": "skipped", "reason": "full_video_bvid_not_found" if not full_bvid else "timeline_comment_empty"}
|
||||
reason = "full_video_bvid_not_found" if not full_bvid else "timeline_comment_empty"
|
||||
full_result = {"status": "skipped", "reason": reason}
|
||||
full_done = True
|
||||
(session_dir / "comment_full_done.flag").touch()
|
||||
elif not full_done:
|
||||
@ -92,44 +91,35 @@ class LegacyBilibiliTopCommentProvider:
|
||||
|
||||
def _post_and_top_comment(
|
||||
self,
|
||||
session: requests.Session,
|
||||
session,
|
||||
csrf: str,
|
||||
bvid: str,
|
||||
content: str,
|
||||
target: str,
|
||||
) -> dict[str, object]:
|
||||
view = session.get("https://api.bilibili.com/x/web-interface/view", params={"bvid": bvid}, timeout=15).json()
|
||||
if view.get("code") != 0:
|
||||
raise ModuleError(
|
||||
code="COMMENT_VIEW_FAILED",
|
||||
message=f"获取{target}视频信息失败: {view.get('message')}",
|
||||
retryable=True,
|
||||
)
|
||||
aid = view["data"]["aid"]
|
||||
add_res = session.post(
|
||||
"https://api.bilibili.com/x/v2/reply/add",
|
||||
data={"type": 1, "oid": aid, "message": content, "plat": 1, "csrf": csrf},
|
||||
timeout=15,
|
||||
).json()
|
||||
if add_res.get("code") != 0:
|
||||
raise ModuleError(
|
||||
code="COMMENT_POST_FAILED",
|
||||
message=f"发布{target}评论失败: {add_res.get('message')}",
|
||||
retryable=True,
|
||||
)
|
||||
rpid = add_res["data"]["rpid"]
|
||||
view = self.bilibili_api.get_video_view(
|
||||
session,
|
||||
bvid,
|
||||
error_code="COMMENT_VIEW_FAILED",
|
||||
error_message=f"获取{target}视频信息失败",
|
||||
)
|
||||
aid = int(view["aid"])
|
||||
add_res = self.bilibili_api.add_reply(
|
||||
session,
|
||||
csrf=csrf,
|
||||
aid=aid,
|
||||
content=content,
|
||||
error_message=f"发布{target}评论失败",
|
||||
)
|
||||
rpid = int(add_res["rpid"])
|
||||
time.sleep(3)
|
||||
top_res = session.post(
|
||||
"https://api.bilibili.com/x/v2/reply/top",
|
||||
data={"type": 1, "oid": aid, "rpid": rpid, "action": 1, "csrf": csrf},
|
||||
timeout=15,
|
||||
).json()
|
||||
if top_res.get("code") != 0:
|
||||
raise ModuleError(
|
||||
code="COMMENT_TOP_FAILED",
|
||||
message=f"置顶{target}评论失败: {top_res.get('message')}",
|
||||
retryable=True,
|
||||
)
|
||||
self.bilibili_api.top_reply(
|
||||
session,
|
||||
csrf=csrf,
|
||||
aid=aid,
|
||||
rpid=rpid,
|
||||
error_message=f"置顶{target}评论失败",
|
||||
)
|
||||
return {"status": "ok", "bvid": bvid, "aid": aid, "rpid": rpid}
|
||||
|
||||
@staticmethod
|
||||
@ -161,14 +151,6 @@ class LegacyBilibiliTopCommentProvider:
|
||||
return "\n".join(lines)
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _load_cookies(path: Path) -> dict[str, str]:
|
||||
with path.open("r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
if "cookie_info" in data:
|
||||
return {c["name"]: c["value"] for c in data.get("cookie_info", {}).get("cookies", [])}
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def _touch_comment_flags(session_dir: Path, *, split_done: bool, full_done: bool) -> None:
|
||||
if split_done:
|
||||
@ -1,26 +1,51 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from biliup_next.core.errors import ModuleError
|
||||
from biliup_next.core.models import Artifact, Task, TaskStep, utc_now_iso
|
||||
from biliup_next.core.models import Artifact, Task, TaskContext, TaskStep, utc_now_iso
|
||||
from biliup_next.core.registry import Registry
|
||||
from biliup_next.infra.task_repository import TaskRepository
|
||||
|
||||
SHANGHAI_TZ = ZoneInfo("Asia/Shanghai")
|
||||
TITLE_PATTERN = re.compile(
|
||||
r"^(?P<streamer>.+?)\s+(?P<month>\d{2})月(?P<day>\d{2})日\s+(?P<hour>\d{2})时(?P<minute>\d{2})分"
|
||||
)
|
||||
|
||||
|
||||
class IngestService:
|
||||
def __init__(self, registry: Registry, repo: TaskRepository):
|
||||
self.registry = registry
|
||||
self.repo = repo
|
||||
|
||||
def create_task_from_file(self, source_path: Path, settings: dict[str, object]) -> Task:
|
||||
def create_task_from_file(
|
||||
self,
|
||||
source_path: Path,
|
||||
settings: dict[str, object],
|
||||
*,
|
||||
context_payload: dict[str, object] | None = None,
|
||||
) -> Task:
|
||||
provider_id = str(settings.get("provider", "local_file"))
|
||||
provider = self.registry.get("ingest_provider", provider_id)
|
||||
provider.validate_source(source_path, settings)
|
||||
source_path = source_path.resolve()
|
||||
session_dir = Path(str(settings["session_dir"])).resolve()
|
||||
try:
|
||||
source_path.relative_to(session_dir)
|
||||
except ValueError as exc:
|
||||
raise ModuleError(
|
||||
code="SOURCE_OUTSIDE_WORKSPACE",
|
||||
message=f"源文件不在 session 工作区内: {source_path}",
|
||||
retryable=False,
|
||||
details={"session_dir": str(session_dir), "hint": "请先使用 stage/import 或 stage/upload 导入文件"},
|
||||
) from exc
|
||||
|
||||
task_id = source_path.stem
|
||||
if self.repo.get_task(task_id):
|
||||
@ -31,10 +56,11 @@ class IngestService:
|
||||
)
|
||||
|
||||
now = utc_now_iso()
|
||||
context_payload = context_payload or {}
|
||||
task = Task(
|
||||
id=task_id,
|
||||
source_type="local_file",
|
||||
source_path=str(source_path.resolve()),
|
||||
source_path=str(source_path),
|
||||
title=source_path.stem,
|
||||
status="created",
|
||||
created_at=now,
|
||||
@ -59,11 +85,22 @@ class IngestService:
|
||||
id=None,
|
||||
task_id=task_id,
|
||||
artifact_type="source_video",
|
||||
path=str(source_path.resolve()),
|
||||
path=str(source_path),
|
||||
metadata_json=json.dumps({"provider": provider_id}),
|
||||
created_at=now,
|
||||
)
|
||||
)
|
||||
context = self._build_task_context(
|
||||
task,
|
||||
context_payload,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
session_gap_minutes=int(settings.get("session_gap_minutes", 60)),
|
||||
)
|
||||
self.repo.upsert_task_context(context)
|
||||
full_video_bvid = (context.full_video_bvid or "").strip()
|
||||
if full_video_bvid.startswith("BV"):
|
||||
(source_path.parent / "full_video_bvid.txt").write_text(full_video_bvid, encoding="utf-8")
|
||||
return task
|
||||
|
||||
def scan_stage(self, settings: dict[str, object]) -> dict[str, object]:
|
||||
@ -123,10 +160,27 @@ class IngestService:
|
||||
)
|
||||
continue
|
||||
|
||||
sidecar_meta = self._load_sidecar_metadata(
|
||||
source_path,
|
||||
enabled=bool(settings.get("meta_sidecar_enabled", True)),
|
||||
suffix=str(settings.get("meta_sidecar_suffix", ".meta.json")),
|
||||
)
|
||||
task_dir = session_dir / task_id
|
||||
task_dir.mkdir(parents=True, exist_ok=True)
|
||||
target_source = self._move_to_directory(source_path, task_dir)
|
||||
task = self.create_task_from_file(target_source, settings)
|
||||
if sidecar_meta["meta_path"] is not None:
|
||||
self._move_optional_metadata_file(sidecar_meta["meta_path"], task_dir)
|
||||
context_payload = {
|
||||
"source_title": source_path.stem,
|
||||
"segment_duration_seconds": duration_seconds,
|
||||
"segment_started_at": sidecar_meta["payload"].get("segment_started_at"),
|
||||
"streamer": sidecar_meta["payload"].get("streamer"),
|
||||
"room_id": sidecar_meta["payload"].get("room_id"),
|
||||
"session_key": sidecar_meta["payload"].get("session_key"),
|
||||
"full_video_bvid": sidecar_meta["payload"].get("full_video_bvid"),
|
||||
"reference_timestamp": sidecar_meta["payload"].get("reference_timestamp") or source_path.stat().st_mtime,
|
||||
}
|
||||
task = self.create_task_from_file(target_source, settings, context_payload=context_payload)
|
||||
accepted.append(
|
||||
{
|
||||
"task_id": task.id,
|
||||
@ -199,3 +253,202 @@ class IngestService:
|
||||
if not candidate.exists():
|
||||
return candidate
|
||||
index += 1
|
||||
|
||||
@staticmethod
|
||||
def _load_sidecar_metadata(source_path: Path, *, enabled: bool, suffix: str) -> dict[str, object]:
|
||||
if not enabled:
|
||||
return {"meta_path": None, "payload": {}}
|
||||
suffix = suffix.strip() or ".meta.json"
|
||||
meta_path = source_path.with_name(f"{source_path.stem}{suffix}")
|
||||
payload: dict[str, object] = {}
|
||||
if meta_path.exists():
|
||||
try:
|
||||
payload = json.loads(meta_path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ModuleError(
|
||||
code="STAGE_META_INVALID",
|
||||
message=f"元数据文件不是合法 JSON: {meta_path.name}",
|
||||
retryable=False,
|
||||
) from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise ModuleError(
|
||||
code="STAGE_META_INVALID",
|
||||
message=f"元数据文件必须是对象: {meta_path.name}",
|
||||
retryable=False,
|
||||
)
|
||||
return {"meta_path": meta_path if meta_path.exists() else None, "payload": payload}
|
||||
|
||||
def _move_optional_metadata_file(self, meta_path: Path, task_dir: Path) -> None:
|
||||
if not meta_path.exists():
|
||||
return
|
||||
self._move_to_directory(meta_path, task_dir)
|
||||
|
||||
def _build_task_context(
|
||||
self,
|
||||
task: Task,
|
||||
context_payload: dict[str, object],
|
||||
*,
|
||||
created_at: str,
|
||||
updated_at: str,
|
||||
session_gap_minutes: int,
|
||||
) -> TaskContext:
|
||||
source_title = self._clean_text(context_payload.get("source_title")) or task.title
|
||||
streamer = self._clean_text(context_payload.get("streamer"))
|
||||
room_id = self._clean_text(context_payload.get("room_id"))
|
||||
session_key = self._clean_text(context_payload.get("session_key"))
|
||||
full_video_bvid = self._clean_bvid(context_payload.get("full_video_bvid"))
|
||||
segment_duration = self._coerce_float(context_payload.get("segment_duration_seconds"))
|
||||
segment_started_at = self._coerce_iso_datetime(context_payload.get("segment_started_at"))
|
||||
|
||||
if streamer is None or segment_started_at is None:
|
||||
inferred = self._infer_from_title(
|
||||
source_title,
|
||||
reference_timestamp=context_payload.get("reference_timestamp"),
|
||||
)
|
||||
if streamer is None:
|
||||
streamer = inferred.get("streamer")
|
||||
if segment_started_at is None:
|
||||
segment_started_at = inferred.get("segment_started_at")
|
||||
|
||||
if session_key is None:
|
||||
session_key, inherited_bvid = self._infer_session_key(
|
||||
streamer=streamer,
|
||||
room_id=room_id,
|
||||
segment_started_at=segment_started_at,
|
||||
segment_duration_seconds=segment_duration,
|
||||
fallback_task_id=task.id,
|
||||
gap_minutes=session_gap_minutes,
|
||||
)
|
||||
if full_video_bvid is None:
|
||||
full_video_bvid = inherited_bvid
|
||||
elif full_video_bvid is None:
|
||||
full_video_bvid = self._find_full_video_bvid_by_session_key(session_key)
|
||||
|
||||
if full_video_bvid is None:
|
||||
binding = self.repo.get_session_binding(session_key=session_key, source_title=source_title)
|
||||
if binding is not None:
|
||||
if session_key is None and binding.session_key:
|
||||
session_key = binding.session_key
|
||||
full_video_bvid = self._clean_bvid(binding.full_video_bvid)
|
||||
|
||||
if session_key is None:
|
||||
session_key = f"task:{task.id}"
|
||||
|
||||
return TaskContext(
|
||||
id=None,
|
||||
task_id=task.id,
|
||||
session_key=session_key,
|
||||
streamer=streamer,
|
||||
room_id=room_id,
|
||||
source_title=source_title,
|
||||
segment_started_at=segment_started_at,
|
||||
segment_duration_seconds=segment_duration,
|
||||
full_video_bvid=full_video_bvid,
|
||||
created_at=created_at,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _clean_text(value: object) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
text = str(value).strip()
|
||||
return text or None
|
||||
|
||||
@staticmethod
|
||||
def _clean_bvid(value: object) -> str | None:
|
||||
text = IngestService._clean_text(value)
|
||||
if text and text.startswith("BV"):
|
||||
return text
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _coerce_float(value: object) -> float | None:
|
||||
if value is None or value == "":
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _coerce_iso_datetime(value: object) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(text).astimezone(SHANGHAI_TZ).isoformat()
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
def _infer_from_title(self, title: str, *, reference_timestamp: object) -> dict[str, str | None]:
|
||||
match = TITLE_PATTERN.match(title)
|
||||
if not match:
|
||||
return {"streamer": None, "segment_started_at": None}
|
||||
reference_dt = self._reference_datetime(reference_timestamp)
|
||||
month = int(match.group("month"))
|
||||
day = int(match.group("day"))
|
||||
hour = int(match.group("hour"))
|
||||
minute = int(match.group("minute"))
|
||||
year = reference_dt.year
|
||||
if (month, day) > (reference_dt.month, reference_dt.day):
|
||||
year -= 1
|
||||
started_at = datetime(year, month, day, hour, minute, tzinfo=SHANGHAI_TZ)
|
||||
return {
|
||||
"streamer": match.group("streamer").strip(),
|
||||
"segment_started_at": started_at.isoformat(),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _reference_datetime(reference_timestamp: object) -> datetime:
|
||||
if isinstance(reference_timestamp, (int, float)):
|
||||
return datetime.fromtimestamp(float(reference_timestamp), tz=SHANGHAI_TZ)
|
||||
return datetime.now(tz=SHANGHAI_TZ)
|
||||
|
||||
def _infer_session_key(
|
||||
self,
|
||||
*,
|
||||
streamer: str | None,
|
||||
room_id: str | None,
|
||||
segment_started_at: str | None,
|
||||
segment_duration_seconds: float | None,
|
||||
fallback_task_id: str,
|
||||
gap_minutes: int,
|
||||
) -> tuple[str | None, str | None]:
|
||||
if not streamer or not segment_started_at:
|
||||
return None, None
|
||||
try:
|
||||
segment_start = datetime.fromisoformat(segment_started_at)
|
||||
except ValueError:
|
||||
return None, None
|
||||
|
||||
tolerance = timedelta(minutes=max(gap_minutes, 0))
|
||||
for context in self.repo.find_recent_task_contexts(streamer):
|
||||
if room_id and context.room_id and room_id != context.room_id:
|
||||
continue
|
||||
candidate_end = self._context_end_time(context)
|
||||
if candidate_end is None:
|
||||
continue
|
||||
if segment_start >= candidate_end and segment_start - candidate_end <= tolerance:
|
||||
return context.session_key, context.full_video_bvid
|
||||
date_tag = segment_start.astimezone(SHANGHAI_TZ).strftime("%Y%m%dT%H%M")
|
||||
return f"{streamer}:{date_tag}", None
|
||||
|
||||
@staticmethod
|
||||
def _context_end_time(context: TaskContext) -> datetime | None:
|
||||
if not context.segment_started_at or context.segment_duration_seconds is None:
|
||||
return None
|
||||
try:
|
||||
started_at = datetime.fromisoformat(context.segment_started_at)
|
||||
except ValueError:
|
||||
return None
|
||||
return started_at + timedelta(seconds=float(context.segment_duration_seconds))
|
||||
|
||||
def _find_full_video_bvid_by_session_key(self, session_key: str) -> str | None:
|
||||
for context in self.repo.list_task_contexts_by_session_key(session_key):
|
||||
bvid = self._clean_bvid(context.full_video_bvid)
|
||||
if bvid:
|
||||
return bvid
|
||||
return None
|
||||
|
||||
247
src/biliup_next/modules/publish/providers/biliup_cli.py
Normal file
247
src/biliup_next/modules/publish/providers/biliup_cli.py
Normal file
@ -0,0 +1,247 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from biliup_next.core.errors import ModuleError
|
||||
from biliup_next.core.models import PublishRecord, Task, utc_now_iso
|
||||
from biliup_next.core.providers import ProviderManifest
|
||||
from biliup_next.infra.adapters.biliup_cli import BiliupCliAdapter
|
||||
|
||||
|
||||
class BiliupCliPublishProvider:
|
||||
def __init__(self, adapter: BiliupCliAdapter | None = None) -> None:
|
||||
self.adapter = adapter or BiliupCliAdapter()
|
||||
|
||||
manifest = ProviderManifest(
|
||||
id="biliup_cli",
|
||||
name="biliup CLI Publish Provider",
|
||||
version="0.1.0",
|
||||
provider_type="publish_provider",
|
||||
entrypoint="biliup_next.modules.publish.providers.biliup_cli:BiliupCliPublishProvider",
|
||||
capabilities=["publish"],
|
||||
enabled_by_default=True,
|
||||
)
|
||||
|
||||
def publish(self, task: Task, clip_videos: list, settings: dict[str, Any]) -> PublishRecord:
|
||||
work_dir = Path(str(settings["session_dir"])) / task.title
|
||||
bvid_file = work_dir / "bvid.txt"
|
||||
upload_done = work_dir / "upload_done.flag"
|
||||
config = self._load_upload_config(Path(str(settings["upload_config_file"])))
|
||||
|
||||
video_files = [artifact.path for artifact in clip_videos]
|
||||
if not video_files:
|
||||
raise ModuleError(
|
||||
code="PUBLISH_NO_CLIPS",
|
||||
message=f"没有可上传的切片: {task.id}",
|
||||
retryable=False,
|
||||
)
|
||||
|
||||
parsed = self._parse_filename(task.title, config)
|
||||
streamer = parsed.get("streamer", task.title)
|
||||
date = parsed.get("date", "")
|
||||
|
||||
songs_txt = work_dir / "songs.txt"
|
||||
songs_json = work_dir / "songs.json"
|
||||
songs_list = songs_txt.read_text(encoding="utf-8").strip() if songs_txt.exists() else ""
|
||||
song_count = 0
|
||||
if songs_json.exists():
|
||||
song_count = len(json.loads(songs_json.read_text(encoding="utf-8")).get("songs", []))
|
||||
|
||||
quote = self._get_random_quote(config)
|
||||
template_vars = {
|
||||
"streamer": streamer,
|
||||
"date": date,
|
||||
"song_count": song_count,
|
||||
"songs_list": songs_list,
|
||||
"daily_quote": quote.get("text", ""),
|
||||
"quote_author": quote.get("author", ""),
|
||||
}
|
||||
template = config.get("template", {})
|
||||
title = template.get("title", "{streamer}_{date}").format(**template_vars)
|
||||
description = template.get("description", "{songs_list}").format(**template_vars)
|
||||
dynamic = template.get("dynamic", "").format(**template_vars)
|
||||
tags = template.get("tag", "翻唱,唱歌,音乐").format(**template_vars)
|
||||
streamer_cfg = config.get("streamers", {})
|
||||
if streamer in streamer_cfg:
|
||||
tags = streamer_cfg[streamer].get("tags", tags)
|
||||
|
||||
upload_settings = config.get("upload_settings", {})
|
||||
tid = upload_settings.get("tid", 31)
|
||||
biliup_path = str(settings["biliup_path"])
|
||||
cookie_file = str(settings["cookie_file"])
|
||||
retry_count = max(1, int(settings.get("retry_count", 5)))
|
||||
|
||||
self.adapter.run_optional([biliup_path, "-u", cookie_file, "renew"])
|
||||
|
||||
first_batch = video_files[:5]
|
||||
remaining_batches = [video_files[i:i + 5] for i in range(5, len(video_files), 5)]
|
||||
|
||||
existing_bvid = bvid_file.read_text(encoding="utf-8").strip() if bvid_file.exists() else ""
|
||||
if upload_done.exists() and existing_bvid.startswith("BV"):
|
||||
return PublishRecord(
|
||||
id=None,
|
||||
task_id=task.id,
|
||||
platform="bilibili",
|
||||
aid=None,
|
||||
bvid=existing_bvid,
|
||||
title=title,
|
||||
published_at=utc_now_iso(),
|
||||
)
|
||||
|
||||
bvid = existing_bvid if existing_bvid.startswith("BV") else self._upload_first_batch(
|
||||
biliup_path=biliup_path,
|
||||
cookie_file=cookie_file,
|
||||
first_batch=first_batch,
|
||||
title=title,
|
||||
tid=tid,
|
||||
tags=tags,
|
||||
description=description,
|
||||
dynamic=dynamic,
|
||||
upload_settings=upload_settings,
|
||||
retry_count=retry_count,
|
||||
)
|
||||
bvid_file.write_text(bvid, encoding="utf-8")
|
||||
|
||||
for batch_index, batch in enumerate(remaining_batches, start=2):
|
||||
self._append_batch(
|
||||
biliup_path=biliup_path,
|
||||
cookie_file=cookie_file,
|
||||
bvid=bvid,
|
||||
batch=batch,
|
||||
batch_index=batch_index,
|
||||
retry_count=retry_count,
|
||||
)
|
||||
|
||||
upload_done.touch()
|
||||
return PublishRecord(
|
||||
id=None,
|
||||
task_id=task.id,
|
||||
platform="bilibili",
|
||||
aid=None,
|
||||
bvid=bvid,
|
||||
title=title,
|
||||
published_at=utc_now_iso(),
|
||||
)
|
||||
|
||||
def _upload_first_batch(
|
||||
self,
|
||||
*,
|
||||
biliup_path: str,
|
||||
cookie_file: str,
|
||||
first_batch: list[str],
|
||||
title: str,
|
||||
tid: int,
|
||||
tags: str,
|
||||
description: str,
|
||||
dynamic: str,
|
||||
upload_settings: dict[str, Any],
|
||||
retry_count: int,
|
||||
) -> str:
|
||||
upload_cmd = [
|
||||
biliup_path,
|
||||
"-u",
|
||||
cookie_file,
|
||||
"upload",
|
||||
*first_batch,
|
||||
"--title",
|
||||
title,
|
||||
"--tid",
|
||||
str(tid),
|
||||
"--tag",
|
||||
tags,
|
||||
"--copyright",
|
||||
str(upload_settings.get("copyright", 2)),
|
||||
"--source",
|
||||
str(upload_settings.get("source", "直播回放")),
|
||||
"--desc",
|
||||
description,
|
||||
]
|
||||
if dynamic:
|
||||
upload_cmd.extend(["--dynamic", dynamic])
|
||||
cover = str(upload_settings.get("cover", "")).strip()
|
||||
if cover and Path(cover).exists():
|
||||
upload_cmd.extend(["--cover", cover])
|
||||
|
||||
for attempt in range(1, retry_count + 1):
|
||||
result = self.adapter.run(upload_cmd, label=f"首批上传[{attempt}/{retry_count}]")
|
||||
if result.returncode == 0:
|
||||
match = re.search(r'"bvid":"(BV[A-Za-z0-9]+)"', result.stdout) or re.search(r"(BV[A-Za-z0-9]+)", result.stdout)
|
||||
if match:
|
||||
return match.group(1)
|
||||
if attempt < retry_count:
|
||||
time.sleep(self._wait_seconds(attempt - 1))
|
||||
continue
|
||||
raise ModuleError(
|
||||
code="PUBLISH_UPLOAD_FAILED",
|
||||
message="首批上传失败",
|
||||
retryable=True,
|
||||
details={"stdout": result.stdout[-2000:], "stderr": result.stderr[-2000:]},
|
||||
)
|
||||
raise AssertionError("unreachable")
|
||||
|
||||
def _append_batch(
|
||||
self,
|
||||
*,
|
||||
biliup_path: str,
|
||||
cookie_file: str,
|
||||
bvid: str,
|
||||
batch: list[str],
|
||||
batch_index: int,
|
||||
retry_count: int,
|
||||
) -> None:
|
||||
time.sleep(45)
|
||||
append_cmd = [biliup_path, "-u", cookie_file, "append", "--vid", bvid, *batch]
|
||||
for attempt in range(1, retry_count + 1):
|
||||
result = self.adapter.run(append_cmd, label=f"追加第{batch_index}批[{attempt}/{retry_count}]")
|
||||
if result.returncode == 0:
|
||||
return
|
||||
if attempt < retry_count:
|
||||
time.sleep(self._wait_seconds(attempt - 1))
|
||||
continue
|
||||
raise ModuleError(
|
||||
code="PUBLISH_APPEND_FAILED",
|
||||
message=f"追加第 {batch_index} 批失败",
|
||||
retryable=True,
|
||||
details={"stdout": result.stdout[-2000:], "stderr": result.stderr[-2000:]},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _wait_seconds(retry_index: int) -> int:
|
||||
return min(300 * (2**retry_index), 3600)
|
||||
|
||||
@staticmethod
|
||||
def _load_upload_config(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
@staticmethod
|
||||
def _parse_filename(filename: str, config: dict[str, Any] | None = None) -> dict[str, str]:
|
||||
config = config or {}
|
||||
patterns = config.get("filename_patterns", {}).get("patterns", [])
|
||||
for pattern_config in patterns:
|
||||
regex = pattern_config.get("regex")
|
||||
if not regex:
|
||||
continue
|
||||
match = re.match(regex, filename)
|
||||
if match:
|
||||
data = match.groupdict()
|
||||
date_format = pattern_config.get("date_format", "{date}")
|
||||
try:
|
||||
data["date"] = date_format.format(**data)
|
||||
except KeyError:
|
||||
pass
|
||||
return data
|
||||
return {"streamer": filename, "date": ""}
|
||||
|
||||
@staticmethod
|
||||
def _get_random_quote(config: dict[str, Any]) -> dict[str, str]:
|
||||
quotes = config.get("quotes", [])
|
||||
if not quotes:
|
||||
return {"text": "", "author": ""}
|
||||
return random.choice(quotes)
|
||||
@ -1,15 +1,13 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from biliup_next.core.errors import ModuleError
|
||||
from biliup_next.core.models import Artifact, Task, utc_now_iso
|
||||
from biliup_next.core.providers import ProviderManifest
|
||||
from biliup_next.infra.legacy_paths import legacy_project_root
|
||||
from biliup_next.infra.adapters.codex_cli import CodexCliAdapter
|
||||
|
||||
SONG_SCHEMA = {
|
||||
"type": "object",
|
||||
@ -24,15 +22,15 @@ SONG_SCHEMA = {
|
||||
"title": {"type": "string"},
|
||||
"artist": {"type": "string"},
|
||||
"confidence": {"type": "number"},
|
||||
"evidence": {"type": "string"}
|
||||
"evidence": {"type": "string"},
|
||||
},
|
||||
"required": ["start", "end", "title", "artist", "confidence", "evidence"],
|
||||
"additionalProperties": False
|
||||
}
|
||||
"additionalProperties": False,
|
||||
},
|
||||
}
|
||||
},
|
||||
"required": ["songs"],
|
||||
"additionalProperties": False
|
||||
"additionalProperties": False,
|
||||
}
|
||||
|
||||
TASK_PROMPT = """你是音乐片段识别助手。当前目录下有一个字幕文件。
|
||||
@ -57,47 +55,34 @@ TASK_PROMPT = """你是音乐片段识别助手。当前目录下有一个字幕
|
||||
最后请严格按照 Schema 生成 JSON 数据。"""
|
||||
|
||||
|
||||
class LegacyCodexSongDetector:
|
||||
class CodexSongDetector:
|
||||
def __init__(self, adapter: CodexCliAdapter | None = None) -> None:
|
||||
self.adapter = adapter or CodexCliAdapter()
|
||||
|
||||
manifest = ProviderManifest(
|
||||
id="codex",
|
||||
name="Legacy Codex Song Detector",
|
||||
name="Codex Song Detector",
|
||||
version="0.1.0",
|
||||
provider_type="song_detector",
|
||||
entrypoint="biliup_next.infra.adapters.codex_legacy:LegacyCodexSongDetector",
|
||||
entrypoint="biliup_next.modules.song_detect.providers.codex:CodexSongDetector",
|
||||
capabilities=["song_detect"],
|
||||
enabled_by_default=True,
|
||||
)
|
||||
|
||||
def __init__(self, next_root: Path):
|
||||
self.next_root = next_root
|
||||
self.legacy_root = legacy_project_root(next_root)
|
||||
|
||||
def detect(self, task: Task, subtitle_srt: Artifact, settings: dict[str, Any]) -> tuple[Artifact, Artifact]:
|
||||
work_dir = Path(subtitle_srt.path).parent
|
||||
work_dir = Path(subtitle_srt.path).resolve().parent
|
||||
schema_path = work_dir / "song_schema.json"
|
||||
songs_json_path = work_dir / "songs.json"
|
||||
songs_txt_path = work_dir / "songs.txt"
|
||||
schema_path.write_text(json.dumps(SONG_SCHEMA, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
env = {
|
||||
**os.environ,
|
||||
"CODEX_CMD": str(settings.get("codex_cmd", "codex")),
|
||||
}
|
||||
cmd = [
|
||||
str(settings.get("codex_cmd", "codex")),
|
||||
"exec",
|
||||
TASK_PROMPT.replace("\n", " "),
|
||||
"--full-auto",
|
||||
"--sandbox", "workspace-write",
|
||||
"--output-schema", "./song_schema.json",
|
||||
"-o", "songs.json",
|
||||
"--skip-git-repo-check",
|
||||
"--json",
|
||||
]
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
cwd=str(work_dir),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
|
||||
codex_cmd = str(settings.get("codex_cmd", "codex"))
|
||||
result = self.adapter.run_song_detect(
|
||||
codex_cmd=codex_cmd,
|
||||
work_dir=work_dir,
|
||||
prompt=TASK_PROMPT,
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
raise ModuleError(
|
||||
code="SONG_DETECT_FAILED",
|
||||
@ -105,36 +90,49 @@ class LegacyCodexSongDetector:
|
||||
retryable=True,
|
||||
details={"stdout": result.stdout[-2000:], "stderr": result.stderr[-2000:]},
|
||||
)
|
||||
songs_json = work_dir / "songs.json"
|
||||
songs_txt = work_dir / "songs.txt"
|
||||
if songs_json.exists() and not songs_txt.exists():
|
||||
data = json.loads(songs_json.read_text(encoding="utf-8"))
|
||||
with songs_txt.open("w", encoding="utf-8") as f:
|
||||
for song in data.get("songs", []):
|
||||
start_time = song["start"].split(",")[0].split(".")[0]
|
||||
f.write(f"{start_time} {song['title']} — {song['artist']}\n")
|
||||
if not songs_json.exists() or not songs_txt.exists():
|
||||
|
||||
if songs_json_path.exists() and not songs_txt_path.exists():
|
||||
self._generate_txt_fallback(songs_json_path, songs_txt_path)
|
||||
|
||||
if not songs_json_path.exists() or not songs_txt_path.exists():
|
||||
raise ModuleError(
|
||||
code="SONG_DETECT_OUTPUT_MISSING",
|
||||
message=f"未生成 songs.json/songs.txt: {work_dir}",
|
||||
retryable=True,
|
||||
details={"stdout": result.stdout[-2000:], "stderr": result.stderr[-2000:]},
|
||||
)
|
||||
|
||||
return (
|
||||
Artifact(
|
||||
id=None,
|
||||
task_id=task.id,
|
||||
artifact_type="songs_json",
|
||||
path=str(songs_json),
|
||||
metadata_json=json.dumps({"provider": "codex_legacy"}),
|
||||
path=str(songs_json_path.resolve()),
|
||||
metadata_json=json.dumps({"provider": "codex"}),
|
||||
created_at=utc_now_iso(),
|
||||
),
|
||||
Artifact(
|
||||
id=None,
|
||||
task_id=task.id,
|
||||
artifact_type="songs_txt",
|
||||
path=str(songs_txt),
|
||||
metadata_json=json.dumps({"provider": "codex_legacy"}),
|
||||
path=str(songs_txt_path.resolve()),
|
||||
metadata_json=json.dumps({"provider": "codex"}),
|
||||
created_at=utc_now_iso(),
|
||||
),
|
||||
)
|
||||
|
||||
def _generate_txt_fallback(self, songs_json_path: Path, songs_txt_path: Path) -> None:
|
||||
try:
|
||||
data = json.loads(songs_json_path.read_text(encoding="utf-8"))
|
||||
songs = data.get("songs", [])
|
||||
with songs_txt_path.open("w", encoding="utf-8") as file_handle:
|
||||
for song in songs:
|
||||
start_time = str(song["start"]).split(",")[0].split(".")[0]
|
||||
file_handle.write(f"{start_time} {song['title']} — {song['artist']}\n")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise ModuleError(
|
||||
code="SONGS_TXT_GENERATE_FAILED",
|
||||
message=f"生成 songs.txt 失败: {songs_txt_path}",
|
||||
retryable=False,
|
||||
details={"error": str(exc)},
|
||||
) from exc
|
||||
@ -8,33 +8,28 @@ from typing import Any
|
||||
from biliup_next.core.errors import ModuleError
|
||||
from biliup_next.core.models import Artifact, Task, utc_now_iso
|
||||
from biliup_next.core.providers import ProviderManifest
|
||||
from biliup_next.infra.legacy_paths import legacy_project_root
|
||||
|
||||
|
||||
class LegacyFfmpegSplitProvider:
|
||||
class FfmpegCopySplitProvider:
|
||||
manifest = ProviderManifest(
|
||||
id="ffmpeg_copy",
|
||||
name="Legacy FFmpeg Split Provider",
|
||||
name="FFmpeg Copy Split Provider",
|
||||
version="0.1.0",
|
||||
provider_type="split_provider",
|
||||
entrypoint="biliup_next.infra.adapters.ffmpeg_split_legacy:LegacyFfmpegSplitProvider",
|
||||
entrypoint="biliup_next.modules.split.providers.ffmpeg_copy:FfmpegCopySplitProvider",
|
||||
capabilities=["split"],
|
||||
enabled_by_default=True,
|
||||
)
|
||||
|
||||
def __init__(self, next_root: Path):
|
||||
self.next_root = next_root
|
||||
self.legacy_root = legacy_project_root(next_root)
|
||||
|
||||
def split(self, task: Task, songs_json: Artifact, source_video: Artifact, settings: dict[str, Any]) -> list[Artifact]:
|
||||
work_dir = Path(songs_json.path).parent
|
||||
work_dir = Path(songs_json.path).resolve().parent
|
||||
split_dir = work_dir / "split_video"
|
||||
split_done = work_dir / "split_done.flag"
|
||||
if split_done.exists() and split_dir.exists():
|
||||
return self._collect_existing_clips(task.id, split_dir)
|
||||
|
||||
with Path(songs_json.path).open("r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
with Path(songs_json.path).open("r", encoding="utf-8") as file_handle:
|
||||
data = json.load(file_handle)
|
||||
songs = data.get("songs", [])
|
||||
if not songs:
|
||||
raise ModuleError(
|
||||
@ -45,32 +40,45 @@ class LegacyFfmpegSplitProvider:
|
||||
|
||||
split_dir.mkdir(parents=True, exist_ok=True)
|
||||
ffmpeg_bin = str(settings.get("ffmpeg_bin", "ffmpeg"))
|
||||
video_path = Path(source_video.path)
|
||||
for idx, song in enumerate(songs, 1):
|
||||
video_path = Path(source_video.path).resolve()
|
||||
|
||||
for index, song in enumerate(songs, 1):
|
||||
start = str(song.get("start", "00:00:00,000")).replace(",", ".")
|
||||
end = str(song.get("end", "00:00:00,000")).replace(",", ".")
|
||||
title = str(song.get("title", "UNKNOWN")).replace("/", "_").replace("\\", "_")
|
||||
output_path = split_dir / f"{idx:02d}_{title}{video_path.suffix}"
|
||||
output_path = split_dir / f"{index:02d}_{title}{video_path.suffix}"
|
||||
if output_path.exists():
|
||||
continue
|
||||
cmd = [
|
||||
ffmpeg_bin,
|
||||
"-y",
|
||||
"-ss", start,
|
||||
"-to", end,
|
||||
"-i", str(video_path),
|
||||
"-c", "copy",
|
||||
"-map_metadata", "0",
|
||||
"-ss",
|
||||
start,
|
||||
"-to",
|
||||
end,
|
||||
"-i",
|
||||
str(video_path),
|
||||
"-c",
|
||||
"copy",
|
||||
"-map_metadata",
|
||||
"0",
|
||||
str(output_path),
|
||||
]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
try:
|
||||
subprocess.run(cmd, capture_output=True, text=True, check=True)
|
||||
except FileNotFoundError as exc:
|
||||
raise ModuleError(
|
||||
code="FFMPEG_NOT_FOUND",
|
||||
message=f"找不到 ffmpeg: {ffmpeg_bin}",
|
||||
retryable=False,
|
||||
) from exc
|
||||
except subprocess.CalledProcessError as exc:
|
||||
raise ModuleError(
|
||||
code="SPLIT_FFMPEG_FAILED",
|
||||
message=f"ffmpeg 切割失败: {output_path.name}",
|
||||
retryable=True,
|
||||
details={"stderr": result.stderr[-2000:]},
|
||||
)
|
||||
details={"stderr": exc.stderr[-2000:], "stdout": exc.stdout[-2000:]},
|
||||
) from exc
|
||||
|
||||
split_done.touch()
|
||||
return self._collect_existing_clips(task.id, split_dir)
|
||||
@ -78,15 +86,16 @@ class LegacyFfmpegSplitProvider:
|
||||
def _collect_existing_clips(self, task_id: str, split_dir: Path) -> list[Artifact]:
|
||||
artifacts: list[Artifact] = []
|
||||
for path in sorted(split_dir.iterdir()):
|
||||
if path.is_file():
|
||||
artifacts.append(
|
||||
Artifact(
|
||||
id=None,
|
||||
task_id=task_id,
|
||||
artifact_type="clip_video",
|
||||
path=str(path),
|
||||
metadata_json=json.dumps({"provider": "ffmpeg_copy"}),
|
||||
created_at=utc_now_iso(),
|
||||
)
|
||||
if not path.is_file():
|
||||
continue
|
||||
artifacts.append(
|
||||
Artifact(
|
||||
id=None,
|
||||
task_id=task_id,
|
||||
artifact_type="clip_video",
|
||||
path=str(path.resolve()),
|
||||
metadata_json=json.dumps({"provider": "ffmpeg_copy"}),
|
||||
created_at=utc_now_iso(),
|
||||
)
|
||||
)
|
||||
return artifacts
|
||||
191
src/biliup_next/modules/transcribe/providers/groq.py
Normal file
191
src/biliup_next/modules/transcribe/providers/groq.py
Normal file
@ -0,0 +1,191 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from biliup_next.core.errors import ModuleError
|
||||
from biliup_next.core.models import Artifact, Task, utc_now_iso
|
||||
from biliup_next.core.providers import ProviderManifest
|
||||
|
||||
|
||||
LANGUAGE = "zh"
|
||||
BITRATE_KBPS = 64
|
||||
MODEL_NAME = "whisper-large-v3-turbo"
|
||||
|
||||
|
||||
class GroqTranscribeProvider:
|
||||
manifest = ProviderManifest(
|
||||
id="groq",
|
||||
name="Groq Transcribe Provider",
|
||||
version="0.1.0",
|
||||
provider_type="transcribe_provider",
|
||||
entrypoint="biliup_next.modules.transcribe.providers.groq:GroqTranscribeProvider",
|
||||
capabilities=["transcribe"],
|
||||
enabled_by_default=True,
|
||||
)
|
||||
|
||||
def transcribe(self, task: Task, source_video: Artifact, settings: dict[str, Any]) -> Artifact:
|
||||
groq_api_key = str(settings.get("groq_api_key", "")).strip()
|
||||
if not groq_api_key:
|
||||
raise ModuleError(
|
||||
code="GROQ_API_KEY_MISSING",
|
||||
message="未配置 transcribe.groq_api_key",
|
||||
retryable=False,
|
||||
)
|
||||
try:
|
||||
from groq import Groq
|
||||
except ModuleNotFoundError as exc:
|
||||
raise ModuleError(
|
||||
code="GROQ_DEPENDENCY_MISSING",
|
||||
message="未安装 groq 依赖,请在 biliup-next 环境中执行 pip install -e .",
|
||||
retryable=False,
|
||||
) from exc
|
||||
|
||||
source_path = Path(source_video.path).resolve()
|
||||
if not source_path.exists():
|
||||
raise ModuleError(
|
||||
code="TRANSCRIBE_SOURCE_MISSING",
|
||||
message=f"源视频不存在: {source_path}",
|
||||
retryable=False,
|
||||
)
|
||||
|
||||
ffmpeg_bin = str(settings.get("ffmpeg_bin", "ffmpeg"))
|
||||
max_file_size_mb = int(settings.get("max_file_size_mb", 23))
|
||||
work_dir = source_path.parent
|
||||
temp_audio_dir = work_dir / "temp_audio"
|
||||
temp_audio_dir.mkdir(parents=True, exist_ok=True)
|
||||
segment_duration = max(1, math.floor((max_file_size_mb * 8 * 1024) / BITRATE_KBPS))
|
||||
output_pattern = temp_audio_dir / "part_%03d.mp3"
|
||||
|
||||
self._extract_audio_segments(
|
||||
ffmpeg_bin=ffmpeg_bin,
|
||||
source_path=source_path,
|
||||
output_pattern=output_pattern,
|
||||
segment_duration=segment_duration,
|
||||
)
|
||||
|
||||
segments = sorted(temp_audio_dir.glob("part_*.mp3"))
|
||||
if not segments:
|
||||
raise ModuleError(
|
||||
code="TRANSCRIBE_AUDIO_SEGMENTS_MISSING",
|
||||
message=f"未生成音频分片: {source_path.name}",
|
||||
retryable=False,
|
||||
)
|
||||
|
||||
client = Groq(api_key=groq_api_key)
|
||||
srt_path = work_dir / f"{task.title}.srt"
|
||||
global_idx = 1
|
||||
|
||||
try:
|
||||
with srt_path.open("w", encoding="utf-8") as srt_file:
|
||||
for index, segment in enumerate(segments):
|
||||
offset_seconds = index * segment_duration
|
||||
segment_data = self._transcribe_with_retry(client, segment)
|
||||
for chunk in segment_data:
|
||||
start = self._format_srt_time(float(chunk["start"]) + offset_seconds)
|
||||
end = self._format_srt_time(float(chunk["end"]) + offset_seconds)
|
||||
text = str(chunk["text"]).strip()
|
||||
srt_file.write(f"{global_idx}\n{start} --> {end}\n{text}\n\n")
|
||||
global_idx += 1
|
||||
finally:
|
||||
shutil.rmtree(temp_audio_dir, ignore_errors=True)
|
||||
|
||||
return Artifact(
|
||||
id=None,
|
||||
task_id=task.id,
|
||||
artifact_type="subtitle_srt",
|
||||
path=str(srt_path.resolve()),
|
||||
metadata_json=json.dumps(
|
||||
{
|
||||
"provider": "groq",
|
||||
"model": MODEL_NAME,
|
||||
"segment_duration_seconds": segment_duration,
|
||||
}
|
||||
),
|
||||
created_at=utc_now_iso(),
|
||||
)
|
||||
|
||||
def _extract_audio_segments(
|
||||
self,
|
||||
*,
|
||||
ffmpeg_bin: str,
|
||||
source_path: Path,
|
||||
output_pattern: Path,
|
||||
segment_duration: int,
|
||||
) -> None:
|
||||
cmd = [
|
||||
ffmpeg_bin,
|
||||
"-y",
|
||||
"-i",
|
||||
str(source_path),
|
||||
"-vn",
|
||||
"-acodec",
|
||||
"libmp3lame",
|
||||
"-b:a",
|
||||
f"{BITRATE_KBPS}k",
|
||||
"-ac",
|
||||
"1",
|
||||
"-ar",
|
||||
"22050",
|
||||
"-f",
|
||||
"segment",
|
||||
"-segment_time",
|
||||
str(segment_duration),
|
||||
"-reset_timestamps",
|
||||
"1",
|
||||
str(output_pattern),
|
||||
]
|
||||
try:
|
||||
subprocess.run(cmd, check=True, capture_output=True, text=True)
|
||||
except FileNotFoundError as exc:
|
||||
raise ModuleError(
|
||||
code="FFMPEG_NOT_FOUND",
|
||||
message=f"找不到 ffmpeg: {ffmpeg_bin}",
|
||||
retryable=False,
|
||||
) from exc
|
||||
except subprocess.CalledProcessError as exc:
|
||||
raise ModuleError(
|
||||
code="FFMPEG_AUDIO_EXTRACT_FAILED",
|
||||
message=f"音频提取失败: {source_path.name}",
|
||||
retryable=True,
|
||||
details={"stderr": exc.stderr[-2000:], "stdout": exc.stdout[-2000:]},
|
||||
) from exc
|
||||
|
||||
def _transcribe_with_retry(self, client: Any, audio_file: Path) -> list[dict[str, Any]]:
|
||||
retry_count = 0
|
||||
while True:
|
||||
try:
|
||||
with audio_file.open("rb") as file_handle:
|
||||
response = client.audio.transcriptions.create(
|
||||
file=(audio_file.name, file_handle.read()),
|
||||
model=MODEL_NAME,
|
||||
response_format="verbose_json",
|
||||
language=LANGUAGE,
|
||||
temperature=0.0,
|
||||
)
|
||||
return [dict(segment) for segment in response.segments]
|
||||
except Exception as exc: # noqa: BLE001
|
||||
retry_count += 1
|
||||
err_str = str(exc)
|
||||
if "429" in err_str or "rate_limit" in err_str.lower():
|
||||
time.sleep(25)
|
||||
continue
|
||||
raise ModuleError(
|
||||
code="GROQ_TRANSCRIBE_FAILED",
|
||||
message=f"Groq 转录失败: {audio_file.name}",
|
||||
retryable=True,
|
||||
details={"error": err_str, "retry_count": retry_count},
|
||||
) from exc
|
||||
|
||||
@staticmethod
|
||||
def _format_srt_time(seconds: float) -> str:
|
||||
td_hours = int(seconds // 3600)
|
||||
td_mins = int((seconds % 3600) // 60)
|
||||
td_secs = int(seconds % 60)
|
||||
td_millis = int((seconds - int(seconds)) * 1000)
|
||||
return f"{td_hours:02}:{td_mins:02}:{td_secs:02},{td_millis:03}"
|
||||
@ -1,9 +1,9 @@
|
||||
{
|
||||
"id": "bilibili_collection",
|
||||
"name": "Legacy Bilibili Collection Provider",
|
||||
"name": "Bilibili Collection Provider",
|
||||
"version": "0.1.0",
|
||||
"provider_type": "collection_provider",
|
||||
"entrypoint": "biliup_next.infra.adapters.bilibili_collection_legacy:LegacyBilibiliCollectionProvider",
|
||||
"entrypoint": "biliup_next.modules.collection.providers.bilibili_collection:BilibiliCollectionProvider",
|
||||
"capabilities": ["collection"],
|
||||
"enabled_by_default": true
|
||||
}
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
{
|
||||
"id": "bilibili_top_comment",
|
||||
"name": "Legacy Bilibili Top Comment Provider",
|
||||
"name": "Bilibili Top Comment Provider",
|
||||
"version": "0.1.0",
|
||||
"provider_type": "comment_provider",
|
||||
"entrypoint": "biliup_next.infra.adapters.bilibili_top_comment_legacy:LegacyBilibiliTopCommentProvider",
|
||||
"entrypoint": "biliup_next.modules.comment.providers.bilibili_top_comment:BilibiliTopCommentProvider",
|
||||
"capabilities": ["comment"],
|
||||
"enabled_by_default": true
|
||||
}
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
"name": "biliup CLI Publish Provider",
|
||||
"version": "0.1.0",
|
||||
"provider_type": "publish_provider",
|
||||
"entrypoint": "biliup_next.infra.adapters.biliup_publish_legacy:LegacyBiliupPublishProvider",
|
||||
"entrypoint": "biliup_next.modules.publish.providers.biliup_cli:BiliupCliPublishProvider",
|
||||
"capabilities": ["publish"],
|
||||
"enabled_by_default": true
|
||||
}
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
"name": "Codex Song Detector",
|
||||
"version": "0.1.0",
|
||||
"provider_type": "song_detector",
|
||||
"entrypoint": "biliup_next.infra.adapters.codex_legacy:LegacyCodexSongDetector",
|
||||
"entrypoint": "biliup_next.modules.song_detect.providers.codex:CodexSongDetector",
|
||||
"capabilities": ["song_detect"],
|
||||
"enabled_by_default": true
|
||||
}
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
"name": "FFmpeg Copy Split Provider",
|
||||
"version": "0.1.0",
|
||||
"provider_type": "split_provider",
|
||||
"entrypoint": "biliup_next.infra.adapters.ffmpeg_split_legacy:LegacyFfmpegSplitProvider",
|
||||
"entrypoint": "biliup_next.modules.split.providers.ffmpeg_copy:FfmpegCopySplitProvider",
|
||||
"capabilities": ["split"],
|
||||
"enabled_by_default": true
|
||||
}
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
"name": "Groq Transcribe Provider",
|
||||
"version": "0.1.0",
|
||||
"provider_type": "transcribe_provider",
|
||||
"entrypoint": "biliup_next.infra.adapters.groq_legacy:LegacyGroqTranscribeProvider",
|
||||
"entrypoint": "biliup_next.modules.transcribe.providers.groq:GroqTranscribeProvider",
|
||||
"capabilities": ["transcribe"],
|
||||
"enabled_by_default": true
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user