theshy/biliup-next
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: professionalize control plane and standalone delivery

Browse Source
This commit is contained in:
theshy
2026-04-07 10:46:30 +08:00
parent d0cf1fd0df
commit 862db502b0
100 changed files with 8313 additions and 1483 deletions
Download Patch File Download Diff File Expand all files Collapse all files

432
src/biliup_next/app/api_server.py
View File

@ -8,13 +8,21 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from urllib.parse import parse_qs, unquote, urlparse
from biliup_next.app.task_actions import bind_full_video_action
from biliup_next.app.task_actions import merge_session_action
from biliup_next.app.task_actions import receive_full_video_webhook
from biliup_next.app.task_actions import rebind_session_full_video_action
from biliup_next.app.task_actions import reset_to_step_action
from biliup_next.app.task_actions import retry_step_action
from biliup_next.app.task_actions import run_task_action
from biliup_next.app.bootstrap import ensure_initialized
from biliup_next.app.bootstrap import reset_initialized_state
from biliup_next.app.control_plane_get_dispatcher import ControlPlaneGetDispatcher
from biliup_next.app.dashboard import render_dashboard_html
from biliup_next.app.control_plane_post_dispatcher import ControlPlanePostDispatcher
from biliup_next.app.retry_meta import retry_meta_for_step
from biliup_next.app.scheduler import build_scheduler_preview
from biliup_next.app.serializers import ControlPlaneSerializer
from biliup_next.app.worker import run_once
from biliup_next.core.config import SettingsService
from biliup_next.core.models import ActionRecord, utc_now_iso
@ -28,61 +36,32 @@ from biliup_next.infra.systemd_runtime import SystemdRuntime
class ApiHandler(BaseHTTPRequestHandler):
server_version = "biliup-next/0.1"
def _task_payload(self, task_id: str, state: dict[str, object]) -> dict[str, object] | None:
task = state["repo"].get_task(task_id)
if task is None:
return None
payload = task.to_dict()
retry_state = self._task_retry_state(task_id, state)
if retry_state:
payload["retry_state"] = retry_state
payload["delivery_state"] = self._task_delivery_state(task_id, state)
return payload
@staticmethod
def _attention_state(task_payload: dict[str, object]) -> str:
if task_payload.get("status") == "failed_manual":
return "manual_now"
retry_state = task_payload.get("retry_state")
if isinstance(retry_state, dict) and retry_state.get("retry_due"):
return "retry_now"
if task_payload.get("status") == "failed_retryable" and isinstance(retry_state, dict) and retry_state.get("next_retry_at"):
return "waiting_retry"
if task_payload.get("status") == "running":
return "running"
return "stable"
@staticmethod
def _delivery_state_label(task_payload: dict[str, object]) -> str:
delivery_state = task_payload.get("delivery_state")
if not isinstance(delivery_state, dict):
return "stable"
if delivery_state.get("split_comment") == "pending" or delivery_state.get("full_video_timeline_comment") == "pending":
return "pending_comment"
if delivery_state.get("source_video_present") is False or delivery_state.get("split_videos_present") is False:
return "cleanup_removed"
return "stable"
def _step_payload(self, step, state: dict[str, object]) -> dict[str, object]: # type: ignore[no-untyped-def]
payload = step.to_dict()
retry_meta = retry_meta_for_step(step, state["settings"])
if retry_meta:
payload.update(retry_meta)
return payload
def _task_retry_state(self, task_id: str, state: dict[str, object]) -> dict[str, object] | None:
for step in state["repo"].list_steps(task_id):
retry_meta = retry_meta_for_step(step, state["settings"])
if retry_meta:
return {"step_name": step.step_name, **retry_meta}
return None
def _task_delivery_state(self, task_id: str, state: dict[str, object]) -> dict[str, object]:
task = state["repo"].get_task(task_id)
if task is None:
return {}
session_dir = Path(str(state["settings"]["paths"]["session_dir"])) / task.title
source_path = Path(task.source_path)
split_dir = session_dir / "split_video"
legacy_comment_done = (session_dir / "comment_done.flag").exists()
def comment_status(flag_name: str, *, enabled: bool) -> str:
if not enabled:
return "disabled"
if flag_name == "comment_full_done.flag" and legacy_comment_done and not (session_dir / flag_name).exists():
return "legacy_untracked"
return "done" if (session_dir / flag_name).exists() else "pending"
return {
"split_comment": comment_status("comment_split_done.flag", enabled=state["settings"]["comment"].get("post_split_comment", True)),
"full_video_timeline_comment": comment_status(
"comment_full_done.flag",
enabled=state["settings"]["comment"].get("post_full_video_timeline_comment", True),
),
"full_video_bvid_resolved": (session_dir / "full_video_bvid.txt").exists(),
"source_video_present": source_path.exists(),
"split_videos_present": split_dir.exists(),
"cleanup_enabled": {
"delete_source_video_after_collection_synced": state["settings"].get("cleanup", {}).get("delete_source_video_after_collection_synced", False),
"delete_split_videos_after_collection_synced": state["settings"].get("cleanup", {}).get("delete_split_videos_after_collection_synced", False),
},
}
return ControlPlaneSerializer(state).step_payload(step)
def _serve_asset(self, asset_name: str) -> None:
root = ensure_initialized()["root"]
@ -116,10 +95,22 @@ class ApiHandler(BaseHTTPRequestHandler):
dist = self._frontend_dist_dir()
if not (dist / "index.html").exists():
return False
if parsed_path in {"/ui", "/ui/"}:
if parsed_path in {"/", "/ui", "/ui/"}:
self._html((dist / "index.html").read_text(encoding="utf-8"))
return True
if parsed_path.startswith("/assets/"):
relative = parsed_path.removeprefix("/")
asset_path = dist / relative
if asset_path.exists() and asset_path.is_file():
body = asset_path.read_bytes()
self.send_response(HTTPStatus.OK)
self.send_header("Content-Type", self._guess_content_type(asset_path))
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
return True
if not parsed_path.startswith("/ui/"):
return False
@ -143,13 +134,16 @@ class ApiHandler(BaseHTTPRequestHandler):
def do_GET(self) -> None: # noqa: N802
parsed = urlparse(self.path)
if parsed.path.startswith("/ui") and self._serve_frontend_dist(parsed.path):
if (parsed.path == "/" or parsed.path.startswith("/ui") or parsed.path.startswith("/assets/")) and self._serve_frontend_dist(parsed.path):
return
if not self._check_auth(parsed.path):
return
if parsed.path.startswith("/assets/"):
self._serve_asset(parsed.path.removeprefix("/assets/"))
return
if parsed.path == "/classic":
self._html(render_dashboard_html())
return
if parsed.path == "/":
self._html(render_dashboard_html())
return
@ -158,16 +152,23 @@ class ApiHandler(BaseHTTPRequestHandler):
self._json({"ok": True})
return
state = ensure_initialized()
get_dispatcher = ControlPlaneGetDispatcher(
state,
attention_state_fn=self._attention_state,
delivery_state_label_fn=self._delivery_state_label,
build_scheduler_preview_fn=build_scheduler_preview,
settings_service_factory=SettingsService,
)
if parsed.path == "/settings":
state = ensure_initialized()
service = SettingsService(state["root"])
self._json(service.load_redacted().settings)
body, status = get_dispatcher.handle_settings()
self._json(body, status=status)
return
if parsed.path == "/settings/schema":
state = ensure_initialized()
service = SettingsService(state["root"])
self._json(service.load().schema)
body, status = get_dispatcher.handle_settings_schema()
self._json(body, status=status)
return
if parsed.path == "/doctor":
@ -180,8 +181,8 @@ class ApiHandler(BaseHTTPRequestHandler):
return
if parsed.path == "/scheduler/preview":
state = ensure_initialized()
self._json(build_scheduler_preview(state, include_stage_scan=False, limit=200))
body, status = get_dispatcher.handle_scheduler_preview()
self._json(body, status=status)
return
if parsed.path == "/logs":
@ -196,146 +197,78 @@ class ApiHandler(BaseHTTPRequestHandler):
return
if parsed.path == "/history":
state = ensure_initialized()
query = parse_qs(parsed.query)
limit = int(query.get("limit", ["100"])[0])
task_id = query.get("task_id", [None])[0]
action_name = query.get("action_name", [None])[0]
status = query.get("status", [None])[0]
items = [
item.to_dict()
for item in state["repo"].list_action_records(
task_id=task_id,
limit=limit,
action_name=action_name,
status=status,
)
]
self._json({"items": items})
body, http_status = get_dispatcher.handle_history(
limit=limit,
task_id=task_id,
action_name=action_name,
status=status,
)
self._json(body, status=http_status)
return
if parsed.path == "/modules":
state = ensure_initialized()
self._json({"items": state["registry"].list_manifests(), "discovered_manifests": state["manifests"]})
body, status = get_dispatcher.handle_modules()
self._json(body, status=status)
return
if parsed.path == "/tasks":
state = ensure_initialized()
query = parse_qs(parsed.query)
limit = int(query.get("limit", ["100"])[0])
tasks = [self._task_payload(task.id, state) for task in state["repo"].list_tasks(limit=limit)]
self._json({"items": tasks})
offset = int(query.get("offset", ["0"])[0])
status = query.get("status", [None])[0]
search = query.get("search", [None])[0]
sort = query.get("sort", ["updated_desc"])[0]
attention = query.get("attention", [None])[0]
delivery = query.get("delivery", [None])[0]
body, http_status = get_dispatcher.handle_tasks(
limit=limit,
offset=offset,
status=status,
search=search,
sort=sort,
attention=attention,
delivery=delivery,
)
self._json(body, status=http_status)
return
if parsed.path.startswith("/tasks/"):
state = ensure_initialized()
if parsed.path.startswith("/sessions/"):
parts = [unquote(p) for p in parsed.path.split("/") if p]
if len(parts) == 2:
task = self._task_payload(parts[1], state)
if task is None:
self._json({"error": "task not found"}, status=HTTPStatus.NOT_FOUND)
return
self._json(task)
body, status = get_dispatcher.handle_session(parts[1])
self._json(body, status=status)
return
if parsed.path.startswith("/tasks/"):
parts = [unquote(p) for p in parsed.path.split("/") if p]
if len(parts) == 2:
body, status = get_dispatcher.handle_task(parts[1])
self._json(body, status=status)
return
if len(parts) == 3 and parts[2] == "steps":
steps = [self._step_payload(step, state) for step in state["repo"].list_steps(parts[1])]
self._json({"items": steps})
body, status = get_dispatcher.handle_task_steps(parts[1])
self._json(body, status=status)
return
if len(parts) == 3 and parts[2] == "context":
body, status = get_dispatcher.handle_task_context(parts[1])
self._json(body, status=status)
return
if len(parts) == 3 and parts[2] == "artifacts":
artifacts = [artifact.to_dict() for artifact in state["repo"].list_artifacts(parts[1])]
self._json({"items": artifacts})
body, status = get_dispatcher.handle_task_artifacts(parts[1])
self._json(body, status=status)
return
if len(parts) == 3 and parts[2] == "history":
actions = [item.to_dict() for item in state["repo"].list_action_records(parts[1], limit=100)]
self._json({"items": actions})
body, status = get_dispatcher.handle_task_history(parts[1])
self._json(body, status=status)
return
if len(parts) == 3 and parts[2] == "timeline":
task = state["repo"].get_task(parts[1])
if task is None:
self._json({"error": "task not found"}, status=HTTPStatus.NOT_FOUND)
return
steps = state["repo"].list_steps(parts[1])
artifacts = state["repo"].list_artifacts(parts[1])
actions = state["repo"].list_action_records(parts[1], limit=200)
items: list[dict[str, object]] = []
if task.created_at:
items.append({
"kind": "task",
"time": task.created_at,
"title": "Task Created",
"summary": task.title,
"status": task.status,
})
if task.updated_at and task.updated_at != task.created_at:
items.append({
"kind": "task",
"time": task.updated_at,
"title": "Task Updated",
"summary": task.status,
"status": task.status,
})
for step in steps:
if step.started_at:
items.append({
"kind": "step",
"time": step.started_at,
"title": f"{step.step_name} started",
"summary": step.status,
"status": step.status,
})
if step.finished_at:
retry_meta = retry_meta_for_step(step, state["settings"])
retry_note = ""
if retry_meta and retry_meta.get("next_retry_at"):
retry_note = f" | next retry: {retry_meta['next_retry_at']}"
items.append({
"kind": "step",
"time": step.finished_at,
"title": f"{step.step_name} finished",
"summary": f"{step.error_message or step.status}{retry_note}",
"status": step.status,
"retry_state": retry_meta,
})
for artifact in artifacts:
if artifact.created_at:
items.append({
"kind": "artifact",
"time": artifact.created_at,
"title": artifact.artifact_type,
"summary": artifact.path,
"status": "created",
})
for action in actions:
summary = action.summary
try:
details = json.loads(action.details_json or "{}")
except json.JSONDecodeError:
details = {}
if action.action_name == "comment" and isinstance(details, dict):
split_status = details.get("split", {}).get("status")
full_status = details.get("full", {}).get("status")
fragments = []
if split_status:
fragments.append(f"split={split_status}")
if full_status:
fragments.append(f"full={full_status}")
if fragments:
summary = f"{summary} | {' '.join(fragments)}"
if action.action_name in {"collection_a", "collection_b"} and isinstance(details, dict):
cleanup = details.get("result", {}).get("cleanup") or details.get("cleanup")
if isinstance(cleanup, dict):
removed = cleanup.get("removed") or []
if removed:
summary = f"{summary} | cleanup removed={len(removed)}"
items.append({
"kind": "action",
"time": action.created_at,
"title": action.action_name,
"summary": summary,
"status": action.status,
})
items.sort(key=lambda item: str(item["time"]), reverse=True)
self._json({"items": items})
body, status = get_dispatcher.handle_task_timeline(parts[1])
self._json(body, status=status)
return
self._json({"error": "not found"}, status=HTTPStatus.NOT_FOUND)
@ -353,74 +286,86 @@ class ApiHandler(BaseHTTPRequestHandler):
service = SettingsService(root)
service.save_staged_from_redacted(payload)
service.promote_staged()
reset_initialized_state()
ensure_initialized()
self._json({"ok": True})
def do_POST(self) -> None: # noqa: N802
parsed = urlparse(self.path)
if not self._check_auth(parsed.path):
return
state = ensure_initialized()
dispatcher = ControlPlanePostDispatcher(
state,
bind_full_video_action=bind_full_video_action,
merge_session_action=merge_session_action,
receive_full_video_webhook=receive_full_video_webhook,
rebind_session_full_video_action=rebind_session_full_video_action,
reset_to_step_action=reset_to_step_action,
retry_step_action=retry_step_action,
run_task_action=run_task_action,
run_once=run_once,
stage_importer_factory=StageImporter,
systemd_runtime_factory=SystemdRuntime,
)
if parsed.path == "/webhooks/full-video-uploaded":
length = int(self.headers.get("Content-Length", "0"))
payload = json.loads(self.rfile.read(length) or b"{}")
body, status = dispatcher.handle_webhook_full_video(payload)
self._json(body, status=status)
return
if parsed.path != "/tasks":
if parsed.path.startswith("/sessions/"):
parts = [unquote(p) for p in parsed.path.split("/") if p]
if len(parts) == 3 and parts[0] == "sessions" and parts[2] == "merge":
session_key = parts[1]
length = int(self.headers.get("Content-Length", "0"))
payload = json.loads(self.rfile.read(length) or b"{}")
body, status = dispatcher.handle_session_merge(session_key, payload)
self._json(body, status=status)
return
if len(parts) == 3 and parts[0] == "sessions" and parts[2] == "rebind":
session_key = parts[1]
length = int(self.headers.get("Content-Length", "0"))
payload = json.loads(self.rfile.read(length) or b"{}")
body, status = dispatcher.handle_session_rebind(session_key, payload)
self._json(body, status=status)
return
if parsed.path.startswith("/tasks/"):
parts = [unquote(p) for p in parsed.path.split("/") if p]
if len(parts) == 3 and parts[0] == "tasks" and parts[2] == "bind-full-video":
task_id = parts[1]
length = int(self.headers.get("Content-Length", "0"))
payload = json.loads(self.rfile.read(length) or b"{}")
body, status = dispatcher.handle_bind_full_video(task_id, payload)
self._json(body, status=status)
return
if len(parts) == 4 and parts[0] == "tasks" and parts[2] == "actions":
task_id = parts[1]
action = parts[3]
if action == "run":
result = run_task_action(task_id)
self._json(result, status=HTTPStatus.ACCEPTED)
return
if action == "retry-step":
length = int(self.headers.get("Content-Length", "0"))
payload = json.loads(self.rfile.read(length) or b"{}")
step_name = payload.get("step_name")
if not step_name:
self._json({"error": "missing step_name"}, status=HTTPStatus.BAD_REQUEST)
return
result = retry_step_action(task_id, step_name)
self._json(result, status=HTTPStatus.ACCEPTED)
return
if action == "reset-to-step":
length = int(self.headers.get("Content-Length", "0"))
payload = json.loads(self.rfile.read(length) or b"{}")
step_name = payload.get("step_name")
if not step_name:
self._json({"error": "missing step_name"}, status=HTTPStatus.BAD_REQUEST)
return
result = reset_to_step_action(task_id, step_name)
self._json(result, status=HTTPStatus.ACCEPTED)
if action in {"run", "retry-step", "reset-to-step"}:
payload = {}
if action != "run":
length = int(self.headers.get("Content-Length", "0"))
payload = json.loads(self.rfile.read(length) or b"{}")
body, status = dispatcher.handle_task_action(task_id, action, payload)
self._json(body, status=status)
return
if parsed.path == "/worker/run-once":
payload = run_once()
self._record_action(None, "worker_run_once", "ok", "worker run once invoked", payload)
self._json(payload, status=HTTPStatus.ACCEPTED)
body, status = dispatcher.handle_worker_run_once()
self._json(body, status=status)
return
if parsed.path.startswith("/runtime/services/"):
parts = [unquote(p) for p in parsed.path.split("/") if p]
if len(parts) == 4 and parts[0] == "runtime" and parts[1] == "services":
try:
payload = SystemdRuntime().act(parts[2], parts[3])
except ValueError as exc:
self._json({"error": str(exc)}, status=HTTPStatus.BAD_REQUEST)
return
self._record_action(None, "service_action", "ok" if payload.get("command_ok") else "error", f"{parts[3]} {parts[2]}", payload)
self._json(payload, status=HTTPStatus.ACCEPTED)
body, status = dispatcher.handle_runtime_service_action(parts[2], parts[3])
self._json(body, status=status)
return
if parsed.path == "/stage/import":
length = int(self.headers.get("Content-Length", "0"))
payload = json.loads(self.rfile.read(length) or b"{}")
source_path = payload.get("source_path")
if not source_path:
self._json({"error": "missing source_path"}, status=HTTPStatus.BAD_REQUEST)
return
state = ensure_initialized()
stage_dir = Path(state["settings"]["paths"]["stage_dir"])
try:
result = StageImporter().import_file(Path(source_path), stage_dir)
except Exception as exc:
self._json({"error": str(exc)}, status=HTTPStatus.BAD_REQUEST)
return
self._record_action(None, "stage_import", "ok", "imported file into stage", result)
self._json(result, status=HTTPStatus.CREATED)
body, status = dispatcher.handle_stage_import(payload)
self._json(body, status=status)
return
if parsed.path == "/stage/upload":
content_type = self.headers.get("Content-Type", "")
@ -437,44 +382,19 @@ class ApiHandler(BaseHTTPRequestHandler):
},
)
file_item = form["file"] if "file" in form else None
if file_item is None or not getattr(file_item, "filename", None):
self._json({"error": "missing file"}, status=HTTPStatus.BAD_REQUEST)
return
state = ensure_initialized()
stage_dir = Path(state["settings"]["paths"]["stage_dir"])
try:
result = StageImporter().import_upload(file_item.filename, file_item.file, stage_dir)
except Exception as exc:
self._json({"error": str(exc)}, status=HTTPStatus.BAD_REQUEST)
return
self._record_action(None, "stage_upload", "ok", "uploaded file into stage", result)
self._json(result, status=HTTPStatus.CREATED)
body, status = dispatcher.handle_stage_upload(file_item)
self._json(body, status=status)
return
if parsed.path == "/scheduler/run-once":
result = run_once()
self._record_action(None, "scheduler_run_once", "ok", "scheduler run once completed", result.get("scheduler", {}))
self._json(result, status=HTTPStatus.ACCEPTED)
body, status = dispatcher.handle_scheduler_run_once()
self._json(body, status=status)
return
self._json({"error": "not found"}, status=HTTPStatus.NOT_FOUND)
return
length = int(self.headers.get("Content-Length", "0"))
payload = json.loads(self.rfile.read(length) or b"{}")
source_path = payload.get("source_path")
if not source_path:
self._json({"error": "missing source_path"}, status=HTTPStatus.BAD_REQUEST)
return
state = ensure_initialized()
try:
task = state["ingest_service"].create_task_from_file(
Path(source_path),
state["settings"]["ingest"],
)
except Exception as exc: # keep API small for now
status = HTTPStatus.CONFLICT if exc.__class__.__name__ == "ModuleError" else HTTPStatus.INTERNAL_SERVER_ERROR
payload = exc.to_dict() if hasattr(exc, "to_dict") else {"error": str(exc)}
self._json(payload, status=status)
return
self._json(task.to_dict(), status=HTTPStatus.CREATED)
body, status = dispatcher.handle_create_task(payload)
self._json(body, status=status)
def log_message(self, format: str, *args) -> None: # noqa: A003
return
@ -510,7 +430,7 @@ class ApiHandler(BaseHTTPRequestHandler):
)
def _check_auth(self, path: str) -> bool:
if path in {"/", "/health", "/ui", "/ui/"} or path.startswith("/assets/") or path.startswith("/ui/assets/"):
if path in {"/", "/health", "/ui", "/ui/", "/classic"} or path.startswith("/assets/") or path.startswith("/ui/assets/"):
return True
state = ensure_initialized()
expected = str(state["settings"]["runtime"].get("control_token", "")).strip()