Merge branch 'main' into feat/agent-tls-direct
# Conflicts: # internal/cmd/daemon.go
This commit is contained in:
Deivid Soto
commit
b0637f266b
42 changed files with 2862 additions and 340 deletions
|
|
@ -139,10 +139,11 @@ func (c *Client) ReportUpgradeResult(ctx context.Context, agentID string, succes
|
|||
// will reach the same conclusion via HEAD probes anyway if this call
|
||||
// fails. We log the error in the caller but don't retry — by the time
|
||||
// a retry would land the user is likely already playing.
|
||||
func (c *Client) MarkSessionReady(ctx context.Context, sessionID string) error {
|
||||
func (c *Client) MarkSessionReady(ctx context.Context, sessionID string, health *SessionHealth) error {
|
||||
req := struct {
|
||||
SessionID string `json:"sessionId"`
|
||||
}{SessionID: sessionID}
|
||||
SessionID string `json:"sessionId"`
|
||||
Health *SessionHealth `json:"health,omitempty"`
|
||||
}{SessionID: sessionID, Health: health}
|
||||
var resp StatusResponse
|
||||
if err := c.doPost(ctx, "/api/internal/agent/session-ready", req, &resp); err != nil {
|
||||
return fmt.Errorf("mark session ready: %w", err)
|
||||
|
|
@ -150,6 +151,46 @@ func (c *Client) MarkSessionReady(ctx context.Context, sessionID string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// ReportSessionError is the failure-path counterpart of MarkSessionReady: it
|
||||
// tells the web a streaming session can NOT start (file gone, path rejected,
|
||||
// ffmpeg missing, spawn failure…). The web marks the session failed, pushes an
|
||||
// SSE "failed" event so the player stops probing a playlist that will never
|
||||
// exist, and self-heals stale library state on code "file_missing".
|
||||
//
|
||||
// code is one of the stable machine codes the web understands:
|
||||
// "file_missing" | "path_rejected" | "no_video_file" | "ffmpeg_unavailable" |
|
||||
// "start_failed". message is free-form detail for diagnostics.
|
||||
//
|
||||
// Best-effort like MarkSessionReady: on older web deployments without the
|
||||
// endpoint this 404s — the caller logs and the player falls back to its
|
||||
// probe-deadline behaviour, exactly as before this channel existed.
|
||||
func (c *Client) ReportSessionError(ctx context.Context, sessionID, code, message string) error {
|
||||
req := struct {
|
||||
SessionID string `json:"sessionId"`
|
||||
Code string `json:"code"`
|
||||
Message string `json:"message,omitempty"`
|
||||
}{SessionID: sessionID, Code: code, Message: message}
|
||||
var resp StatusResponse
|
||||
if err := c.doPost(ctx, "/api/internal/agent/session-error", req, &resp); err != nil {
|
||||
return fmt.Errorf("report session error: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SessionHealth is an OPTIONAL live-transcode health snapshot attached to a
|
||||
// session-ready report (F3). A nil *SessionHealth means the agent has no
|
||||
// telemetry to share (cache hit, direct-play, or progress not yet stable) and
|
||||
// the web side keeps its stall-shape heuristic. Old web replicas ignore the
|
||||
// extra field; old agents simply never send it.
|
||||
type SessionHealth struct {
|
||||
// "ok" (≥ realtime) | "marginal" (keeps up barely) | "struggling" (can't).
|
||||
Health string `json:"health"`
|
||||
// ffmpeg speed= EWMA: 1.0 = exactly realtime, < 1.0 = slower than playback.
|
||||
RealtimeRatio float64 `json:"realtimeRatio"`
|
||||
// "realtime" | "transcode" (encoder is the wall) | "input_bound" (source read).
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
// RefreshStreamURL re-resolves a fresh debrid direct URL for a live streaming
|
||||
// session (hueco #2 / 2c). Called by the daemon when a debrid source expires
|
||||
// mid-stream (the link is time-limited; the content is still cached). Returns
|
||||
|
|
|
|||
|
|
@ -56,6 +56,11 @@ type Daemon struct {
|
|||
OnStreamSession func(sess StreamSession)
|
||||
OnControlAction func(action, taskID string, deleteFiles bool)
|
||||
GetActiveCount func() int // returns number of active downloads (wired from manager)
|
||||
// OnAgentKeyMinted fires when a register reply carries a freshly-minted
|
||||
// per-machine key (the daemon registered with a general/legacy key). cmd
|
||||
// persists it so the next start authenticates with the bound agent key —
|
||||
// migrating legacy agents and stopping the per-restart re-mint.
|
||||
OnAgentKeyMinted func(newKey string)
|
||||
|
||||
// State
|
||||
User UserInfo
|
||||
|
|
@ -186,6 +191,12 @@ func (d *Daemon) Register(ctx context.Context) error {
|
|||
return fmt.Errorf("register: %w (after %d retries)", err, maxRetries)
|
||||
}
|
||||
|
||||
// Registered with a general/legacy key → the server minted a per-machine key.
|
||||
// Persist it (cmd wires the callback) so the next start uses the bound key.
|
||||
if resp.AgentKey != "" && d.OnAgentKeyMinted != nil {
|
||||
d.OnAgentKeyMinted(resp.AgentKey)
|
||||
}
|
||||
|
||||
d.User = resp.User
|
||||
d.Features = resp.Features
|
||||
now := time.Now()
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import (
|
|||
type MirrorEntry struct {
|
||||
URL string `json:"url"`
|
||||
Label string `json:"label"`
|
||||
Kind string `json:"kind"` // "clearnet" | "tor"
|
||||
Kind string `json:"kind"` // "clearnet" | "tor"
|
||||
Primary bool `json:"primary"`
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -66,6 +66,18 @@ type SyncClient struct {
|
|||
// It should delete the files and return the IDs of successfully deleted items.
|
||||
OnDeleteFiles func(items []LibraryDeleteRequest) []int
|
||||
|
||||
// OnSubtitleFetch is called when the server requests on-demand subtitle
|
||||
// downloads. It should download each (from req.URL, already VTT), write a
|
||||
// sidecar next to req.FilePath, and return the IDs successfully fetched plus
|
||||
// the ones that failed (so the web can mark them errored).
|
||||
OnSubtitleFetch func(reqs []SubtitleFetchRequest) ([]int, []SubtitleFetchError)
|
||||
|
||||
// OnRevoked is called when a sync is rejected because this agent's credential
|
||||
// was revoked (the user deleted the agent from the dashboard). The daemon
|
||||
// wires this to wipe the stored key + stop — it must NOT keep retrying or the
|
||||
// server will reject every sync forever.
|
||||
OnRevoked func(err error)
|
||||
|
||||
// SyncNow triggers an immediate sync (e.g., on task completion).
|
||||
SyncNow chan struct{}
|
||||
|
||||
|
|
@ -83,6 +95,11 @@ type SyncClient struct {
|
|||
// deleteInFlight tracks item IDs currently being processed or awaiting confirmation.
|
||||
// Prevents the same file from being passed to OnDeleteFiles multiple times.
|
||||
deleteInFlight map[int]struct{}
|
||||
|
||||
// Subtitle-fetch jobs awaiting confirmation + dedup (guarded by pendingDeleteMu).
|
||||
pendingSubtitlesFetched []int
|
||||
pendingSubtitlesFailed []SubtitleFetchError
|
||||
subtitleInFlight map[int]struct{}
|
||||
}
|
||||
|
||||
// NewSyncClient creates a sync client.
|
||||
|
|
@ -152,6 +169,12 @@ func (sc *SyncClient) doSync(ctx context.Context) {
|
|||
resp, err := sc.client.Sync(ctx, req)
|
||||
if err != nil {
|
||||
if ctx.Err() == nil {
|
||||
// Credential revoked (agent deleted from the dashboard) → stop; don't
|
||||
// spam a sync the server will reject forever.
|
||||
if IsRevoked(err) && sc.OnRevoked != nil {
|
||||
sc.OnRevoked(err)
|
||||
return
|
||||
}
|
||||
log.Printf("sync failed: %v", err)
|
||||
}
|
||||
return
|
||||
|
|
@ -208,6 +231,20 @@ func (sc *SyncClient) buildRequest() SyncRequest {
|
|||
}
|
||||
sc.pendingDeleteConfirmed = nil
|
||||
}
|
||||
if len(sc.pendingSubtitlesFetched) > 0 {
|
||||
req.SubtitlesFetched = sc.pendingSubtitlesFetched
|
||||
for _, id := range sc.pendingSubtitlesFetched {
|
||||
delete(sc.subtitleInFlight, id)
|
||||
}
|
||||
sc.pendingSubtitlesFetched = nil
|
||||
}
|
||||
if len(sc.pendingSubtitlesFailed) > 0 {
|
||||
req.SubtitlesFailed = sc.pendingSubtitlesFailed
|
||||
for _, f := range sc.pendingSubtitlesFailed {
|
||||
delete(sc.subtitleInFlight, f.ID)
|
||||
}
|
||||
sc.pendingSubtitlesFailed = nil
|
||||
}
|
||||
sc.pendingDeleteMu.Unlock()
|
||||
return req
|
||||
}
|
||||
|
|
@ -279,6 +316,37 @@ func (sc *SyncClient) processResponse(resp *SyncResponse) {
|
|||
}(newItems)
|
||||
}
|
||||
}
|
||||
|
||||
// On-demand subtitle fetches — dedup against in-flight, run off the sync
|
||||
// goroutine (network + disk I/O), confirm on the next cycle.
|
||||
if len(resp.SubtitleFetches) > 0 && sc.OnSubtitleFetch != nil {
|
||||
sc.pendingDeleteMu.Lock()
|
||||
if sc.subtitleInFlight == nil {
|
||||
sc.subtitleInFlight = make(map[int]struct{})
|
||||
}
|
||||
var newReqs []SubtitleFetchRequest
|
||||
for _, r := range resp.SubtitleFetches {
|
||||
if _, inFlight := sc.subtitleInFlight[r.ID]; !inFlight {
|
||||
newReqs = append(newReqs, r)
|
||||
sc.subtitleInFlight[r.ID] = struct{}{}
|
||||
}
|
||||
}
|
||||
sc.pendingDeleteMu.Unlock()
|
||||
|
||||
if len(newReqs) > 0 {
|
||||
go func(reqs []SubtitleFetchRequest) {
|
||||
done, failed := sc.OnSubtitleFetch(reqs)
|
||||
// Both done and failed are reported on the next uplink; buildRequest
|
||||
// clears them from subtitleInFlight when it flushes them. A failure
|
||||
// becomes status='error' on the web (no silent infinite retry — the
|
||||
// user re-requests, which creates a fresh row).
|
||||
sc.pendingDeleteMu.Lock()
|
||||
sc.pendingSubtitlesFetched = append(sc.pendingSubtitlesFetched, done...)
|
||||
sc.pendingSubtitlesFailed = append(sc.pendingSubtitlesFailed, failed...)
|
||||
sc.pendingDeleteMu.Unlock()
|
||||
}(newReqs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// runWakeListener holds a long-poll connection to /api/internal/agent/wake.
|
||||
|
|
|
|||
|
|
@ -1,7 +1,10 @@
|
|||
package agent
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
|
|
@ -65,7 +68,13 @@ type RegisterRequest struct {
|
|||
|
||||
// RegisterResponse is returned by the server after registration.
|
||||
type RegisterResponse struct {
|
||||
Success bool `json:"success"`
|
||||
Success bool `json:"success"`
|
||||
// AgentKey is a freshly-minted per-machine API key, present only when the
|
||||
// CLI registered with the user's general key (manual-paste bootstrap). The
|
||||
// CLI must persist it and authenticate with it from then on, discarding the
|
||||
// general key. Empty in the browser-authorize path (the token already IS the
|
||||
// agent key) and on every later register.
|
||||
AgentKey string `json:"agentKey,omitempty"`
|
||||
User UserInfo `json:"user"`
|
||||
Features FeatureFlags `json:"features"`
|
||||
}
|
||||
|
|
@ -198,6 +207,32 @@ func (e *HTTPError) Error() string {
|
|||
return fmt.Sprintf("API error %d: %s", e.StatusCode, e.Message)
|
||||
}
|
||||
|
||||
// IsRevoked reports whether an error is an EXPLICIT server revocation signal —
|
||||
// the user deleted this agent from the dashboard. The server sends 410
|
||||
// agent_revoked (the registration is tombstoned OR the per-machine key was
|
||||
// revoked — the auth layer maps a revoked agent key to 410, not 401) or 403
|
||||
// agent_key_mismatch (the key belongs to another machine). On these the daemon
|
||||
// wipes its credential and requires a fresh `unarr login`.
|
||||
//
|
||||
// A BARE 401 is deliberately NOT treated as revoked: it's ambiguous (a deploy
|
||||
// blip, a load-balancer hiccup, a transient auth error) and must never wipe a
|
||||
// working agent's credential. The retry/log paths handle a transient 401; a
|
||||
// genuine revocation always arrives as 410.
|
||||
func IsRevoked(err error) bool {
|
||||
var he *HTTPError
|
||||
if !errors.As(err, &he) {
|
||||
return false
|
||||
}
|
||||
if he.StatusCode == http.StatusGone {
|
||||
return true
|
||||
}
|
||||
if he.StatusCode == http.StatusForbidden &&
|
||||
strings.Contains(he.Message, "agent_key_mismatch") {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// AgentInfo holds metadata about the running agent for display.
|
||||
type AgentInfo struct {
|
||||
ID string
|
||||
|
|
@ -331,6 +366,17 @@ type LibrarySyncRequest struct {
|
|||
AgentID string `json:"agentId,omitempty"` // lets the server scope stale-cleanup per agent
|
||||
IsLastBatch bool `json:"isLastBatch"`
|
||||
SyncStartedAt string `json:"syncStartedAt,omitempty"` // ISO-8601; same for all batches in a session
|
||||
// ScanRoots lists EVERY root this sync session covered (a session spans all
|
||||
// roots since 1.0.9 — one syncStartedAt, one isLastBatch). The server scopes
|
||||
// stale-row cleanup of a partial session to these prefixes. Older servers
|
||||
// ignore the field and fall back to ScanPath.
|
||||
ScanRoots []string `json:"scanRoots,omitempty"`
|
||||
// FullCycle marks a session that covered every root the agent scans
|
||||
// (daemon auto-scan, `unarr scan` without args). The server may then reap
|
||||
// unseen rows REGARDLESS of path prefix — old-base-path ghost rows
|
||||
// included. Must stay false for a manual subtree scan or when any root's
|
||||
// scan failed, or the cleanup would reap rows the session never visited.
|
||||
FullCycle bool `json:"fullCycle,omitempty"`
|
||||
}
|
||||
|
||||
// LibrarySyncItem is a single scanned media file with ffprobe metadata.
|
||||
|
|
@ -398,6 +444,11 @@ type SyncRequest struct {
|
|||
Tasks []TaskState `json:"tasks"`
|
||||
CanDelete bool `json:"canDelete"` // library.allow_delete is enabled
|
||||
DeleteConfirmed []int `json:"deleteConfirmed,omitempty"` // library item IDs successfully deleted from disk
|
||||
// Subtitle-fetch job IDs the agent completed (sidecar written to disk).
|
||||
SubtitlesFetched []int `json:"subtitlesFetched,omitempty"`
|
||||
// Subtitle-fetch jobs that permanently failed (download/write error) — the web
|
||||
// marks them errored so the UI fails fast instead of waiting for a timeout.
|
||||
SubtitlesFailed []SubtitleFetchError `json:"subtitlesFailed,omitempty"`
|
||||
// Live managed-VPN split-tunnel state, sent every sync so the web sees the
|
||||
// WireGuard slot owner update in near-realtime (vs. register, once at startup).
|
||||
// VPNActive has no omitempty: false (tunnel down) must reach the server so it
|
||||
|
|
@ -450,6 +501,20 @@ type StreamSession struct {
|
|||
// omitted. Forces a full video re-encode (the overlay can't ride a copy
|
||||
// path), so the web only sends it when the user picks a bitmap sub.
|
||||
BurnSubtitleIndex *int `json:"burnSubtitleIndex,omitempty"`
|
||||
// StartSec is the playback position (seconds) the viewer opens at — the
|
||||
// saved resume point, or the current position on a quality/audio switch.
|
||||
// HLS sessions spawn the FIRST ffmpeg already seeked there instead of
|
||||
// encoding from segment 0 and immediately seek-restarting (double spawn,
|
||||
// slow resume). 0/omitted = start at the beginning. Older daemons simply
|
||||
// don't decode the field and keep the old start-at-0 behaviour.
|
||||
StartSec float64 `json:"startSec,omitempty"`
|
||||
// Prewarm marks a background cache-fill session (next-episode prewarm,
|
||||
// hover prewarm): the daemon must encode it WITHOUT displacing the
|
||||
// viewer's live session — it waits until the active encode finishes and
|
||||
// registers alongside instead of evicting (Register kills every other
|
||||
// session; a prewarm claimed mid-playback used to kill the stream the
|
||||
// user was watching). False/omitted = a real viewer session.
|
||||
Prewarm bool `json:"prewarm,omitempty"`
|
||||
// PlayMethod is how the daemon should serve this session:
|
||||
// "" — default (HLS transcode); also what legacy servers send.
|
||||
// "direct" — the source is already browser-native (the web decided this
|
||||
|
|
@ -470,14 +535,31 @@ type StreamSession struct {
|
|||
|
||||
// SyncResponse is returned by the server with all pending actions for the CLI.
|
||||
type SyncResponse struct {
|
||||
NewTasks []Task `json:"newTasks,omitempty"`
|
||||
Controls []ControlAction `json:"controls,omitempty"`
|
||||
StreamRequests []StreamRequest `json:"streamRequests,omitempty"`
|
||||
StreamSessions []StreamSession `json:"streamSessions,omitempty"`
|
||||
Watching bool `json:"watching"`
|
||||
Upgrade *UpgradeSignal `json:"upgrade,omitempty"`
|
||||
Scan bool `json:"scan,omitempty"`
|
||||
FilesToDelete []LibraryDeleteRequest `json:"filesToDelete,omitempty"`
|
||||
NewTasks []Task `json:"newTasks,omitempty"`
|
||||
Controls []ControlAction `json:"controls,omitempty"`
|
||||
StreamRequests []StreamRequest `json:"streamRequests,omitempty"`
|
||||
StreamSessions []StreamSession `json:"streamSessions,omitempty"`
|
||||
Watching bool `json:"watching"`
|
||||
Upgrade *UpgradeSignal `json:"upgrade,omitempty"`
|
||||
Scan bool `json:"scan,omitempty"`
|
||||
FilesToDelete []LibraryDeleteRequest `json:"filesToDelete,omitempty"`
|
||||
SubtitleFetches []SubtitleFetchRequest `json:"subtitleFetches,omitempty"`
|
||||
}
|
||||
|
||||
// SubtitleFetchRequest is a server-side request to download a subtitle (from our
|
||||
// proxy URL, already charset-fixed + VTT) and save it as a sidecar next to a
|
||||
// media file. URL is the absolute /api/internal/subtitles/proxy URL.
|
||||
type SubtitleFetchRequest struct {
|
||||
ID int `json:"id"`
|
||||
FilePath string `json:"filePath"`
|
||||
Lang string `json:"lang"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
// SubtitleFetchError reports a permanently-failed subtitle fetch back to the web.
|
||||
type SubtitleFetchError struct {
|
||||
ID int `json:"id"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ const browserAuthTimeout = 60 * time.Second
|
|||
// 3. User logs in and clicks "Authorize" on the web page
|
||||
// 4. Web redirects to localhost:{port}/callback?token=tc_...&state={state}
|
||||
// 5. CLI validates state, extracts token, closes server
|
||||
func browserAuth(apiURL string) (string, error) {
|
||||
func browserAuth(apiURL, agentID string) (string, error) {
|
||||
// Validate apiURL is a well-formed HTTP(S) URL
|
||||
parsed, err := url.Parse(apiURL)
|
||||
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" {
|
||||
|
|
@ -96,8 +96,12 @@ func browserAuth(apiURL string) (string, error) {
|
|||
}
|
||||
}()
|
||||
|
||||
// Open browser
|
||||
// Open browser. Forward the agentId so the server mints a per-machine key
|
||||
// bound to it (omitted → server falls back to the legacy general key).
|
||||
authURL := fmt.Sprintf("%s/unarr/auth?state=%s&port=%d", apiURL, url.QueryEscape(state), port)
|
||||
if agentID != "" {
|
||||
authURL += "&agentId=" + url.QueryEscape(agentID)
|
||||
}
|
||||
openBrowser(authURL)
|
||||
|
||||
// Listen for Enter key to skip to manual fallback
|
||||
|
|
|
|||
|
|
@ -653,6 +653,14 @@ func runDaemonStart() error {
|
|||
}
|
||||
}
|
||||
|
||||
// Wire: sync receives on-demand subtitle-fetch jobs (write VTT sidecars).
|
||||
// Always available (additive, no deletion) as long as we have scan paths.
|
||||
if len(daemonCfg.ScanPaths) > 0 {
|
||||
sc.OnSubtitleFetch = func(reqs []agent.SubtitleFetchRequest) ([]int, []agent.SubtitleFetchError) {
|
||||
return library.FetchSubtitles(reqs, daemonCfg.ScanPaths)
|
||||
}
|
||||
}
|
||||
|
||||
// Wire: sync receives stream requests for completed downloads
|
||||
d.OnStreamRequested = func(sr agent.StreamRequest) {
|
||||
if streamSrv.CurrentTaskID() == sr.TaskID {
|
||||
|
|
@ -683,65 +691,19 @@ func runDaemonStart() error {
|
|||
}()
|
||||
}
|
||||
|
||||
allowedRoots := streamAllowedRoots(cfg)
|
||||
|
||||
filePath := filepath.Clean(sr.FilePath)
|
||||
// Self-heal a base-path mismatch: the web may hand us a path under an old
|
||||
// root (e.g. /mnt/nas/peliculas/… from before a binary→docker move) that
|
||||
// is now outside our allowed dirs but whose file still exists under a
|
||||
// current root (/downloads/…). Remap the path's tail onto an allowed root
|
||||
// so playback works immediately; the next re-scan persists the fix to the
|
||||
// DB. See docs/plans/unarr-path-resilience.md.
|
||||
if !isAllowedStreamPath(filePath, allowedRoots...) {
|
||||
if remapped := relocateUnreachable(filePath, allowedRoots); remapped != "" {
|
||||
log.Printf("[%s] stream self-heal: remapped %s → %s", agent.ShortID(sr.TaskID), filePath, remapped)
|
||||
filePath = remapped
|
||||
} else {
|
||||
log.Printf("[%s] stream request rejected: path outside allowed dirs: %s", agent.ShortID(sr.TaskID), filePath)
|
||||
reportStreamError(fmt.Sprintf("path outside allowed dirs: %s", filePath))
|
||||
return
|
||||
}
|
||||
}
|
||||
// os.Stat over NFS can transiently fail (ESTALE/EAGAIN/timeout) right
|
||||
// after a remount or under load. Retry a few times before giving up so
|
||||
// a hiccup doesn't surface as a spurious "file not found" — this is the
|
||||
// root of the intermittent "works on the 3rd try" stream failures.
|
||||
var info os.FileInfo
|
||||
var statErr error
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
if info, statErr = os.Stat(filePath); statErr == nil {
|
||||
break
|
||||
}
|
||||
if attempt < 2 {
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
if statErr != nil {
|
||||
// Last resort before failing: the file may simply have moved within
|
||||
// an allowed root — try to relocate it by path tail.
|
||||
if remapped := relocateUnreachable(filePath, allowedRoots); remapped != "" {
|
||||
log.Printf("[%s] stream self-heal: relocated missing %s → %s", agent.ShortID(sr.TaskID), filePath, remapped)
|
||||
filePath = remapped
|
||||
info, statErr = os.Stat(filePath)
|
||||
}
|
||||
}
|
||||
if statErr != nil {
|
||||
log.Printf("[%s] stream request: file not found after retries: %s (%v)", agent.ShortID(sr.TaskID), filePath, statErr)
|
||||
reportStreamError(fmt.Sprintf("file not found: %s", filePath))
|
||||
// current root (/downloads/…). resolvePlayableFile remaps, stat-retries
|
||||
// (NFS) and resolves directories; the next re-scan persists the fix to
|
||||
// the DB. See docs/plans/unarr-path-resilience.md.
|
||||
filePath, errCode, perr := resolvePlayableFile(sr.FilePath, streamAllowedRoots(cfg), agent.ShortID(sr.TaskID))
|
||||
if perr != nil {
|
||||
log.Printf("[%s] stream request rejected (%s): %v", agent.ShortID(sr.TaskID), errCode, perr)
|
||||
reportStreamError(perr.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if info.IsDir() {
|
||||
found := engine.FindVideoFile(filePath)
|
||||
if found == "" {
|
||||
log.Printf("[%s] stream request: no video file in directory: %s", agent.ShortID(sr.TaskID), filePath)
|
||||
reportStreamError(fmt.Sprintf("no video file in directory: %s", filePath))
|
||||
return
|
||||
}
|
||||
filePath = found
|
||||
log.Printf("[%s] resolved directory to video file: %s", agent.ShortID(sr.TaskID), filepath.Base(filePath))
|
||||
}
|
||||
|
||||
cancelStreamContexts()
|
||||
streamSrv.SetFile(engine.NewDiskFileProvider(filePath), sr.TaskID)
|
||||
log.Printf("[%s] streaming from disk: %s → %s", agent.ShortID(sr.TaskID), filepath.Base(filePath), streamSrv.URL())
|
||||
|
|
@ -771,20 +733,104 @@ func runDaemonStart() error {
|
|||
return // already running
|
||||
}
|
||||
|
||||
// failSession logs AND reports a startup failure to the web — every
|
||||
// abort path in this handler must go through it. A silent `return`
|
||||
// here left the player probing a playlist that would never exist
|
||||
// until its 30s deadline (incident 2026-06-10: deleted file + stale
|
||||
// library row = eternal "Preparando sesión"). Best-effort: on old web
|
||||
// deployments the endpoint 404s and the player falls back to the
|
||||
// probe deadline, exactly as before.
|
||||
failSession := func(sessionID, code, message string) {
|
||||
log.Printf("[hls %s] failed (%s): %s", agent.ShortID(sessionID), code, message)
|
||||
go func() {
|
||||
// Fresh context on purpose: failures cluster exactly when the
|
||||
// daemon ctx is being cancelled (shutdown kills in-flight
|
||||
// session starts), and a report derived from it would die
|
||||
// before reaching the web. The 10s cap still bounds it.
|
||||
rctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if err := agentClient.ReportSessionError(rctx, sessionID, code, message); err != nil {
|
||||
log.Printf("[hls %s] session error report failed: %v", agent.ShortID(sessionID), err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// markReady reports "first bytes are servable" for the no-transcode
|
||||
// paths (direct-play, remux, debrid direct) — one place instead of a
|
||||
// copy per branch. HLS sessions report via watchSessionReady instead
|
||||
// (they wait for seg-0 + attach a health snapshot).
|
||||
markReady := func(sessionID string) {
|
||||
go func() {
|
||||
rctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := agentClient.MarkSessionReady(rctx, sessionID, nil); err != nil {
|
||||
log.Printf("[stream %s] mark-ready failed: %v", agent.ShortID(sessionID), err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// startHLSPlayback starts an HLS encode (local file or debrid URL) and
|
||||
// wires it into the StreamServer. Shared by the local-file HLS path and
|
||||
// the debrid HLS-from-URL path (hueco #2 / 2b) so both register, probe
|
||||
// off the sync loop, and report readiness identically.
|
||||
//
|
||||
// Prewarm sessions (background cache-fill: next-episode, hover) take a
|
||||
// deferential path: wait until no live encode is running (never steal
|
||||
// the encoder from the viewer), then register WITHOUT displacing other
|
||||
// sessions. Before this, a prewarm claimed mid-playback went through
|
||||
// Register() and KILLED the stream the user was watching (verified
|
||||
// 2026-06-10: prewarm started → live session "closed (cache
|
||||
// discarded)" → player 404).
|
||||
startHLSPlayback := func(hlsCfg engine.HLSSessionConfig, hlsCtx context.Context, hlsCancel context.CancelFunc) {
|
||||
playerSessionRegistry.add(hlsCfg.SessionID, hlsCancel)
|
||||
prewarm := sess.Prewarm
|
||||
go func() {
|
||||
if prewarm {
|
||||
// Defer until the encoder is free. Poll is cheap (10 s);
|
||||
// cap the wait at 30 min — a prewarm that can't start
|
||||
// within an episode's runtime has lost its purpose.
|
||||
deadline := time.Now().Add(30 * time.Minute)
|
||||
for streamSrv.HLS().HasLiveEncode() {
|
||||
if time.Now().After(deadline) || hlsCtx.Err() != nil {
|
||||
playerSessionRegistry.remove(hlsCfg.SessionID)
|
||||
hlsCancel()
|
||||
log.Printf("[hls %s] prewarm abandoned (encoder busy %s)",
|
||||
agent.ShortID(hlsCfg.SessionID), "30m")
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-hlsCtx.Done():
|
||||
playerSessionRegistry.remove(hlsCfg.SessionID)
|
||||
return
|
||||
case <-time.After(10 * time.Second):
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// REAL session: reap in-flight prewarm encodes BEFORE
|
||||
// StartHLSSession so the per-key cache writer-lock is
|
||||
// free and the viewer's encode lands in the persistent
|
||||
// cache (not an uncached tmpdir). A SEALED prewarm is
|
||||
// unaffected — this session simply cache-HITs it.
|
||||
if n := streamSrv.HLS().CloseWhere(func(s *engine.HLSSession) bool { return s.IsPrewarm() }); n > 0 {
|
||||
log.Printf("[hls %s] reaped %d in-flight prewarm(s) for the viewer session",
|
||||
agent.ShortID(hlsCfg.SessionID), n)
|
||||
}
|
||||
}
|
||||
hsess, err := engine.StartHLSSession(hlsCtx, hlsCfg)
|
||||
if err != nil {
|
||||
playerSessionRegistry.remove(hlsCfg.SessionID)
|
||||
hlsCancel()
|
||||
log.Printf("[hls %s] start failed: %v", agent.ShortID(hlsCfg.SessionID), err)
|
||||
failSession(hlsCfg.SessionID, sessErrStartFailed, err.Error())
|
||||
return
|
||||
}
|
||||
if prewarm {
|
||||
// Side-by-side: never evict the viewer's session. A later
|
||||
// REAL session still evicts this one via Register — by
|
||||
// then the encode is usually sealed in the segment cache.
|
||||
streamSrv.HLS().RegisterKeep(hsess)
|
||||
log.Printf("[hls %s] prewarm encoding: %s", agent.ShortID(hlsCfg.SessionID), hlsCfg.FileName)
|
||||
return // no viewer waiting → no ready-watcher
|
||||
}
|
||||
streamSrv.HLS().Register(hsess)
|
||||
go watchSessionReady(hlsCtx, agentClient, hsess, hlsCfg.SessionID)
|
||||
}()
|
||||
|
|
@ -814,17 +860,13 @@ func runDaemonStart() error {
|
|||
provider, perr := engine.NewDebridFileProvider(bctx, sess.DirectURL, sess.FileName, sess.FileSize, refresh)
|
||||
if perr != nil {
|
||||
playerSessionRegistry.remove(sess.SessionID)
|
||||
log.Printf("[stream %s] debrid provider failed: %v", agent.ShortID(sess.SessionID), perr)
|
||||
failSession(sess.SessionID, sessErrStartFailed, fmt.Sprintf("debrid provider: %v", perr))
|
||||
return
|
||||
}
|
||||
streamSrv.SetFile(provider, sess.TaskID)
|
||||
log.Printf("[stream %s] debrid direct-play: %s (%d bytes)",
|
||||
agent.ShortID(sess.SessionID), provider.FileName(), provider.FileSize())
|
||||
rctx, rcancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer rcancel()
|
||||
if err := agentClient.MarkSessionReady(rctx, sess.SessionID); err != nil {
|
||||
log.Printf("[stream %s] mark-ready failed: %v", agent.ShortID(sess.SessionID), err)
|
||||
}
|
||||
markReady(sess.SessionID)
|
||||
}()
|
||||
return
|
||||
}
|
||||
|
|
@ -837,7 +879,7 @@ func runDaemonStart() error {
|
|||
if sess.DirectURL != "" { // playMethod == "hls" implied (2a returned above)
|
||||
tcRuntime := buildTranscodeRuntime(ctx, cfg)
|
||||
if tcRuntime.FFmpegPath == "" || tcRuntime.FFprobePath == "" {
|
||||
log.Printf("[hls %s] rejected: ffmpeg/ffprobe unavailable (debrid HLS)", agent.ShortID(sess.SessionID))
|
||||
failSession(sess.SessionID, sessErrFfmpegMissing, "ffmpeg/ffprobe unavailable (debrid HLS)")
|
||||
return
|
||||
}
|
||||
hlsCtx, hlsCancel := context.WithCancel(ctx)
|
||||
|
|
@ -849,6 +891,8 @@ func runDaemonStart() error {
|
|||
Quality: sess.Quality,
|
||||
AudioIndex: sess.AudioIndex,
|
||||
BurnSubtitleIndex: sess.BurnSubtitleIndex,
|
||||
StartSec: sess.StartSec,
|
||||
Prewarm: sess.Prewarm,
|
||||
Transcode: tcRuntime,
|
||||
Cache: hlsCache,
|
||||
// 2c: refresh the debrid link if it expires mid-transcode; the
|
||||
|
|
@ -861,43 +905,22 @@ func runDaemonStart() error {
|
|||
return
|
||||
}
|
||||
|
||||
filePath := sess.FilePath
|
||||
if filePath == "" {
|
||||
log.Printf("[hls %s] rejected: empty file path", agent.ShortID(sess.SessionID))
|
||||
if sess.FilePath == "" {
|
||||
failSession(sess.SessionID, sessErrStartFailed, "empty file path")
|
||||
return
|
||||
}
|
||||
filePath = filepath.Clean(filePath)
|
||||
// Apply the SAME base-path self-heal remap as the raw /stream handler
|
||||
// (OnStreamRequest above). Without it, a path under an old/host base
|
||||
// SAME base-path self-heal + stat-retry + dir resolution as the raw
|
||||
// /stream handler (resolvePlayableFile). A path under an old/host base
|
||||
// (e.g. /mnt/nas/peliculas/… handed by the web while this docker agent
|
||||
// mounts that media at /downloads) is rejected here even though the raw
|
||||
// path self-heals it — so the web silently falls back to the raw stream
|
||||
// and HLS/remux never runs (no transcode, slow funnel start). NOTE: this
|
||||
// replicates only the lexical-remap; the raw handler additionally retries
|
||||
// os.Stat for transient NFS errors. The HLS dir-check below proceeds (not
|
||||
// rejects) on a stat error, so it tolerates an NFS blip differently.
|
||||
// mounts that media at /downloads) remaps onto the current root; a path
|
||||
// whose file is genuinely gone fails fast as "file_missing" so the web
|
||||
// can prune the stale library row and the player can fall back, instead
|
||||
// of the player probing a playlist that will never exist.
|
||||
// See docs/plans/unarr-path-resilience.md.
|
||||
hlsAllowedRoots := streamAllowedRoots(cfg)
|
||||
if !isAllowedStreamPath(filePath, hlsAllowedRoots...) {
|
||||
if remapped := relocateUnreachable(filePath, hlsAllowedRoots); remapped != "" {
|
||||
log.Printf("[hls %s] self-heal: remapped %s → %s",
|
||||
agent.ShortID(sess.SessionID), filePath, remapped)
|
||||
filePath = remapped
|
||||
} else {
|
||||
log.Printf("[hls %s] rejected: path outside allowed dirs: %s",
|
||||
agent.ShortID(sess.SessionID), filePath)
|
||||
return
|
||||
}
|
||||
}
|
||||
// Resolve directory → first video file (matches StreamRequest behavior).
|
||||
if info, err := os.Stat(filePath); err == nil && info.IsDir() {
|
||||
found := engine.FindVideoFile(filePath)
|
||||
if found == "" {
|
||||
log.Printf("[hls %s] rejected: no video file in dir %s",
|
||||
agent.ShortID(sess.SessionID), filePath)
|
||||
return
|
||||
}
|
||||
filePath = found
|
||||
filePath, errCode, perr := resolvePlayableFile(sess.FilePath, streamAllowedRoots(cfg), "hls "+agent.ShortID(sess.SessionID))
|
||||
if perr != nil {
|
||||
failSession(sess.SessionID, errCode, perr.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Direct-play (hueco #3 / 3a): the web decided this source is already
|
||||
|
|
@ -914,19 +937,13 @@ func runDaemonStart() error {
|
|||
log.Printf("[stream %s] direct-play: %s", agent.ShortID(sess.SessionID), filepath.Base(filePath))
|
||||
// File is on disk → ready immediately. Tell the web so the player
|
||||
// attaches <video src> without burning its HEAD-probe retry budget.
|
||||
go func() {
|
||||
rctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := agentClient.MarkSessionReady(rctx, sess.SessionID); err != nil {
|
||||
log.Printf("[stream %s] mark-ready failed: %v", agent.ShortID(sess.SessionID), err)
|
||||
}
|
||||
}()
|
||||
markReady(sess.SessionID)
|
||||
return
|
||||
}
|
||||
|
||||
tcRuntime := buildTranscodeRuntime(ctx, cfg)
|
||||
if tcRuntime.FFmpegPath == "" || tcRuntime.FFprobePath == "" {
|
||||
log.Printf("[hls %s] rejected: ffmpeg/ffprobe unavailable", agent.ShortID(sess.SessionID))
|
||||
failSession(sess.SessionID, sessErrFfmpegMissing, "ffmpeg/ffprobe unavailable")
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -940,7 +957,7 @@ func runDaemonStart() error {
|
|||
probe, perr := engine.ProbeFile(probeCtx, tcRuntime.FFprobePath, filePath)
|
||||
cancelProbe()
|
||||
if perr != nil {
|
||||
log.Printf("[stream %s] remux probe failed: %v", agent.ShortID(sess.SessionID), perr)
|
||||
failSession(sess.SessionID, sessErrStartFailed, fmt.Sprintf("remux probe: %v", perr))
|
||||
return
|
||||
}
|
||||
tProbe := time.Now()
|
||||
|
|
@ -948,7 +965,7 @@ func runDaemonStart() error {
|
|||
src, serr := engine.NewRemuxSource(remuxCtx, filePath, probe, tcRuntime.FFmpegPath, sess.FileName)
|
||||
if serr != nil {
|
||||
remuxCancel()
|
||||
log.Printf("[stream %s] remux start failed: %v", agent.ShortID(sess.SessionID), serr)
|
||||
failSession(sess.SessionID, sessErrStartFailed, fmt.Sprintf("remux start: %v", serr))
|
||||
return
|
||||
}
|
||||
streamSrv.SetGrowingFile(src, sess.TaskID)
|
||||
|
|
@ -962,13 +979,7 @@ func runDaemonStart() error {
|
|||
log.Printf("[stream %s] remux (copy) → fMP4: %s [probe=%v spawn=%v]",
|
||||
agent.ShortID(sess.SessionID), filepath.Base(filePath),
|
||||
tProbe.Sub(tStart).Round(time.Millisecond), time.Since(tProbe).Round(time.Millisecond))
|
||||
go func() {
|
||||
rctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := agentClient.MarkSessionReady(rctx, sess.SessionID); err != nil {
|
||||
log.Printf("[stream %s] mark-ready failed: %v", agent.ShortID(sess.SessionID), err)
|
||||
}
|
||||
}()
|
||||
markReady(sess.SessionID)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -984,6 +995,8 @@ func runDaemonStart() error {
|
|||
Quality: sess.Quality,
|
||||
AudioIndex: sess.AudioIndex,
|
||||
BurnSubtitleIndex: sess.BurnSubtitleIndex,
|
||||
StartSec: sess.StartSec,
|
||||
Prewarm: sess.Prewarm,
|
||||
Transcode: tcRuntime,
|
||||
Cache: hlsCache,
|
||||
}, hlsCtx, hlsCancel)
|
||||
|
|
@ -1037,6 +1050,26 @@ func runDaemonStart() error {
|
|||
// Start reporter only for stream task handling
|
||||
go reporter.Run(ctx)
|
||||
|
||||
// Credential revoked mid-run (agent deleted from the dashboard): wipe the
|
||||
// stored key + agentId so a supervisor restart can't loop on a rejected
|
||||
// identity, then stop the daemon. Reconnecting needs a fresh `unarr login`.
|
||||
d.SyncClient().OnRevoked = func(err error) {
|
||||
reportAgentRevoked(cfg, err)
|
||||
cancel()
|
||||
}
|
||||
|
||||
// Legacy bootstrap: if register hands back a per-machine key, persist it so
|
||||
// the next start authenticates with the bound agent key (one-time migration;
|
||||
// also stops the server re-minting on every restart).
|
||||
d.OnAgentKeyMinted = func(newKey string) {
|
||||
cfg.Auth.APIKey = newKey
|
||||
if serr := config.Save(cfg, resolvedConfigPath()); serr != nil {
|
||||
log.Printf("[agent] could not persist per-machine key: %v", serr)
|
||||
} else {
|
||||
log.Printf("[agent] migrated to a per-machine agent key")
|
||||
}
|
||||
}
|
||||
|
||||
// Start daemon (blocks — runs sync loop)
|
||||
errCh := make(chan error, 1)
|
||||
go func() {
|
||||
|
|
@ -1076,10 +1109,34 @@ func runDaemonStart() error {
|
|||
cancelAllPlayerSessions()
|
||||
streamSrv.Shutdown(context.Background())
|
||||
cancel()
|
||||
// Registration was rejected because this agent's credential is revoked
|
||||
// (deleted from the dashboard). Wipe it and exit cleanly so the service
|
||||
// supervisor doesn't restart-loop against a 410; user must re-login.
|
||||
if agent.IsRevoked(err) {
|
||||
reportAgentRevoked(cfg, err)
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// reportAgentRevoked tells the user their agent was removed and wipes the
|
||||
// stored credential (api key + agentId) so the next start requires a fresh
|
||||
// `unarr login` (which mints a new per-machine key bound to a new agentId)
|
||||
// instead of looping against a server that keeps rejecting the old identity.
|
||||
func reportAgentRevoked(cfg config.Config, err error) {
|
||||
log.Printf("[agent] credential revoked by server (%v) — this machine was removed from your account", err)
|
||||
cfg.Auth.APIKey = ""
|
||||
cfg.Agent.ID = ""
|
||||
if serr := config.Save(cfg, resolvedConfigPath()); serr != nil {
|
||||
log.Printf("[agent] could not clear stored credential: %v", serr)
|
||||
}
|
||||
fmt.Println()
|
||||
fmt.Println(" This agent was removed from your account.")
|
||||
fmt.Println(" Run `unarr login` on this machine to reconnect it.")
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
// isAllowedStreamPath checks that filePath is within one of the directories
|
||||
// the daemon is configured to manage. This defends against a compromised API
|
||||
// server sending a path traversal payload (e.g. /etc/passwd) in StreamRequest.
|
||||
|
|
@ -1156,6 +1213,82 @@ func relocateUnreachable(filePath string, allowedRoots []string) string {
|
|||
return ""
|
||||
}
|
||||
|
||||
// Stable machine codes for the web's session-error channel
|
||||
// (POST /api/internal/agent/session-error) — mirrored by
|
||||
// SESSION_ERROR_CODES in the web repo. Only "file_missing" triggers
|
||||
// destructive self-heal on the web (it prunes the stale library row + task
|
||||
// pointer), so the resolver must never return it while the file may exist.
|
||||
const (
|
||||
pathErrRejected = "path_rejected"
|
||||
pathErrMissing = "file_missing"
|
||||
pathErrNoVideo = "no_video_file"
|
||||
sessErrFfmpegMissing = "ffmpeg_unavailable"
|
||||
sessErrStartFailed = "start_failed"
|
||||
)
|
||||
|
||||
// resolvePlayableFile validates and self-heals a web-provided source path into
|
||||
// a playable on-disk video file. Shared by the raw /stream handler and every
|
||||
// session transport (HLS / remux / direct-play) so they all behave
|
||||
// identically — before this, the HLS path replicated only the lexical remap
|
||||
// and silently diverged on stat retries (docs/plans/unarr-path-resilience.md):
|
||||
//
|
||||
// 1. Containment: the cleaned path must live under an allowed root; if not,
|
||||
// relocate it by path tail (old base path → current mount).
|
||||
// 2. Existence: os.Stat with retries (NFS can transiently fail right after a
|
||||
// remount or under load — the root of the "works on the 3rd try" stream
|
||||
// failures), then one last relocate for files that moved within a root.
|
||||
// 3. Directories resolve to their first contained video file.
|
||||
//
|
||||
// On failure returns a stable errCode: "path_rejected" means the file EXISTS
|
||||
// at the original path but outside every allowed root (an agent config
|
||||
// problem — the web must NOT prune library rows over it); "file_missing"
|
||||
// means no readable file was found anywhere; "no_video_file" is a directory
|
||||
// with nothing playable inside.
|
||||
func resolvePlayableFile(rawPath string, allowedRoots []string, logLabel string) (string, string, error) {
|
||||
filePath := filepath.Clean(rawPath)
|
||||
if !isAllowedStreamPath(filePath, allowedRoots...) {
|
||||
if remapped := relocateUnreachable(filePath, allowedRoots); remapped != "" {
|
||||
log.Printf("[%s] stream self-heal: remapped %s → %s", logLabel, filePath, remapped)
|
||||
filePath = remapped
|
||||
} else if _, err := os.Stat(filePath); err == nil {
|
||||
return "", pathErrRejected, fmt.Errorf("path outside allowed dirs: %s", filePath)
|
||||
} else {
|
||||
return "", pathErrMissing, fmt.Errorf("file not found under any allowed root: %s", filePath)
|
||||
}
|
||||
}
|
||||
var info os.FileInfo
|
||||
var statErr error
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
if info, statErr = os.Stat(filePath); statErr == nil {
|
||||
break
|
||||
}
|
||||
if attempt < 2 {
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
if statErr != nil {
|
||||
// Last resort before failing: the file may simply have moved within
|
||||
// an allowed root — try to relocate it by path tail.
|
||||
if remapped := relocateUnreachable(filePath, allowedRoots); remapped != "" {
|
||||
log.Printf("[%s] stream self-heal: relocated missing %s → %s", logLabel, filePath, remapped)
|
||||
filePath = remapped
|
||||
info, statErr = os.Stat(filePath)
|
||||
}
|
||||
}
|
||||
if statErr != nil {
|
||||
return "", pathErrMissing, fmt.Errorf("file not found after retries: %s (%v)", filePath, statErr)
|
||||
}
|
||||
if info.IsDir() {
|
||||
found := engine.FindVideoFile(filePath)
|
||||
if found == "" {
|
||||
return "", pathErrNoVideo, fmt.Errorf("no video file in directory: %s", filePath)
|
||||
}
|
||||
log.Printf("[%s] resolved directory to video file: %s", logLabel, filepath.Base(found))
|
||||
filePath = found
|
||||
}
|
||||
return filePath, "", nil
|
||||
}
|
||||
|
||||
func formatSpeedLog(bps int64) string {
|
||||
switch {
|
||||
case bps >= 1024*1024*1024:
|
||||
|
|
@ -1246,19 +1379,26 @@ func runAutoScan(ctx context.Context, cfg config.Config, interval time.Duration,
|
|||
}
|
||||
}
|
||||
|
||||
// Scan each path independently and sync per path so the server can
|
||||
// scope stale-item deletion to the correct directory prefix.
|
||||
const batchSize = 100
|
||||
totalSynced := 0
|
||||
// Scan every path, then sync ALL of them as ONE session (single
|
||||
// syncStartedAt + final isLastBatch via library.SyncBatches). Per-root
|
||||
// sessions let the server's per-agent stale cleanup reap rows of roots
|
||||
// a session never visited; one full-cycle session makes the cleanup
|
||||
// sound AND lets it reap old-base-path ghost rows (fullCycle=true —
|
||||
// only when every root scanned cleanly).
|
||||
var syncItems []agent.LibrarySyncItem
|
||||
var coveredRoots []string
|
||||
fullCycle := true
|
||||
var mergedItems []library.LibraryItem
|
||||
|
||||
for _, scanPath := range scanPaths {
|
||||
cache, err := library.Scan(ctx, scanPath, existing, scanOpts)
|
||||
if err != nil {
|
||||
log.Printf("[auto-scan] scan failed for %s: %v", scanPath, err)
|
||||
fullCycle = false
|
||||
continue
|
||||
}
|
||||
mergedItems = append(mergedItems, cache.Items...)
|
||||
coveredRoots = append(coveredRoots, scanPath)
|
||||
|
||||
if prewarmFFmpeg != "" {
|
||||
library.PrewarmSidecars(ctx, cache, library.PrewarmOptions{
|
||||
|
|
@ -1278,28 +1418,28 @@ func runAutoScan(ctx context.Context, cfg config.Config, interval time.Duration,
|
|||
log.Printf("[auto-scan] no items under %s", scanPath)
|
||||
continue
|
||||
}
|
||||
syncItems = append(syncItems, items...)
|
||||
}
|
||||
|
||||
syncStartedAt := time.Now().UTC().Format(time.RFC3339)
|
||||
for i := 0; i < len(items); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(items) {
|
||||
end = len(items)
|
||||
}
|
||||
isLast := end >= len(items)
|
||||
|
||||
_, err := ac.SyncLibrary(ctx, agent.LibrarySyncRequest{
|
||||
Items: items[i:end],
|
||||
ScanPath: scanPath,
|
||||
AgentID: cfg.Agent.ID,
|
||||
IsLastBatch: isLast,
|
||||
SyncStartedAt: syncStartedAt,
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("[auto-scan] sync failed for %s: %v", scanPath, err)
|
||||
break
|
||||
}
|
||||
totalSynced := 0
|
||||
if len(syncItems) > 0 {
|
||||
res, err := library.SyncBatches(ctx, ac, syncItems, library.SyncOptions{
|
||||
AgentID: cfg.Agent.ID,
|
||||
ScanPath: coveredRoots[0],
|
||||
ScanRoots: coveredRoots,
|
||||
FullCycle: fullCycle,
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("[auto-scan] sync failed: %v", err)
|
||||
} else if res.Removed > 0 {
|
||||
log.Printf("[auto-scan] server removed %d stale item(s)", res.Removed)
|
||||
}
|
||||
totalSynced += len(items)
|
||||
totalSynced = res.Synced
|
||||
} else {
|
||||
// An entirely-empty library can't open a sync session (the server
|
||||
// requires ≥1 item per batch), so stale rows survive until a file
|
||||
// reappears — same trade-off as before, now explicit.
|
||||
log.Printf("[auto-scan] no items under any scan path — skipping sync")
|
||||
}
|
||||
|
||||
// Save merged cache for incremental scanning next time.
|
||||
|
|
@ -1445,6 +1585,17 @@ func watchSessionReady(ctx context.Context, client *agent.Client, hsess *engine.
|
|||
deadline := time.Now().Add(60 * time.Second)
|
||||
ticker := time.NewTicker(200 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
readyPosted := false
|
||||
postReady := func(health *agent.SessionHealth) {
|
||||
// Parent ctx so a session cancel mid-POST (user closed tab, daemon
|
||||
// shutdown) tears down the in-flight webhook instead of blocking the
|
||||
// goroutine for up to 10 s on a now-orphan call.
|
||||
rctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
if err := client.MarkSessionReady(rctx, sessionID, health); err != nil {
|
||||
log.Printf("[hls %s] mark-ready failed: %v", agent.ShortID(sessionID), err)
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
for {
|
||||
// Session torn down through a path that didn't cancel ctx (registry
|
||||
// replace, idle sweep, internal kill). Bail before polling further —
|
||||
|
|
@ -1453,17 +1604,29 @@ func watchSessionReady(ctx context.Context, client *agent.Client, hsess *engine.
|
|||
if hsess.IsClosed() {
|
||||
return
|
||||
}
|
||||
// Cache HIT or seg-0 ready → notify + done.
|
||||
if hsess.FromCache() || hsess.ReadyCount() >= 1 {
|
||||
// Parent ctx so a session cancel mid-POST (user closed tab,
|
||||
// daemon shutdown) tears down the in-flight webhook instead of
|
||||
// blocking the goroutine for up to 10 s on a now-orphan call.
|
||||
rctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
if err := client.MarkSessionReady(rctx, sessionID); err != nil {
|
||||
log.Printf("[hls %s] mark-ready failed: %v", agent.ShortID(sessionID), err)
|
||||
// Phase 1: cache HIT or first segment ready → flip the "Preparando…"
|
||||
// UI now. Compare against WriterStartIdx, not `>= 1`: a resume
|
||||
// session (StartSec) pre-seeds readyMax to the start index, so
|
||||
// ReadyCount() is ≥ 1 before ffmpeg has written a single byte —
|
||||
// `>= 1` would fire "ready" instantly and freeze the player waiting
|
||||
// on a segment that doesn't exist yet.
|
||||
if !readyPosted && (hsess.FromCache() || hsess.ReadyCount() > hsess.WriterStartIdx()) {
|
||||
postReady(nil)
|
||||
readyPosted = true
|
||||
// Cache replay has no live encode → no telemetry to report, done.
|
||||
if hsess.FromCache() {
|
||||
return
|
||||
}
|
||||
}
|
||||
// Phase 2 (F3): once enough -stats samples accumulated (encoder past
|
||||
// its cold ramp), report ONE live-health snapshot so the player can
|
||||
// name a too-slow transcode in ~4s instead of inferring it from stalls.
|
||||
// >=4 samples ≈ 2s of encoding past seg-0; the EWMA has settled by then.
|
||||
if readyPosted {
|
||||
if st := hsess.GetTranscodeStats(); st.Samples >= 4 {
|
||||
postReady(classifyAgentHealth(st))
|
||||
return
|
||||
}
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
|
|
@ -1471,7 +1634,15 @@ func watchSessionReady(ctx context.Context, client *agent.Client, hsess *engine.
|
|||
case <-ticker.C:
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
log.Printf("[hls %s] mark-ready: timeout waiting for seg-0", agent.ShortID(sessionID))
|
||||
if !readyPosted {
|
||||
log.Printf("[hls %s] mark-ready: timeout waiting for seg-0", agent.ShortID(sessionID))
|
||||
return
|
||||
}
|
||||
// Ready but never got stable telemetry — report whatever we have so
|
||||
// the player isn't left without a verdict (better partial than none).
|
||||
if st := hsess.GetTranscodeStats(); st.Samples > 0 {
|
||||
postReady(classifyAgentHealth(st))
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
|
@ -1514,3 +1685,36 @@ func fetchAgentCert(ctx context.Context, client *agent.Client, hash string) {
|
|||
}
|
||||
log.Printf("[acme] installed cert for *.%s.%s", hash, base)
|
||||
}
|
||||
|
||||
// Realtime-ratio cutoffs for classifyAgentHealth. This is a cross-repo contract
|
||||
// with the web bottleneck classifier (src/lib/stream/bottleneck-classifier.ts):
|
||||
// - ≥ realtimeFloor → "ok" (encoder keeps up)
|
||||
// - [strugglingFloor,..) → "marginal" (barely)
|
||||
// - < strugglingFloor → "struggling" (can't) — the web fast-path commits
|
||||
// the honest overlay + pauses on this WITHOUT waiting for a stall, so the
|
||||
// floor is intentionally conservative (the web uses a looser 0.85 only once
|
||||
// a stall has already corroborated the slowdown).
|
||||
const (
|
||||
agentRealtimeFloor = 0.95
|
||||
agentStrugglingFloor = 0.75
|
||||
)
|
||||
|
||||
// classifyAgentHealth turns a live ffmpeg telemetry snapshot into the health
|
||||
// report the web side consumes (F3). The ×realtime speed is the load-bearing
|
||||
// signal: < 1.0 means the encode can't keep up with playback. An input-bound
|
||||
// hint (source read error) reclassifies the cause as the link, not the encoder.
|
||||
func classifyAgentHealth(st engine.TranscodeStats) *agent.SessionHealth {
|
||||
ratio := st.SpeedX
|
||||
var health, reason string
|
||||
switch {
|
||||
case st.InputBound && ratio < agentRealtimeFloor:
|
||||
health, reason = "struggling", "input_bound"
|
||||
case ratio >= agentRealtimeFloor:
|
||||
health, reason = "ok", "realtime"
|
||||
case ratio >= agentStrugglingFloor:
|
||||
health, reason = "marginal", "transcode"
|
||||
default:
|
||||
health, reason = "struggling", "transcode"
|
||||
}
|
||||
return &agent.SessionHealth{Health: health, RealtimeRatio: ratio, Reason: reason}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -75,12 +75,19 @@ func runInit(apiURLOverride string) error {
|
|||
|
||||
apiKey := cfg.Auth.APIKey
|
||||
|
||||
// Resolve the agentId up front so browser-authorize can bind the minted
|
||||
// per-machine key to it.
|
||||
agentID := cfg.Agent.ID
|
||||
if agentID == "" {
|
||||
agentID = uuid.New().String()
|
||||
}
|
||||
|
||||
if apiKey == "" {
|
||||
// Try browser-based auth first (like Claude Code / GitHub CLI)
|
||||
fmt.Println(" Opening browser to connect your account...")
|
||||
fmt.Println()
|
||||
|
||||
browserKey, browserErr := browserAuth(apiURL)
|
||||
browserKey, browserErr := browserAuth(apiURL, agentID)
|
||||
if browserErr == nil && strings.HasPrefix(browserKey, "tc_") {
|
||||
apiKey = browserKey
|
||||
green.Println(" ✓ Connected via browser")
|
||||
|
|
@ -127,11 +134,6 @@ func runInit(apiURLOverride string) error {
|
|||
// Validate API key by registering with the server
|
||||
fmt.Print(" Verifying API key... ")
|
||||
|
||||
agentID := cfg.Agent.ID
|
||||
if agentID == "" {
|
||||
agentID = uuid.New().String()
|
||||
}
|
||||
|
||||
hostname, _ := os.Hostname()
|
||||
agentName := cfg.Agent.Name
|
||||
if agentName == "" {
|
||||
|
|
@ -150,9 +152,21 @@ func runInit(apiURLOverride string) error {
|
|||
if err != nil {
|
||||
color.Red("FAILED")
|
||||
fmt.Println()
|
||||
// Stored credential was revoked (machine deleted from the dashboard) —
|
||||
// drop it so a re-run mints a fresh identity.
|
||||
if agent.IsRevoked(err) {
|
||||
clearRevokedIdentity(cfg, "init")
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("API key validation failed: %w", err)
|
||||
}
|
||||
|
||||
// Manual-paste bootstrap: swap to the minted per-machine key, discard the
|
||||
// general key the user pasted.
|
||||
if resp.AgentKey != "" {
|
||||
apiKey = resp.AgentKey
|
||||
}
|
||||
|
||||
green.Println("OK")
|
||||
fmt.Printf(" Connected as %s (%s) [%s]\n", resp.User.Name, resp.User.Email, strings.ToUpper(resp.User.Plan))
|
||||
fmt.Println()
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
|
@ -16,6 +17,20 @@ import (
|
|||
"github.com/torrentclaw/unarr/internal/config"
|
||||
)
|
||||
|
||||
// clearRevokedIdentity wipes the stored credential (api key + agentId) after the
|
||||
// server reports this machine's registration was revoked, so a re-run of the
|
||||
// given command mints a fresh identity instead of looping against a dead key.
|
||||
func clearRevokedIdentity(cfg config.Config, retryCmd string) {
|
||||
cfg.Auth.APIKey = ""
|
||||
cfg.Agent.ID = ""
|
||||
if err := config.Save(cfg, resolvedConfigPath()); err != nil {
|
||||
log.Printf("could not clear revoked credential: %v", err)
|
||||
}
|
||||
fmt.Println(" This machine's previous registration was removed from your account.")
|
||||
fmt.Printf(" Run `unarr %s` again to reconnect it as a new agent.\n", retryCmd)
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
func newLoginCmd() *cobra.Command {
|
||||
var apiURL string
|
||||
|
||||
|
|
@ -70,11 +85,18 @@ func runLogin(apiURLOverride string) error {
|
|||
|
||||
var apiKey string
|
||||
|
||||
// Resolve the agentId up front so the browser-authorize flow can bind the
|
||||
// minted per-machine key to it.
|
||||
agentID := cfg.Agent.ID
|
||||
if agentID == "" {
|
||||
agentID = uuid.New().String()
|
||||
}
|
||||
|
||||
// Try browser-based auth first
|
||||
fmt.Println(" Opening browser to connect your account...")
|
||||
fmt.Println()
|
||||
|
||||
browserKey, browserErr := browserAuth(apiURL)
|
||||
browserKey, browserErr := browserAuth(apiURL, agentID)
|
||||
if browserErr == nil && strings.HasPrefix(browserKey, "tc_") {
|
||||
apiKey = browserKey
|
||||
green.Println(" ✓ Connected via browser")
|
||||
|
|
@ -120,11 +142,6 @@ func runLogin(apiURLOverride string) error {
|
|||
|
||||
fmt.Print(" Verifying API key... ")
|
||||
|
||||
agentID := cfg.Agent.ID
|
||||
if agentID == "" {
|
||||
agentID = uuid.New().String()
|
||||
}
|
||||
|
||||
hostname, _ := os.Hostname()
|
||||
agentName := cfg.Agent.Name
|
||||
if agentName == "" {
|
||||
|
|
@ -143,9 +160,21 @@ func runLogin(apiURLOverride string) error {
|
|||
if err != nil {
|
||||
color.Red("FAILED")
|
||||
fmt.Println()
|
||||
// The stored credential was revoked (this machine was deleted from the
|
||||
// dashboard). Drop it so the next run mints a fresh identity.
|
||||
if agent.IsRevoked(err) {
|
||||
clearRevokedIdentity(cfg, "login")
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("API key validation failed: %w", err)
|
||||
}
|
||||
|
||||
// Manual-paste bootstrap: the server minted a per-machine key bound to this
|
||||
// agentId. Swap to it and discard the general key the user pasted.
|
||||
if resp.AgentKey != "" {
|
||||
apiKey = resp.AgentKey
|
||||
}
|
||||
|
||||
green.Println("OK")
|
||||
fmt.Printf(" Connected as %s (%s) [%s]\n", resp.User.Name, resp.User.Email, strings.ToUpper(resp.User.Plan))
|
||||
fmt.Println()
|
||||
|
|
|
|||
98
internal/cmd/resolve_playable_test.go
Normal file
98
internal/cmd/resolve_playable_test.go
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
package cmd
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestResolvePlayableFile(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mkfile(t, filepath.Join(root, "Acme Show", "Season 01", "ep.mkv"))
|
||||
roots := []string{root}
|
||||
|
||||
t.Run("allowed path resolves to itself", func(t *testing.T) {
|
||||
want := filepath.Join(root, "Acme Show", "Season 01", "ep.mkv")
|
||||
got, code, err := resolvePlayableFile(want, roots, "test")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error (%s): %v", code, err)
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("got %q want %q", got, want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("old base path relocates onto current root", func(t *testing.T) {
|
||||
got, code, err := resolvePlayableFile("/old/base/Acme Show/Season 01/ep.mkv", roots, "test")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error (%s): %v", code, err)
|
||||
}
|
||||
want := filepath.Join(root, "Acme Show", "Season 01", "ep.mkv")
|
||||
if got != want {
|
||||
t.Errorf("got %q want %q", got, want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("deleted file under old base is file_missing, never path_rejected", func(t *testing.T) {
|
||||
// The incident shape (2026-06-10): web hands a stale host path
|
||||
// (/mnt/nas/…) whose file was deleted — the docker agent can't see the
|
||||
// original path AND no tail relocates. file_missing tells the web to
|
||||
// prune the stale row; path_rejected would block that self-heal.
|
||||
_, code, err := resolvePlayableFile("/old/base/Acme Show/Season 01/gone.mkv", roots, "test")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for deleted file")
|
||||
}
|
||||
if code != pathErrMissing {
|
||||
t.Errorf("code = %q, want %q", code, pathErrMissing)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("existing file outside roots is path_rejected", func(t *testing.T) {
|
||||
outside := t.TempDir()
|
||||
// 1-segment-deep on purpose: a ≥3-segment tail could legitimately
|
||||
// relocate INTO the root if a same-named file existed there.
|
||||
mkfile(t, filepath.Join(outside, "leak.mkv"))
|
||||
_, code, err := resolvePlayableFile(filepath.Join(outside, "leak.mkv"), roots, "test")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for out-of-root file")
|
||||
}
|
||||
if code != pathErrRejected {
|
||||
t.Errorf("code = %q, want %q", code, pathErrRejected)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing file inside an allowed root is file_missing", func(t *testing.T) {
|
||||
_, code, err := resolvePlayableFile(filepath.Join(root, "Acme Show", "Season 01", "gone.mkv"), roots, "test")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for missing file")
|
||||
}
|
||||
if code != pathErrMissing {
|
||||
t.Errorf("code = %q, want %q", code, pathErrMissing)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("directory resolves to its video file", func(t *testing.T) {
|
||||
got, code, err := resolvePlayableFile(filepath.Join(root, "Acme Show", "Season 01"), roots, "test")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error (%s): %v", code, err)
|
||||
}
|
||||
want := filepath.Join(root, "Acme Show", "Season 01", "ep.mkv")
|
||||
if got != want {
|
||||
t.Errorf("got %q want %q", got, want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("directory without video is no_video_file", func(t *testing.T) {
|
||||
empty := filepath.Join(root, "Empty Show")
|
||||
if err := os.MkdirAll(empty, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, code, err := resolvePlayableFile(empty, roots, "test")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty directory")
|
||||
}
|
||||
if code != pathErrNoVideo {
|
||||
t.Errorf("code = %q, want %q", code, pathErrNoVideo)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
@ -192,6 +192,17 @@ func Execute() {
|
|||
}
|
||||
|
||||
// loadConfig loads config once (lazy initialization).
|
||||
// resolvedConfigPath returns the config file the CLI actually reads/writes,
|
||||
// honouring the global --config flag. Use this for every Save so a revocation
|
||||
// wipe or key migration lands in the right file (e.g. the dev-local agent's
|
||||
// ~/.config/unarr-dev/config.toml), not always the default path.
|
||||
func resolvedConfigPath() string {
|
||||
if cfgFile != "" {
|
||||
return cfgFile
|
||||
}
|
||||
return config.FilePath()
|
||||
}
|
||||
|
||||
func loadConfig() config.Config {
|
||||
if cfgLoaded {
|
||||
return appCfg
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ import (
|
|||
"sort"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/fatih/color"
|
||||
"github.com/spf13/cobra"
|
||||
|
|
@ -40,20 +39,40 @@ to see available quality upgrades.`,
|
|||
if showStatus {
|
||||
return runScanStatus()
|
||||
}
|
||||
cfg := loadConfig()
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
// All scanned roots feed ONE sync session (single syncStartedAt +
|
||||
// final isLastBatch) so the server's stale-row cleanup sees the
|
||||
// whole cycle at once. fullCycle only without an explicit path —
|
||||
// a subtree scan must never let the server reap outside it.
|
||||
if len(args) == 0 {
|
||||
cfg := loadConfig()
|
||||
paths := library.ResolveScanPaths(cfg.Download.Dir, cfg.Organize.MoviesDir, cfg.Organize.TVShowsDir, cfg.Library.ScanPath)
|
||||
if len(paths) == 0 {
|
||||
return fmt.Errorf("usage: unarr scan <path>\n\nNo scan paths configured. Provide a path or set up downloads.dir via 'unarr init'")
|
||||
}
|
||||
var items []agent.LibrarySyncItem
|
||||
for _, p := range paths {
|
||||
if err := runScan(p, workers, ffprobe, noSync); err != nil {
|
||||
cache, err := runScan(ctx, cfg, p, workers, ffprobe)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
items = append(items, library.BuildSyncItems(cache)...)
|
||||
}
|
||||
if noSync || jsonOut {
|
||||
return nil
|
||||
}
|
||||
return syncToServer(ctx, cfg, items, paths, true)
|
||||
}
|
||||
cache, err := runScan(ctx, cfg, args[0], workers, ffprobe)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if noSync || jsonOut {
|
||||
return nil
|
||||
}
|
||||
return runScan(args[0], workers, ffprobe, noSync)
|
||||
return syncToServer(ctx, cfg, library.BuildSyncItems(cache), []string{args[0]}, false)
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -65,18 +84,20 @@ to see available quality upgrades.`,
|
|||
return cmd
|
||||
}
|
||||
|
||||
func runScan(dirPath string, workers int, ffprobePath string, noSync bool) error {
|
||||
// runScan walks one root, saves the cache and prewarms sidecars. Syncing to
|
||||
// the server is the CALLER's job (RunE) — all roots of an invocation feed one
|
||||
// sync session via syncToServer, so per-root sessions can't trick the server
|
||||
// into reaping rows of roots the session never visited.
|
||||
func runScan(ctx context.Context, cfg config.Config, dirPath string, workers int, ffprobePath string) (*library.LibraryCache, error) {
|
||||
// Validate path
|
||||
info, err := os.Stat(dirPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("path not found: %s", dirPath)
|
||||
return nil, fmt.Errorf("path not found: %s", dirPath)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return fmt.Errorf("not a directory: %s", dirPath)
|
||||
return nil, fmt.Errorf("not a directory: %s", dirPath)
|
||||
}
|
||||
|
||||
cfg := loadConfig()
|
||||
|
||||
// Resolve workers: flag → config → default 8
|
||||
if workers == 0 {
|
||||
workers = cfg.Library.Workers
|
||||
|
|
@ -93,10 +114,6 @@ func runScan(dirPath string, workers int, ffprobePath string, noSync bool) error
|
|||
// Load existing cache for incremental scanning
|
||||
existing, _ := library.LoadCache()
|
||||
|
||||
// Context with signal handling
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
bold := color.New(color.Bold)
|
||||
bold.Printf("\n Scanning %s...\n\n", dirPath)
|
||||
|
||||
|
|
@ -114,14 +131,14 @@ func runScan(dirPath string, workers int, ffprobePath string, noSync bool) error
|
|||
},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("scan failed: %w", err)
|
||||
return nil, fmt.Errorf("scan failed: %w", err)
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "\r\033[K") // clear progress line
|
||||
|
||||
// Save cache
|
||||
if err := library.SaveCache(cache); err != nil {
|
||||
return fmt.Errorf("save cache: %w", err)
|
||||
return nil, fmt.Errorf("save cache: %w", err)
|
||||
}
|
||||
|
||||
// Remember scan path in config
|
||||
|
|
@ -133,11 +150,12 @@ func runScan(dirPath string, workers int, ffprobePath string, noSync bool) error
|
|||
// Print summary
|
||||
printScanSummary(cache)
|
||||
|
||||
// JSON output mode
|
||||
// JSON output mode — emit the cache and skip the prewarm (the caller skips
|
||||
// the sync via the same flag).
|
||||
if jsonOut {
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
return enc.Encode(cache)
|
||||
return cache, enc.Encode(cache)
|
||||
}
|
||||
|
||||
// Pre-extract sidecars (text subs → WebVTT, panel frames → JPEG) into a hidden
|
||||
|
|
@ -162,15 +180,14 @@ func runScan(dirPath string, workers int, ffprobePath string, noSync bool) error
|
|||
}
|
||||
}
|
||||
|
||||
// Sync to server
|
||||
if !noSync {
|
||||
return syncToServer(ctx, cfg, cache)
|
||||
}
|
||||
|
||||
return nil
|
||||
return cache, nil
|
||||
}
|
||||
|
||||
func syncToServer(ctx context.Context, cfg config.Config, cache *library.LibraryCache) error {
|
||||
// syncToServer uploads the scanned items of THIS invocation as one sync
|
||||
// session. roots lists every root the invocation scanned; fullCycle marks a
|
||||
// no-args run that covered all configured roots (the server may then reap
|
||||
// stale rows regardless of prefix — see LibrarySyncRequest.FullCycle).
|
||||
func syncToServer(ctx context.Context, cfg config.Config, items []agent.LibrarySyncItem, roots []string, fullCycle bool) error {
|
||||
apiKey := apiKeyFlag
|
||||
if apiKey == "" {
|
||||
apiKey = cfg.Auth.APIKey
|
||||
|
|
@ -182,50 +199,28 @@ func syncToServer(ctx context.Context, cfg config.Config, cache *library.Library
|
|||
|
||||
ac := agent.NewClient(cfg.Auth.APIURL, apiKey, "unarr/"+Version)
|
||||
|
||||
items := library.BuildSyncItems(cache)
|
||||
|
||||
if len(items) == 0 {
|
||||
color.Yellow("\n No valid items to sync.")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Send in batches of 100
|
||||
const batchSize = 100
|
||||
totalSynced := 0
|
||||
totalMatched := 0
|
||||
totalRemoved := 0
|
||||
syncStartedAt := time.Now().UTC().Format(time.RFC3339)
|
||||
|
||||
for i := 0; i < len(items); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(items) {
|
||||
end = len(items)
|
||||
}
|
||||
batch := items[i:end]
|
||||
isLast := end >= len(items)
|
||||
|
||||
fmt.Fprintf(os.Stderr, "\r Syncing %d/%d items...\033[K", end, len(items))
|
||||
|
||||
resp, err := ac.SyncLibrary(ctx, agent.LibrarySyncRequest{
|
||||
Items: batch,
|
||||
ScanPath: cache.Path,
|
||||
AgentID: cfg.Agent.ID,
|
||||
IsLastBatch: isLast,
|
||||
SyncStartedAt: syncStartedAt,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("sync failed: %w", err)
|
||||
}
|
||||
|
||||
totalSynced += resp.Synced
|
||||
totalMatched += resp.Matched
|
||||
totalRemoved += resp.Removed
|
||||
res, err := library.SyncBatches(ctx, ac, items, library.SyncOptions{
|
||||
AgentID: cfg.Agent.ID,
|
||||
ScanPath: roots[0],
|
||||
ScanRoots: roots,
|
||||
FullCycle: fullCycle,
|
||||
OnProgress: func(sent, total int) {
|
||||
fmt.Fprintf(os.Stderr, "\r Syncing %d/%d items...\033[K", sent, total)
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("sync failed: %w", err)
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "\r\033[K")
|
||||
|
||||
green := color.New(color.FgGreen)
|
||||
green.Printf("\n ✓ Synced %d items (%d matched, %d removed)\n", totalSynced, totalMatched, totalRemoved)
|
||||
green.Printf("\n ✓ Synced %d items (%d matched, %d removed)\n", res.Synced, res.Matched, res.Removed)
|
||||
|
||||
apiURL := strings.TrimSuffix(cfg.Auth.APIURL, "/")
|
||||
fmt.Printf(" → View upgrades at %s/library\n\n", apiURL)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
package cmd
|
||||
|
||||
// Version is the CLI version. Overridden by goreleaser ldflags at release time.
|
||||
var Version = "1.0.4-beta"
|
||||
var Version = "1.0.9-beta"
|
||||
|
|
|
|||
|
|
@ -216,6 +216,21 @@ type LibraryConfig struct {
|
|||
// generation never saturates the machine or the NAS. Default 0.7; 0 falls back
|
||||
// to the default. Linux-only (no load reading elsewhere → unthrottled).
|
||||
PrewarmMaxLoadRatio float64 `toml:"prewarm_max_load_ratio"`
|
||||
|
||||
// On-demand / automatic subtitle fetching from the web (Wyzie aggregator,
|
||||
// PRO). The web can always push a hot request (library/player button); this
|
||||
// section only controls SCAN-TIME auto-fetch, which is OFF by default.
|
||||
Subtitles SubtitlesConfig `toml:"subtitles"`
|
||||
}
|
||||
|
||||
// SubtitlesConfig controls scan-time subtitle auto-fetch.
|
||||
type SubtitlesConfig struct {
|
||||
// AutoFetch: during a library scan, fetch missing subtitles for the preferred
|
||||
// languages and write them as sidecars. Default false (opt-in).
|
||||
AutoFetch bool `toml:"auto_fetch"`
|
||||
// Languages: preferred subtitle languages (ISO 639-1) to ensure exist, in
|
||||
// priority order, e.g. ["es", "en"]. Empty → auto-fetch does nothing.
|
||||
Languages []string `toml:"languages"`
|
||||
}
|
||||
|
||||
// TrickplayConfig controls scan-time trickplay sprite generation.
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ import (
|
|||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
|
@ -65,6 +66,17 @@ func segmentStartSec(idx int) float64 {
|
|||
return float64(idx * hlsSegmentDuration)
|
||||
}
|
||||
|
||||
// segmentIdxForTime returns the index of the segment containing second `sec`
|
||||
// of the timeline — the inverse of segmentStartSec. Used to translate a
|
||||
// session's StartSec (resume position) into the segment the FIRST ffmpeg
|
||||
// should start writing from.
|
||||
func segmentIdxForTime(sec float64) int {
|
||||
if sec <= 0 {
|
||||
return 0
|
||||
}
|
||||
return int(sec / float64(hlsSegmentDuration))
|
||||
}
|
||||
|
||||
// segmentCountForDuration returns how many segments cover a source of the
|
||||
// given duration. Always returns at least 1.
|
||||
func segmentCountForDuration(dur float64) int {
|
||||
|
|
@ -159,7 +171,21 @@ type HLSSessionConfig struct {
|
|||
// with the clean one. Forces the video re-encode the HLS path already does
|
||||
// to also composite the subtitle overlay.
|
||||
BurnSubtitleIndex *int
|
||||
Transcode TranscodeRuntime
|
||||
// StartSec is the playback position (seconds) the viewer will start at —
|
||||
// the saved resume point, or the current position on a quality/audio
|
||||
// switch. When > 0 the FIRST ffmpeg spawns already seeked there
|
||||
// (`-ss` + `-output_ts_offset` + `-start_number`, the same flags as a
|
||||
// seek-restart), instead of encoding from segment 0 only to be
|
||||
// killed by an immediate seek-restart when the player asks for the resume
|
||||
// segment (double spawn, slow resume). 0 = start at the beginning.
|
||||
// Ignored on a cache HIT (every segment is already on disk).
|
||||
StartSec float64
|
||||
// Prewarm marks a background cache-fill session. The daemon defers its
|
||||
// encode until no live encode runs and registers it via RegisterKeep
|
||||
// (never evicting the viewer). It also lets a REAL session close stale
|
||||
// prewarms up front so the cache writer-lock is free for the viewer.
|
||||
Prewarm bool
|
||||
Transcode TranscodeRuntime
|
||||
// Cache is an optional persistent segment cache keyed by (source, quality,
|
||||
// audio). When set, completed encodes are kept across sessions so re-plays
|
||||
// of the same file at the same quality skip ffmpeg entirely. nil disables
|
||||
|
|
@ -254,6 +280,21 @@ type HLSSession struct {
|
|||
cacheKey string
|
||||
fromCache bool
|
||||
writerLockHeld bool
|
||||
|
||||
// Live transcode telemetry (F3). ffmpeg's -stats progress line is parsed
|
||||
// in hlsStderrCapture.Write into an EWMA of speed= (×realtime) + fps=, plus
|
||||
// an input-bound hint set when the SOURCE read errors (slow/broken pull vs a
|
||||
// too-slow encode). GetTranscodeStats() snapshots this so the ready-watcher
|
||||
// can report a real measurement to the web side — letting the player name a
|
||||
// too-slow transcode honestly in ~4s instead of inferring it from stall
|
||||
// shape over 15-30s. Guarded by statsMu (the stderr goroutine writes; the
|
||||
// watcher goroutine reads).
|
||||
statsMu sync.Mutex
|
||||
speedEWMA float64
|
||||
fpsEWMA float64
|
||||
speedSamples int
|
||||
warmupSeen int // cold-start frames discarded before the EWMA is trusted
|
||||
inputBound bool
|
||||
}
|
||||
|
||||
// hlsSeekAhead is how many segments past the writer's current position the
|
||||
|
|
@ -305,6 +346,63 @@ func (r *HLSSessionRegistry) Register(s *HLSSession) {
|
|||
}
|
||||
}
|
||||
|
||||
// CloseWhere closes + removes every registered session matching pred. Used
|
||||
// by the REAL-session path to reap stale prewarm encodes BEFORE its own
|
||||
// StartHLSSession runs — that frees the per-key cache writer-lock, so the
|
||||
// viewer's encode lands in the persistent cache instead of falling back to
|
||||
// an uncached per-session tmpdir (and a SEALED prewarm survives as a cache
|
||||
// HIT: closing a from-cache reader never invalidates the entry).
|
||||
func (r *HLSSessionRegistry) CloseWhere(pred func(*HLSSession) bool) int {
|
||||
r.mu.Lock()
|
||||
victims := make([]*HLSSession, 0, len(r.sessions))
|
||||
for id, s := range r.sessions {
|
||||
if pred(s) {
|
||||
victims = append(victims, s)
|
||||
delete(r.sessions, id)
|
||||
}
|
||||
}
|
||||
r.mu.Unlock()
|
||||
for _, s := range victims {
|
||||
_ = s.Close()
|
||||
}
|
||||
return len(victims)
|
||||
}
|
||||
|
||||
// IsPrewarm reports whether this session was started as a background
|
||||
// cache-fill (HLSSessionConfig.Prewarm). cfg is immutable after construction.
|
||||
func (s *HLSSession) IsPrewarm() bool { return s.cfg.Prewarm }
|
||||
|
||||
// RegisterKeep adds a session WITHOUT displacing the others — the prewarm
|
||||
// path: a background cache-fill encode must not evict the viewer's live
|
||||
// session (Register's eviction killed the stream being watched when the
|
||||
// next-episode prewarm got claimed mid-playback). It still replaces (and
|
||||
// closes) a previous session with the SAME ID. A later Register() of a real
|
||||
// viewer session evicts prewarms like any other session — a completed
|
||||
// (sealed) prewarm survives in the segment cache either way.
|
||||
func (r *HLSSessionRegistry) RegisterKeep(s *HLSSession) {
|
||||
r.mu.Lock()
|
||||
prev := r.sessions[s.cfg.SessionID]
|
||||
r.sessions[s.cfg.SessionID] = s
|
||||
r.mu.Unlock()
|
||||
if prev != nil && prev != s {
|
||||
_ = prev.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// HasLiveEncode reports whether any registered session still has a RUNNING
|
||||
// ffmpeg (encode not finished). Used to defer prewarm encodes so they never
|
||||
// compete with the viewer's live transcode for the encoder.
|
||||
func (r *HLSSessionRegistry) HasLiveEncode() bool {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
for _, s := range r.sessions {
|
||||
if !s.EncodeExited() {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Remove drops a session from the registry without closing it.
|
||||
func (r *HLSSessionRegistry) Remove(id string) {
|
||||
r.mu.Lock()
|
||||
|
|
@ -487,11 +585,38 @@ func StartHLSSession(ctx context.Context, cfg HLSSessionConfig) (*HLSSession, er
|
|||
return s, nil
|
||||
}
|
||||
|
||||
// Resume-aware first spawn: when the session carries a StartSec (resume
|
||||
// point / position on a quality switch), launch ffmpeg already seeked at
|
||||
// the segment containing it. The web player opens playback at the same
|
||||
// position (hls.js startPosition), so segment 0 would never be requested —
|
||||
// encoding from 0 just to seek-restart milliseconds later wasted a full
|
||||
// ffmpeg spawn and doubled the resume latency. Earlier segments simply
|
||||
// don't exist on disk; ServeSegment's `idx < segStart` branch restarts the
|
||||
// encoder if the user later scrubs back before the resume point. A partial
|
||||
// encode never seals the cache (allSegmentsPresent checks 0..N), matching
|
||||
// today's post-seek behaviour.
|
||||
startIdx := 0
|
||||
if cfg.StartSec > 0 && cfg.StartSec < probe.DurationSec {
|
||||
startIdx = segmentIdxForTime(cfg.StartSec)
|
||||
if startIdx > segCount-1 {
|
||||
startIdx = segCount - 1
|
||||
}
|
||||
} else if cfg.StartSec >= probe.DurationSec && cfg.StartSec > 0 {
|
||||
// Stale resume beyond this source's duration (the file was replaced by
|
||||
// a shorter cut, or progress was saved against another release). Start
|
||||
// from the beginning instead of encoding only the final segment, which
|
||||
// would "end" the video seconds after it starts.
|
||||
log.Printf("[hls %s] startSec %.0f ≥ duration %.0f — starting from 0",
|
||||
shortHLSID(cfg.SessionID), cfg.StartSec, probe.DurationSec)
|
||||
}
|
||||
s.ffmpegSegStart = startIdx
|
||||
s.readyMax = startIdx
|
||||
|
||||
// Spawn ffmpeg under a dedicated context so Close() can kill it without
|
||||
// touching the parent ctx.
|
||||
ffCtx, cancel := context.WithCancel(context.Background())
|
||||
s.cancel = cancel
|
||||
args := buildHLSFFmpegArgs(cfg, probe, tmpDir)
|
||||
args := buildHLSFFmpegArgsAt(cfg, probe, tmpDir, startIdx, segmentStartSec(startIdx))
|
||||
cmd := exec.CommandContext(ffCtx, cfg.Transcode.FFmpegPath, args...)
|
||||
cmd.Stderr = &hlsStderrCapture{owner: s}
|
||||
if err := cmd.Start(); err != nil {
|
||||
|
|
@ -524,10 +649,14 @@ func StartHLSSession(ctx context.Context, cfg HLSSessionConfig) (*HLSSession, er
|
|||
if profile.Preset != "" {
|
||||
presetNote = " preset=" + profile.Preset
|
||||
}
|
||||
log.Printf("[hls %s] started: %s, %.1fs, %d segs (quality=%s, encoder=%s accel=%s%s)%s",
|
||||
startNote := ""
|
||||
if startIdx > 0 {
|
||||
startNote = fmt.Sprintf(" start=seg-%d@%.0fs", startIdx, segmentStartSec(startIdx))
|
||||
}
|
||||
log.Printf("[hls %s] started: %s, %.1fs, %d segs (quality=%s, encoder=%s accel=%s%s)%s%s",
|
||||
shortHLSID(cfg.SessionID), cfg.logName(),
|
||||
probe.DurationSec, segCount, coalesce(cfg.Quality, "auto"),
|
||||
profile.Codec, string(cfg.Transcode.HWAccel), presetNote, cachedNote)
|
||||
profile.Codec, string(cfg.Transcode.HWAccel), presetNote, cachedNote, startNote)
|
||||
return s, nil
|
||||
}
|
||||
|
||||
|
|
@ -558,13 +687,18 @@ func (s *HLSSession) ProbeInfo() map[string]any {
|
|||
}
|
||||
subs := make([]map[string]any, 0, len(s.probe.SubtitleTracks))
|
||||
for _, sb := range s.probe.SubtitleTracks {
|
||||
// `external`/`path` let the stream server attach a tokened /sub vttUrl
|
||||
// (path-addressed for sidecars, index-addressed for embedded). `path` is
|
||||
// stripped after the URL is built so the raw path isn't doubled in JSON.
|
||||
subs = append(subs, map[string]any{
|
||||
"index": sb.Index,
|
||||
"lang": sb.Lang,
|
||||
"codec": sb.Codec,
|
||||
"title": sb.Title,
|
||||
"forced": sb.Forced,
|
||||
"text": sb.IsTextSubtitle(),
|
||||
"index": sb.Index,
|
||||
"lang": sb.Lang,
|
||||
"codec": sb.Codec,
|
||||
"title": sb.Title,
|
||||
"forced": sb.Forced,
|
||||
"text": sb.IsTextSubtitle(),
|
||||
"external": sb.External,
|
||||
"path": sb.Path,
|
||||
})
|
||||
}
|
||||
return map[string]any{
|
||||
|
|
@ -580,21 +714,106 @@ func (s *HLSSession) ProbeInfo() map[string]any {
|
|||
}
|
||||
}
|
||||
|
||||
// ReadyCount returns how many segments are currently fully on disk.
|
||||
// Caller can `>= 1` it to check whether seg-0 has landed (and so the
|
||||
// player can be told to attach). For cache-HIT sessions this is always
|
||||
// `segmentCount` from the moment StartHLSSession returns.
|
||||
// ReadyCount returns the session's readyMax watermark: segment idx is on disk
|
||||
// iff idx < ReadyCount() AND idx >= WriterStartIdx(). For a from-zero encode
|
||||
// this is simply "how many segments are on disk"; for a resume session
|
||||
// (StartSec > 0) readyMax is pre-seeded to the start index, so the FIRST real
|
||||
// segment has landed only once ReadyCount() > WriterStartIdx() — use that
|
||||
// comparison, not `>= 1`, to flip the player's "Preparando…" UI. For
|
||||
// cache-HIT sessions this is always `segmentCount` from the moment
|
||||
// StartHLSSession returns.
|
||||
func (s *HLSSession) ReadyCount() int {
|
||||
s.readyMu.Lock()
|
||||
defer s.readyMu.Unlock()
|
||||
return s.readyMax
|
||||
}
|
||||
|
||||
// EncodeExited reports whether this session's ffmpeg has finished (clean or
|
||||
// crashed) or never ran (cache HIT). False while an encode is producing
|
||||
// segments. Used by HasLiveEncode to defer prewarm work.
|
||||
func (s *HLSSession) EncodeExited() bool {
|
||||
s.readyMu.Lock()
|
||||
defer s.readyMu.Unlock()
|
||||
return s.exited
|
||||
}
|
||||
|
||||
// WriterStartIdx returns the segment index the CURRENT ffmpeg writer started
|
||||
// at: 0 for a from-the-beginning encode, the resume segment for a StartSec
|
||||
// session, the seek target after a seek-restart. See ReadyCount for the
|
||||
// "first segment landed" comparison.
|
||||
func (s *HLSSession) WriterStartIdx() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.ffmpegSegStart
|
||||
}
|
||||
|
||||
// FromCache reports whether this session was served from the HLS cache
|
||||
// (no ffmpeg subprocess spawned). Used by ready-watcher logic to short-
|
||||
// circuit polling — a cache HIT is ready the moment we return.
|
||||
func (s *HLSSession) FromCache() bool { return s.fromCache }
|
||||
|
||||
// TranscodeStats is a point-in-time snapshot of live ffmpeg progress for one
|
||||
// HLS session (F3). SpeedX < 1.0 means the encode runs slower than realtime —
|
||||
// the player can't sustain playback without buffering. Samples==0 means no
|
||||
// -stats line has been parsed yet (the watcher keeps waiting before reporting).
|
||||
type TranscodeStats struct {
|
||||
SpeedX float64 // EWMA of ffmpeg speed= (×realtime; 1.0 = exactly realtime)
|
||||
Fps float64 // EWMA of ffmpeg fps=
|
||||
Samples int // progress lines parsed so far (0 = no telemetry yet)
|
||||
InputBound bool // source read hit I/O errors (slow/broken pull, not encode)
|
||||
FromCache bool // replayed from cache → no live encode, stats meaningless
|
||||
}
|
||||
|
||||
// GetTranscodeStats returns a snapshot of the parsed ffmpeg progress EWMAs.
|
||||
func (s *HLSSession) GetTranscodeStats() TranscodeStats {
|
||||
s.statsMu.Lock()
|
||||
defer s.statsMu.Unlock()
|
||||
return TranscodeStats{
|
||||
SpeedX: s.speedEWMA,
|
||||
Fps: s.fpsEWMA,
|
||||
Samples: s.speedSamples,
|
||||
InputBound: s.inputBound,
|
||||
FromCache: s.fromCache,
|
||||
}
|
||||
}
|
||||
|
||||
// hlsStatsWarmupSkip is how many leading -stats frames to discard before
|
||||
// trusting the EWMA. ffmpeg's first readings reflect the pipeline filling
|
||||
// (often speed=0.0x) and would otherwise drag a healthy encoder into a false
|
||||
// "struggling" verdict that pauses a stream which plays fine once warmed up.
|
||||
const hlsStatsWarmupSkip = 2
|
||||
|
||||
// recordProgress folds one parsed ffmpeg -stats sample into the session EWMAs.
|
||||
// alpha=0.3 smooths the noisy per-line numbers while still tracking a sustained
|
||||
// slowdown within a few samples (~2s of encoding).
|
||||
func (s *HLSSession) recordProgress(speedX, fps float64) {
|
||||
s.statsMu.Lock()
|
||||
defer s.statsMu.Unlock()
|
||||
// Drop the cold-start frames so a steady-state slowdown — not the encoder
|
||||
// spin-up — is what the watcher reports.
|
||||
if s.warmupSeen < hlsStatsWarmupSkip {
|
||||
s.warmupSeen++
|
||||
return
|
||||
}
|
||||
const alpha = 0.3
|
||||
if s.speedSamples == 0 {
|
||||
s.speedEWMA = speedX
|
||||
s.fpsEWMA = fps
|
||||
} else {
|
||||
s.speedEWMA = alpha*speedX + (1-alpha)*s.speedEWMA
|
||||
s.fpsEWMA = alpha*fps + (1-alpha)*s.fpsEWMA
|
||||
}
|
||||
s.speedSamples++
|
||||
}
|
||||
|
||||
// markInputBound flags that ffmpeg reported a source-read error — the wall is
|
||||
// the input pull (slow debrid link / dropped torrent peer), not the encoder.
|
||||
func (s *HLSSession) markInputBound() {
|
||||
s.statsMu.Lock()
|
||||
s.inputBound = true
|
||||
s.statsMu.Unlock()
|
||||
}
|
||||
|
||||
// IsClosed reports whether Close() has been invoked. Exposed (vs the
|
||||
// internal isClosed) so external watchers — the ready-webhook
|
||||
// goroutine in cmd/daemon.go — can short-circuit polling on a session
|
||||
|
|
@ -1076,11 +1295,6 @@ func (s *HLSSession) restartFromSegment(targetIdx int) error {
|
|||
|
||||
// ---- ffmpeg argument builders ----
|
||||
|
||||
// buildHLSFFmpegArgs returns the argv for the initial HLS encode (start at 0).
|
||||
func buildHLSFFmpegArgs(cfg HLSSessionConfig, probe *StreamProbe, tmpDir string) []string {
|
||||
return buildHLSFFmpegArgsAt(cfg, probe, tmpDir, 0, 0)
|
||||
}
|
||||
|
||||
// EncoderProfile names the codec + preset + decoder hint combination the HLS
|
||||
// pipeline picks for the given hardware backend + transcode config. Exposed
|
||||
// so callers can log the chosen encoder before ffmpeg launches and so both
|
||||
|
|
@ -1140,7 +1354,10 @@ func ResolveEncoderProfile(hw HWAccel, configuredPreset string) EncoderProfile {
|
|||
// `-output_ts_offset` keeps the segment PTS aligned with manifest timeline.
|
||||
func buildHLSFFmpegArgsAt(cfg HLSSessionConfig, probe *StreamProbe, tmpDir string, startIdx int, startSec float64) []string {
|
||||
profile := ResolveEncoderProfile(cfg.Transcode.HWAccel, cfg.Transcode.Preset)
|
||||
args := []string{"-y", "-hide_banner", "-loglevel", "warning"}
|
||||
// -stats forces ffmpeg to emit the frame=/fps=/speed= progress line to
|
||||
// stderr even at -loglevel warning; hlsStderrCapture parses it for live
|
||||
// transcode telemetry (F3) without logging it.
|
||||
args := []string{"-y", "-hide_banner", "-loglevel", "warning", "-stats"}
|
||||
|
||||
// Demuxer-side HW-decode hint. Sourced from the profile so a future
|
||||
// codec/hint mismatch is impossible — the encoder + decode hint are
|
||||
|
|
@ -1266,12 +1483,21 @@ func buildHLSFFmpegArgsAt(cfg HLSSessionConfig, probe *StreamProbe, tmpDir strin
|
|||
// scene-cut). No B-frame reorder → monotonic DTS → uniform segments, no
|
||||
// "Packet duration is out of range" flood. Safe with -force_key_frames
|
||||
// (unlike -tune ll, which broke per-segment cuts — see note above).
|
||||
args = append(args, "-preset", profile.Preset, "-rc", "vbr", "-bf", "0", "-no-scenecut", "1")
|
||||
// -forced-idr 1 is LOAD-BEARING: NVENC emits -force_key_frames frames
|
||||
// as plain (non-IDR) I-frames on current ffmpeg/driver combos, the HLS
|
||||
// muxer only cuts on IDR, and every segment silently stretches to the
|
||||
// default GOP (250 frames ≈ 10.4 s @24fps) while the server-rendered
|
||||
// playlist still promises hlsSegmentDuration. The PTS↔playlist mismatch
|
||||
// breaks seeks and desyncs subtitles (measured 2026-06-10: 3 segments
|
||||
// per 30 s instead of 15; with -forced-idr exactly 15).
|
||||
args = append(args, "-preset", profile.Preset, "-rc", "vbr", "-bf", "0", "-no-scenecut", "1", "-forced-idr", "1")
|
||||
case "h264_qsv":
|
||||
// veryfast is the fastest realistic QSV preset; medium was too
|
||||
// conservative for first-start. look_ahead=0 keeps the encoder
|
||||
// truly low-latency (no rate-control look-ahead window).
|
||||
args = append(args, "-preset", profile.Preset, "-look_ahead", "0")
|
||||
// -forced_idr: same non-IDR forced-keyframe failure mode as NVENC (see
|
||||
// above) — QSV's AVOption spells it with an underscore.
|
||||
args = append(args, "-preset", profile.Preset, "-look_ahead", "0", "-forced_idr", "1")
|
||||
case "h264_videotoolbox":
|
||||
// VideoToolbox has no "preset" knob; `-realtime` flips into the
|
||||
// low-latency path used by FaceTime. We let the `-b:v / -maxrate
|
||||
|
|
@ -1332,7 +1558,31 @@ func buildHLSFFmpegArgsAt(cfg HLSSessionConfig, probe *StreamProbe, tmpDir strin
|
|||
if bitrate == "" {
|
||||
bitrate = "5M"
|
||||
}
|
||||
args = append(args, "-b:v", bitrate, "-maxrate", bitrate, "-bufsize", bitrate)
|
||||
// Rate control: capped constant-quality where the encoder supports it well
|
||||
// (libx264 CRF, NVENC CQ), plain CBR-ish elsewhere. Constant quality is the
|
||||
// on-the-fly analogue of per-title encoding: easy scenes (dialogue, anime
|
||||
// flats) emit FAR fewer bits than the fixed target — which is what keeps a
|
||||
// funnel/LTE link from stalling — while complex scenes can still use up to
|
||||
// `-maxrate` (the same ceiling as before, so worst-case quality and the
|
||||
// level-derived VBV pair are unchanged). `-bufsize 2×maxrate` gives the VBV
|
||||
// a standard one-segment window to absorb spikes; the old 1× window forced
|
||||
// the encoder to flatline at the cap. CPB stays far below every H.264
|
||||
// level's limit (level 3.1 allows 14 Mbps CPB vs our 3M at 480p).
|
||||
switch codec {
|
||||
case "libx264":
|
||||
// Capped CRF: no -b:v (CRF drives quality), -maxrate/-bufsize cap it.
|
||||
args = append(args, "-crf", "23", "-maxrate", bitrate, "-bufsize", doubleBitrate(bitrate))
|
||||
case "h264_nvenc":
|
||||
// NVENC constant-quality VBR: -cq targets quality, -b:v 0 disables the
|
||||
// default 2M average-bitrate target that would otherwise fight it.
|
||||
args = append(args, "-cq", "23", "-b:v", "0", "-maxrate", bitrate, "-bufsize", doubleBitrate(bitrate))
|
||||
default:
|
||||
// QSV / VideoToolbox / VAAPI: keep the proven fixed-bitrate triple —
|
||||
// their constant-quality knobs (ICQ, -q:v) have vendor-specific gotchas
|
||||
// (VideoToolbox ignores -q:v when -b:v is set; QSV ICQ conflicts with
|
||||
// look_ahead=0) and we can't regression-test them here.
|
||||
args = append(args, "-b:v", bitrate, "-maxrate", bitrate, "-bufsize", bitrate)
|
||||
}
|
||||
|
||||
// Force keyframe alignment with segment boundaries.
|
||||
args = append(args, "-force_key_frames", fmt.Sprintf("expr:gte(t,n_forced*%d)", hlsSegmentDuration))
|
||||
|
|
@ -1581,6 +1831,46 @@ type hlsStderrCapture struct {
|
|||
|
||||
const maxStderrBuf = 64 * 1024
|
||||
|
||||
// ffmpeg -stats progress lines look like:
|
||||
//
|
||||
// frame= 123 fps= 30 q=28.0 size= 456kB time=00:00:08.00 speed=1.05x
|
||||
//
|
||||
// emitted with a trailing \r (overwrite-in-place), once per ~0.5s. We parse
|
||||
// speed=/fps= out of them for live transcode telemetry (F3) and DON'T log them
|
||||
// (one per 0.5s would drown the daemon log) — only \n-terminated warning/error
|
||||
// lines reach log.Printf below.
|
||||
var (
|
||||
reFFmpegSpeed = regexp.MustCompile(`speed=\s*([0-9.]+)x`)
|
||||
reFFmpegFps = regexp.MustCompile(`fps=\s*([0-9.]+)`)
|
||||
)
|
||||
|
||||
func parseFFmpegProgress(line string) (speedX, fps float64, ok bool) {
|
||||
m := reFFmpegSpeed.FindStringSubmatch(line)
|
||||
if m == nil {
|
||||
return 0, 0, false
|
||||
}
|
||||
v, err := strconv.ParseFloat(m[1], 64)
|
||||
if err != nil {
|
||||
return 0, 0, false
|
||||
}
|
||||
if fm := reFFmpegFps.FindStringSubmatch(line); fm != nil {
|
||||
fps, _ = strconv.ParseFloat(fm[1], 64)
|
||||
}
|
||||
return v, fps, true
|
||||
}
|
||||
|
||||
// isInputBoundLine spots ffmpeg stderr that means the SOURCE read failed (slow
|
||||
// debrid link, dropped torrent peer, network timeout) rather than the encoder
|
||||
// being too slow — so the player names the bottleneck as the link, not the GPU.
|
||||
func isInputBoundLine(line string) bool {
|
||||
l := strings.ToLower(line)
|
||||
return strings.Contains(l, "i/o error") ||
|
||||
strings.Contains(l, "connection reset") ||
|
||||
strings.Contains(l, "rw_timeout") ||
|
||||
strings.Contains(l, "error in the pull function") ||
|
||||
strings.Contains(l, "connection timed out")
|
||||
}
|
||||
|
||||
func (c *hlsStderrCapture) Write(p []byte) (int, error) {
|
||||
// If the incoming chunk alone exceeds the cap (very long unterminated
|
||||
// line), drop the buffered prefix AND truncate p so a single multi-MB
|
||||
|
|
@ -1589,20 +1879,33 @@ func (c *hlsStderrCapture) Write(p []byte) (int, error) {
|
|||
c.buf.Reset()
|
||||
p = p[len(p)-maxStderrBuf:]
|
||||
} else if c.buf.Len()+len(p) > maxStderrBuf {
|
||||
// Drop the unterminated partial line; we'll resync on the next \n.
|
||||
// Drop the unterminated partial line; we'll resync on the next \r/\n.
|
||||
c.buf.Reset()
|
||||
}
|
||||
c.buf.Write(p)
|
||||
// Frame on \r OR \n: ffmpeg's progress line is \r-terminated, warnings are
|
||||
// \n-terminated. Parsing progress per-frame keeps the EWMA fresh; logging
|
||||
// only the \n lines keeps the log readable.
|
||||
for {
|
||||
line, rest, ok := strings.Cut(c.buf.String(), "\n")
|
||||
if !ok {
|
||||
s := c.buf.String()
|
||||
idx := strings.IndexAny(s, "\r\n")
|
||||
if idx < 0 {
|
||||
break
|
||||
}
|
||||
line := strings.TrimSpace(s[:idx])
|
||||
c.buf.Reset()
|
||||
c.buf.WriteString(rest)
|
||||
if line = strings.TrimSpace(line); line != "" {
|
||||
log.Printf("[hls %s] ffmpeg: %s", shortHLSID(c.owner.cfg.SessionID), line)
|
||||
c.buf.WriteString(s[idx+1:])
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
if speedX, fps, ok := parseFFmpegProgress(line); ok {
|
||||
c.owner.recordProgress(speedX, fps)
|
||||
continue // progress line — telemetry only, never logged
|
||||
}
|
||||
if isInputBoundLine(line) {
|
||||
c.owner.markInputBound()
|
||||
}
|
||||
log.Printf("[hls %s] ffmpeg: %s", shortHLSID(c.owner.cfg.SessionID), line)
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
|
|
|||
103
internal/engine/hls_progress_test.go
Normal file
103
internal/engine/hls_progress_test.go
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
package engine
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseFFmpegProgress(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
line string
|
||||
wantSpeed float64
|
||||
wantFps float64
|
||||
wantOK bool
|
||||
}{
|
||||
{"realtime", "frame= 123 fps= 30 q=28.0 size= 456kB time=00:00:08.00 bitrate=467.0kbits/s speed=1.05x", 1.05, 30, true},
|
||||
{"slow", "frame= 12 fps=2.4 q=-1.0 size= 40kB time=00:00:00.40 speed=0.18x", 0.18, 2.4, true},
|
||||
{"tight_spacing", "speed=2x", 2, 0, true},
|
||||
{"no_speed", "[libplacebo @ 0x55] Spent 2657ms on a slow shader", 0, 0, false},
|
||||
{"warning_line", "[hevc @ 0x7f] Could not find ref with POC 12", 0, 0, false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
sp, fps, ok := parseFFmpegProgress(c.line)
|
||||
if ok != c.wantOK {
|
||||
t.Fatalf("ok=%v want %v", ok, c.wantOK)
|
||||
}
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if math.Abs(sp-c.wantSpeed) > 1e-9 {
|
||||
t.Errorf("speed=%v want %v", sp, c.wantSpeed)
|
||||
}
|
||||
if math.Abs(fps-c.wantFps) > 1e-9 {
|
||||
t.Errorf("fps=%v want %v", fps, c.wantFps)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsInputBoundLine(t *testing.T) {
|
||||
bound := []string{
|
||||
"[http @ 0x55] HTTP error: Connection reset by peer",
|
||||
"rw_timeout reached, aborting",
|
||||
"Error in the pull function.",
|
||||
"tcp://: I/O error",
|
||||
}
|
||||
for _, l := range bound {
|
||||
if !isInputBoundLine(l) {
|
||||
t.Errorf("expected input-bound: %q", l)
|
||||
}
|
||||
}
|
||||
notBound := []string{
|
||||
"frame= 1 fps=30 speed=1.0x",
|
||||
"[libplacebo] slow shader",
|
||||
}
|
||||
for _, l := range notBound {
|
||||
if isInputBoundLine(l) {
|
||||
t.Errorf("expected NOT input-bound: %q", l)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// hlsStderrCapture must frame on \r (progress) as well as \n (warnings),
|
||||
// fold progress into the EWMA, and surface a sustained slow encode as < 1.0x.
|
||||
func TestHlsStderrCaptureProgressEWMA(t *testing.T) {
|
||||
s := &HLSSession{}
|
||||
s.cfg.SessionID = "test-session-00000000"
|
||||
c := &hlsStderrCapture{owner: s}
|
||||
|
||||
// Cold-start frames ffmpeg emits while the pipeline fills — must be skipped
|
||||
// (hlsStatsWarmupSkip) so they don't drag the EWMA into a false struggle.
|
||||
warmup := "frame=0 fps=0 speed=0.01x\r" +
|
||||
"frame=0 fps=0 speed=0.04x\r"
|
||||
// A burst of \r-terminated steady-state progress lines, like real ffmpeg.
|
||||
chunk := "frame=1 fps=2 speed=0.20x\r" +
|
||||
"frame=2 fps=2 speed=0.21x\r" +
|
||||
"frame=3 fps=2 speed=0.19x\r" +
|
||||
"frame=4 fps=2 speed=0.20x\r" +
|
||||
"frame=5 fps=2 speed=0.20x\r"
|
||||
if _, err := c.Write([]byte(warmup + chunk)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := s.GetTranscodeStats()
|
||||
// 7 progress lines written, first hlsStatsWarmupSkip(2) discarded → 5 counted.
|
||||
if st.Samples != 5 {
|
||||
t.Fatalf("samples=%d want 5 (7 lines - 2 warmup)", st.Samples)
|
||||
}
|
||||
if st.SpeedX > 0.5 || st.SpeedX < 0.1 {
|
||||
t.Errorf("speedX EWMA=%v, want ~0.2 (sustained slow encode)", st.SpeedX)
|
||||
}
|
||||
if st.InputBound {
|
||||
t.Error("not input-bound for a pure slow encode")
|
||||
}
|
||||
|
||||
// A \n-terminated I/O error line flips input-bound.
|
||||
if _, err := c.Write([]byte("tcp://: I/O error\n")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !s.GetTranscodeStats().InputBound {
|
||||
t.Error("expected input-bound after I/O error line")
|
||||
}
|
||||
}
|
||||
127
internal/engine/hls_ratecontrol_test.go
Normal file
127
internal/engine/hls_ratecontrol_test.go
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
package engine
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDoubleBitrate(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"6000k": "12000k",
|
||||
"25000k": "50000k",
|
||||
"1500k": "3000k",
|
||||
"5M": "10M",
|
||||
"1.5M": "3M",
|
||||
"2.5m": "5m",
|
||||
"800000": "1600000",
|
||||
"": "",
|
||||
"garbage": "garbage", // unparseable → unchanged (1× bufsize fallback)
|
||||
"-5M": "-5M", // non-positive → unchanged
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := doubleBitrate(in); got != want {
|
||||
t.Errorf("doubleBitrate(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// segmentIdxForTime must be the exact inverse of segmentStartSec so the
|
||||
// resume-aware first spawn (HLSSessionConfig.StartSec) lands on the same
|
||||
// segment the player's hls.js startPosition will request.
|
||||
func TestSegmentIdxForTime(t *testing.T) {
|
||||
cases := map[float64]int{
|
||||
0: 0,
|
||||
-3: 0,
|
||||
0.5: 0,
|
||||
1.99: 0,
|
||||
2: 1,
|
||||
3.9: 1,
|
||||
60: 30,
|
||||
3599.9: 1799,
|
||||
}
|
||||
for sec, want := range cases {
|
||||
if got := segmentIdxForTime(sec); got != want {
|
||||
t.Errorf("segmentIdxForTime(%v) = %d, want %d", sec, got, want)
|
||||
}
|
||||
}
|
||||
// Round-trip: the start time of the segment we resolve must never be
|
||||
// AFTER the requested position (the player would miss its first frames).
|
||||
for _, sec := range []float64{0, 1, 2, 7.3, 119.9, 4321} {
|
||||
idx := segmentIdxForTime(sec)
|
||||
if start := segmentStartSec(idx); start > sec {
|
||||
t.Errorf("segmentStartSec(segmentIdxForTime(%v)) = %v > %v", sec, start, sec)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Capped constant-quality rate control: libx264 gets -crf (no -b:v), NVENC
|
||||
// gets -cq with -b:v 0, both keep -maxrate at the level-coherent cap and a
|
||||
// 2× -bufsize. VAAPI (and the other vendor encoders) keep the proven
|
||||
// fixed-bitrate triple untouched.
|
||||
func TestBuildHLSFFmpegArgsRateControl(t *testing.T) {
|
||||
probe := &StreamProbe{Width: 1920, Height: 1080, DurationSec: 100}
|
||||
base := HLSSessionConfig{
|
||||
SessionID: "test",
|
||||
SourcePath: "/media/Movie.mkv",
|
||||
Quality: "1080p",
|
||||
Transcode: TranscodeRuntime{
|
||||
FFmpegPath: "/usr/bin/ffmpeg",
|
||||
FFprobePath: "/usr/bin/ffprobe",
|
||||
},
|
||||
}
|
||||
|
||||
t.Run("libx264 capped CRF", func(t *testing.T) {
|
||||
cfg := base
|
||||
cfg.Transcode.HWAccel = HWAccelNone
|
||||
got := strings.Join(buildHLSFFmpegArgsAt(cfg, probe, "/tmp/tmpdir", 0, 0), " ")
|
||||
for _, want := range []string{"-crf 23", "-maxrate 6000k", "-bufsize 12000k"} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("libx264 argv missing %q\n%s", want, got)
|
||||
}
|
||||
}
|
||||
if strings.Contains(got, "-b:v 6000k") {
|
||||
t.Errorf("libx264 argv must not carry -b:v alongside -crf\n%s", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("nvenc constant-quality VBR", func(t *testing.T) {
|
||||
cfg := base
|
||||
cfg.Transcode.HWAccel = HWAccelNVENC
|
||||
got := strings.Join(buildHLSFFmpegArgsAt(cfg, probe, "/tmp/tmpdir", 0, 0), " ")
|
||||
// -forced-idr 1 is load-bearing: without it NVENC emits the forced
|
||||
// keyframes as non-IDR and every HLS segment stretches to the full
|
||||
// GOP, desyncing the playlist timeline (subs/seeks).
|
||||
for _, want := range []string{"-rc vbr", "-cq 23", "-b:v 0", "-maxrate 6000k", "-bufsize 12000k", "-forced-idr 1"} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("nvenc argv missing %q\n%s", want, got)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("qsv keeps bitrate + forced_idr", func(t *testing.T) {
|
||||
cfg := base
|
||||
cfg.Transcode.HWAccel = HWAccelQSV
|
||||
got := strings.Join(buildHLSFFmpegArgsAt(cfg, probe, "/tmp/tmpdir", 0, 0), " ")
|
||||
// -forced_idr 1 (QSV's spelling): same non-IDR forced-keyframe failure
|
||||
// mode as NVENC — without it segments stretch to the full GOP.
|
||||
for _, want := range []string{"-look_ahead 0", "-forced_idr 1", "-b:v 6000k"} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("qsv argv missing %q\n%s", want, got)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("vaapi keeps fixed-bitrate triple", func(t *testing.T) {
|
||||
cfg := base
|
||||
cfg.Transcode.HWAccel = HWAccelVAAPI
|
||||
got := strings.Join(buildHLSFFmpegArgsAt(cfg, probe, "/tmp/tmpdir", 0, 0), " ")
|
||||
for _, want := range []string{"-b:v 6000k", "-maxrate 6000k", "-bufsize 6000k"} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("vaapi argv missing %q\n%s", want, got)
|
||||
}
|
||||
}
|
||||
if strings.Contains(got, "-crf") || strings.Contains(got, "-cq") {
|
||||
t.Errorf("vaapi argv must not carry constant-quality flags\n%s", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
80
internal/engine/hls_registry_prewarm_test.go
Normal file
80
internal/engine/hls_registry_prewarm_test.go
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
package engine
|
||||
|
||||
import "testing"
|
||||
|
||||
// bare session: no ffmpeg, no tmpdir — exercises pure registry semantics.
|
||||
func bareSession(id string, prewarm bool, exited bool) *HLSSession {
|
||||
s := &HLSSession{cfg: HLSSessionConfig{SessionID: id, Prewarm: prewarm}}
|
||||
s.exited = exited
|
||||
return s
|
||||
}
|
||||
|
||||
// A prewarm registered via RegisterKeep must NOT evict the viewer's live
|
||||
// session (the old Register-for-everything path killed the stream being
|
||||
// watched when the next-episode prewarm got claimed mid-playback).
|
||||
func TestRegisterKeepDoesNotEvict(t *testing.T) {
|
||||
r := NewHLSSessionRegistry()
|
||||
live := bareSession("live", false, false)
|
||||
r.Register(live)
|
||||
|
||||
pre := bareSession("pre", true, false)
|
||||
r.RegisterKeep(pre)
|
||||
|
||||
if r.Get("live") == nil {
|
||||
t.Fatal("RegisterKeep evicted the live session")
|
||||
}
|
||||
if r.Get("pre") == nil {
|
||||
t.Fatal("RegisterKeep did not register the prewarm")
|
||||
}
|
||||
if live.isClosed() {
|
||||
t.Fatal("RegisterKeep closed the live session")
|
||||
}
|
||||
|
||||
// A REAL session via Register still evicts everything (single viewer).
|
||||
real2 := bareSession("real2", false, false)
|
||||
r.Register(real2)
|
||||
if r.Get("live") != nil || r.Get("pre") != nil {
|
||||
t.Fatal("Register must evict every other session")
|
||||
}
|
||||
if !live.isClosed() || !pre.isClosed() {
|
||||
t.Fatal("Register must close the evicted sessions")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloseWherePrewarmsOnly(t *testing.T) {
|
||||
r := NewHLSSessionRegistry()
|
||||
live := bareSession("live", false, false)
|
||||
pre1 := bareSession("pre1", true, false)
|
||||
pre2 := bareSession("pre2", true, true)
|
||||
r.Register(live)
|
||||
r.RegisterKeep(pre1)
|
||||
r.RegisterKeep(pre2)
|
||||
|
||||
n := r.CloseWhere(func(s *HLSSession) bool { return s.IsPrewarm() })
|
||||
if n != 2 {
|
||||
t.Fatalf("CloseWhere closed %d sessions, want 2", n)
|
||||
}
|
||||
if r.Get("live") == nil || live.isClosed() {
|
||||
t.Fatal("CloseWhere must not touch the live session")
|
||||
}
|
||||
if r.Get("pre1") != nil || r.Get("pre2") != nil {
|
||||
t.Fatal("CloseWhere must remove the prewarms from the registry")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasLiveEncode(t *testing.T) {
|
||||
r := NewHLSSessionRegistry()
|
||||
if r.HasLiveEncode() {
|
||||
t.Fatal("empty registry must report no live encode")
|
||||
}
|
||||
done := bareSession("done", false, true) // encode finished / cache HIT
|
||||
r.Register(done)
|
||||
if r.HasLiveEncode() {
|
||||
t.Fatal("an exited encode must not count as live")
|
||||
}
|
||||
running := bareSession("running", true, false)
|
||||
r.RegisterKeep(running)
|
||||
if !r.HasLiveEncode() {
|
||||
t.Fatal("a running encode must count as live")
|
||||
}
|
||||
}
|
||||
|
|
@ -16,7 +16,7 @@ type fakePersister struct {
|
|||
tasks map[string]bool
|
||||
}
|
||||
|
||||
func newFakePersister() *fakePersister { return &fakePersister{tasks: map[string]bool{}} }
|
||||
func newFakePersister() *fakePersister { return &fakePersister{tasks: map[string]bool{}} }
|
||||
func (f *fakePersister) Add(t agent.Task) { f.mu.Lock(); f.tasks[t.ID] = true; f.mu.Unlock() }
|
||||
func (f *fakePersister) Remove(id string) { f.mu.Lock(); delete(f.tasks, id); f.mu.Unlock() }
|
||||
func (f *fakePersister) has(id string) bool { f.mu.Lock(); defer f.mu.Unlock(); return f.tasks[id] }
|
||||
|
|
|
|||
|
|
@ -50,11 +50,15 @@ type ProbeAudioTrack struct {
|
|||
// Codec discriminates text (srt/ass/webvtt → extract to WebVTT) vs bitmap
|
||||
// (pgs/dvbsub → require burn-in).
|
||||
type ProbeSubtitleTrack struct {
|
||||
Index int // 0-based subtitle stream index (ffmpeg -map 0:s:Index)
|
||||
Index int // 0-based EMBEDDED subtitle stream index (ffmpeg -map 0:s:Index). Unused when External.
|
||||
Lang string // ISO 639-1
|
||||
Codec string // lowercased — "subrip", "ass", "webvtt", "hdmv_pgs_subtitle", ...
|
||||
Title string
|
||||
Forced bool
|
||||
// External marks a sidecar file (served via /sub?p=<Path>&i=-1) rather than
|
||||
// an embedded stream. Path is its absolute filesystem path (External only).
|
||||
External bool
|
||||
Path string
|
||||
}
|
||||
|
||||
// IsTextSubtitle reports whether a subtitle codec can be extracted to WebVTT
|
||||
|
|
@ -134,14 +138,27 @@ func ProbeFile(ctx context.Context, ffprobePath, filePath string) (*StreamProbe,
|
|||
}
|
||||
if len(mi.Subtitles) > 0 {
|
||||
probe.SubtitleTracks = make([]ProbeSubtitleTrack, 0, len(mi.Subtitles))
|
||||
for i, s := range mi.Subtitles {
|
||||
probe.SubtitleTracks = append(probe.SubtitleTracks, ProbeSubtitleTrack{
|
||||
Index: i,
|
||||
Lang: s.Lang,
|
||||
Codec: strings.ToLower(s.Codec),
|
||||
Title: s.Title,
|
||||
Forced: s.Forced,
|
||||
})
|
||||
// Embedded streams come first (ffprobe order); external sidecars are
|
||||
// appended after. Count embedded separately so each embedded track's
|
||||
// Index is its true `0:s:N` value regardless of how many externals trail
|
||||
// it; externals get Index=-1 and address by Path instead.
|
||||
embeddedIdx := 0
|
||||
for _, s := range mi.Subtitles {
|
||||
t := ProbeSubtitleTrack{
|
||||
Lang: s.Lang,
|
||||
Codec: strings.ToLower(s.Codec),
|
||||
Title: s.Title,
|
||||
Forced: s.Forced,
|
||||
External: s.External,
|
||||
Path: s.Path,
|
||||
}
|
||||
if s.External {
|
||||
t.Index = -1
|
||||
} else {
|
||||
t.Index = embeddedIdx
|
||||
embeddedIdx++
|
||||
}
|
||||
probe.SubtitleTracks = append(probe.SubtitleTracks, t)
|
||||
}
|
||||
}
|
||||
storeProbeCache(filePath, probe)
|
||||
|
|
|
|||
|
|
@ -10,10 +10,10 @@ func TestDynamicReadahead(t *testing.T) {
|
|||
}{
|
||||
{"unknown bitrate → default", 0, defaultReadahead},
|
||||
{"negative → default", -1, defaultReadahead},
|
||||
{"low bitrate clamps to min", 1_000_000, minReadahead}, // 1 Mbps → ~3.75 MiB < 8 MiB
|
||||
{"mid bitrate scales", 5_000_000, 5_000_000 / 8 * readaheadSeconds}, // 5 Mbps → ~18.75 MiB
|
||||
{"low bitrate clamps to min", 1_000_000, minReadahead}, // 1 Mbps → ~3.75 MiB < 8 MiB
|
||||
{"mid bitrate scales", 5_000_000, 5_000_000 / 8 * readaheadSeconds}, // 5 Mbps → ~18.75 MiB
|
||||
{"high bitrate within range", 25_000_000, 25_000_000 / 8 * readaheadSeconds}, // 4K ~25 Mbps → ~93.75 MiB
|
||||
{"very high clamps to max", 80_000_000, maxReadahead}, // 80 Mbps → 300 MiB > cap
|
||||
{"very high clamps to max", 80_000_000, maxReadahead}, // 80 Mbps → 300 MiB > cap
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -135,9 +135,12 @@ func TestServeGrowing_BoundedRange(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestServeGrowing_EstimateUsedWhileNotFinal(t *testing.T) {
|
||||
// Not final: only 8 bytes produced, but estimate says 100. The advertised
|
||||
// total is the estimate (scrubber timeline); body is what exists so far.
|
||||
func TestServeGrowing_UnknownTotalWhileNotFinal(t *testing.T) {
|
||||
// Not final: only 8 bytes produced, estimate says 100. The instance length
|
||||
// is genuinely unknown while the remux grows, so we advertise "/*" (RFC 7233
|
||||
// §4.2) instead of a total the native player would map its timeline onto and
|
||||
// re-seek against (the playback loop). The estimate is only an upper-bound
|
||||
// hint for `end`; body is what exists so far.
|
||||
src := &fakeGrowing{data: []byte("01234567"), final: false, est: 100}
|
||||
ss := &StreamServer{}
|
||||
|
||||
|
|
@ -149,8 +152,8 @@ func TestServeGrowing_EstimateUsedWhileNotFinal(t *testing.T) {
|
|||
if res.StatusCode != http.StatusPartialContent {
|
||||
t.Fatalf("status = %d, want 206", res.StatusCode)
|
||||
}
|
||||
if got := res.Header.Get("Content-Range"); got != "bytes 0-99/100" {
|
||||
t.Errorf("Content-Range = %q, want bytes 0-99/100 (estimate)", got)
|
||||
if got := res.Header.Get("Content-Range"); got != "bytes 0-99/*" {
|
||||
t.Errorf("Content-Range = %q, want bytes 0-99/* (unknown total)", got)
|
||||
}
|
||||
// Not final → no exact Content-Length (chunked) so we never promise bytes
|
||||
// a still-running remux might not produce.
|
||||
|
|
@ -163,6 +166,8 @@ func TestServeGrowing_EstimateUsedWhileNotFinal(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestServeGrowing_HeadProbe(t *testing.T) {
|
||||
// HEAD while growing: total is unknown, so no Content-Length is promised
|
||||
// (advertising the estimate is the bug this fix removes).
|
||||
src := &fakeGrowing{data: make([]byte, 0), final: false, est: 4242}
|
||||
ss := &StreamServer{}
|
||||
|
||||
|
|
@ -174,14 +179,32 @@ func TestServeGrowing_HeadProbe(t *testing.T) {
|
|||
if res.StatusCode != http.StatusOK {
|
||||
t.Fatalf("HEAD status = %d, want 200", res.StatusCode)
|
||||
}
|
||||
if got := res.Header.Get("Content-Length"); got != "4242" {
|
||||
t.Errorf("HEAD Content-Length = %q, want 4242", got)
|
||||
if got := res.Header.Get("Content-Length"); got != "" {
|
||||
t.Errorf("HEAD Content-Length = %q, want empty (unknown total while growing)", got)
|
||||
}
|
||||
if rec.Body.Len() != 0 {
|
||||
t.Errorf("HEAD body = %d bytes, want 0", rec.Body.Len())
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeGrowing_HeadProbeFinal(t *testing.T) {
|
||||
// HEAD once final: the true total IS known, so advertise it.
|
||||
src := &fakeGrowing{data: make([]byte, 4242), final: true}
|
||||
ss := &StreamServer{}
|
||||
|
||||
req := httptest.NewRequest(http.MethodHead, "/stream", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
ss.serveGrowing(rec, req, src)
|
||||
|
||||
res := rec.Result()
|
||||
if res.StatusCode != http.StatusOK {
|
||||
t.Fatalf("HEAD status = %d, want 200", res.StatusCode)
|
||||
}
|
||||
if got := res.Header.Get("Content-Length"); got != "4242" {
|
||||
t.Errorf("HEAD Content-Length = %q, want 4242 (final size known)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeGrowing_RangeBeyondTotal(t *testing.T) {
|
||||
src := &fakeGrowing{data: []byte("0123456789"), final: true}
|
||||
ss := &StreamServer{}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import (
|
|||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
|
|
@ -743,7 +744,9 @@ func (ss *StreamServer) hlsHandler(w http.ResponseWriter, r *http.Request) {
|
|||
case resource == "probe.json":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
_ = json.NewEncoder(w).Encode(session.ProbeInfo())
|
||||
info := session.ProbeInfo()
|
||||
ss.attachSubtitleVTTURLs(info, session.cfg.sourceRef())
|
||||
_ = json.NewEncoder(w).Encode(info)
|
||||
case resource == "video/index.m3u8":
|
||||
session.ServeVideoPlaylist(w, r)
|
||||
case resource == "video/init.mp4":
|
||||
|
|
@ -1234,8 +1237,11 @@ func (ss *StreamServer) subtitleHandler(w http.ResponseWriter, r *http.Request)
|
|||
http.Error(w, "missing path", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
// index >= 0 → EMBEDDED stream index (-map 0:s:N) of the media at `p`.
|
||||
// index < 0 → EXTERNAL sidecar: `p` IS the subtitle file; the whole file is
|
||||
// the track. Both bind the token to (path, index) so a tampered p/i fails.
|
||||
index, err := strconv.Atoi(q.Get("i"))
|
||||
if err != nil || index < 0 {
|
||||
if err != nil {
|
||||
http.Error(w, "bad index", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
|
@ -1245,21 +1251,30 @@ func (ss *StreamServer) subtitleHandler(w http.ResponseWriter, r *http.Request)
|
|||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
rawPath = ss.healMediaPath(rawPath) // host→container base-path skew (see /thumbnail)
|
||||
if fi, statErr := os.Stat(rawPath); statErr != nil || !fi.Mode().IsRegular() {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
// Cache hit: serve a fresh sidecar (written by the scan-time prewarm or a
|
||||
// prior request) instantly, skipping ffmpeg. This is also what makes huge
|
||||
// remuxes work — the prewarm extracts without the on-demand HTTP timeout
|
||||
// below, so by play time the hit avoids the 60s ceiling that was returning
|
||||
// 500s on 50GB+ files. Checked BEFORE the ffmpeg guard so a pre-warmed track
|
||||
// is still serveable even if ffmpeg was removed after the cache was filled.
|
||||
if vtt, ok := mediainfo.ReadCachedSubtitle(rawPath, index); ok {
|
||||
ss.writeVTT(w, vtt)
|
||||
return
|
||||
external := index < 0
|
||||
// A debrid/HLS-from-URL source has no local file — ffmpeg reads the URL
|
||||
// directly. Skip the path heal + regular-file stat + on-disk cache for those;
|
||||
// only local files get the sidecar cache.
|
||||
isURL := strings.Contains(rawPath, "://")
|
||||
langHint := q.Get("l") // ISO 639-1 charset hint for external sidecar decoding
|
||||
|
||||
if !isURL {
|
||||
rawPath = ss.healMediaPath(rawPath) // host→container base-path skew (see /thumbnail)
|
||||
if fi, statErr := os.Stat(rawPath); statErr != nil || !fi.Mode().IsRegular() {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
// Cache hit: serve a fresh sidecar (written by the scan-time prewarm or a
|
||||
// prior request) instantly, skipping ffmpeg. This is also what makes huge
|
||||
// remuxes work — the prewarm extracts without the on-demand HTTP timeout
|
||||
// below, so by play time the hit avoids the 60s ceiling that was returning
|
||||
// 500s on 50GB+ files. Checked BEFORE the ffmpeg guard so a pre-warmed track
|
||||
// is still serveable even if ffmpeg was removed after the cache was filled.
|
||||
if vtt, ok := mediainfo.ReadCachedSubtitle(rawPath, index); ok {
|
||||
ss.writeVTT(w, vtt)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Beyond here we must extract on demand, which needs ffmpeg.
|
||||
|
|
@ -1275,15 +1290,23 @@ func (ss *StreamServer) subtitleHandler(w http.ResponseWriter, r *http.Request)
|
|||
ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
out, err := mediainfo.ExtractSubtitleVTT(ctx, ss.ffmpegPath, rawPath, index)
|
||||
var out []byte
|
||||
if external {
|
||||
// Standalone sidecar file: transcode charset → UTF-8 (langHint guides the
|
||||
// code-page guess) then ffmpeg → WebVTT.
|
||||
out, err = mediainfo.ExtractExternalSubtitleVTT(ctx, ss.ffmpegPath, rawPath, langHint)
|
||||
} else {
|
||||
out, err = mediainfo.ExtractSubtitleVTT(ctx, ss.ffmpegPath, rawPath, index)
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("[sub] extract failed (i=%d path=%q): %v", index, rawPath, err)
|
||||
log.Printf("[sub] extract failed (i=%d path=%q external=%v url=%v): %v", index, rawPath, external, isURL, err)
|
||||
http.Error(w, "subtitle extract failed", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
// Write-through so the next request is a cache hit. Best-effort: a read-only
|
||||
// media mount just logs and serves the in-memory bytes.
|
||||
if ss.cacheSubtitles {
|
||||
// media mount just logs and serves the in-memory bytes. URL sources have no
|
||||
// stable on-disk anchor for the sidecar cache → skip.
|
||||
if ss.cacheSubtitles && !isURL {
|
||||
if werr := mediainfo.WriteCachedSubtitle(rawPath, index, out); werr != nil {
|
||||
log.Printf("[sub] cache write skipped (i=%d path=%q): %v", index, rawPath, werr)
|
||||
}
|
||||
|
|
@ -1291,6 +1314,60 @@ func (ss *StreamServer) subtitleHandler(w http.ResponseWriter, r *http.Request)
|
|||
ss.writeVTT(w, out)
|
||||
}
|
||||
|
||||
// attachSubtitleVTTURLs enriches a ProbeInfo map's "subtitles" entries with a
|
||||
// ready-to-use, tokened `vttUrl` for every TEXT track, so the web player can
|
||||
// attach <track>s for ANY play method (torrent/debrid HLS included) without the
|
||||
// server needing the source path — it's the single subtitle wiring path that
|
||||
// makes embedded subs work on streams that were never library-scanned.
|
||||
//
|
||||
// - embedded (external=false): /sub?p=<srcRef>&i=<index>&t=<tok>
|
||||
// - external (external=true) : /sub?p=<sidecar path>&i=-1&t=<tok>&l=<lang>
|
||||
//
|
||||
// The token uses the SAME streamScopeSub(path,index) the web mints with, so a
|
||||
// library-scanned track and a probe-derived one address identically. The raw
|
||||
// "path" key is removed after the URL is built (it's encoded in the URL already).
|
||||
// URLs are root-relative; the player resolves them against the funnel origin it
|
||||
// fetched probe.json from. Bitmap tracks get no vttUrl (burn-in only).
|
||||
func (ss *StreamServer) attachSubtitleVTTURLs(info map[string]any, srcRef string) {
|
||||
subsAny, ok := info["subtitles"].([]map[string]any)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
for _, sb := range subsAny {
|
||||
isText, _ := sb["text"].(bool)
|
||||
if !isText {
|
||||
delete(sb, "path")
|
||||
continue
|
||||
}
|
||||
external, _ := sb["external"].(bool)
|
||||
var p string
|
||||
var idx int
|
||||
if external {
|
||||
p, _ = sb["path"].(string)
|
||||
idx = -1
|
||||
} else {
|
||||
p = srcRef
|
||||
if iv, ok := sb["index"].(int); ok {
|
||||
idx = iv
|
||||
}
|
||||
}
|
||||
if p == "" {
|
||||
delete(sb, "path")
|
||||
continue
|
||||
}
|
||||
tok := mintStreamToken(ss.streamSecret, streamScopeSub(p, idx), now)
|
||||
u := "/sub?p=" + url.QueryEscape(p) + "&i=" + strconv.Itoa(idx) + "&t=" + tok
|
||||
if external {
|
||||
if lang, _ := sb["lang"].(string); lang != "" && lang != "und" {
|
||||
u += "&l=" + url.QueryEscape(lang)
|
||||
}
|
||||
}
|
||||
sb["vttUrl"] = u
|
||||
delete(sb, "path")
|
||||
}
|
||||
}
|
||||
|
||||
// writeVTT writes the standard WebVTT response headers + body for both the
|
||||
// cache-hit and freshly-extracted paths of subtitleHandler.
|
||||
func (ss *StreamServer) writeVTT(w http.ResponseWriter, vtt []byte) {
|
||||
|
|
@ -1400,25 +1477,38 @@ func (ss *StreamServer) serveGrowing(w http.ResponseWriter, r *http.Request, src
|
|||
w.Header().Set("Content-Type", "video/mp4")
|
||||
w.Header().Set("Content-Disposition", fmt.Sprintf("inline; filename=%q", src.FileName()))
|
||||
|
||||
// Total to advertise: exact when ffmpeg has exited, else the estimate.
|
||||
total := src.EstimatedSize()
|
||||
if src.Final() {
|
||||
total = src.Size()
|
||||
// The instance length is KNOWN only once ffmpeg has exited. While the remux
|
||||
// is still growing, the final size is genuinely unknown — the source MKV
|
||||
// size is NOT it (the audio re-encode to AAC + fMP4 fragmentation change the
|
||||
// byte count). Advertising that wrong total made the native <video> map its
|
||||
// timeline onto a bogus length, request byte offsets that didn't line up,
|
||||
// re-seek, and reopen the connection hundreds of times a second (the remux
|
||||
// playback loop). Per RFC 7233 §4.2 we now send "/*" (unknown total) while
|
||||
// growing, so the player streams sequentially instead of re-seeking against
|
||||
// a fake size. `end` uses the estimate only as an upper-bound hint.
|
||||
final := src.Final()
|
||||
total := src.Size()
|
||||
if !final {
|
||||
total = src.EstimatedSize()
|
||||
}
|
||||
if total <= 0 {
|
||||
total = src.Size()
|
||||
}
|
||||
|
||||
start, explicitEnd := parseByteRange(r.Header.Get("Range"))
|
||||
if total > 0 && start >= total {
|
||||
// Range beyond what we expect to produce — let the browser recover.
|
||||
// A 416 is only sound against a KNOWN total. While growing we can't say a
|
||||
// start is unsatisfiable (more bytes are still coming), so only guard when
|
||||
// final.
|
||||
if final && total > 0 && start >= total {
|
||||
// Range beyond the real end — let the browser recover.
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", total))
|
||||
http.Error(w, "range not satisfiable", http.StatusRequestedRangeNotSatisfiable)
|
||||
return
|
||||
}
|
||||
|
||||
if r.Method == http.MethodHead {
|
||||
if total > 0 {
|
||||
// Only promise a length we actually know (final). While growing, omit it.
|
||||
if final && total > 0 {
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(total, 10))
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
|
@ -1429,13 +1519,23 @@ func (ss *StreamServer) serveGrowing(w http.ResponseWriter, r *http.Request, src
|
|||
if explicitEnd >= 0 && explicitEnd < end {
|
||||
end = explicitEnd
|
||||
}
|
||||
if total > 0 {
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, total))
|
||||
if end < start {
|
||||
end = start
|
||||
}
|
||||
// Exact Content-Length only when the source is final (true size known) so
|
||||
// we never promise bytes a still-running remux might not produce.
|
||||
if src.Final() && explicitEnd < 0 {
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(src.Size()-start, 10))
|
||||
if final {
|
||||
if total > 0 {
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, total))
|
||||
}
|
||||
// Exact Content-Length only when final (true size known) so we never
|
||||
// promise bytes a still-running remux might not produce.
|
||||
if explicitEnd < 0 {
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(src.Size()-start, 10))
|
||||
}
|
||||
} else {
|
||||
// Growing: honest "unknown total" so the player doesn't re-seek against
|
||||
// a wrong size. No Content-Length (chunked) — bytes flow as ffmpeg makes
|
||||
// them and the read loop below blocks at the live edge.
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/*", start, end))
|
||||
}
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
|
||||
|
|
|
|||
|
|
@ -23,6 +23,23 @@ import (
|
|||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
// portfwdFilterHandler wraps anacrolix/log handlers and drops the noisy
|
||||
// UPnP/NAT-PMP port-mapping warnings (e.g. "error: AddPortMapping: 500 Internal
|
||||
// Server Error") that home routers emit when they reject the mapping. Everything
|
||||
// else passes through unchanged.
|
||||
type portfwdFilterHandler struct {
|
||||
inner []alog.Handler
|
||||
}
|
||||
|
||||
func (h portfwdFilterHandler) Handle(r alog.Record) {
|
||||
if strings.Contains(r.Text(), "AddPortMapping") {
|
||||
return
|
||||
}
|
||||
for _, inner := range h.inner {
|
||||
inner.Handle(r)
|
||||
}
|
||||
}
|
||||
|
||||
var defaultTrackers = []string{
|
||||
// Tier 1: ngosang/trackerslist "best" + newtrackon "stable"
|
||||
"udp://tracker.opentrackr.org:1337/announce",
|
||||
|
|
@ -126,6 +143,16 @@ func NewTorrentDownloader(cfg TorrentConfig) (*TorrentDownloader, error) {
|
|||
tcfg.Seed = cfg.SeedEnabled
|
||||
tcfg.NoUpload = !cfg.SeedEnabled
|
||||
tcfg.Logger = alog.Default.FilterLevel(alog.Warning)
|
||||
// Drop the noisy UPnP/NAT-PMP port-mapping warnings. The library attempts to
|
||||
// map the listen port on the router for inbound peers (best-effort, only
|
||||
// helps on routers that support it). Many home routers reject AddPortMapping
|
||||
// with "500 Internal Server Error" and the lib retries on every lease cycle,
|
||||
// spamming the log. The rejection is harmless (download works over DHT +
|
||||
// outbound peers), so suppress just that line while keeping the attempts for
|
||||
// routers that do support it.
|
||||
tcfg.Logger.SetHandlers(portfwdFilterHandler{
|
||||
inner: append([]alog.Handler(nil), alog.Default.Handlers...),
|
||||
})
|
||||
|
||||
// No browser-facing WebTorrent peer; daemon never seeds via WSS.
|
||||
tcfg.DisableWebtorrent = true
|
||||
|
|
|
|||
|
|
@ -1,5 +1,10 @@
|
|||
package engine
|
||||
|
||||
import (
|
||||
"math"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// TranscodeRuntime carries the resolved ffmpeg/ffprobe paths + tunables so
|
||||
// each session can decide whether to passthrough or pipe through ffmpeg.
|
||||
type TranscodeRuntime struct {
|
||||
|
|
@ -48,6 +53,35 @@ func resolveQualityCap(label string) qualityCap {
|
|||
}
|
||||
}
|
||||
|
||||
// doubleBitrate returns an ffmpeg bitrate string with twice the value of the
|
||||
// input ("6000k" → "12000k", "1.5M" → "3M", "5M" → "10M"). Used to size
|
||||
// `-bufsize` at the standard 2× of `-maxrate` for capped-CRF/CQ rate control.
|
||||
// An unparseable string falls back to the input unchanged (1× bufsize — the
|
||||
// pre-CRF behaviour, safe just suboptimal). The doubled CPB stays far below
|
||||
// every H.264 level's limit for the (level, maxrate) pairs this package emits
|
||||
// (worst case: 1080p level 4.1 → 12000k bufsize vs 62500k allowed).
|
||||
func doubleBitrate(b string) string {
|
||||
if b == "" {
|
||||
return b
|
||||
}
|
||||
num := b
|
||||
suffix := ""
|
||||
switch b[len(b)-1] {
|
||||
case 'k', 'K', 'm', 'M':
|
||||
num = b[:len(b)-1]
|
||||
suffix = string(b[len(b)-1])
|
||||
}
|
||||
v, err := strconv.ParseFloat(num, 64)
|
||||
if err != nil || v <= 0 {
|
||||
return b
|
||||
}
|
||||
d := v * 2
|
||||
if d == math.Trunc(d) {
|
||||
return strconv.FormatFloat(d, 'f', 0, 64) + suffix
|
||||
}
|
||||
return strconv.FormatFloat(d, 'f', -1, 64) + suffix
|
||||
}
|
||||
|
||||
// capForHeight returns the bitrate-cap pair appropriate for an effective
|
||||
// output height. Used after clamping outputHeight to the source's resolution:
|
||||
// asking ffmpeg for "2160p" bitrate (25 Mbps) on a 1080p source overshoots
|
||||
|
|
|
|||
|
|
@ -12,9 +12,9 @@
|
|||
//
|
||||
// Lifecycle:
|
||||
//
|
||||
// t, err := funnel.Start(ctx, funnel.Config{Port: 11819})
|
||||
// defer t.Close()
|
||||
// url, err := t.WaitURL(30 * time.Second) // blocks until cloudflared emits the URL
|
||||
// t, err := funnel.Start(ctx, funnel.Config{Port: 11819})
|
||||
// defer t.Close()
|
||||
// url, err := t.WaitURL(30 * time.Second) // blocks until cloudflared emits the URL
|
||||
//
|
||||
// The tunnel runs until the context is cancelled or t.Close() is called.
|
||||
package funnel
|
||||
|
|
@ -200,4 +200,3 @@ func (t *Tunnel) scanStderr(r io.Reader) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
139
internal/library/mediainfo/charset.go
Normal file
139
internal/library/mediainfo/charset.go
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
package mediainfo
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"golang.org/x/text/encoding"
|
||||
"golang.org/x/text/encoding/charmap"
|
||||
"golang.org/x/text/encoding/japanese"
|
||||
"golang.org/x/text/encoding/korean"
|
||||
"golang.org/x/text/encoding/simplifiedchinese"
|
||||
"golang.org/x/text/encoding/traditionalchinese"
|
||||
"golang.org/x/text/encoding/unicode"
|
||||
"golang.org/x/text/transform"
|
||||
)
|
||||
|
||||
// Subtitle charset normalisation.
|
||||
//
|
||||
// External subtitle files are routinely NOT UTF-8: legacy .srt files come in the
|
||||
// uploader's local code page (Windows-1252 Western, Windows-1256 Arabic, GBK
|
||||
// Chinese, Shift-JIS Japanese, …). Feeding those raw to ffmpeg → WebVTT yields
|
||||
// mojibake. We detect the encoding and transcode to UTF-8 before extraction.
|
||||
//
|
||||
// Detection order: BOM (authoritative) → valid UTF-8 → a code page chosen from
|
||||
// the track's declared language (from its filename, e.g. ".ar.srt"). The
|
||||
// language hint is the reliable signal we have without a full statistical
|
||||
// detector: an Arabic sub that isn't UTF-8 is almost certainly Windows-1256, a
|
||||
// Russian one Windows-1251, and so on. Western European is the safe default.
|
||||
|
||||
// legacyEncodingForLang returns the most likely single-byte / CJK encoding for a
|
||||
// non-UTF-8 subtitle in the given language hint. The hint is normally an ISO
|
||||
// 639-1 code, but Chinese carries a script suffix ("zh-hant" / "zh-tw") so a
|
||||
// Traditional sidecar decodes as Big5 instead of GBK (decoding Big5 bytes as GBK
|
||||
// is mojibake — and anime fansubs routinely ship both chs AND cht). Default:
|
||||
// Windows-1252.
|
||||
func legacyEncodingForLang(lang string) encoding.Encoding {
|
||||
switch strings.ToLower(strings.TrimSpace(lang)) {
|
||||
case "ar", "fa", "ur": // Arabic script
|
||||
return charmap.Windows1256
|
||||
case "ru", "uk", "bg", "sr", "mk": // Cyrillic
|
||||
return charmap.Windows1251
|
||||
case "el": // Greek
|
||||
return charmap.Windows1253
|
||||
case "he": // Hebrew
|
||||
return charmap.Windows1255
|
||||
case "tr": // Turkish
|
||||
return charmap.Windows1254
|
||||
case "th": // Thai
|
||||
return charmap.Windows874
|
||||
case "zh-hant", "zh_hant", "zh-tw", "zh-hk", "zhtw": // Traditional Chinese
|
||||
return traditionalchinese.Big5
|
||||
case "zh", "zh-hans", "zh-cn": // Simplified Chinese (covers most pirate releases)
|
||||
return simplifiedchinese.GBK
|
||||
case "ja": // Japanese
|
||||
return japanese.ShiftJIS
|
||||
case "ko": // Korean
|
||||
return korean.EUCKR
|
||||
case "vi": // Vietnamese
|
||||
return charmap.Windows1258
|
||||
case "pl", "cs", "sk", "hu", "ro", "hr", "sl": // Central European
|
||||
return charmap.Windows1250
|
||||
case "lt", "lv", "et": // Baltic
|
||||
return charmap.Windows1257
|
||||
default: // Western European + everything else
|
||||
return charmap.Windows1252
|
||||
}
|
||||
}
|
||||
|
||||
// DecodeSubtitleToUTF8 returns the bytes as UTF-8, transcoding from a detected
|
||||
// legacy encoding when needed. The returned name is for logging ("utf-8",
|
||||
// "bom-utf16le", "windows-1256", …). Never fails: a transcode error falls back
|
||||
// to the original bytes (ffmpeg may still cope).
|
||||
func DecodeSubtitleToUTF8(data []byte, langHint string) ([]byte, string) {
|
||||
// BOM wins — it's unambiguous.
|
||||
switch {
|
||||
case bytes.HasPrefix(data, []byte{0xEF, 0xBB, 0xBF}):
|
||||
return data[3:], "bom-utf8"
|
||||
case bytes.HasPrefix(data, []byte{0xFF, 0xFE}):
|
||||
return decodeWith(data, unicode.UTF16(unicode.LittleEndian, unicode.UseBOM), "bom-utf16le")
|
||||
case bytes.HasPrefix(data, []byte{0xFE, 0xFF}):
|
||||
return decodeWith(data, unicode.UTF16(unicode.BigEndian, unicode.UseBOM), "bom-utf16be")
|
||||
}
|
||||
// Already valid UTF-8 → no transcode (ASCII is a subset, so plain English
|
||||
// srt files hit this).
|
||||
if utf8.Valid(data) {
|
||||
return data, "utf-8"
|
||||
}
|
||||
// Non-UTF-8: transcode from the language's likely code page.
|
||||
enc := legacyEncodingForLang(langHint)
|
||||
out, name := decodeWith(data, enc, encodingName(enc))
|
||||
return out, name
|
||||
}
|
||||
|
||||
// decodeWith transforms data through enc's decoder to UTF-8. On error returns the
|
||||
// original bytes (best-effort) with the name suffixed "(raw)".
|
||||
func decodeWith(data []byte, enc encoding.Encoding, name string) ([]byte, string) {
|
||||
out, _, err := transform.Bytes(enc.NewDecoder(), data)
|
||||
if err != nil || len(out) == 0 {
|
||||
return data, name + "(raw)"
|
||||
}
|
||||
return out, name
|
||||
}
|
||||
|
||||
// encodingName maps a known encoding back to a short label for logs.
|
||||
func encodingName(enc encoding.Encoding) string {
|
||||
switch enc {
|
||||
case charmap.Windows1250:
|
||||
return "windows-1250"
|
||||
case charmap.Windows1251:
|
||||
return "windows-1251"
|
||||
case charmap.Windows1252:
|
||||
return "windows-1252"
|
||||
case charmap.Windows1253:
|
||||
return "windows-1253"
|
||||
case charmap.Windows1254:
|
||||
return "windows-1254"
|
||||
case charmap.Windows1255:
|
||||
return "windows-1255"
|
||||
case charmap.Windows1256:
|
||||
return "windows-1256"
|
||||
case charmap.Windows1257:
|
||||
return "windows-1257"
|
||||
case charmap.Windows1258:
|
||||
return "windows-1258"
|
||||
case charmap.Windows874:
|
||||
return "windows-874"
|
||||
case simplifiedchinese.GBK:
|
||||
return "gbk"
|
||||
case traditionalchinese.Big5:
|
||||
return "big5"
|
||||
case japanese.ShiftJIS:
|
||||
return "shift-jis"
|
||||
case korean.EUCKR:
|
||||
return "euc-kr"
|
||||
default:
|
||||
return "legacy"
|
||||
}
|
||||
}
|
||||
64
internal/library/mediainfo/charset_test.go
Normal file
64
internal/library/mediainfo/charset_test.go
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
package mediainfo
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"golang.org/x/text/encoding/charmap"
|
||||
"golang.org/x/text/transform"
|
||||
)
|
||||
|
||||
func TestDecodeSubtitleToUTF8_PlainASCII(t *testing.T) {
|
||||
in := []byte("Hello world")
|
||||
out, name := DecodeSubtitleToUTF8(in, "en")
|
||||
if string(out) != "Hello world" || name != "utf-8" {
|
||||
t.Fatalf("ASCII passthrough failed: %q %s", out, name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeSubtitleToUTF8_BOMStripped(t *testing.T) {
|
||||
in := append([]byte{0xEF, 0xBB, 0xBF}, []byte("café")...)
|
||||
out, name := DecodeSubtitleToUTF8(in, "fr")
|
||||
if string(out) != "café" || name != "bom-utf8" {
|
||||
t.Fatalf("UTF-8 BOM strip failed: %q %s", out, name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeSubtitleToUTF8_Windows1252(t *testing.T) {
|
||||
// "café" encoded in Windows-1252 (é = 0xE9) is NOT valid UTF-8.
|
||||
enc1252, _, err := transform.Bytes(charmap.Windows1252.NewEncoder(), []byte("café"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, name := DecodeSubtitleToUTF8(enc1252, "fr")
|
||||
if string(out) != "café" {
|
||||
t.Fatalf("Windows-1252 decode failed: got %q (%s)", out, name)
|
||||
}
|
||||
if name != "windows-1252" {
|
||||
t.Fatalf("expected windows-1252, got %s", name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeSubtitleToUTF8_TraditionalChineseBig5(t *testing.T) {
|
||||
// 繁 (U+7E41) in Big5 is 0xC1 0x63. Decoding it as GBK would be mojibake, so
|
||||
// the zh-Hant hint must route to Big5.
|
||||
in := []byte{0xC1, 0x63}
|
||||
out, name := DecodeSubtitleToUTF8(in, "zh-Hant")
|
||||
if name != "big5" {
|
||||
t.Fatalf("expected big5 for zh-Hant, got %s", name)
|
||||
}
|
||||
if string(out) != "繁" {
|
||||
t.Fatalf("Big5 decode failed: got %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeSubtitleToUTF8_ArabicByLang(t *testing.T) {
|
||||
// Arabic letter ا (U+0627) is 0xC7 in Windows-1256.
|
||||
in := []byte{0xC7}
|
||||
out, name := DecodeSubtitleToUTF8(in, "ar")
|
||||
if name != "windows-1256" {
|
||||
t.Fatalf("expected windows-1256 for Arabic, got %s", name)
|
||||
}
|
||||
if string(out) != "ا" {
|
||||
t.Fatalf("Arabic decode failed: got %q", out)
|
||||
}
|
||||
}
|
||||
|
|
@ -95,6 +95,16 @@ func ExtractMediaInfo(ctx context.Context, ffprobePath, filePath string) (*Media
|
|||
if integ := assessIntegrity(stderr.String(), mi); integ != nil {
|
||||
mi.Integrity = integ
|
||||
}
|
||||
// Append external sidecar subtitles (a .srt/.ass next to the video, or a
|
||||
// Subs/ bundle) AFTER the embedded streams, so embedded keep slice positions
|
||||
// == their 0:s:N index. Local files only — a remote URL has no directory to
|
||||
// scan (debrid streams rely on embedded subs from the URL). Best-effort:
|
||||
// DiscoverSidecarSubtitles returns nil on an unreadable dir.
|
||||
if !strings.Contains(filePath, "://") {
|
||||
if ext := DiscoverSidecarSubtitles(filePath); len(ext) > 0 {
|
||||
mi.Subtitles = append(mi.Subtitles, ext...)
|
||||
}
|
||||
}
|
||||
return mi, nil
|
||||
}
|
||||
|
||||
|
|
|
|||
206
internal/library/mediainfo/gallery_real_test.go
Normal file
206
internal/library/mediainfo/gallery_real_test.go
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
package mediainfo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestGalleryReal is a manual end-to-end harness against a REAL media library.
|
||||
// It is skipped unless GALLERY_DIR is set, so it never runs in CI.
|
||||
//
|
||||
// GALLERY_DIR=/mnt/nas/peliculas go test ./internal/library/mediainfo/ \
|
||||
// -run TestGalleryReal -v -timeout 30m
|
||||
//
|
||||
// It surveys every video file (embedded subs via ffprobe + discovered sidecars),
|
||||
// then actually extracts WebVTT for one representative of each kind and checks the
|
||||
// output is a valid, non-empty WEBVTT document.
|
||||
func TestGalleryReal(t *testing.T) {
|
||||
dir := os.Getenv("GALLERY_DIR")
|
||||
if dir == "" {
|
||||
t.Skip("set GALLERY_DIR to run the real-gallery survey")
|
||||
}
|
||||
ffprobe := envOr("FFPROBE", "ffprobe")
|
||||
ffmpeg := envOr("FFMPEG", "ffmpeg")
|
||||
|
||||
videoExt := map[string]bool{".mkv": true, ".mp4": true, ".avi": true, ".m4v": true, ".webm": true, ".mov": true, ".ts": true}
|
||||
var videos []string
|
||||
_ = filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if strings.Contains(p, "/.unarr/") || strings.Contains(p, "/.Trash") || strings.Contains(p, "/@eaDir/") {
|
||||
return nil
|
||||
}
|
||||
if videoExt[strings.ToLower(filepath.Ext(p))] {
|
||||
videos = append(videos, p)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
sort.Strings(videos)
|
||||
t.Logf("found %d video files under %s", len(videos), dir)
|
||||
|
||||
type cat struct {
|
||||
embTextCodecs map[string]int // codec → count of files
|
||||
embBitmap map[string]int
|
||||
extCodecs map[string]int
|
||||
filesEmbText []string
|
||||
filesEmbBitmap []string
|
||||
filesExt []string
|
||||
errs int
|
||||
}
|
||||
c := cat{embTextCodecs: map[string]int{}, embBitmap: map[string]int{}, extCodecs: map[string]int{}}
|
||||
|
||||
for _, v := range videos {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
mi, err := ExtractMediaInfo(ctx, ffprobe, v)
|
||||
cancel()
|
||||
if err != nil {
|
||||
c.errs++
|
||||
t.Logf("PROBE ERR %s: %v", filepath.Base(v), err)
|
||||
continue
|
||||
}
|
||||
var sawEmbText, sawEmbBitmap, sawExt bool
|
||||
for _, s := range mi.Subtitles {
|
||||
codec := strings.ToLower(s.Codec)
|
||||
switch {
|
||||
case s.External:
|
||||
c.extCodecs[codec]++
|
||||
sawExt = true
|
||||
case IsTextSubtitleCodec(codec):
|
||||
c.embTextCodecs[codec]++
|
||||
sawEmbText = true
|
||||
default:
|
||||
c.embBitmap[codec]++
|
||||
sawEmbBitmap = true
|
||||
}
|
||||
}
|
||||
if sawEmbText {
|
||||
c.filesEmbText = append(c.filesEmbText, v)
|
||||
}
|
||||
if sawEmbBitmap {
|
||||
c.filesEmbBitmap = append(c.filesEmbBitmap, v)
|
||||
}
|
||||
if sawExt {
|
||||
c.filesExt = append(c.filesExt, v)
|
||||
}
|
||||
}
|
||||
|
||||
t.Logf("=== CENSUS ===")
|
||||
t.Logf("probe errors: %d", c.errs)
|
||||
t.Logf("embedded TEXT codecs (files w/ track): %v", c.embTextCodecs)
|
||||
t.Logf("embedded BITMAP codecs (burn-in only): %v", c.embBitmap)
|
||||
t.Logf("external SIDECAR codecs: %v", c.extCodecs)
|
||||
t.Logf("files w/ embedded text: %d | w/ embedded bitmap: %d | w/ external sidecar: %d",
|
||||
len(c.filesEmbText), len(c.filesEmbBitmap), len(c.filesExt))
|
||||
|
||||
// --- Real extraction checks ---
|
||||
validVTT := func(b []byte) bool {
|
||||
return len(b) > 0 && strings.HasPrefix(strings.TrimSpace(string(b)), "WEBVTT")
|
||||
}
|
||||
|
||||
// Embedded text: extract index 0 of the first such file.
|
||||
if len(c.filesEmbText) > 0 {
|
||||
f := c.filesEmbText[0]
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
out, err := ExtractSubtitleVTT(ctx, ffmpeg, f, 0)
|
||||
cancel()
|
||||
if err != nil || !validVTT(out) {
|
||||
t.Errorf("EMBEDDED extract FAILED for %s: err=%v len=%d", filepath.Base(f), err, len(out))
|
||||
} else {
|
||||
t.Logf("EMBEDDED extract OK: %s → %d bytes WebVTT", filepath.Base(f), len(out))
|
||||
}
|
||||
}
|
||||
|
||||
// External sidecar: find one and extract it via the path-addressed function.
|
||||
if len(c.filesExt) > 0 {
|
||||
f := c.filesExt[0]
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
mi, _ := ExtractMediaInfo(ctx, ffprobe, f)
|
||||
cancel()
|
||||
var subPath, lang string
|
||||
for _, s := range mi.Subtitles {
|
||||
if s.External {
|
||||
subPath, lang = s.Path, s.Lang
|
||||
break
|
||||
}
|
||||
}
|
||||
ctx2, cancel2 := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
out, err := ExtractExternalSubtitleVTT(ctx2, ffmpeg, subPath, lang)
|
||||
cancel2()
|
||||
if err != nil || !validVTT(out) {
|
||||
t.Errorf("EXTERNAL extract FAILED for %s: err=%v len=%d", filepath.Base(subPath), err, len(out))
|
||||
} else {
|
||||
t.Logf("EXTERNAL extract OK: %s (lang=%s) → %d bytes WebVTT", filepath.Base(subPath), lang, len(out))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func envOr(k, def string) string {
|
||||
if v := os.Getenv(k); v != "" {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// TestGalleryExtractAllSidecars extracts EVERY discovered sidecar in the gallery
|
||||
// and reports any that fail — the real proof the external path is robust across
|
||||
// formats/charsets. Skipped unless GALLERY_DIR is set.
|
||||
func TestGalleryExtractAllSidecars(t *testing.T) {
|
||||
dir := os.Getenv("GALLERY_DIR")
|
||||
if dir == "" {
|
||||
t.Skip("set GALLERY_DIR")
|
||||
}
|
||||
ffmpeg := envOr("FFMPEG", "ffmpeg")
|
||||
var subs []SubtitleTrack
|
||||
_ = filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() || strings.Contains(p, "/.unarr/") || strings.Contains(p, "/.Trash") || strings.Contains(p, "/@eaDir/") {
|
||||
return nil
|
||||
}
|
||||
ext := strings.ToLower(filepath.Ext(p))
|
||||
if videoOf(ext) {
|
||||
subs = append(subs, DiscoverSidecarSubtitles(p)...)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
// Dedupe by path.
|
||||
seen := map[string]bool{}
|
||||
var uniq []SubtitleTrack
|
||||
for _, s := range subs {
|
||||
if !seen[s.Path] {
|
||||
seen[s.Path] = true
|
||||
uniq = append(uniq, s)
|
||||
}
|
||||
}
|
||||
t.Logf("discovered %d unique sidecar subtitle files", len(uniq))
|
||||
fails := 0
|
||||
for _, s := range uniq {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
out, err := ExtractExternalSubtitleVTT(ctx, ffmpeg, s.Path, s.Lang)
|
||||
cancel()
|
||||
ok := len(out) > 0 && strings.HasPrefix(strings.TrimSpace(string(out)), "WEBVTT")
|
||||
if err != nil || !ok {
|
||||
fails++
|
||||
t.Errorf("FAIL %s (lang=%s codec=%s): err=%v len=%d", filepath.Base(s.Path), s.Lang, s.Codec, err, len(out))
|
||||
} else {
|
||||
t.Logf("OK %s (lang=%s codec=%s) → %d bytes", filepath.Base(s.Path), s.Lang, s.Codec, len(out))
|
||||
}
|
||||
}
|
||||
if fails > 0 {
|
||||
t.Errorf("%d/%d sidecar extractions failed", fails, len(uniq))
|
||||
} else {
|
||||
t.Logf("all %d sidecar extractions produced valid WebVTT", len(uniq))
|
||||
}
|
||||
}
|
||||
|
||||
func videoOf(ext string) bool {
|
||||
switch ext {
|
||||
case ".mkv", ".mp4", ".avi", ".m4v", ".webm", ".mov", ".ts":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
|
@ -64,7 +64,13 @@ var langNormalize = map[string]string{
|
|||
"mlt": "mt", "mt": "mt",
|
||||
"swa": "sw", "sw": "sw",
|
||||
"afr": "af", "af": "af",
|
||||
"lat": "la", "la": "la",
|
||||
"kan": "kn", "kn": "kn",
|
||||
"mal": "ml", "ml": "ml",
|
||||
"mar": "mr", "mr": "mr",
|
||||
"pan": "pa", "pa": "pa",
|
||||
"guj": "gu", "gu": "gu",
|
||||
"kann": "kn",
|
||||
"lat": "la", "la": "la",
|
||||
|
||||
// Full English names (ffprobe sometimes returns these instead of codes)
|
||||
"english": "en", "spanish": "es", "french": "fr", "german": "de",
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import (
|
|||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"math"
|
||||
"os"
|
||||
"os/exec"
|
||||
|
|
@ -148,6 +149,66 @@ func ExtractSubtitleVTT(ctx context.Context, ffmpegPath, mediaPath string, index
|
|||
return out, nil
|
||||
}
|
||||
|
||||
// ExtractExternalSubtitleVTT converts a STANDALONE sidecar subtitle file (a
|
||||
// .srt/.ass/.ssa/.vtt sitting next to the media) to WebVTT. Unlike the embedded
|
||||
// path it has no stream index — the whole file is the track. It first transcodes
|
||||
// the bytes to UTF-8 (legacy code pages → mojibake otherwise; see charset.go)
|
||||
// using the track's language as the detection hint, then runs ffmpeg to emit
|
||||
// WebVTT. The UTF-8 bytes go through a temp file with the ORIGINAL extension so
|
||||
// ffmpeg selects the right demuxer (.srt→subrip, .ass→ass, .vtt→webvtt), and
|
||||
// `-sub_charenc UTF-8` stops ffmpeg from re-guessing what we already decoded.
|
||||
func ExtractExternalSubtitleVTT(ctx context.Context, ffmpegPath, subPath, langHint string) ([]byte, error) {
|
||||
raw, err := os.ReadFile(subPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read sidecar subtitle: %w", err)
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return nil, errors.New("sidecar subtitle is empty")
|
||||
}
|
||||
utf8Bytes, encName := DecodeSubtitleToUTF8(raw, langHint)
|
||||
// A "(raw)" suffix means the legacy transcode failed and we're passing the
|
||||
// original bytes through — the likeliest cause of user-visible mojibake, so
|
||||
// leave a trail to diagnose it in the field.
|
||||
if strings.HasSuffix(encName, "(raw)") {
|
||||
log.Printf("[sub] external charset transcode fell back to raw bytes (%s, lang=%q): possible mojibake", filepath.Base(subPath), langHint)
|
||||
}
|
||||
|
||||
ext := strings.ToLower(filepath.Ext(subPath))
|
||||
if ext == "" {
|
||||
ext = ".srt"
|
||||
}
|
||||
tmpDir, err := os.MkdirTemp("", "unarr-extsub-")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = os.RemoveAll(tmpDir) }()
|
||||
tmpIn := filepath.Join(tmpDir, "in"+ext)
|
||||
if werr := os.WriteFile(tmpIn, utf8Bytes, 0o600); werr != nil {
|
||||
return nil, werr
|
||||
}
|
||||
|
||||
args := []string{
|
||||
"-nostdin",
|
||||
"-loglevel", "error",
|
||||
"-sub_charenc", "UTF-8",
|
||||
"-i", tmpIn,
|
||||
"-c:s", "webvtt",
|
||||
"-f", "webvtt",
|
||||
"-",
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, ffmpegPath, args...)
|
||||
var stderr strings.Builder
|
||||
cmd.Stderr = &stderr
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ffmpeg external subtitle extract: %w: %s", err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil, errors.New("ffmpeg produced no subtitle output")
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ExtractSubtitlesVTTMulti extracts several text subtitle streams in a SINGLE
|
||||
// ffmpeg pass. The expensive part of subtitle extraction is demuxing the whole
|
||||
// container (subtitle packets are interleaved across the runtime), so a 60GB
|
||||
|
|
|
|||
207
internal/library/mediainfo/sidecar_subs.go
Normal file
207
internal/library/mediainfo/sidecar_subs.go
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
package mediainfo
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// External (sidecar) subtitle discovery.
|
||||
//
|
||||
// A huge share of torrents — anime fansubs especially — ship subtitles as
|
||||
// SEPARATE files, not embedded streams: a `.srt`/`.ass` named after the video,
|
||||
// or a bundle inside a `Subs/` (or `Subtitles/`) subfolder. ffprobe on the video
|
||||
// container never sees these, so the scan recorded zero subtitles for them
|
||||
// (e.g. ToonsHub "MSubs" releases). This module finds those files so they become
|
||||
// real, selectable tracks served via the /sub endpoint (path-based, i=-1).
|
||||
//
|
||||
// Only TEXT formats are surfaced (srt/ass/ssa/vtt, and a lone .sub). VobSub
|
||||
// (.idx + .sub) is bitmap — no text form — so it's skipped here; bitmap subs are
|
||||
// burn-in only and external bitmap burn-in isn't wired.
|
||||
|
||||
// subFolderNames are common subfolder names that hold a release's subtitle
|
||||
// bundle. Matched case-insensitively. Files inside belong to the sibling media.
|
||||
var subFolderNames = map[string]bool{
|
||||
"subs": true, "subtitles": true, "sub": true, "subtitle": true,
|
||||
}
|
||||
|
||||
// sidecarSubExts maps a subtitle file extension to its ffmpeg-style codec name.
|
||||
// The codec drives the web's text-vs-bitmap classification (isTextSubtitleCodec).
|
||||
var sidecarSubExts = map[string]string{
|
||||
".srt": "subrip",
|
||||
".ass": "ass",
|
||||
".ssa": "ssa",
|
||||
".vtt": "webvtt",
|
||||
".sub": "subrip", // MicroDVD/text — UNLESS paired with a .idx (VobSub, handled below)
|
||||
}
|
||||
|
||||
// forcedTokens / sdhTokens are filename markers that refine a sidecar's role.
|
||||
var forcedTokens = map[string]bool{"forced": true, "forzado": true, "forces": true}
|
||||
var sdhTokens = map[string]bool{"sdh": true, "cc": true, "hi": false} // "hi" is also Hindi → don't treat as SDH
|
||||
|
||||
// sidecarLangAliases maps RELEASE-NAMING subtitle tokens (fansub/scene shorthand
|
||||
// NOT covered by the ISO 639-1/2 normaliser) to a language hint. Two things make
|
||||
// this necessary beyond NormalizeLang:
|
||||
// - Chinese SCRIPT matters for charset: Simplified (chs/sc/gb) is GBK,
|
||||
// Traditional (cht/tc/big5) is Big5 — decoding one as the other is mojibake.
|
||||
// We keep the script in the hint ("zh" vs "zh-Hant") so legacyEncodingForLang
|
||||
// picks the right code page. Anime fansubs routinely ship both.
|
||||
// - lat/latino/vostfr etc. aren't ISO at all and would fall to "und".
|
||||
//
|
||||
// Applied ONLY to sidecar filenames, not ffprobe metadata, so it can't clash with
|
||||
// the global langNormalize ("lat"→Latin there). Plain ISO codes (eng/spa/…) are
|
||||
// intentionally left to NormalizeLang.
|
||||
var sidecarLangAliases = map[string]string{
|
||||
"chs": "zh", "sc": "zh", "gb": "zh", "gbk": "zh", "hans": "zh", // Simplified → GBK
|
||||
"cht": "zh-Hant", "tc": "zh-Hant", "big5": "zh-Hant", "hant": "zh-Hant", // Traditional → Big5
|
||||
"lat": "es", "latino": "es", "esp": "es", "español": "es", "espanol": "es",
|
||||
"vostfr": "fr", "vff": "fr", "vf": "fr",
|
||||
"ptbr": "pt", "pt-br": "pt", "bra": "pt",
|
||||
}
|
||||
|
||||
// DiscoverSidecarSubtitles finds external subtitle files for a local media file:
|
||||
// siblings named after the video, plus everything in a Subs/Subtitles subfolder.
|
||||
// Returns text tracks only, each with External=true and an absolute Path. Safe on
|
||||
// any path — returns nil if the directory can't be read (best-effort, like the
|
||||
// rest of the scan). Never call for a remote URL source (no local directory).
|
||||
//
|
||||
// NOTE: discovered sidecars are NOT deduped against embedded streams of the same
|
||||
// language. That's deliberate — a `Movie.en.srt` next to a video that also has an
|
||||
// embedded English stream is usually a DIFFERENT track (full vs SDH, retimed, or
|
||||
// a better translation), so silently dropping either would hide a choice the user
|
||||
// may want. Both surface as separate, distinctly-labelled entries.
|
||||
func DiscoverSidecarSubtitles(mediaPath string) []SubtitleTrack {
|
||||
if mediaPath == "" || strings.Contains(mediaPath, "://") {
|
||||
return nil
|
||||
}
|
||||
dir := filepath.Dir(mediaPath)
|
||||
videoBase := strings.TrimSuffix(filepath.Base(mediaPath), filepath.Ext(mediaPath))
|
||||
videoBaseLower := strings.ToLower(videoBase)
|
||||
|
||||
var out []SubtitleTrack
|
||||
seen := make(map[string]bool) // absolute path dedupe
|
||||
|
||||
// 1. Siblings in the media's own directory whose name starts with the video
|
||||
// base name: "Movie.srt", "Movie.en.srt", "Movie.en.forced.ass", …
|
||||
addFromDir(dir, func(name string) bool {
|
||||
return strings.HasPrefix(strings.ToLower(name), videoBaseLower)
|
||||
}, videoBase, &out, seen)
|
||||
|
||||
// 2. A Subs/Subtitles subfolder: take EVERY subtitle file (the whole folder
|
||||
// belongs to this release). Filenames there are usually language-named
|
||||
// ("2_English.srt", "spa.ass") with no video-base prefix.
|
||||
if entries, err := os.ReadDir(dir); err == nil {
|
||||
for _, e := range entries {
|
||||
if e.IsDir() && subFolderNames[strings.ToLower(e.Name())] {
|
||||
addFromDir(filepath.Join(dir, e.Name()), func(string) bool { return true }, "", &out, seen)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// addFromDir scans one directory, emitting a SubtitleTrack for each text sidecar
|
||||
// whose name passes `match`. stripPrefix (the video base, may be "") is removed
|
||||
// before parsing language/role tokens so "Movie.en.forced.srt" parses as "en"+forced.
|
||||
func addFromDir(dir string, match func(name string) bool, stripPrefix string, out *[]SubtitleTrack, seen map[string]bool) {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// Pre-index .idx files so a paired .sub is recognised as VobSub (bitmap) and skipped.
|
||||
idxBases := make(map[string]bool)
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() && strings.EqualFold(filepath.Ext(e.Name()), ".idx") {
|
||||
idxBases[strings.ToLower(strings.TrimSuffix(e.Name(), filepath.Ext(e.Name())))] = true
|
||||
}
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
name := e.Name()
|
||||
ext := strings.ToLower(filepath.Ext(name))
|
||||
codec, ok := sidecarSubExts[ext]
|
||||
if !ok || !match(name) {
|
||||
continue
|
||||
}
|
||||
// VobSub: a .sub paired with a same-named .idx is bitmap, not text. Skip.
|
||||
if ext == ".sub" && idxBases[strings.ToLower(strings.TrimSuffix(name, ext))] {
|
||||
continue
|
||||
}
|
||||
abs := filepath.Join(dir, name)
|
||||
if seen[abs] {
|
||||
continue
|
||||
}
|
||||
seen[abs] = true
|
||||
|
||||
lang, forced, title := parseSidecarName(name, ext, stripPrefix)
|
||||
*out = append(*out, SubtitleTrack{
|
||||
Lang: lang,
|
||||
Codec: codec,
|
||||
Title: title,
|
||||
Forced: forced,
|
||||
External: true,
|
||||
Path: abs,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// parseSidecarName extracts (lang, forced, title) from a subtitle filename.
|
||||
// stripPrefix (the video base) is removed first; the remainder is tokenised on
|
||||
// common separators and scanned for a language code + role markers. Unknown →
|
||||
// lang "und". The title is a human hint ("Forced", "SDH") or "".
|
||||
func parseSidecarName(name, ext, stripPrefix string) (lang string, forced bool, title string) {
|
||||
stem := strings.TrimSuffix(name, filepath.Ext(name))
|
||||
if stripPrefix != "" && len(stem) >= len(stripPrefix) &&
|
||||
strings.EqualFold(stem[:len(stripPrefix)], stripPrefix) {
|
||||
stem = stem[len(stripPrefix):]
|
||||
}
|
||||
lang = "und"
|
||||
var roles []string
|
||||
for _, tok := range strings.FieldsFunc(stem, func(r rune) bool {
|
||||
return r == '.' || r == '_' || r == '-' || r == ' ' || r == '[' || r == ']' || r == '(' || r == ')'
|
||||
}) {
|
||||
low := strings.ToLower(strings.TrimSpace(tok))
|
||||
if low == "" {
|
||||
continue
|
||||
}
|
||||
if forcedTokens[low] {
|
||||
forced = true
|
||||
roles = append(roles, "Forced")
|
||||
continue
|
||||
}
|
||||
if v, isSDH := sdhTokens[low]; isSDH && v {
|
||||
roles = append(roles, "SDH")
|
||||
continue
|
||||
}
|
||||
// First token that maps to a real language wins. Try release-naming
|
||||
// aliases (chs/lat/…) first, then the standard ISO normaliser. NormalizeLang
|
||||
// echoes unknown input back lowercased, so accept only a mapped result
|
||||
// (different from the raw token, or already a known 2-letter code).
|
||||
if lang == "und" {
|
||||
if alias, ok := sidecarLangAliases[low]; ok {
|
||||
lang = alias
|
||||
continue
|
||||
}
|
||||
if norm := NormalizeLang(low); norm != "und" && (norm != low || len(low) == 2) && isKnownLang(norm) {
|
||||
lang = norm
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
title = strings.Join(roles, " ")
|
||||
return lang, forced, title
|
||||
}
|
||||
|
||||
// isKnownLang reports whether code is a value present in langNormalize (i.e. a
|
||||
// real ISO 639-1 we recognise) — guards against treating a random filename token
|
||||
// ("web", "dl") as a language.
|
||||
func isKnownLang(code string) bool {
|
||||
for _, v := range langNormalize {
|
||||
if v == code {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
113
internal/library/mediainfo/sidecar_subs_test.go
Normal file
113
internal/library/mediainfo/sidecar_subs_test.go
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
package mediainfo
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func writeFile(t *testing.T, path, content string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func findTrack(tracks []SubtitleTrack, base string) *SubtitleTrack {
|
||||
for i := range tracks {
|
||||
if filepath.Base(tracks[i].Path) == base {
|
||||
return &tracks[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestDiscoverSidecarSubtitles_Siblings(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
video := filepath.Join(dir, "Witch.Hat.Atelier.S01E10.mkv")
|
||||
writeFile(t, video, "x")
|
||||
writeFile(t, filepath.Join(dir, "Witch.Hat.Atelier.S01E10.srt"), "1\n00:00:01,000 --> 00:00:02,000\nhi\n")
|
||||
writeFile(t, filepath.Join(dir, "Witch.Hat.Atelier.S01E10.es.ass"), "[Script Info]")
|
||||
writeFile(t, filepath.Join(dir, "Witch.Hat.Atelier.S01E10.en.forced.srt"), "x")
|
||||
// Unrelated file with a different base must NOT be matched as a sibling.
|
||||
writeFile(t, filepath.Join(dir, "Other.Movie.srt"), "x")
|
||||
|
||||
tracks := DiscoverSidecarSubtitles(video)
|
||||
if len(tracks) != 3 {
|
||||
t.Fatalf("want 3 sibling tracks, got %d: %+v", len(tracks), tracks)
|
||||
}
|
||||
for _, tr := range tracks {
|
||||
if !tr.External || tr.Path == "" {
|
||||
t.Errorf("track not marked external w/ path: %+v", tr)
|
||||
}
|
||||
}
|
||||
if es := findTrack(tracks, "Witch.Hat.Atelier.S01E10.es.ass"); es == nil || es.Lang != "es" || es.Codec != "ass" {
|
||||
t.Errorf("es.ass mis-parsed: %+v", es)
|
||||
}
|
||||
if fr := findTrack(tracks, "Witch.Hat.Atelier.S01E10.en.forced.srt"); fr == nil || fr.Lang != "en" || !fr.Forced {
|
||||
t.Errorf("forced track mis-parsed: %+v", fr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoverSidecarSubtitles_SubsFolder(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
video := filepath.Join(dir, "Movie.2024.1080p.mkv")
|
||||
writeFile(t, video, "x")
|
||||
subs := filepath.Join(dir, "Subs")
|
||||
if err := os.Mkdir(subs, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeFile(t, filepath.Join(subs, "2_English.srt"), "x")
|
||||
writeFile(t, filepath.Join(subs, "spa.ass"), "x")
|
||||
|
||||
tracks := DiscoverSidecarSubtitles(video)
|
||||
if len(tracks) != 2 {
|
||||
t.Fatalf("want 2 Subs/ tracks, got %d: %+v", len(tracks), tracks)
|
||||
}
|
||||
if en := findTrack(tracks, "2_English.srt"); en == nil || en.Lang != "en" {
|
||||
t.Errorf("English mis-parsed: %+v", en)
|
||||
}
|
||||
if es := findTrack(tracks, "spa.ass"); es == nil || es.Lang != "es" {
|
||||
t.Errorf("spa mis-parsed: %+v", es)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSidecarName_ReleaseAliases(t *testing.T) {
|
||||
cases := []struct {
|
||||
name, ext, prefix, wantLang string
|
||||
}{
|
||||
{"[DMG] Orange [01].chs.ass", ".ass", "", "zh"}, // Chinese Simplified fansub code → GBK
|
||||
{"Show.cht.srt", ".srt", "Show", "zh-Hant"}, // Chinese Traditional → Big5
|
||||
{"Movie.big5.srt", ".srt", "Movie", "zh-Hant"}, // Traditional via codepage token
|
||||
{"Movie.lat.srt", ".srt", "Movie", "es"}, // Latin-American Spanish
|
||||
{"Movie.latino.srt", ".srt", "Movie", "es"}, //
|
||||
{"Pelicula.esp.srt", ".srt", "Pelicula", "es"}, //
|
||||
{"Anime.VOSTFR.ass", ".ass", "Anime", "fr"}, // French fansub
|
||||
{"X.kan.srt", ".srt", "X", "kn"}, // Kannada via langNormalize add
|
||||
{"X.mal.srt", ".srt", "X", "ml"}, // Malayalam
|
||||
}
|
||||
for _, c := range cases {
|
||||
lang, _, _ := parseSidecarName(c.name, c.ext, c.prefix)
|
||||
if lang != c.wantLang {
|
||||
t.Errorf("%s: got lang %q, want %q", c.name, lang, c.wantLang)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoverSidecarSubtitles_VobSubSkipped(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
video := filepath.Join(dir, "Film.mkv")
|
||||
writeFile(t, video, "x")
|
||||
writeFile(t, filepath.Join(dir, "Film.idx"), "x")
|
||||
writeFile(t, filepath.Join(dir, "Film.sub"), "x") // VobSub bitmap → skip
|
||||
tracks := DiscoverSidecarSubtitles(video)
|
||||
if len(tracks) != 0 {
|
||||
t.Fatalf("VobSub .sub+.idx must be skipped, got %d: %+v", len(tracks), tracks)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoverSidecarSubtitles_RemoteURLNoop(t *testing.T) {
|
||||
if tracks := DiscoverSidecarSubtitles("https://example.com/movie.mkv"); tracks != nil {
|
||||
t.Fatalf("remote URL must yield no sidecars, got %+v", tracks)
|
||||
}
|
||||
}
|
||||
|
|
@ -107,9 +107,16 @@ func ReadCachedTrickplay(mediaPath string, width int) (TrickplayManifest, bool)
|
|||
// GenerateTrickplay builds the montage sprite + manifest for mediaPath and caches
|
||||
// them in the sidecar dir. ONE ffmpeg pass samples a frame every intervalSec
|
||||
// (fps=1/interval), scales each to width (even height), and tiles them into a
|
||||
// single JPEG. The whole file is decoded once — slow but a one-time, cached,
|
||||
// scan-time cost (run with idle I/O priority by the prewarm), and it removes ALL
|
||||
// live extraction during playback (no contention with the active stream).
|
||||
// single JPEG.
|
||||
//
|
||||
// `-skip_frame nokey` makes the decoder touch ONLY keyframes — ~12× less CPU
|
||||
// than the old full decode (measured 233 s → 19 s CPU on a 24-min 1080p
|
||||
// episode), which matters because this runs alongside live streaming on the
|
||||
// same box. The fps filter still emits one frame per UNIFORM tick (it
|
||||
// repeats the latest keyframe for ticks between keyframes), so the manifest
|
||||
// contract — tileIndex = floor(t / IntervalSec) — is unchanged and cached
|
||||
// clients keep working; each tile just shows the nearest keyframe ≤ its
|
||||
// tick (≤ one GOP off, invisible at 240-320 px scrub size).
|
||||
//
|
||||
// durationSec drives the grid size; pass the probed duration (0 → error, nothing
|
||||
// to sample). The caller owns the ctx deadline (generous at scan time).
|
||||
|
|
@ -179,10 +186,18 @@ func GenerateTrickplay(ctx context.Context, ffmpegPath, mediaPath string, interv
|
|||
tmpSprite := spritePath + ".tmp"
|
||||
|
||||
// fps filter wants a rational; format 1/effInterval with enough precision.
|
||||
// eof_action=pass: with -skip_frame nokey a short/all-inter clip can decode
|
||||
// to a SINGLE keyframe, and fps's default eof handling emits zero frames
|
||||
// from a one-frame stream (it never sees a later PTS to close the first
|
||||
// tick) → "Nothing was written into output". pass flushes the last frame
|
||||
// at EOF instead; on normal media it only matters at the very end, where
|
||||
// -frames:v 1 + the tile grid already bound the output.
|
||||
fps := fmt.Sprintf("1/%s", strconv.FormatFloat(effInterval, 'f', 3, 64))
|
||||
vf := fmt.Sprintf("fps=%s,scale=%d:-2,tile=%dx%d", fps, width, cols, rows)
|
||||
vf := fmt.Sprintf("fps=%s:eof_action=pass,scale=%d:-2,tile=%dx%d", fps, width, cols, rows)
|
||||
args := []string{
|
||||
"-nostdin", "-loglevel", "error", "-y",
|
||||
// Decoder-level keyframe-only mode — must precede -i (input option).
|
||||
"-skip_frame", "nokey",
|
||||
"-i", mediaPath,
|
||||
"-frames:v", "1",
|
||||
"-vf", vf,
|
||||
|
|
|
|||
|
|
@ -42,10 +42,23 @@ type AudioTrack struct {
|
|||
Default bool `json:"default"`
|
||||
}
|
||||
|
||||
// SubtitleTrack represents a single subtitle stream.
|
||||
// SubtitleTrack represents a single subtitle source — either an EMBEDDED stream
|
||||
// (the common case, identified by its ffmpeg `0:s:N` order in the slice) or an
|
||||
// EXTERNAL sidecar file sitting next to the media (Path set, External true).
|
||||
//
|
||||
// External sidecars (a `.srt`/`.ass`/`.vtt` named after the video, or one in a
|
||||
// `Subs/` subfolder) are appended AFTER all embedded tracks so the embedded
|
||||
// tracks keep slice positions equal to their `0:s:N` index — the web's
|
||||
// resolveSubtitleTracks relies on that for embedded, and switches to Path-based
|
||||
// addressing for external (served via /sub?p=<file>&i=-1).
|
||||
type SubtitleTrack struct {
|
||||
Lang string `json:"lang"`
|
||||
Codec string `json:"codec"`
|
||||
Title string `json:"title"`
|
||||
Forced bool `json:"forced"`
|
||||
// External is true for a sidecar file; false (omitted) for an embedded stream.
|
||||
External bool `json:"external,omitempty"`
|
||||
// Path is the absolute filesystem path of the sidecar file (External only).
|
||||
// Empty for embedded streams (those live inside the media container).
|
||||
Path string `json:"path,omitempty"`
|
||||
}
|
||||
|
|
|
|||
142
internal/library/subtitles.go
Normal file
142
internal/library/subtitles.go
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
package library
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/torrentclaw/unarr/internal/agent"
|
||||
)
|
||||
|
||||
// maxSubtitleBytes caps a downloaded subtitle (sane: even a long film SRT is
|
||||
// a few hundred KB; this guards against a misbehaving upstream).
|
||||
const maxSubtitleBytes = 10 << 20 // 10 MiB
|
||||
|
||||
var subtitleLangRe = regexp.MustCompile(`^[a-z]{2,3}$`)
|
||||
|
||||
var subtitleHTTPClient = &http.Client{Timeout: 30 * time.Second}
|
||||
|
||||
// FetchSubtitles downloads each requested subtitle (from our proxy URL, already
|
||||
// charset-fixed WebVTT) and writes it as a sidecar next to the media file:
|
||||
// `<basename>.<lang>.vtt`. Returns the IDs successfully written (or already
|
||||
// present) and the ones that failed (with a short reason) so the web can mark
|
||||
// them errored. Safety mirrors DeleteFiles: the media file must resolve within a
|
||||
// configured scan path before we write beside it.
|
||||
func FetchSubtitles(reqs []agent.SubtitleFetchRequest, scanPaths []string) (done []int, failed []agent.SubtitleFetchError) {
|
||||
// Resolve scan paths through symlinks too, so a symlinked root (e.g. the
|
||||
// docker bind-mount /downloads → /mnt/nas/peliculas) still matches a media
|
||||
// path that EvalSymlinks resolved to the real target. Mirrors the containment
|
||||
// check used for the resolved media path below.
|
||||
safe := make([]string, 0, len(scanPaths))
|
||||
for _, sp := range scanPaths {
|
||||
if !filepath.IsAbs(sp) {
|
||||
log.Printf("library: ignoring non-absolute scan path: %q", sp)
|
||||
continue
|
||||
}
|
||||
if real, err := filepath.EvalSymlinks(sp); err == nil {
|
||||
safe = append(safe, real)
|
||||
} else {
|
||||
safe = append(safe, filepath.Clean(sp))
|
||||
}
|
||||
}
|
||||
if len(safe) == 0 {
|
||||
log.Printf("library: no valid scan paths — refusing to write subtitle sidecars")
|
||||
for _, r := range reqs {
|
||||
failed = append(failed, agent.SubtitleFetchError{ID: r.ID, Error: "no valid scan paths"})
|
||||
}
|
||||
return nil, failed
|
||||
}
|
||||
|
||||
for _, r := range reqs {
|
||||
if err := fetchSubtitleOne(r, safe); err != nil {
|
||||
log.Printf("library: subtitle fetch %d (%q): %v", r.ID, r.FilePath, err)
|
||||
msg := err.Error()
|
||||
if len(msg) > 480 {
|
||||
msg = msg[:480]
|
||||
}
|
||||
failed = append(failed, agent.SubtitleFetchError{ID: r.ID, Error: msg})
|
||||
continue
|
||||
}
|
||||
log.Printf("library: wrote subtitle sidecar for item %d (%s)", r.ID, r.Lang)
|
||||
done = append(done, r.ID)
|
||||
}
|
||||
return done, failed
|
||||
}
|
||||
|
||||
func fetchSubtitleOne(r agent.SubtitleFetchRequest, scanPaths []string) error {
|
||||
if !filepath.IsAbs(r.FilePath) {
|
||||
return fmt.Errorf("path is not absolute: %q", r.FilePath)
|
||||
}
|
||||
lang := strings.ToLower(strings.TrimSpace(r.Lang))
|
||||
if !subtitleLangRe.MatchString(lang) {
|
||||
return fmt.Errorf("invalid language %q", r.Lang)
|
||||
}
|
||||
|
||||
// Resolve the media file (symlinks too) and confine it to a scan path.
|
||||
real, err := filepath.EvalSymlinks(filepath.Clean(r.FilePath))
|
||||
if err != nil {
|
||||
return fmt.Errorf("media file unreachable: %w", err)
|
||||
}
|
||||
if !isWithinScanPaths(real, scanPaths) {
|
||||
return fmt.Errorf("path %q is outside all scan paths", real)
|
||||
}
|
||||
|
||||
ext := filepath.Ext(real)
|
||||
sidecar := strings.TrimSuffix(real, ext) + "." + lang + ".vtt"
|
||||
if _, statErr := os.Stat(sidecar); statErr == nil {
|
||||
return nil // already present — idempotent success
|
||||
}
|
||||
|
||||
data, err := downloadSubtitle(r.URL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Write atomically: temp in the same dir, then rename. Clean up any stale
|
||||
// .tmp from a prior crash first, and on every failure path, so a partial
|
||||
// write (disk full, killed) never lingers.
|
||||
tmp := sidecar + ".tmp"
|
||||
_ = os.Remove(tmp)
|
||||
if err := os.WriteFile(tmp, data, 0o644); err != nil {
|
||||
_ = os.Remove(tmp)
|
||||
return fmt.Errorf("write temp sidecar: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmp, sidecar); err != nil {
|
||||
_ = os.Remove(tmp)
|
||||
return fmt.Errorf("rename sidecar: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func downloadSubtitle(url string) ([]byte, error) {
|
||||
// Our proxy URL is always HTTPS. Restrict to https (allow http only for a
|
||||
// local dev server) so a tampered sync response can't point the agent at an
|
||||
// internal/metadata host.
|
||||
if !strings.HasPrefix(url, "https://") &&
|
||||
!strings.HasPrefix(url, "http://localhost") &&
|
||||
!strings.HasPrefix(url, "http://127.0.0.1") {
|
||||
return nil, fmt.Errorf("subtitle url must be https")
|
||||
}
|
||||
resp, err := subtitleHTTPClient.Get(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("download: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("download status %d", resp.StatusCode)
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(resp.Body, maxSubtitleBytes))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read body: %w", err)
|
||||
}
|
||||
if len(data) == 0 {
|
||||
return nil, fmt.Errorf("empty subtitle")
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
|
@ -1,12 +1,74 @@
|
|||
package library
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/torrentclaw/unarr/internal/agent"
|
||||
)
|
||||
|
||||
// SyncOptions describes ONE library sync session — a set of batches sharing a
|
||||
// single syncStartedAt so the server can reap rows not seen by the session.
|
||||
type SyncOptions struct {
|
||||
AgentID string
|
||||
// ScanPath is the primary root, kept for pre-scanRoots servers.
|
||||
ScanPath string
|
||||
// ScanRoots lists every root this session covers (see LibrarySyncRequest).
|
||||
ScanRoots []string
|
||||
// FullCycle: the session spans every configured root — the server may reap
|
||||
// unseen rows regardless of path prefix. NEVER set it for a subtree scan.
|
||||
FullCycle bool
|
||||
// OnProgress, when non-nil, is called after each batch with (sent, total).
|
||||
OnProgress func(sent, total int)
|
||||
}
|
||||
|
||||
// SyncResult aggregates the per-batch server responses of a session.
|
||||
type SyncResult struct {
|
||||
Synced int
|
||||
Matched int
|
||||
Removed int
|
||||
}
|
||||
|
||||
// SyncBatches uploads items to the server in batches of 100 as ONE sync
|
||||
// session: every batch shares the same syncStartedAt and only the final one
|
||||
// carries isLastBatch, so the server's stale-row cleanup sees the whole cycle
|
||||
// at once. The single source of the batching protocol — shared by `unarr scan`
|
||||
// (cmd/scan.go) and the daemon auto-scan (cmd/daemon.go); before this each
|
||||
// root synced as its own session and the per-agent cleanup could reap rows of
|
||||
// roots the session never visited.
|
||||
func SyncBatches(ctx context.Context, ac *agent.Client, items []agent.LibrarySyncItem, opts SyncOptions) (SyncResult, error) {
|
||||
const batchSize = 100
|
||||
var res SyncResult
|
||||
syncStartedAt := time.Now().UTC().Format(time.RFC3339)
|
||||
for i := 0; i < len(items); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(items) {
|
||||
end = len(items)
|
||||
}
|
||||
resp, err := ac.SyncLibrary(ctx, agent.LibrarySyncRequest{
|
||||
Items: items[i:end],
|
||||
ScanPath: opts.ScanPath,
|
||||
AgentID: opts.AgentID,
|
||||
IsLastBatch: end >= len(items),
|
||||
SyncStartedAt: syncStartedAt,
|
||||
ScanRoots: opts.ScanRoots,
|
||||
FullCycle: opts.FullCycle,
|
||||
})
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
res.Synced += resp.Synced
|
||||
res.Matched += resp.Matched
|
||||
res.Removed += resp.Removed
|
||||
if opts.OnProgress != nil {
|
||||
opts.OnProgress(end, len(items))
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// relToRoot returns the file's path relative to the scan root (forward-slashed),
|
||||
// or "" when it doesn't live under root. The server stores this so streaming can
|
||||
// later reconstruct the absolute path from the agent's *current* root.
|
||||
|
|
|
|||
|
|
@ -174,11 +174,11 @@ type wgConf struct {
|
|||
dns []netip.Addr
|
||||
mtu int
|
||||
|
||||
peerPublicKey string // hex
|
||||
presharedKey string // hex (optional)
|
||||
endpoint string // resolved ip:port
|
||||
allowedIPs []string
|
||||
keepalive int
|
||||
peerPublicKey string // hex
|
||||
presharedKey string // hex (optional)
|
||||
endpoint string // resolved ip:port
|
||||
allowedIPs []string
|
||||
keepalive int
|
||||
}
|
||||
|
||||
func (w *wgConf) uapi() string {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue