diff --git a/go.mod b/go.mod index a47f6e3..f8d42d8 100644 --- a/go.mod +++ b/go.mod @@ -14,8 +14,10 @@ require ( github.com/huin/goupnp v1.3.0 github.com/olekukonko/tablewriter v1.1.4 github.com/spf13/cobra v1.10.2 + github.com/spf13/pflag v1.0.10 github.com/torrentclaw/go-client v0.2.0 golang.org/x/term v0.43.0 + golang.org/x/text v0.37.0 golang.org/x/time v0.15.0 golang.zx2c4.com/wireguard v0.0.0-20250521234502-f333402bd9cb ) @@ -113,7 +115,6 @@ require ( github.com/rivo/uniseg v0.4.7 // indirect github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/spf13/pflag v1.0.10 // indirect github.com/tidwall/btree v1.8.1 // indirect github.com/wlynxg/anet v0.0.5 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect @@ -127,7 +128,6 @@ require ( golang.org/x/net v0.54.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.44.0 // indirect - golang.org/x/text v0.37.0 // indirect golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect gvisor.dev/gvisor v0.0.0-20250503011706-39ed1f5ac29c // indirect lukechampine.com/blake3 v1.4.1 // indirect diff --git a/internal/engine/hls.go b/internal/engine/hls.go index 30261a6..1eab7e0 100644 --- a/internal/engine/hls.go +++ b/internal/engine/hls.go @@ -574,13 +574,18 @@ func (s *HLSSession) ProbeInfo() map[string]any { } subs := make([]map[string]any, 0, len(s.probe.SubtitleTracks)) for _, sb := range s.probe.SubtitleTracks { + // `external`/`path` let the stream server attach a tokened /sub vttUrl + // (path-addressed for sidecars, index-addressed for embedded). `path` is + // stripped after the URL is built so the raw path isn't doubled in JSON. subs = append(subs, map[string]any{ - "index": sb.Index, - "lang": sb.Lang, - "codec": sb.Codec, - "title": sb.Title, - "forced": sb.Forced, - "text": sb.IsTextSubtitle(), + "index": sb.Index, + "lang": sb.Lang, + "codec": sb.Codec, + "title": sb.Title, + "forced": sb.Forced, + "text": sb.IsTextSubtitle(), + "external": sb.External, + "path": sb.Path, }) } return map[string]any{ diff --git a/internal/engine/probe.go b/internal/engine/probe.go index 57d1e5a..70176e5 100644 --- a/internal/engine/probe.go +++ b/internal/engine/probe.go @@ -50,11 +50,15 @@ type ProbeAudioTrack struct { // Codec discriminates text (srt/ass/webvtt → extract to WebVTT) vs bitmap // (pgs/dvbsub → require burn-in). type ProbeSubtitleTrack struct { - Index int // 0-based subtitle stream index (ffmpeg -map 0:s:Index) + Index int // 0-based EMBEDDED subtitle stream index (ffmpeg -map 0:s:Index). Unused when External. Lang string // ISO 639-1 Codec string // lowercased — "subrip", "ass", "webvtt", "hdmv_pgs_subtitle", ... Title string Forced bool + // External marks a sidecar file (served via /sub?p=&i=-1) rather than + // an embedded stream. Path is its absolute filesystem path (External only). + External bool + Path string } // IsTextSubtitle reports whether a subtitle codec can be extracted to WebVTT @@ -134,14 +138,27 @@ func ProbeFile(ctx context.Context, ffprobePath, filePath string) (*StreamProbe, } if len(mi.Subtitles) > 0 { probe.SubtitleTracks = make([]ProbeSubtitleTrack, 0, len(mi.Subtitles)) - for i, s := range mi.Subtitles { - probe.SubtitleTracks = append(probe.SubtitleTracks, ProbeSubtitleTrack{ - Index: i, - Lang: s.Lang, - Codec: strings.ToLower(s.Codec), - Title: s.Title, - Forced: s.Forced, - }) + // Embedded streams come first (ffprobe order); external sidecars are + // appended after. Count embedded separately so each embedded track's + // Index is its true `0:s:N` value regardless of how many externals trail + // it; externals get Index=-1 and address by Path instead. + embeddedIdx := 0 + for _, s := range mi.Subtitles { + t := ProbeSubtitleTrack{ + Lang: s.Lang, + Codec: strings.ToLower(s.Codec), + Title: s.Title, + Forced: s.Forced, + External: s.External, + Path: s.Path, + } + if s.External { + t.Index = -1 + } else { + t.Index = embeddedIdx + embeddedIdx++ + } + probe.SubtitleTracks = append(probe.SubtitleTracks, t) } } storeProbeCache(filePath, probe) diff --git a/internal/engine/stream_server.go b/internal/engine/stream_server.go index 6740246..2d972ad 100644 --- a/internal/engine/stream_server.go +++ b/internal/engine/stream_server.go @@ -10,6 +10,7 @@ import ( "log" "net" "net/http" + "net/url" "os" "os/exec" "path/filepath" @@ -733,7 +734,9 @@ func (ss *StreamServer) hlsHandler(w http.ResponseWriter, r *http.Request) { case resource == "probe.json": w.Header().Set("Content-Type", "application/json") w.Header().Set("Cache-Control", "no-cache") - _ = json.NewEncoder(w).Encode(session.ProbeInfo()) + info := session.ProbeInfo() + ss.attachSubtitleVTTURLs(info, session.cfg.sourceRef()) + _ = json.NewEncoder(w).Encode(info) case resource == "video/index.m3u8": session.ServeVideoPlaylist(w, r) case resource == "video/init.mp4": @@ -1224,8 +1227,11 @@ func (ss *StreamServer) subtitleHandler(w http.ResponseWriter, r *http.Request) http.Error(w, "missing path", http.StatusBadRequest) return } + // index >= 0 → EMBEDDED stream index (-map 0:s:N) of the media at `p`. + // index < 0 → EXTERNAL sidecar: `p` IS the subtitle file; the whole file is + // the track. Both bind the token to (path, index) so a tampered p/i fails. index, err := strconv.Atoi(q.Get("i")) - if err != nil || index < 0 { + if err != nil { http.Error(w, "bad index", http.StatusBadRequest) return } @@ -1235,21 +1241,30 @@ func (ss *StreamServer) subtitleHandler(w http.ResponseWriter, r *http.Request) http.Error(w, "not found", http.StatusNotFound) return } - rawPath = ss.healMediaPath(rawPath) // host→container base-path skew (see /thumbnail) - if fi, statErr := os.Stat(rawPath); statErr != nil || !fi.Mode().IsRegular() { - http.Error(w, "not found", http.StatusNotFound) - return - } - // Cache hit: serve a fresh sidecar (written by the scan-time prewarm or a - // prior request) instantly, skipping ffmpeg. This is also what makes huge - // remuxes work — the prewarm extracts without the on-demand HTTP timeout - // below, so by play time the hit avoids the 60s ceiling that was returning - // 500s on 50GB+ files. Checked BEFORE the ffmpeg guard so a pre-warmed track - // is still serveable even if ffmpeg was removed after the cache was filled. - if vtt, ok := mediainfo.ReadCachedSubtitle(rawPath, index); ok { - ss.writeVTT(w, vtt) - return + external := index < 0 + // A debrid/HLS-from-URL source has no local file — ffmpeg reads the URL + // directly. Skip the path heal + regular-file stat + on-disk cache for those; + // only local files get the sidecar cache. + isURL := strings.Contains(rawPath, "://") + langHint := q.Get("l") // ISO 639-1 charset hint for external sidecar decoding + + if !isURL { + rawPath = ss.healMediaPath(rawPath) // host→container base-path skew (see /thumbnail) + if fi, statErr := os.Stat(rawPath); statErr != nil || !fi.Mode().IsRegular() { + http.Error(w, "not found", http.StatusNotFound) + return + } + // Cache hit: serve a fresh sidecar (written by the scan-time prewarm or a + // prior request) instantly, skipping ffmpeg. This is also what makes huge + // remuxes work — the prewarm extracts without the on-demand HTTP timeout + // below, so by play time the hit avoids the 60s ceiling that was returning + // 500s on 50GB+ files. Checked BEFORE the ffmpeg guard so a pre-warmed track + // is still serveable even if ffmpeg was removed after the cache was filled. + if vtt, ok := mediainfo.ReadCachedSubtitle(rawPath, index); ok { + ss.writeVTT(w, vtt) + return + } } // Beyond here we must extract on demand, which needs ffmpeg. @@ -1265,15 +1280,23 @@ func (ss *StreamServer) subtitleHandler(w http.ResponseWriter, r *http.Request) ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second) defer cancel() - out, err := mediainfo.ExtractSubtitleVTT(ctx, ss.ffmpegPath, rawPath, index) + var out []byte + if external { + // Standalone sidecar file: transcode charset → UTF-8 (langHint guides the + // code-page guess) then ffmpeg → WebVTT. + out, err = mediainfo.ExtractExternalSubtitleVTT(ctx, ss.ffmpegPath, rawPath, langHint) + } else { + out, err = mediainfo.ExtractSubtitleVTT(ctx, ss.ffmpegPath, rawPath, index) + } if err != nil { - log.Printf("[sub] extract failed (i=%d path=%q): %v", index, rawPath, err) + log.Printf("[sub] extract failed (i=%d path=%q external=%v url=%v): %v", index, rawPath, external, isURL, err) http.Error(w, "subtitle extract failed", http.StatusInternalServerError) return } // Write-through so the next request is a cache hit. Best-effort: a read-only - // media mount just logs and serves the in-memory bytes. - if ss.cacheSubtitles { + // media mount just logs and serves the in-memory bytes. URL sources have no + // stable on-disk anchor for the sidecar cache → skip. + if ss.cacheSubtitles && !isURL { if werr := mediainfo.WriteCachedSubtitle(rawPath, index, out); werr != nil { log.Printf("[sub] cache write skipped (i=%d path=%q): %v", index, rawPath, werr) } @@ -1281,6 +1304,60 @@ func (ss *StreamServer) subtitleHandler(w http.ResponseWriter, r *http.Request) ss.writeVTT(w, out) } +// attachSubtitleVTTURLs enriches a ProbeInfo map's "subtitles" entries with a +// ready-to-use, tokened `vttUrl` for every TEXT track, so the web player can +// attach s for ANY play method (torrent/debrid HLS included) without the +// server needing the source path — it's the single subtitle wiring path that +// makes embedded subs work on streams that were never library-scanned. +// +// - embedded (external=false): /sub?p=&i=&t= +// - external (external=true) : /sub?p=&i=-1&t=&l= +// +// The token uses the SAME streamScopeSub(path,index) the web mints with, so a +// library-scanned track and a probe-derived one address identically. The raw +// "path" key is removed after the URL is built (it's encoded in the URL already). +// URLs are root-relative; the player resolves them against the funnel origin it +// fetched probe.json from. Bitmap tracks get no vttUrl (burn-in only). +func (ss *StreamServer) attachSubtitleVTTURLs(info map[string]any, srcRef string) { + subsAny, ok := info["subtitles"].([]map[string]any) + if !ok { + return + } + now := time.Now() + for _, sb := range subsAny { + isText, _ := sb["text"].(bool) + if !isText { + delete(sb, "path") + continue + } + external, _ := sb["external"].(bool) + var p string + var idx int + if external { + p, _ = sb["path"].(string) + idx = -1 + } else { + p = srcRef + if iv, ok := sb["index"].(int); ok { + idx = iv + } + } + if p == "" { + delete(sb, "path") + continue + } + tok := mintStreamToken(ss.streamSecret, streamScopeSub(p, idx), now) + u := "/sub?p=" + url.QueryEscape(p) + "&i=" + strconv.Itoa(idx) + "&t=" + tok + if external { + if lang, _ := sb["lang"].(string); lang != "" && lang != "und" { + u += "&l=" + url.QueryEscape(lang) + } + } + sb["vttUrl"] = u + delete(sb, "path") + } +} + // writeVTT writes the standard WebVTT response headers + body for both the // cache-hit and freshly-extracted paths of subtitleHandler. func (ss *StreamServer) writeVTT(w http.ResponseWriter, vtt []byte) { diff --git a/internal/library/mediainfo/charset.go b/internal/library/mediainfo/charset.go new file mode 100644 index 0000000..68bac3c --- /dev/null +++ b/internal/library/mediainfo/charset.go @@ -0,0 +1,139 @@ +package mediainfo + +import ( + "bytes" + "strings" + "unicode/utf8" + + "golang.org/x/text/encoding" + "golang.org/x/text/encoding/charmap" + "golang.org/x/text/encoding/japanese" + "golang.org/x/text/encoding/korean" + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/encoding/traditionalchinese" + "golang.org/x/text/encoding/unicode" + "golang.org/x/text/transform" +) + +// Subtitle charset normalisation. +// +// External subtitle files are routinely NOT UTF-8: legacy .srt files come in the +// uploader's local code page (Windows-1252 Western, Windows-1256 Arabic, GBK +// Chinese, Shift-JIS Japanese, …). Feeding those raw to ffmpeg → WebVTT yields +// mojibake. We detect the encoding and transcode to UTF-8 before extraction. +// +// Detection order: BOM (authoritative) → valid UTF-8 → a code page chosen from +// the track's declared language (from its filename, e.g. ".ar.srt"). The +// language hint is the reliable signal we have without a full statistical +// detector: an Arabic sub that isn't UTF-8 is almost certainly Windows-1256, a +// Russian one Windows-1251, and so on. Western European is the safe default. + +// legacyEncodingForLang returns the most likely single-byte / CJK encoding for a +// non-UTF-8 subtitle in the given language hint. The hint is normally an ISO +// 639-1 code, but Chinese carries a script suffix ("zh-hant" / "zh-tw") so a +// Traditional sidecar decodes as Big5 instead of GBK (decoding Big5 bytes as GBK +// is mojibake — and anime fansubs routinely ship both chs AND cht). Default: +// Windows-1252. +func legacyEncodingForLang(lang string) encoding.Encoding { + switch strings.ToLower(strings.TrimSpace(lang)) { + case "ar", "fa", "ur": // Arabic script + return charmap.Windows1256 + case "ru", "uk", "bg", "sr", "mk": // Cyrillic + return charmap.Windows1251 + case "el": // Greek + return charmap.Windows1253 + case "he": // Hebrew + return charmap.Windows1255 + case "tr": // Turkish + return charmap.Windows1254 + case "th": // Thai + return charmap.Windows874 + case "zh-hant", "zh_hant", "zh-tw", "zh-hk", "zhtw": // Traditional Chinese + return traditionalchinese.Big5 + case "zh", "zh-hans", "zh-cn": // Simplified Chinese (covers most pirate releases) + return simplifiedchinese.GBK + case "ja": // Japanese + return japanese.ShiftJIS + case "ko": // Korean + return korean.EUCKR + case "vi": // Vietnamese + return charmap.Windows1258 + case "pl", "cs", "sk", "hu", "ro", "hr", "sl": // Central European + return charmap.Windows1250 + case "lt", "lv", "et": // Baltic + return charmap.Windows1257 + default: // Western European + everything else + return charmap.Windows1252 + } +} + +// DecodeSubtitleToUTF8 returns the bytes as UTF-8, transcoding from a detected +// legacy encoding when needed. The returned name is for logging ("utf-8", +// "bom-utf16le", "windows-1256", …). Never fails: a transcode error falls back +// to the original bytes (ffmpeg may still cope). +func DecodeSubtitleToUTF8(data []byte, langHint string) ([]byte, string) { + // BOM wins — it's unambiguous. + switch { + case bytes.HasPrefix(data, []byte{0xEF, 0xBB, 0xBF}): + return data[3:], "bom-utf8" + case bytes.HasPrefix(data, []byte{0xFF, 0xFE}): + return decodeWith(data, unicode.UTF16(unicode.LittleEndian, unicode.UseBOM), "bom-utf16le") + case bytes.HasPrefix(data, []byte{0xFE, 0xFF}): + return decodeWith(data, unicode.UTF16(unicode.BigEndian, unicode.UseBOM), "bom-utf16be") + } + // Already valid UTF-8 → no transcode (ASCII is a subset, so plain English + // srt files hit this). + if utf8.Valid(data) { + return data, "utf-8" + } + // Non-UTF-8: transcode from the language's likely code page. + enc := legacyEncodingForLang(langHint) + out, name := decodeWith(data, enc, encodingName(enc)) + return out, name +} + +// decodeWith transforms data through enc's decoder to UTF-8. On error returns the +// original bytes (best-effort) with the name suffixed "(raw)". +func decodeWith(data []byte, enc encoding.Encoding, name string) ([]byte, string) { + out, _, err := transform.Bytes(enc.NewDecoder(), data) + if err != nil || len(out) == 0 { + return data, name + "(raw)" + } + return out, name +} + +// encodingName maps a known encoding back to a short label for logs. +func encodingName(enc encoding.Encoding) string { + switch enc { + case charmap.Windows1250: + return "windows-1250" + case charmap.Windows1251: + return "windows-1251" + case charmap.Windows1252: + return "windows-1252" + case charmap.Windows1253: + return "windows-1253" + case charmap.Windows1254: + return "windows-1254" + case charmap.Windows1255: + return "windows-1255" + case charmap.Windows1256: + return "windows-1256" + case charmap.Windows1257: + return "windows-1257" + case charmap.Windows1258: + return "windows-1258" + case charmap.Windows874: + return "windows-874" + case simplifiedchinese.GBK: + return "gbk" + case traditionalchinese.Big5: + return "big5" + case japanese.ShiftJIS: + return "shift-jis" + case korean.EUCKR: + return "euc-kr" + default: + return "legacy" + } +} diff --git a/internal/library/mediainfo/charset_test.go b/internal/library/mediainfo/charset_test.go new file mode 100644 index 0000000..c3672b1 --- /dev/null +++ b/internal/library/mediainfo/charset_test.go @@ -0,0 +1,64 @@ +package mediainfo + +import ( + "testing" + + "golang.org/x/text/encoding/charmap" + "golang.org/x/text/transform" +) + +func TestDecodeSubtitleToUTF8_PlainASCII(t *testing.T) { + in := []byte("Hello world") + out, name := DecodeSubtitleToUTF8(in, "en") + if string(out) != "Hello world" || name != "utf-8" { + t.Fatalf("ASCII passthrough failed: %q %s", out, name) + } +} + +func TestDecodeSubtitleToUTF8_BOMStripped(t *testing.T) { + in := append([]byte{0xEF, 0xBB, 0xBF}, []byte("café")...) + out, name := DecodeSubtitleToUTF8(in, "fr") + if string(out) != "café" || name != "bom-utf8" { + t.Fatalf("UTF-8 BOM strip failed: %q %s", out, name) + } +} + +func TestDecodeSubtitleToUTF8_Windows1252(t *testing.T) { + // "café" encoded in Windows-1252 (é = 0xE9) is NOT valid UTF-8. + enc1252, _, err := transform.Bytes(charmap.Windows1252.NewEncoder(), []byte("café")) + if err != nil { + t.Fatal(err) + } + out, name := DecodeSubtitleToUTF8(enc1252, "fr") + if string(out) != "café" { + t.Fatalf("Windows-1252 decode failed: got %q (%s)", out, name) + } + if name != "windows-1252" { + t.Fatalf("expected windows-1252, got %s", name) + } +} + +func TestDecodeSubtitleToUTF8_TraditionalChineseBig5(t *testing.T) { + // 繁 (U+7E41) in Big5 is 0xC1 0x63. Decoding it as GBK would be mojibake, so + // the zh-Hant hint must route to Big5. + in := []byte{0xC1, 0x63} + out, name := DecodeSubtitleToUTF8(in, "zh-Hant") + if name != "big5" { + t.Fatalf("expected big5 for zh-Hant, got %s", name) + } + if string(out) != "繁" { + t.Fatalf("Big5 decode failed: got %q", out) + } +} + +func TestDecodeSubtitleToUTF8_ArabicByLang(t *testing.T) { + // Arabic letter ا (U+0627) is 0xC7 in Windows-1256. + in := []byte{0xC7} + out, name := DecodeSubtitleToUTF8(in, "ar") + if name != "windows-1256" { + t.Fatalf("expected windows-1256 for Arabic, got %s", name) + } + if string(out) != "ا" { + t.Fatalf("Arabic decode failed: got %q", out) + } +} diff --git a/internal/library/mediainfo/ffprobe.go b/internal/library/mediainfo/ffprobe.go index c850029..3f1628f 100644 --- a/internal/library/mediainfo/ffprobe.go +++ b/internal/library/mediainfo/ffprobe.go @@ -95,6 +95,16 @@ func ExtractMediaInfo(ctx context.Context, ffprobePath, filePath string) (*Media if integ := assessIntegrity(stderr.String(), mi); integ != nil { mi.Integrity = integ } + // Append external sidecar subtitles (a .srt/.ass next to the video, or a + // Subs/ bundle) AFTER the embedded streams, so embedded keep slice positions + // == their 0:s:N index. Local files only — a remote URL has no directory to + // scan (debrid streams rely on embedded subs from the URL). Best-effort: + // DiscoverSidecarSubtitles returns nil on an unreadable dir. + if !strings.Contains(filePath, "://") { + if ext := DiscoverSidecarSubtitles(filePath); len(ext) > 0 { + mi.Subtitles = append(mi.Subtitles, ext...) + } + } return mi, nil } diff --git a/internal/library/mediainfo/gallery_real_test.go b/internal/library/mediainfo/gallery_real_test.go new file mode 100644 index 0000000..4aa9280 --- /dev/null +++ b/internal/library/mediainfo/gallery_real_test.go @@ -0,0 +1,206 @@ +package mediainfo + +import ( + "context" + "os" + "path/filepath" + "sort" + "strings" + "testing" + "time" +) + +// TestGalleryReal is a manual end-to-end harness against a REAL media library. +// It is skipped unless GALLERY_DIR is set, so it never runs in CI. +// +// GALLERY_DIR=/mnt/nas/peliculas go test ./internal/library/mediainfo/ \ +// -run TestGalleryReal -v -timeout 30m +// +// It surveys every video file (embedded subs via ffprobe + discovered sidecars), +// then actually extracts WebVTT for one representative of each kind and checks the +// output is a valid, non-empty WEBVTT document. +func TestGalleryReal(t *testing.T) { + dir := os.Getenv("GALLERY_DIR") + if dir == "" { + t.Skip("set GALLERY_DIR to run the real-gallery survey") + } + ffprobe := envOr("FFPROBE", "ffprobe") + ffmpeg := envOr("FFMPEG", "ffmpeg") + + videoExt := map[string]bool{".mkv": true, ".mp4": true, ".avi": true, ".m4v": true, ".webm": true, ".mov": true, ".ts": true} + var videos []string + _ = filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + if strings.Contains(p, "/.unarr/") || strings.Contains(p, "/.Trash") || strings.Contains(p, "/@eaDir/") { + return nil + } + if videoExt[strings.ToLower(filepath.Ext(p))] { + videos = append(videos, p) + } + return nil + }) + sort.Strings(videos) + t.Logf("found %d video files under %s", len(videos), dir) + + type cat struct { + embTextCodecs map[string]int // codec → count of files + embBitmap map[string]int + extCodecs map[string]int + filesEmbText []string + filesEmbBitmap []string + filesExt []string + errs int + } + c := cat{embTextCodecs: map[string]int{}, embBitmap: map[string]int{}, extCodecs: map[string]int{}} + + for _, v := range videos { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + mi, err := ExtractMediaInfo(ctx, ffprobe, v) + cancel() + if err != nil { + c.errs++ + t.Logf("PROBE ERR %s: %v", filepath.Base(v), err) + continue + } + var sawEmbText, sawEmbBitmap, sawExt bool + for _, s := range mi.Subtitles { + codec := strings.ToLower(s.Codec) + switch { + case s.External: + c.extCodecs[codec]++ + sawExt = true + case IsTextSubtitleCodec(codec): + c.embTextCodecs[codec]++ + sawEmbText = true + default: + c.embBitmap[codec]++ + sawEmbBitmap = true + } + } + if sawEmbText { + c.filesEmbText = append(c.filesEmbText, v) + } + if sawEmbBitmap { + c.filesEmbBitmap = append(c.filesEmbBitmap, v) + } + if sawExt { + c.filesExt = append(c.filesExt, v) + } + } + + t.Logf("=== CENSUS ===") + t.Logf("probe errors: %d", c.errs) + t.Logf("embedded TEXT codecs (files w/ track): %v", c.embTextCodecs) + t.Logf("embedded BITMAP codecs (burn-in only): %v", c.embBitmap) + t.Logf("external SIDECAR codecs: %v", c.extCodecs) + t.Logf("files w/ embedded text: %d | w/ embedded bitmap: %d | w/ external sidecar: %d", + len(c.filesEmbText), len(c.filesEmbBitmap), len(c.filesExt)) + + // --- Real extraction checks --- + validVTT := func(b []byte) bool { + return len(b) > 0 && strings.HasPrefix(strings.TrimSpace(string(b)), "WEBVTT") + } + + // Embedded text: extract index 0 of the first such file. + if len(c.filesEmbText) > 0 { + f := c.filesEmbText[0] + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + out, err := ExtractSubtitleVTT(ctx, ffmpeg, f, 0) + cancel() + if err != nil || !validVTT(out) { + t.Errorf("EMBEDDED extract FAILED for %s: err=%v len=%d", filepath.Base(f), err, len(out)) + } else { + t.Logf("EMBEDDED extract OK: %s → %d bytes WebVTT", filepath.Base(f), len(out)) + } + } + + // External sidecar: find one and extract it via the path-addressed function. + if len(c.filesExt) > 0 { + f := c.filesExt[0] + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + mi, _ := ExtractMediaInfo(ctx, ffprobe, f) + cancel() + var subPath, lang string + for _, s := range mi.Subtitles { + if s.External { + subPath, lang = s.Path, s.Lang + break + } + } + ctx2, cancel2 := context.WithTimeout(context.Background(), 60*time.Second) + out, err := ExtractExternalSubtitleVTT(ctx2, ffmpeg, subPath, lang) + cancel2() + if err != nil || !validVTT(out) { + t.Errorf("EXTERNAL extract FAILED for %s: err=%v len=%d", filepath.Base(subPath), err, len(out)) + } else { + t.Logf("EXTERNAL extract OK: %s (lang=%s) → %d bytes WebVTT", filepath.Base(subPath), lang, len(out)) + } + } +} + +func envOr(k, def string) string { + if v := os.Getenv(k); v != "" { + return v + } + return def +} + +// TestGalleryExtractAllSidecars extracts EVERY discovered sidecar in the gallery +// and reports any that fail — the real proof the external path is robust across +// formats/charsets. Skipped unless GALLERY_DIR is set. +func TestGalleryExtractAllSidecars(t *testing.T) { + dir := os.Getenv("GALLERY_DIR") + if dir == "" { + t.Skip("set GALLERY_DIR") + } + ffmpeg := envOr("FFMPEG", "ffmpeg") + var subs []SubtitleTrack + _ = filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() || strings.Contains(p, "/.unarr/") || strings.Contains(p, "/.Trash") || strings.Contains(p, "/@eaDir/") { + return nil + } + ext := strings.ToLower(filepath.Ext(p)) + if videoOf(ext) { + subs = append(subs, DiscoverSidecarSubtitles(p)...) + } + return nil + }) + // Dedupe by path. + seen := map[string]bool{} + var uniq []SubtitleTrack + for _, s := range subs { + if !seen[s.Path] { + seen[s.Path] = true + uniq = append(uniq, s) + } + } + t.Logf("discovered %d unique sidecar subtitle files", len(uniq)) + fails := 0 + for _, s := range uniq { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + out, err := ExtractExternalSubtitleVTT(ctx, ffmpeg, s.Path, s.Lang) + cancel() + ok := len(out) > 0 && strings.HasPrefix(strings.TrimSpace(string(out)), "WEBVTT") + if err != nil || !ok { + fails++ + t.Errorf("FAIL %s (lang=%s codec=%s): err=%v len=%d", filepath.Base(s.Path), s.Lang, s.Codec, err, len(out)) + } else { + t.Logf("OK %s (lang=%s codec=%s) → %d bytes", filepath.Base(s.Path), s.Lang, s.Codec, len(out)) + } + } + if fails > 0 { + t.Errorf("%d/%d sidecar extractions failed", fails, len(uniq)) + } else { + t.Logf("all %d sidecar extractions produced valid WebVTT", len(uniq)) + } +} + +func videoOf(ext string) bool { + switch ext { + case ".mkv", ".mp4", ".avi", ".m4v", ".webm", ".mov", ".ts": + return true + } + return false +} diff --git a/internal/library/mediainfo/lang.go b/internal/library/mediainfo/lang.go index 0a5d42f..10acae8 100644 --- a/internal/library/mediainfo/lang.go +++ b/internal/library/mediainfo/lang.go @@ -64,7 +64,13 @@ var langNormalize = map[string]string{ "mlt": "mt", "mt": "mt", "swa": "sw", "sw": "sw", "afr": "af", "af": "af", - "lat": "la", "la": "la", + "kan": "kn", "kn": "kn", + "mal": "ml", "ml": "ml", + "mar": "mr", "mr": "mr", + "pan": "pa", "pa": "pa", + "guj": "gu", "gu": "gu", + "kann": "kn", + "lat": "la", "la": "la", // Full English names (ffprobe sometimes returns these instead of codes) "english": "en", "spanish": "es", "french": "fr", "german": "de", diff --git a/internal/library/mediainfo/sidecar.go b/internal/library/mediainfo/sidecar.go index 47bd1a7..de16c57 100644 --- a/internal/library/mediainfo/sidecar.go +++ b/internal/library/mediainfo/sidecar.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "log" "math" "os" "os/exec" @@ -148,6 +149,66 @@ func ExtractSubtitleVTT(ctx context.Context, ffmpegPath, mediaPath string, index return out, nil } +// ExtractExternalSubtitleVTT converts a STANDALONE sidecar subtitle file (a +// .srt/.ass/.ssa/.vtt sitting next to the media) to WebVTT. Unlike the embedded +// path it has no stream index — the whole file is the track. It first transcodes +// the bytes to UTF-8 (legacy code pages → mojibake otherwise; see charset.go) +// using the track's language as the detection hint, then runs ffmpeg to emit +// WebVTT. The UTF-8 bytes go through a temp file with the ORIGINAL extension so +// ffmpeg selects the right demuxer (.srt→subrip, .ass→ass, .vtt→webvtt), and +// `-sub_charenc UTF-8` stops ffmpeg from re-guessing what we already decoded. +func ExtractExternalSubtitleVTT(ctx context.Context, ffmpegPath, subPath, langHint string) ([]byte, error) { + raw, err := os.ReadFile(subPath) + if err != nil { + return nil, fmt.Errorf("read sidecar subtitle: %w", err) + } + if len(raw) == 0 { + return nil, errors.New("sidecar subtitle is empty") + } + utf8Bytes, encName := DecodeSubtitleToUTF8(raw, langHint) + // A "(raw)" suffix means the legacy transcode failed and we're passing the + // original bytes through — the likeliest cause of user-visible mojibake, so + // leave a trail to diagnose it in the field. + if strings.HasSuffix(encName, "(raw)") { + log.Printf("[sub] external charset transcode fell back to raw bytes (%s, lang=%q): possible mojibake", filepath.Base(subPath), langHint) + } + + ext := strings.ToLower(filepath.Ext(subPath)) + if ext == "" { + ext = ".srt" + } + tmpDir, err := os.MkdirTemp("", "unarr-extsub-") + if err != nil { + return nil, err + } + defer func() { _ = os.RemoveAll(tmpDir) }() + tmpIn := filepath.Join(tmpDir, "in"+ext) + if werr := os.WriteFile(tmpIn, utf8Bytes, 0o600); werr != nil { + return nil, werr + } + + args := []string{ + "-nostdin", + "-loglevel", "error", + "-sub_charenc", "UTF-8", + "-i", tmpIn, + "-c:s", "webvtt", + "-f", "webvtt", + "-", + } + cmd := exec.CommandContext(ctx, ffmpegPath, args...) + var stderr strings.Builder + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("ffmpeg external subtitle extract: %w: %s", err, strings.TrimSpace(stderr.String())) + } + if len(out) == 0 { + return nil, errors.New("ffmpeg produced no subtitle output") + } + return out, nil +} + // ExtractSubtitlesVTTMulti extracts several text subtitle streams in a SINGLE // ffmpeg pass. The expensive part of subtitle extraction is demuxing the whole // container (subtitle packets are interleaved across the runtime), so a 60GB diff --git a/internal/library/mediainfo/sidecar_subs.go b/internal/library/mediainfo/sidecar_subs.go new file mode 100644 index 0000000..94ec84b --- /dev/null +++ b/internal/library/mediainfo/sidecar_subs.go @@ -0,0 +1,207 @@ +package mediainfo + +import ( + "os" + "path/filepath" + "strings" +) + +// External (sidecar) subtitle discovery. +// +// A huge share of torrents — anime fansubs especially — ship subtitles as +// SEPARATE files, not embedded streams: a `.srt`/`.ass` named after the video, +// or a bundle inside a `Subs/` (or `Subtitles/`) subfolder. ffprobe on the video +// container never sees these, so the scan recorded zero subtitles for them +// (e.g. ToonsHub "MSubs" releases). This module finds those files so they become +// real, selectable tracks served via the /sub endpoint (path-based, i=-1). +// +// Only TEXT formats are surfaced (srt/ass/ssa/vtt, and a lone .sub). VobSub +// (.idx + .sub) is bitmap — no text form — so it's skipped here; bitmap subs are +// burn-in only and external bitmap burn-in isn't wired. + +// subFolderNames are common subfolder names that hold a release's subtitle +// bundle. Matched case-insensitively. Files inside belong to the sibling media. +var subFolderNames = map[string]bool{ + "subs": true, "subtitles": true, "sub": true, "subtitle": true, +} + +// sidecarSubExts maps a subtitle file extension to its ffmpeg-style codec name. +// The codec drives the web's text-vs-bitmap classification (isTextSubtitleCodec). +var sidecarSubExts = map[string]string{ + ".srt": "subrip", + ".ass": "ass", + ".ssa": "ssa", + ".vtt": "webvtt", + ".sub": "subrip", // MicroDVD/text — UNLESS paired with a .idx (VobSub, handled below) +} + +// forcedTokens / sdhTokens are filename markers that refine a sidecar's role. +var forcedTokens = map[string]bool{"forced": true, "forzado": true, "forces": true} +var sdhTokens = map[string]bool{"sdh": true, "cc": true, "hi": false} // "hi" is also Hindi → don't treat as SDH + +// sidecarLangAliases maps RELEASE-NAMING subtitle tokens (fansub/scene shorthand +// NOT covered by the ISO 639-1/2 normaliser) to a language hint. Two things make +// this necessary beyond NormalizeLang: +// - Chinese SCRIPT matters for charset: Simplified (chs/sc/gb) is GBK, +// Traditional (cht/tc/big5) is Big5 — decoding one as the other is mojibake. +// We keep the script in the hint ("zh" vs "zh-Hant") so legacyEncodingForLang +// picks the right code page. Anime fansubs routinely ship both. +// - lat/latino/vostfr etc. aren't ISO at all and would fall to "und". +// +// Applied ONLY to sidecar filenames, not ffprobe metadata, so it can't clash with +// the global langNormalize ("lat"→Latin there). Plain ISO codes (eng/spa/…) are +// intentionally left to NormalizeLang. +var sidecarLangAliases = map[string]string{ + "chs": "zh", "sc": "zh", "gb": "zh", "gbk": "zh", "hans": "zh", // Simplified → GBK + "cht": "zh-Hant", "tc": "zh-Hant", "big5": "zh-Hant", "hant": "zh-Hant", // Traditional → Big5 + "lat": "es", "latino": "es", "esp": "es", "español": "es", "espanol": "es", + "vostfr": "fr", "vff": "fr", "vf": "fr", + "ptbr": "pt", "pt-br": "pt", "bra": "pt", +} + +// DiscoverSidecarSubtitles finds external subtitle files for a local media file: +// siblings named after the video, plus everything in a Subs/Subtitles subfolder. +// Returns text tracks only, each with External=true and an absolute Path. Safe on +// any path — returns nil if the directory can't be read (best-effort, like the +// rest of the scan). Never call for a remote URL source (no local directory). +// +// NOTE: discovered sidecars are NOT deduped against embedded streams of the same +// language. That's deliberate — a `Movie.en.srt` next to a video that also has an +// embedded English stream is usually a DIFFERENT track (full vs SDH, retimed, or +// a better translation), so silently dropping either would hide a choice the user +// may want. Both surface as separate, distinctly-labelled entries. +func DiscoverSidecarSubtitles(mediaPath string) []SubtitleTrack { + if mediaPath == "" || strings.Contains(mediaPath, "://") { + return nil + } + dir := filepath.Dir(mediaPath) + videoBase := strings.TrimSuffix(filepath.Base(mediaPath), filepath.Ext(mediaPath)) + videoBaseLower := strings.ToLower(videoBase) + + var out []SubtitleTrack + seen := make(map[string]bool) // absolute path dedupe + + // 1. Siblings in the media's own directory whose name starts with the video + // base name: "Movie.srt", "Movie.en.srt", "Movie.en.forced.ass", … + addFromDir(dir, func(name string) bool { + return strings.HasPrefix(strings.ToLower(name), videoBaseLower) + }, videoBase, &out, seen) + + // 2. A Subs/Subtitles subfolder: take EVERY subtitle file (the whole folder + // belongs to this release). Filenames there are usually language-named + // ("2_English.srt", "spa.ass") with no video-base prefix. + if entries, err := os.ReadDir(dir); err == nil { + for _, e := range entries { + if e.IsDir() && subFolderNames[strings.ToLower(e.Name())] { + addFromDir(filepath.Join(dir, e.Name()), func(string) bool { return true }, "", &out, seen) + } + } + } + return out +} + +// addFromDir scans one directory, emitting a SubtitleTrack for each text sidecar +// whose name passes `match`. stripPrefix (the video base, may be "") is removed +// before parsing language/role tokens so "Movie.en.forced.srt" parses as "en"+forced. +func addFromDir(dir string, match func(name string) bool, stripPrefix string, out *[]SubtitleTrack, seen map[string]bool) { + entries, err := os.ReadDir(dir) + if err != nil { + return + } + // Pre-index .idx files so a paired .sub is recognised as VobSub (bitmap) and skipped. + idxBases := make(map[string]bool) + for _, e := range entries { + if !e.IsDir() && strings.EqualFold(filepath.Ext(e.Name()), ".idx") { + idxBases[strings.ToLower(strings.TrimSuffix(e.Name(), filepath.Ext(e.Name())))] = true + } + } + for _, e := range entries { + if e.IsDir() { + continue + } + name := e.Name() + ext := strings.ToLower(filepath.Ext(name)) + codec, ok := sidecarSubExts[ext] + if !ok || !match(name) { + continue + } + // VobSub: a .sub paired with a same-named .idx is bitmap, not text. Skip. + if ext == ".sub" && idxBases[strings.ToLower(strings.TrimSuffix(name, ext))] { + continue + } + abs := filepath.Join(dir, name) + if seen[abs] { + continue + } + seen[abs] = true + + lang, forced, title := parseSidecarName(name, ext, stripPrefix) + *out = append(*out, SubtitleTrack{ + Lang: lang, + Codec: codec, + Title: title, + Forced: forced, + External: true, + Path: abs, + }) + } +} + +// parseSidecarName extracts (lang, forced, title) from a subtitle filename. +// stripPrefix (the video base) is removed first; the remainder is tokenised on +// common separators and scanned for a language code + role markers. Unknown → +// lang "und". The title is a human hint ("Forced", "SDH") or "". +func parseSidecarName(name, ext, stripPrefix string) (lang string, forced bool, title string) { + stem := strings.TrimSuffix(name, filepath.Ext(name)) + if stripPrefix != "" && len(stem) >= len(stripPrefix) && + strings.EqualFold(stem[:len(stripPrefix)], stripPrefix) { + stem = stem[len(stripPrefix):] + } + lang = "und" + var roles []string + for _, tok := range strings.FieldsFunc(stem, func(r rune) bool { + return r == '.' || r == '_' || r == '-' || r == ' ' || r == '[' || r == ']' || r == '(' || r == ')' + }) { + low := strings.ToLower(strings.TrimSpace(tok)) + if low == "" { + continue + } + if forcedTokens[low] { + forced = true + roles = append(roles, "Forced") + continue + } + if v, isSDH := sdhTokens[low]; isSDH && v { + roles = append(roles, "SDH") + continue + } + // First token that maps to a real language wins. Try release-naming + // aliases (chs/lat/…) first, then the standard ISO normaliser. NormalizeLang + // echoes unknown input back lowercased, so accept only a mapped result + // (different from the raw token, or already a known 2-letter code). + if lang == "und" { + if alias, ok := sidecarLangAliases[low]; ok { + lang = alias + continue + } + if norm := NormalizeLang(low); norm != "und" && (norm != low || len(low) == 2) && isKnownLang(norm) { + lang = norm + continue + } + } + } + title = strings.Join(roles, " ") + return lang, forced, title +} + +// isKnownLang reports whether code is a value present in langNormalize (i.e. a +// real ISO 639-1 we recognise) — guards against treating a random filename token +// ("web", "dl") as a language. +func isKnownLang(code string) bool { + for _, v := range langNormalize { + if v == code { + return true + } + } + return false +} diff --git a/internal/library/mediainfo/sidecar_subs_test.go b/internal/library/mediainfo/sidecar_subs_test.go new file mode 100644 index 0000000..073fcbf --- /dev/null +++ b/internal/library/mediainfo/sidecar_subs_test.go @@ -0,0 +1,113 @@ +package mediainfo + +import ( + "os" + "path/filepath" + "testing" +) + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +func findTrack(tracks []SubtitleTrack, base string) *SubtitleTrack { + for i := range tracks { + if filepath.Base(tracks[i].Path) == base { + return &tracks[i] + } + } + return nil +} + +func TestDiscoverSidecarSubtitles_Siblings(t *testing.T) { + dir := t.TempDir() + video := filepath.Join(dir, "Witch.Hat.Atelier.S01E10.mkv") + writeFile(t, video, "x") + writeFile(t, filepath.Join(dir, "Witch.Hat.Atelier.S01E10.srt"), "1\n00:00:01,000 --> 00:00:02,000\nhi\n") + writeFile(t, filepath.Join(dir, "Witch.Hat.Atelier.S01E10.es.ass"), "[Script Info]") + writeFile(t, filepath.Join(dir, "Witch.Hat.Atelier.S01E10.en.forced.srt"), "x") + // Unrelated file with a different base must NOT be matched as a sibling. + writeFile(t, filepath.Join(dir, "Other.Movie.srt"), "x") + + tracks := DiscoverSidecarSubtitles(video) + if len(tracks) != 3 { + t.Fatalf("want 3 sibling tracks, got %d: %+v", len(tracks), tracks) + } + for _, tr := range tracks { + if !tr.External || tr.Path == "" { + t.Errorf("track not marked external w/ path: %+v", tr) + } + } + if es := findTrack(tracks, "Witch.Hat.Atelier.S01E10.es.ass"); es == nil || es.Lang != "es" || es.Codec != "ass" { + t.Errorf("es.ass mis-parsed: %+v", es) + } + if fr := findTrack(tracks, "Witch.Hat.Atelier.S01E10.en.forced.srt"); fr == nil || fr.Lang != "en" || !fr.Forced { + t.Errorf("forced track mis-parsed: %+v", fr) + } +} + +func TestDiscoverSidecarSubtitles_SubsFolder(t *testing.T) { + dir := t.TempDir() + video := filepath.Join(dir, "Movie.2024.1080p.mkv") + writeFile(t, video, "x") + subs := filepath.Join(dir, "Subs") + if err := os.Mkdir(subs, 0o755); err != nil { + t.Fatal(err) + } + writeFile(t, filepath.Join(subs, "2_English.srt"), "x") + writeFile(t, filepath.Join(subs, "spa.ass"), "x") + + tracks := DiscoverSidecarSubtitles(video) + if len(tracks) != 2 { + t.Fatalf("want 2 Subs/ tracks, got %d: %+v", len(tracks), tracks) + } + if en := findTrack(tracks, "2_English.srt"); en == nil || en.Lang != "en" { + t.Errorf("English mis-parsed: %+v", en) + } + if es := findTrack(tracks, "spa.ass"); es == nil || es.Lang != "es" { + t.Errorf("spa mis-parsed: %+v", es) + } +} + +func TestParseSidecarName_ReleaseAliases(t *testing.T) { + cases := []struct { + name, ext, prefix, wantLang string + }{ + {"[DMG] Orange [01].chs.ass", ".ass", "", "zh"}, // Chinese Simplified fansub code → GBK + {"Show.cht.srt", ".srt", "Show", "zh-Hant"}, // Chinese Traditional → Big5 + {"Movie.big5.srt", ".srt", "Movie", "zh-Hant"}, // Traditional via codepage token + {"Movie.lat.srt", ".srt", "Movie", "es"}, // Latin-American Spanish + {"Movie.latino.srt", ".srt", "Movie", "es"}, // + {"Pelicula.esp.srt", ".srt", "Pelicula", "es"}, // + {"Anime.VOSTFR.ass", ".ass", "Anime", "fr"}, // French fansub + {"X.kan.srt", ".srt", "X", "kn"}, // Kannada via langNormalize add + {"X.mal.srt", ".srt", "X", "ml"}, // Malayalam + } + for _, c := range cases { + lang, _, _ := parseSidecarName(c.name, c.ext, c.prefix) + if lang != c.wantLang { + t.Errorf("%s: got lang %q, want %q", c.name, lang, c.wantLang) + } + } +} + +func TestDiscoverSidecarSubtitles_VobSubSkipped(t *testing.T) { + dir := t.TempDir() + video := filepath.Join(dir, "Film.mkv") + writeFile(t, video, "x") + writeFile(t, filepath.Join(dir, "Film.idx"), "x") + writeFile(t, filepath.Join(dir, "Film.sub"), "x") // VobSub bitmap → skip + tracks := DiscoverSidecarSubtitles(video) + if len(tracks) != 0 { + t.Fatalf("VobSub .sub+.idx must be skipped, got %d: %+v", len(tracks), tracks) + } +} + +func TestDiscoverSidecarSubtitles_RemoteURLNoop(t *testing.T) { + if tracks := DiscoverSidecarSubtitles("https://example.com/movie.mkv"); tracks != nil { + t.Fatalf("remote URL must yield no sidecars, got %+v", tracks) + } +} diff --git a/internal/library/mediainfo/types.go b/internal/library/mediainfo/types.go index efa17ca..25153ff 100644 --- a/internal/library/mediainfo/types.go +++ b/internal/library/mediainfo/types.go @@ -42,10 +42,23 @@ type AudioTrack struct { Default bool `json:"default"` } -// SubtitleTrack represents a single subtitle stream. +// SubtitleTrack represents a single subtitle source — either an EMBEDDED stream +// (the common case, identified by its ffmpeg `0:s:N` order in the slice) or an +// EXTERNAL sidecar file sitting next to the media (Path set, External true). +// +// External sidecars (a `.srt`/`.ass`/`.vtt` named after the video, or one in a +// `Subs/` subfolder) are appended AFTER all embedded tracks so the embedded +// tracks keep slice positions equal to their `0:s:N` index — the web's +// resolveSubtitleTracks relies on that for embedded, and switches to Path-based +// addressing for external (served via /sub?p=&i=-1). type SubtitleTrack struct { Lang string `json:"lang"` Codec string `json:"codec"` Title string `json:"title"` Forced bool `json:"forced"` + // External is true for a sidecar file; false (omitted) for an embedded stream. + External bool `json:"external,omitempty"` + // Path is the absolute filesystem path of the sidecar file (External only). + // Empty for embedded streams (those live inside the media container). + Path string `json:"path,omitempty"` }