feat(subs): resilient subtitle extraction — sidecars, charset, torrent/debrid
Close the recurring "video has subtitles but the web player shows none" gap with a source-agnostic pipeline: - Discover EXTERNAL sidecar subs in the scan (Video.es.ass siblings + a Subs/ bundle), parse lang/forced/SDH from the filename, skip VobSub (.sub+.idx). ffprobe-only scanning ignored these (ToonsHub/anime "MSubs" releases). - Transcode sidecar charset -> UTF-8 before WebVTT (BOM/UTF-16/code-page by language). Chinese SCRIPT matters: chs/sc -> GBK, cht/tc/big5 -> Big5 (decoding one as the other is mojibake). - /sub now serves a standalone sidecar file (i=-1, p=file, &l=lang hint) and a remote debrid URL (ffmpeg reads http, no local stat) — not just embedded streams of a local file. - probe.json emits a tokened vttUrl per TEXT track so torrent/debrid HLS streams (never library-scanned) get subtitles too. Embedded index is counted among embedded streams only, so -map 0:s:N stays aligned when sidecars are appended. Tested against a real 347-file gallery: 26/26 sidecars and embedded ass/srt/ mov_text all extract to valid WebVTT; bitmap (pgs/dvd_subtitle) correctly stays burn-in. Manual harness gated behind GALLERY_DIR.
This commit is contained in:
parent
22081cf106
commit
d708ea2360
13 changed files with 957 additions and 39 deletions
4
go.mod
4
go.mod
|
|
@ -14,8 +14,10 @@ require (
|
||||||
github.com/huin/goupnp v1.3.0
|
github.com/huin/goupnp v1.3.0
|
||||||
github.com/olekukonko/tablewriter v1.1.4
|
github.com/olekukonko/tablewriter v1.1.4
|
||||||
github.com/spf13/cobra v1.10.2
|
github.com/spf13/cobra v1.10.2
|
||||||
|
github.com/spf13/pflag v1.0.10
|
||||||
github.com/torrentclaw/go-client v0.2.0
|
github.com/torrentclaw/go-client v0.2.0
|
||||||
golang.org/x/term v0.43.0
|
golang.org/x/term v0.43.0
|
||||||
|
golang.org/x/text v0.37.0
|
||||||
golang.org/x/time v0.15.0
|
golang.org/x/time v0.15.0
|
||||||
golang.zx2c4.com/wireguard v0.0.0-20250521234502-f333402bd9cb
|
golang.zx2c4.com/wireguard v0.0.0-20250521234502-f333402bd9cb
|
||||||
)
|
)
|
||||||
|
|
@ -113,7 +115,6 @@ require (
|
||||||
github.com/rivo/uniseg v0.4.7 // indirect
|
github.com/rivo/uniseg v0.4.7 // indirect
|
||||||
github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 // indirect
|
github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 // indirect
|
||||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||||
github.com/spf13/pflag v1.0.10 // indirect
|
|
||||||
github.com/tidwall/btree v1.8.1 // indirect
|
github.com/tidwall/btree v1.8.1 // indirect
|
||||||
github.com/wlynxg/anet v0.0.5 // indirect
|
github.com/wlynxg/anet v0.0.5 // indirect
|
||||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||||
|
|
@ -127,7 +128,6 @@ require (
|
||||||
golang.org/x/net v0.54.0 // indirect
|
golang.org/x/net v0.54.0 // indirect
|
||||||
golang.org/x/sync v0.20.0 // indirect
|
golang.org/x/sync v0.20.0 // indirect
|
||||||
golang.org/x/sys v0.44.0 // indirect
|
golang.org/x/sys v0.44.0 // indirect
|
||||||
golang.org/x/text v0.37.0 // indirect
|
|
||||||
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect
|
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect
|
||||||
gvisor.dev/gvisor v0.0.0-20250503011706-39ed1f5ac29c // indirect
|
gvisor.dev/gvisor v0.0.0-20250503011706-39ed1f5ac29c // indirect
|
||||||
lukechampine.com/blake3 v1.4.1 // indirect
|
lukechampine.com/blake3 v1.4.1 // indirect
|
||||||
|
|
|
||||||
|
|
@ -574,6 +574,9 @@ func (s *HLSSession) ProbeInfo() map[string]any {
|
||||||
}
|
}
|
||||||
subs := make([]map[string]any, 0, len(s.probe.SubtitleTracks))
|
subs := make([]map[string]any, 0, len(s.probe.SubtitleTracks))
|
||||||
for _, sb := range s.probe.SubtitleTracks {
|
for _, sb := range s.probe.SubtitleTracks {
|
||||||
|
// `external`/`path` let the stream server attach a tokened /sub vttUrl
|
||||||
|
// (path-addressed for sidecars, index-addressed for embedded). `path` is
|
||||||
|
// stripped after the URL is built so the raw path isn't doubled in JSON.
|
||||||
subs = append(subs, map[string]any{
|
subs = append(subs, map[string]any{
|
||||||
"index": sb.Index,
|
"index": sb.Index,
|
||||||
"lang": sb.Lang,
|
"lang": sb.Lang,
|
||||||
|
|
@ -581,6 +584,8 @@ func (s *HLSSession) ProbeInfo() map[string]any {
|
||||||
"title": sb.Title,
|
"title": sb.Title,
|
||||||
"forced": sb.Forced,
|
"forced": sb.Forced,
|
||||||
"text": sb.IsTextSubtitle(),
|
"text": sb.IsTextSubtitle(),
|
||||||
|
"external": sb.External,
|
||||||
|
"path": sb.Path,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return map[string]any{
|
return map[string]any{
|
||||||
|
|
|
||||||
|
|
@ -50,11 +50,15 @@ type ProbeAudioTrack struct {
|
||||||
// Codec discriminates text (srt/ass/webvtt → extract to WebVTT) vs bitmap
|
// Codec discriminates text (srt/ass/webvtt → extract to WebVTT) vs bitmap
|
||||||
// (pgs/dvbsub → require burn-in).
|
// (pgs/dvbsub → require burn-in).
|
||||||
type ProbeSubtitleTrack struct {
|
type ProbeSubtitleTrack struct {
|
||||||
Index int // 0-based subtitle stream index (ffmpeg -map 0:s:Index)
|
Index int // 0-based EMBEDDED subtitle stream index (ffmpeg -map 0:s:Index). Unused when External.
|
||||||
Lang string // ISO 639-1
|
Lang string // ISO 639-1
|
||||||
Codec string // lowercased — "subrip", "ass", "webvtt", "hdmv_pgs_subtitle", ...
|
Codec string // lowercased — "subrip", "ass", "webvtt", "hdmv_pgs_subtitle", ...
|
||||||
Title string
|
Title string
|
||||||
Forced bool
|
Forced bool
|
||||||
|
// External marks a sidecar file (served via /sub?p=<Path>&i=-1) rather than
|
||||||
|
// an embedded stream. Path is its absolute filesystem path (External only).
|
||||||
|
External bool
|
||||||
|
Path string
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsTextSubtitle reports whether a subtitle codec can be extracted to WebVTT
|
// IsTextSubtitle reports whether a subtitle codec can be extracted to WebVTT
|
||||||
|
|
@ -134,14 +138,27 @@ func ProbeFile(ctx context.Context, ffprobePath, filePath string) (*StreamProbe,
|
||||||
}
|
}
|
||||||
if len(mi.Subtitles) > 0 {
|
if len(mi.Subtitles) > 0 {
|
||||||
probe.SubtitleTracks = make([]ProbeSubtitleTrack, 0, len(mi.Subtitles))
|
probe.SubtitleTracks = make([]ProbeSubtitleTrack, 0, len(mi.Subtitles))
|
||||||
for i, s := range mi.Subtitles {
|
// Embedded streams come first (ffprobe order); external sidecars are
|
||||||
probe.SubtitleTracks = append(probe.SubtitleTracks, ProbeSubtitleTrack{
|
// appended after. Count embedded separately so each embedded track's
|
||||||
Index: i,
|
// Index is its true `0:s:N` value regardless of how many externals trail
|
||||||
|
// it; externals get Index=-1 and address by Path instead.
|
||||||
|
embeddedIdx := 0
|
||||||
|
for _, s := range mi.Subtitles {
|
||||||
|
t := ProbeSubtitleTrack{
|
||||||
Lang: s.Lang,
|
Lang: s.Lang,
|
||||||
Codec: strings.ToLower(s.Codec),
|
Codec: strings.ToLower(s.Codec),
|
||||||
Title: s.Title,
|
Title: s.Title,
|
||||||
Forced: s.Forced,
|
Forced: s.Forced,
|
||||||
})
|
External: s.External,
|
||||||
|
Path: s.Path,
|
||||||
|
}
|
||||||
|
if s.External {
|
||||||
|
t.Index = -1
|
||||||
|
} else {
|
||||||
|
t.Index = embeddedIdx
|
||||||
|
embeddedIdx++
|
||||||
|
}
|
||||||
|
probe.SubtitleTracks = append(probe.SubtitleTracks, t)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
storeProbeCache(filePath, probe)
|
storeProbeCache(filePath, probe)
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,7 @@ import (
|
||||||
"log"
|
"log"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
|
@ -733,7 +734,9 @@ func (ss *StreamServer) hlsHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
case resource == "probe.json":
|
case resource == "probe.json":
|
||||||
w.Header().Set("Content-Type", "application/json")
|
w.Header().Set("Content-Type", "application/json")
|
||||||
w.Header().Set("Cache-Control", "no-cache")
|
w.Header().Set("Cache-Control", "no-cache")
|
||||||
_ = json.NewEncoder(w).Encode(session.ProbeInfo())
|
info := session.ProbeInfo()
|
||||||
|
ss.attachSubtitleVTTURLs(info, session.cfg.sourceRef())
|
||||||
|
_ = json.NewEncoder(w).Encode(info)
|
||||||
case resource == "video/index.m3u8":
|
case resource == "video/index.m3u8":
|
||||||
session.ServeVideoPlaylist(w, r)
|
session.ServeVideoPlaylist(w, r)
|
||||||
case resource == "video/init.mp4":
|
case resource == "video/init.mp4":
|
||||||
|
|
@ -1224,8 +1227,11 @@ func (ss *StreamServer) subtitleHandler(w http.ResponseWriter, r *http.Request)
|
||||||
http.Error(w, "missing path", http.StatusBadRequest)
|
http.Error(w, "missing path", http.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
// index >= 0 → EMBEDDED stream index (-map 0:s:N) of the media at `p`.
|
||||||
|
// index < 0 → EXTERNAL sidecar: `p` IS the subtitle file; the whole file is
|
||||||
|
// the track. Both bind the token to (path, index) so a tampered p/i fails.
|
||||||
index, err := strconv.Atoi(q.Get("i"))
|
index, err := strconv.Atoi(q.Get("i"))
|
||||||
if err != nil || index < 0 {
|
if err != nil {
|
||||||
http.Error(w, "bad index", http.StatusBadRequest)
|
http.Error(w, "bad index", http.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
@ -1235,12 +1241,20 @@ func (ss *StreamServer) subtitleHandler(w http.ResponseWriter, r *http.Request)
|
||||||
http.Error(w, "not found", http.StatusNotFound)
|
http.Error(w, "not found", http.StatusNotFound)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
external := index < 0
|
||||||
|
// A debrid/HLS-from-URL source has no local file — ffmpeg reads the URL
|
||||||
|
// directly. Skip the path heal + regular-file stat + on-disk cache for those;
|
||||||
|
// only local files get the sidecar cache.
|
||||||
|
isURL := strings.Contains(rawPath, "://")
|
||||||
|
langHint := q.Get("l") // ISO 639-1 charset hint for external sidecar decoding
|
||||||
|
|
||||||
|
if !isURL {
|
||||||
rawPath = ss.healMediaPath(rawPath) // host→container base-path skew (see /thumbnail)
|
rawPath = ss.healMediaPath(rawPath) // host→container base-path skew (see /thumbnail)
|
||||||
if fi, statErr := os.Stat(rawPath); statErr != nil || !fi.Mode().IsRegular() {
|
if fi, statErr := os.Stat(rawPath); statErr != nil || !fi.Mode().IsRegular() {
|
||||||
http.Error(w, "not found", http.StatusNotFound)
|
http.Error(w, "not found", http.StatusNotFound)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Cache hit: serve a fresh sidecar (written by the scan-time prewarm or a
|
// Cache hit: serve a fresh sidecar (written by the scan-time prewarm or a
|
||||||
// prior request) instantly, skipping ffmpeg. This is also what makes huge
|
// prior request) instantly, skipping ffmpeg. This is also what makes huge
|
||||||
// remuxes work — the prewarm extracts without the on-demand HTTP timeout
|
// remuxes work — the prewarm extracts without the on-demand HTTP timeout
|
||||||
|
|
@ -1251,6 +1265,7 @@ func (ss *StreamServer) subtitleHandler(w http.ResponseWriter, r *http.Request)
|
||||||
ss.writeVTT(w, vtt)
|
ss.writeVTT(w, vtt)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Beyond here we must extract on demand, which needs ffmpeg.
|
// Beyond here we must extract on demand, which needs ffmpeg.
|
||||||
if ss.ffmpegPath == "" {
|
if ss.ffmpegPath == "" {
|
||||||
|
|
@ -1265,15 +1280,23 @@ func (ss *StreamServer) subtitleHandler(w http.ResponseWriter, r *http.Request)
|
||||||
ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
|
ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
out, err := mediainfo.ExtractSubtitleVTT(ctx, ss.ffmpegPath, rawPath, index)
|
var out []byte
|
||||||
|
if external {
|
||||||
|
// Standalone sidecar file: transcode charset → UTF-8 (langHint guides the
|
||||||
|
// code-page guess) then ffmpeg → WebVTT.
|
||||||
|
out, err = mediainfo.ExtractExternalSubtitleVTT(ctx, ss.ffmpegPath, rawPath, langHint)
|
||||||
|
} else {
|
||||||
|
out, err = mediainfo.ExtractSubtitleVTT(ctx, ss.ffmpegPath, rawPath, index)
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("[sub] extract failed (i=%d path=%q): %v", index, rawPath, err)
|
log.Printf("[sub] extract failed (i=%d path=%q external=%v url=%v): %v", index, rawPath, external, isURL, err)
|
||||||
http.Error(w, "subtitle extract failed", http.StatusInternalServerError)
|
http.Error(w, "subtitle extract failed", http.StatusInternalServerError)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
// Write-through so the next request is a cache hit. Best-effort: a read-only
|
// Write-through so the next request is a cache hit. Best-effort: a read-only
|
||||||
// media mount just logs and serves the in-memory bytes.
|
// media mount just logs and serves the in-memory bytes. URL sources have no
|
||||||
if ss.cacheSubtitles {
|
// stable on-disk anchor for the sidecar cache → skip.
|
||||||
|
if ss.cacheSubtitles && !isURL {
|
||||||
if werr := mediainfo.WriteCachedSubtitle(rawPath, index, out); werr != nil {
|
if werr := mediainfo.WriteCachedSubtitle(rawPath, index, out); werr != nil {
|
||||||
log.Printf("[sub] cache write skipped (i=%d path=%q): %v", index, rawPath, werr)
|
log.Printf("[sub] cache write skipped (i=%d path=%q): %v", index, rawPath, werr)
|
||||||
}
|
}
|
||||||
|
|
@ -1281,6 +1304,60 @@ func (ss *StreamServer) subtitleHandler(w http.ResponseWriter, r *http.Request)
|
||||||
ss.writeVTT(w, out)
|
ss.writeVTT(w, out)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// attachSubtitleVTTURLs enriches a ProbeInfo map's "subtitles" entries with a
|
||||||
|
// ready-to-use, tokened `vttUrl` for every TEXT track, so the web player can
|
||||||
|
// attach <track>s for ANY play method (torrent/debrid HLS included) without the
|
||||||
|
// server needing the source path — it's the single subtitle wiring path that
|
||||||
|
// makes embedded subs work on streams that were never library-scanned.
|
||||||
|
//
|
||||||
|
// - embedded (external=false): /sub?p=<srcRef>&i=<index>&t=<tok>
|
||||||
|
// - external (external=true) : /sub?p=<sidecar path>&i=-1&t=<tok>&l=<lang>
|
||||||
|
//
|
||||||
|
// The token uses the SAME streamScopeSub(path,index) the web mints with, so a
|
||||||
|
// library-scanned track and a probe-derived one address identically. The raw
|
||||||
|
// "path" key is removed after the URL is built (it's encoded in the URL already).
|
||||||
|
// URLs are root-relative; the player resolves them against the funnel origin it
|
||||||
|
// fetched probe.json from. Bitmap tracks get no vttUrl (burn-in only).
|
||||||
|
func (ss *StreamServer) attachSubtitleVTTURLs(info map[string]any, srcRef string) {
|
||||||
|
subsAny, ok := info["subtitles"].([]map[string]any)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
for _, sb := range subsAny {
|
||||||
|
isText, _ := sb["text"].(bool)
|
||||||
|
if !isText {
|
||||||
|
delete(sb, "path")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
external, _ := sb["external"].(bool)
|
||||||
|
var p string
|
||||||
|
var idx int
|
||||||
|
if external {
|
||||||
|
p, _ = sb["path"].(string)
|
||||||
|
idx = -1
|
||||||
|
} else {
|
||||||
|
p = srcRef
|
||||||
|
if iv, ok := sb["index"].(int); ok {
|
||||||
|
idx = iv
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if p == "" {
|
||||||
|
delete(sb, "path")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
tok := mintStreamToken(ss.streamSecret, streamScopeSub(p, idx), now)
|
||||||
|
u := "/sub?p=" + url.QueryEscape(p) + "&i=" + strconv.Itoa(idx) + "&t=" + tok
|
||||||
|
if external {
|
||||||
|
if lang, _ := sb["lang"].(string); lang != "" && lang != "und" {
|
||||||
|
u += "&l=" + url.QueryEscape(lang)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sb["vttUrl"] = u
|
||||||
|
delete(sb, "path")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// writeVTT writes the standard WebVTT response headers + body for both the
|
// writeVTT writes the standard WebVTT response headers + body for both the
|
||||||
// cache-hit and freshly-extracted paths of subtitleHandler.
|
// cache-hit and freshly-extracted paths of subtitleHandler.
|
||||||
func (ss *StreamServer) writeVTT(w http.ResponseWriter, vtt []byte) {
|
func (ss *StreamServer) writeVTT(w http.ResponseWriter, vtt []byte) {
|
||||||
|
|
|
||||||
139
internal/library/mediainfo/charset.go
Normal file
139
internal/library/mediainfo/charset.go
Normal file
|
|
@ -0,0 +1,139 @@
|
||||||
|
package mediainfo
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"strings"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"golang.org/x/text/encoding"
|
||||||
|
"golang.org/x/text/encoding/charmap"
|
||||||
|
"golang.org/x/text/encoding/japanese"
|
||||||
|
"golang.org/x/text/encoding/korean"
|
||||||
|
"golang.org/x/text/encoding/simplifiedchinese"
|
||||||
|
"golang.org/x/text/encoding/traditionalchinese"
|
||||||
|
"golang.org/x/text/encoding/unicode"
|
||||||
|
"golang.org/x/text/transform"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Subtitle charset normalisation.
|
||||||
|
//
|
||||||
|
// External subtitle files are routinely NOT UTF-8: legacy .srt files come in the
|
||||||
|
// uploader's local code page (Windows-1252 Western, Windows-1256 Arabic, GBK
|
||||||
|
// Chinese, Shift-JIS Japanese, …). Feeding those raw to ffmpeg → WebVTT yields
|
||||||
|
// mojibake. We detect the encoding and transcode to UTF-8 before extraction.
|
||||||
|
//
|
||||||
|
// Detection order: BOM (authoritative) → valid UTF-8 → a code page chosen from
|
||||||
|
// the track's declared language (from its filename, e.g. ".ar.srt"). The
|
||||||
|
// language hint is the reliable signal we have without a full statistical
|
||||||
|
// detector: an Arabic sub that isn't UTF-8 is almost certainly Windows-1256, a
|
||||||
|
// Russian one Windows-1251, and so on. Western European is the safe default.
|
||||||
|
|
||||||
|
// legacyEncodingForLang returns the most likely single-byte / CJK encoding for a
|
||||||
|
// non-UTF-8 subtitle in the given language hint. The hint is normally an ISO
|
||||||
|
// 639-1 code, but Chinese carries a script suffix ("zh-hant" / "zh-tw") so a
|
||||||
|
// Traditional sidecar decodes as Big5 instead of GBK (decoding Big5 bytes as GBK
|
||||||
|
// is mojibake — and anime fansubs routinely ship both chs AND cht). Default:
|
||||||
|
// Windows-1252.
|
||||||
|
func legacyEncodingForLang(lang string) encoding.Encoding {
|
||||||
|
switch strings.ToLower(strings.TrimSpace(lang)) {
|
||||||
|
case "ar", "fa", "ur": // Arabic script
|
||||||
|
return charmap.Windows1256
|
||||||
|
case "ru", "uk", "bg", "sr", "mk": // Cyrillic
|
||||||
|
return charmap.Windows1251
|
||||||
|
case "el": // Greek
|
||||||
|
return charmap.Windows1253
|
||||||
|
case "he": // Hebrew
|
||||||
|
return charmap.Windows1255
|
||||||
|
case "tr": // Turkish
|
||||||
|
return charmap.Windows1254
|
||||||
|
case "th": // Thai
|
||||||
|
return charmap.Windows874
|
||||||
|
case "zh-hant", "zh_hant", "zh-tw", "zh-hk", "zhtw": // Traditional Chinese
|
||||||
|
return traditionalchinese.Big5
|
||||||
|
case "zh", "zh-hans", "zh-cn": // Simplified Chinese (covers most pirate releases)
|
||||||
|
return simplifiedchinese.GBK
|
||||||
|
case "ja": // Japanese
|
||||||
|
return japanese.ShiftJIS
|
||||||
|
case "ko": // Korean
|
||||||
|
return korean.EUCKR
|
||||||
|
case "vi": // Vietnamese
|
||||||
|
return charmap.Windows1258
|
||||||
|
case "pl", "cs", "sk", "hu", "ro", "hr", "sl": // Central European
|
||||||
|
return charmap.Windows1250
|
||||||
|
case "lt", "lv", "et": // Baltic
|
||||||
|
return charmap.Windows1257
|
||||||
|
default: // Western European + everything else
|
||||||
|
return charmap.Windows1252
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// DecodeSubtitleToUTF8 returns the bytes as UTF-8, transcoding from a detected
|
||||||
|
// legacy encoding when needed. The returned name is for logging ("utf-8",
|
||||||
|
// "bom-utf16le", "windows-1256", …). Never fails: a transcode error falls back
|
||||||
|
// to the original bytes (ffmpeg may still cope).
|
||||||
|
func DecodeSubtitleToUTF8(data []byte, langHint string) ([]byte, string) {
|
||||||
|
// BOM wins — it's unambiguous.
|
||||||
|
switch {
|
||||||
|
case bytes.HasPrefix(data, []byte{0xEF, 0xBB, 0xBF}):
|
||||||
|
return data[3:], "bom-utf8"
|
||||||
|
case bytes.HasPrefix(data, []byte{0xFF, 0xFE}):
|
||||||
|
return decodeWith(data, unicode.UTF16(unicode.LittleEndian, unicode.UseBOM), "bom-utf16le")
|
||||||
|
case bytes.HasPrefix(data, []byte{0xFE, 0xFF}):
|
||||||
|
return decodeWith(data, unicode.UTF16(unicode.BigEndian, unicode.UseBOM), "bom-utf16be")
|
||||||
|
}
|
||||||
|
// Already valid UTF-8 → no transcode (ASCII is a subset, so plain English
|
||||||
|
// srt files hit this).
|
||||||
|
if utf8.Valid(data) {
|
||||||
|
return data, "utf-8"
|
||||||
|
}
|
||||||
|
// Non-UTF-8: transcode from the language's likely code page.
|
||||||
|
enc := legacyEncodingForLang(langHint)
|
||||||
|
out, name := decodeWith(data, enc, encodingName(enc))
|
||||||
|
return out, name
|
||||||
|
}
|
||||||
|
|
||||||
|
// decodeWith transforms data through enc's decoder to UTF-8. On error returns the
|
||||||
|
// original bytes (best-effort) with the name suffixed "(raw)".
|
||||||
|
func decodeWith(data []byte, enc encoding.Encoding, name string) ([]byte, string) {
|
||||||
|
out, _, err := transform.Bytes(enc.NewDecoder(), data)
|
||||||
|
if err != nil || len(out) == 0 {
|
||||||
|
return data, name + "(raw)"
|
||||||
|
}
|
||||||
|
return out, name
|
||||||
|
}
|
||||||
|
|
||||||
|
// encodingName maps a known encoding back to a short label for logs.
|
||||||
|
func encodingName(enc encoding.Encoding) string {
|
||||||
|
switch enc {
|
||||||
|
case charmap.Windows1250:
|
||||||
|
return "windows-1250"
|
||||||
|
case charmap.Windows1251:
|
||||||
|
return "windows-1251"
|
||||||
|
case charmap.Windows1252:
|
||||||
|
return "windows-1252"
|
||||||
|
case charmap.Windows1253:
|
||||||
|
return "windows-1253"
|
||||||
|
case charmap.Windows1254:
|
||||||
|
return "windows-1254"
|
||||||
|
case charmap.Windows1255:
|
||||||
|
return "windows-1255"
|
||||||
|
case charmap.Windows1256:
|
||||||
|
return "windows-1256"
|
||||||
|
case charmap.Windows1257:
|
||||||
|
return "windows-1257"
|
||||||
|
case charmap.Windows1258:
|
||||||
|
return "windows-1258"
|
||||||
|
case charmap.Windows874:
|
||||||
|
return "windows-874"
|
||||||
|
case simplifiedchinese.GBK:
|
||||||
|
return "gbk"
|
||||||
|
case traditionalchinese.Big5:
|
||||||
|
return "big5"
|
||||||
|
case japanese.ShiftJIS:
|
||||||
|
return "shift-jis"
|
||||||
|
case korean.EUCKR:
|
||||||
|
return "euc-kr"
|
||||||
|
default:
|
||||||
|
return "legacy"
|
||||||
|
}
|
||||||
|
}
|
||||||
64
internal/library/mediainfo/charset_test.go
Normal file
64
internal/library/mediainfo/charset_test.go
Normal file
|
|
@ -0,0 +1,64 @@
|
||||||
|
package mediainfo
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"golang.org/x/text/encoding/charmap"
|
||||||
|
"golang.org/x/text/transform"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDecodeSubtitleToUTF8_PlainASCII(t *testing.T) {
|
||||||
|
in := []byte("Hello world")
|
||||||
|
out, name := DecodeSubtitleToUTF8(in, "en")
|
||||||
|
if string(out) != "Hello world" || name != "utf-8" {
|
||||||
|
t.Fatalf("ASCII passthrough failed: %q %s", out, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDecodeSubtitleToUTF8_BOMStripped(t *testing.T) {
|
||||||
|
in := append([]byte{0xEF, 0xBB, 0xBF}, []byte("café")...)
|
||||||
|
out, name := DecodeSubtitleToUTF8(in, "fr")
|
||||||
|
if string(out) != "café" || name != "bom-utf8" {
|
||||||
|
t.Fatalf("UTF-8 BOM strip failed: %q %s", out, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDecodeSubtitleToUTF8_Windows1252(t *testing.T) {
|
||||||
|
// "café" encoded in Windows-1252 (é = 0xE9) is NOT valid UTF-8.
|
||||||
|
enc1252, _, err := transform.Bytes(charmap.Windows1252.NewEncoder(), []byte("café"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
out, name := DecodeSubtitleToUTF8(enc1252, "fr")
|
||||||
|
if string(out) != "café" {
|
||||||
|
t.Fatalf("Windows-1252 decode failed: got %q (%s)", out, name)
|
||||||
|
}
|
||||||
|
if name != "windows-1252" {
|
||||||
|
t.Fatalf("expected windows-1252, got %s", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDecodeSubtitleToUTF8_TraditionalChineseBig5(t *testing.T) {
|
||||||
|
// 繁 (U+7E41) in Big5 is 0xC1 0x63. Decoding it as GBK would be mojibake, so
|
||||||
|
// the zh-Hant hint must route to Big5.
|
||||||
|
in := []byte{0xC1, 0x63}
|
||||||
|
out, name := DecodeSubtitleToUTF8(in, "zh-Hant")
|
||||||
|
if name != "big5" {
|
||||||
|
t.Fatalf("expected big5 for zh-Hant, got %s", name)
|
||||||
|
}
|
||||||
|
if string(out) != "繁" {
|
||||||
|
t.Fatalf("Big5 decode failed: got %q", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDecodeSubtitleToUTF8_ArabicByLang(t *testing.T) {
|
||||||
|
// Arabic letter ا (U+0627) is 0xC7 in Windows-1256.
|
||||||
|
in := []byte{0xC7}
|
||||||
|
out, name := DecodeSubtitleToUTF8(in, "ar")
|
||||||
|
if name != "windows-1256" {
|
||||||
|
t.Fatalf("expected windows-1256 for Arabic, got %s", name)
|
||||||
|
}
|
||||||
|
if string(out) != "ا" {
|
||||||
|
t.Fatalf("Arabic decode failed: got %q", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -95,6 +95,16 @@ func ExtractMediaInfo(ctx context.Context, ffprobePath, filePath string) (*Media
|
||||||
if integ := assessIntegrity(stderr.String(), mi); integ != nil {
|
if integ := assessIntegrity(stderr.String(), mi); integ != nil {
|
||||||
mi.Integrity = integ
|
mi.Integrity = integ
|
||||||
}
|
}
|
||||||
|
// Append external sidecar subtitles (a .srt/.ass next to the video, or a
|
||||||
|
// Subs/ bundle) AFTER the embedded streams, so embedded keep slice positions
|
||||||
|
// == their 0:s:N index. Local files only — a remote URL has no directory to
|
||||||
|
// scan (debrid streams rely on embedded subs from the URL). Best-effort:
|
||||||
|
// DiscoverSidecarSubtitles returns nil on an unreadable dir.
|
||||||
|
if !strings.Contains(filePath, "://") {
|
||||||
|
if ext := DiscoverSidecarSubtitles(filePath); len(ext) > 0 {
|
||||||
|
mi.Subtitles = append(mi.Subtitles, ext...)
|
||||||
|
}
|
||||||
|
}
|
||||||
return mi, nil
|
return mi, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
206
internal/library/mediainfo/gallery_real_test.go
Normal file
206
internal/library/mediainfo/gallery_real_test.go
Normal file
|
|
@ -0,0 +1,206 @@
|
||||||
|
package mediainfo
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestGalleryReal is a manual end-to-end harness against a REAL media library.
|
||||||
|
// It is skipped unless GALLERY_DIR is set, so it never runs in CI.
|
||||||
|
//
|
||||||
|
// GALLERY_DIR=/mnt/nas/peliculas go test ./internal/library/mediainfo/ \
|
||||||
|
// -run TestGalleryReal -v -timeout 30m
|
||||||
|
//
|
||||||
|
// It surveys every video file (embedded subs via ffprobe + discovered sidecars),
|
||||||
|
// then actually extracts WebVTT for one representative of each kind and checks the
|
||||||
|
// output is a valid, non-empty WEBVTT document.
|
||||||
|
func TestGalleryReal(t *testing.T) {
|
||||||
|
dir := os.Getenv("GALLERY_DIR")
|
||||||
|
if dir == "" {
|
||||||
|
t.Skip("set GALLERY_DIR to run the real-gallery survey")
|
||||||
|
}
|
||||||
|
ffprobe := envOr("FFPROBE", "ffprobe")
|
||||||
|
ffmpeg := envOr("FFMPEG", "ffmpeg")
|
||||||
|
|
||||||
|
videoExt := map[string]bool{".mkv": true, ".mp4": true, ".avi": true, ".m4v": true, ".webm": true, ".mov": true, ".ts": true}
|
||||||
|
var videos []string
|
||||||
|
_ = filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error {
|
||||||
|
if err != nil || d.IsDir() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if strings.Contains(p, "/.unarr/") || strings.Contains(p, "/.Trash") || strings.Contains(p, "/@eaDir/") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if videoExt[strings.ToLower(filepath.Ext(p))] {
|
||||||
|
videos = append(videos, p)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
sort.Strings(videos)
|
||||||
|
t.Logf("found %d video files under %s", len(videos), dir)
|
||||||
|
|
||||||
|
type cat struct {
|
||||||
|
embTextCodecs map[string]int // codec → count of files
|
||||||
|
embBitmap map[string]int
|
||||||
|
extCodecs map[string]int
|
||||||
|
filesEmbText []string
|
||||||
|
filesEmbBitmap []string
|
||||||
|
filesExt []string
|
||||||
|
errs int
|
||||||
|
}
|
||||||
|
c := cat{embTextCodecs: map[string]int{}, embBitmap: map[string]int{}, extCodecs: map[string]int{}}
|
||||||
|
|
||||||
|
for _, v := range videos {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||||
|
mi, err := ExtractMediaInfo(ctx, ffprobe, v)
|
||||||
|
cancel()
|
||||||
|
if err != nil {
|
||||||
|
c.errs++
|
||||||
|
t.Logf("PROBE ERR %s: %v", filepath.Base(v), err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var sawEmbText, sawEmbBitmap, sawExt bool
|
||||||
|
for _, s := range mi.Subtitles {
|
||||||
|
codec := strings.ToLower(s.Codec)
|
||||||
|
switch {
|
||||||
|
case s.External:
|
||||||
|
c.extCodecs[codec]++
|
||||||
|
sawExt = true
|
||||||
|
case IsTextSubtitleCodec(codec):
|
||||||
|
c.embTextCodecs[codec]++
|
||||||
|
sawEmbText = true
|
||||||
|
default:
|
||||||
|
c.embBitmap[codec]++
|
||||||
|
sawEmbBitmap = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if sawEmbText {
|
||||||
|
c.filesEmbText = append(c.filesEmbText, v)
|
||||||
|
}
|
||||||
|
if sawEmbBitmap {
|
||||||
|
c.filesEmbBitmap = append(c.filesEmbBitmap, v)
|
||||||
|
}
|
||||||
|
if sawExt {
|
||||||
|
c.filesExt = append(c.filesExt, v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("=== CENSUS ===")
|
||||||
|
t.Logf("probe errors: %d", c.errs)
|
||||||
|
t.Logf("embedded TEXT codecs (files w/ track): %v", c.embTextCodecs)
|
||||||
|
t.Logf("embedded BITMAP codecs (burn-in only): %v", c.embBitmap)
|
||||||
|
t.Logf("external SIDECAR codecs: %v", c.extCodecs)
|
||||||
|
t.Logf("files w/ embedded text: %d | w/ embedded bitmap: %d | w/ external sidecar: %d",
|
||||||
|
len(c.filesEmbText), len(c.filesEmbBitmap), len(c.filesExt))
|
||||||
|
|
||||||
|
// --- Real extraction checks ---
|
||||||
|
validVTT := func(b []byte) bool {
|
||||||
|
return len(b) > 0 && strings.HasPrefix(strings.TrimSpace(string(b)), "WEBVTT")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Embedded text: extract index 0 of the first such file.
|
||||||
|
if len(c.filesEmbText) > 0 {
|
||||||
|
f := c.filesEmbText[0]
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||||
|
out, err := ExtractSubtitleVTT(ctx, ffmpeg, f, 0)
|
||||||
|
cancel()
|
||||||
|
if err != nil || !validVTT(out) {
|
||||||
|
t.Errorf("EMBEDDED extract FAILED for %s: err=%v len=%d", filepath.Base(f), err, len(out))
|
||||||
|
} else {
|
||||||
|
t.Logf("EMBEDDED extract OK: %s → %d bytes WebVTT", filepath.Base(f), len(out))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// External sidecar: find one and extract it via the path-addressed function.
|
||||||
|
if len(c.filesExt) > 0 {
|
||||||
|
f := c.filesExt[0]
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||||
|
mi, _ := ExtractMediaInfo(ctx, ffprobe, f)
|
||||||
|
cancel()
|
||||||
|
var subPath, lang string
|
||||||
|
for _, s := range mi.Subtitles {
|
||||||
|
if s.External {
|
||||||
|
subPath, lang = s.Path, s.Lang
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ctx2, cancel2 := context.WithTimeout(context.Background(), 60*time.Second)
|
||||||
|
out, err := ExtractExternalSubtitleVTT(ctx2, ffmpeg, subPath, lang)
|
||||||
|
cancel2()
|
||||||
|
if err != nil || !validVTT(out) {
|
||||||
|
t.Errorf("EXTERNAL extract FAILED for %s: err=%v len=%d", filepath.Base(subPath), err, len(out))
|
||||||
|
} else {
|
||||||
|
t.Logf("EXTERNAL extract OK: %s (lang=%s) → %d bytes WebVTT", filepath.Base(subPath), lang, len(out))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func envOr(k, def string) string {
|
||||||
|
if v := os.Getenv(k); v != "" {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
return def
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestGalleryExtractAllSidecars extracts EVERY discovered sidecar in the gallery
|
||||||
|
// and reports any that fail — the real proof the external path is robust across
|
||||||
|
// formats/charsets. Skipped unless GALLERY_DIR is set.
|
||||||
|
func TestGalleryExtractAllSidecars(t *testing.T) {
|
||||||
|
dir := os.Getenv("GALLERY_DIR")
|
||||||
|
if dir == "" {
|
||||||
|
t.Skip("set GALLERY_DIR")
|
||||||
|
}
|
||||||
|
ffmpeg := envOr("FFMPEG", "ffmpeg")
|
||||||
|
var subs []SubtitleTrack
|
||||||
|
_ = filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error {
|
||||||
|
if err != nil || d.IsDir() || strings.Contains(p, "/.unarr/") || strings.Contains(p, "/.Trash") || strings.Contains(p, "/@eaDir/") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
ext := strings.ToLower(filepath.Ext(p))
|
||||||
|
if videoOf(ext) {
|
||||||
|
subs = append(subs, DiscoverSidecarSubtitles(p)...)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
// Dedupe by path.
|
||||||
|
seen := map[string]bool{}
|
||||||
|
var uniq []SubtitleTrack
|
||||||
|
for _, s := range subs {
|
||||||
|
if !seen[s.Path] {
|
||||||
|
seen[s.Path] = true
|
||||||
|
uniq = append(uniq, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Logf("discovered %d unique sidecar subtitle files", len(uniq))
|
||||||
|
fails := 0
|
||||||
|
for _, s := range uniq {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||||
|
out, err := ExtractExternalSubtitleVTT(ctx, ffmpeg, s.Path, s.Lang)
|
||||||
|
cancel()
|
||||||
|
ok := len(out) > 0 && strings.HasPrefix(strings.TrimSpace(string(out)), "WEBVTT")
|
||||||
|
if err != nil || !ok {
|
||||||
|
fails++
|
||||||
|
t.Errorf("FAIL %s (lang=%s codec=%s): err=%v len=%d", filepath.Base(s.Path), s.Lang, s.Codec, err, len(out))
|
||||||
|
} else {
|
||||||
|
t.Logf("OK %s (lang=%s codec=%s) → %d bytes", filepath.Base(s.Path), s.Lang, s.Codec, len(out))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if fails > 0 {
|
||||||
|
t.Errorf("%d/%d sidecar extractions failed", fails, len(uniq))
|
||||||
|
} else {
|
||||||
|
t.Logf("all %d sidecar extractions produced valid WebVTT", len(uniq))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func videoOf(ext string) bool {
|
||||||
|
switch ext {
|
||||||
|
case ".mkv", ".mp4", ".avi", ".m4v", ".webm", ".mov", ".ts":
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
@ -64,6 +64,12 @@ var langNormalize = map[string]string{
|
||||||
"mlt": "mt", "mt": "mt",
|
"mlt": "mt", "mt": "mt",
|
||||||
"swa": "sw", "sw": "sw",
|
"swa": "sw", "sw": "sw",
|
||||||
"afr": "af", "af": "af",
|
"afr": "af", "af": "af",
|
||||||
|
"kan": "kn", "kn": "kn",
|
||||||
|
"mal": "ml", "ml": "ml",
|
||||||
|
"mar": "mr", "mr": "mr",
|
||||||
|
"pan": "pa", "pa": "pa",
|
||||||
|
"guj": "gu", "gu": "gu",
|
||||||
|
"kann": "kn",
|
||||||
"lat": "la", "la": "la",
|
"lat": "la", "la": "la",
|
||||||
|
|
||||||
// Full English names (ffprobe sometimes returns these instead of codes)
|
// Full English names (ffprobe sometimes returns these instead of codes)
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,7 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"log"
|
||||||
"math"
|
"math"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
|
|
@ -148,6 +149,66 @@ func ExtractSubtitleVTT(ctx context.Context, ffmpegPath, mediaPath string, index
|
||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ExtractExternalSubtitleVTT converts a STANDALONE sidecar subtitle file (a
|
||||||
|
// .srt/.ass/.ssa/.vtt sitting next to the media) to WebVTT. Unlike the embedded
|
||||||
|
// path it has no stream index — the whole file is the track. It first transcodes
|
||||||
|
// the bytes to UTF-8 (legacy code pages → mojibake otherwise; see charset.go)
|
||||||
|
// using the track's language as the detection hint, then runs ffmpeg to emit
|
||||||
|
// WebVTT. The UTF-8 bytes go through a temp file with the ORIGINAL extension so
|
||||||
|
// ffmpeg selects the right demuxer (.srt→subrip, .ass→ass, .vtt→webvtt), and
|
||||||
|
// `-sub_charenc UTF-8` stops ffmpeg from re-guessing what we already decoded.
|
||||||
|
func ExtractExternalSubtitleVTT(ctx context.Context, ffmpegPath, subPath, langHint string) ([]byte, error) {
|
||||||
|
raw, err := os.ReadFile(subPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read sidecar subtitle: %w", err)
|
||||||
|
}
|
||||||
|
if len(raw) == 0 {
|
||||||
|
return nil, errors.New("sidecar subtitle is empty")
|
||||||
|
}
|
||||||
|
utf8Bytes, encName := DecodeSubtitleToUTF8(raw, langHint)
|
||||||
|
// A "(raw)" suffix means the legacy transcode failed and we're passing the
|
||||||
|
// original bytes through — the likeliest cause of user-visible mojibake, so
|
||||||
|
// leave a trail to diagnose it in the field.
|
||||||
|
if strings.HasSuffix(encName, "(raw)") {
|
||||||
|
log.Printf("[sub] external charset transcode fell back to raw bytes (%s, lang=%q): possible mojibake", filepath.Base(subPath), langHint)
|
||||||
|
}
|
||||||
|
|
||||||
|
ext := strings.ToLower(filepath.Ext(subPath))
|
||||||
|
if ext == "" {
|
||||||
|
ext = ".srt"
|
||||||
|
}
|
||||||
|
tmpDir, err := os.MkdirTemp("", "unarr-extsub-")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer func() { _ = os.RemoveAll(tmpDir) }()
|
||||||
|
tmpIn := filepath.Join(tmpDir, "in"+ext)
|
||||||
|
if werr := os.WriteFile(tmpIn, utf8Bytes, 0o600); werr != nil {
|
||||||
|
return nil, werr
|
||||||
|
}
|
||||||
|
|
||||||
|
args := []string{
|
||||||
|
"-nostdin",
|
||||||
|
"-loglevel", "error",
|
||||||
|
"-sub_charenc", "UTF-8",
|
||||||
|
"-i", tmpIn,
|
||||||
|
"-c:s", "webvtt",
|
||||||
|
"-f", "webvtt",
|
||||||
|
"-",
|
||||||
|
}
|
||||||
|
cmd := exec.CommandContext(ctx, ffmpegPath, args...)
|
||||||
|
var stderr strings.Builder
|
||||||
|
cmd.Stderr = &stderr
|
||||||
|
out, err := cmd.Output()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("ffmpeg external subtitle extract: %w: %s", err, strings.TrimSpace(stderr.String()))
|
||||||
|
}
|
||||||
|
if len(out) == 0 {
|
||||||
|
return nil, errors.New("ffmpeg produced no subtitle output")
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
// ExtractSubtitlesVTTMulti extracts several text subtitle streams in a SINGLE
|
// ExtractSubtitlesVTTMulti extracts several text subtitle streams in a SINGLE
|
||||||
// ffmpeg pass. The expensive part of subtitle extraction is demuxing the whole
|
// ffmpeg pass. The expensive part of subtitle extraction is demuxing the whole
|
||||||
// container (subtitle packets are interleaved across the runtime), so a 60GB
|
// container (subtitle packets are interleaved across the runtime), so a 60GB
|
||||||
|
|
|
||||||
207
internal/library/mediainfo/sidecar_subs.go
Normal file
207
internal/library/mediainfo/sidecar_subs.go
Normal file
|
|
@ -0,0 +1,207 @@
|
||||||
|
package mediainfo
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// External (sidecar) subtitle discovery.
|
||||||
|
//
|
||||||
|
// A huge share of torrents — anime fansubs especially — ship subtitles as
|
||||||
|
// SEPARATE files, not embedded streams: a `.srt`/`.ass` named after the video,
|
||||||
|
// or a bundle inside a `Subs/` (or `Subtitles/`) subfolder. ffprobe on the video
|
||||||
|
// container never sees these, so the scan recorded zero subtitles for them
|
||||||
|
// (e.g. ToonsHub "MSubs" releases). This module finds those files so they become
|
||||||
|
// real, selectable tracks served via the /sub endpoint (path-based, i=-1).
|
||||||
|
//
|
||||||
|
// Only TEXT formats are surfaced (srt/ass/ssa/vtt, and a lone .sub). VobSub
|
||||||
|
// (.idx + .sub) is bitmap — no text form — so it's skipped here; bitmap subs are
|
||||||
|
// burn-in only and external bitmap burn-in isn't wired.
|
||||||
|
|
||||||
|
// subFolderNames are common subfolder names that hold a release's subtitle
|
||||||
|
// bundle. Matched case-insensitively. Files inside belong to the sibling media.
|
||||||
|
var subFolderNames = map[string]bool{
|
||||||
|
"subs": true, "subtitles": true, "sub": true, "subtitle": true,
|
||||||
|
}
|
||||||
|
|
||||||
|
// sidecarSubExts maps a subtitle file extension to its ffmpeg-style codec name.
|
||||||
|
// The codec drives the web's text-vs-bitmap classification (isTextSubtitleCodec).
|
||||||
|
var sidecarSubExts = map[string]string{
|
||||||
|
".srt": "subrip",
|
||||||
|
".ass": "ass",
|
||||||
|
".ssa": "ssa",
|
||||||
|
".vtt": "webvtt",
|
||||||
|
".sub": "subrip", // MicroDVD/text — UNLESS paired with a .idx (VobSub, handled below)
|
||||||
|
}
|
||||||
|
|
||||||
|
// forcedTokens / sdhTokens are filename markers that refine a sidecar's role.
|
||||||
|
var forcedTokens = map[string]bool{"forced": true, "forzado": true, "forces": true}
|
||||||
|
var sdhTokens = map[string]bool{"sdh": true, "cc": true, "hi": false} // "hi" is also Hindi → don't treat as SDH
|
||||||
|
|
||||||
|
// sidecarLangAliases maps RELEASE-NAMING subtitle tokens (fansub/scene shorthand
|
||||||
|
// NOT covered by the ISO 639-1/2 normaliser) to a language hint. Two things make
|
||||||
|
// this necessary beyond NormalizeLang:
|
||||||
|
// - Chinese SCRIPT matters for charset: Simplified (chs/sc/gb) is GBK,
|
||||||
|
// Traditional (cht/tc/big5) is Big5 — decoding one as the other is mojibake.
|
||||||
|
// We keep the script in the hint ("zh" vs "zh-Hant") so legacyEncodingForLang
|
||||||
|
// picks the right code page. Anime fansubs routinely ship both.
|
||||||
|
// - lat/latino/vostfr etc. aren't ISO at all and would fall to "und".
|
||||||
|
//
|
||||||
|
// Applied ONLY to sidecar filenames, not ffprobe metadata, so it can't clash with
|
||||||
|
// the global langNormalize ("lat"→Latin there). Plain ISO codes (eng/spa/…) are
|
||||||
|
// intentionally left to NormalizeLang.
|
||||||
|
var sidecarLangAliases = map[string]string{
|
||||||
|
"chs": "zh", "sc": "zh", "gb": "zh", "gbk": "zh", "hans": "zh", // Simplified → GBK
|
||||||
|
"cht": "zh-Hant", "tc": "zh-Hant", "big5": "zh-Hant", "hant": "zh-Hant", // Traditional → Big5
|
||||||
|
"lat": "es", "latino": "es", "esp": "es", "español": "es", "espanol": "es",
|
||||||
|
"vostfr": "fr", "vff": "fr", "vf": "fr",
|
||||||
|
"ptbr": "pt", "pt-br": "pt", "bra": "pt",
|
||||||
|
}
|
||||||
|
|
||||||
|
// DiscoverSidecarSubtitles finds external subtitle files for a local media file:
|
||||||
|
// siblings named after the video, plus everything in a Subs/Subtitles subfolder.
|
||||||
|
// Returns text tracks only, each with External=true and an absolute Path. Safe on
|
||||||
|
// any path — returns nil if the directory can't be read (best-effort, like the
|
||||||
|
// rest of the scan). Never call for a remote URL source (no local directory).
|
||||||
|
//
|
||||||
|
// NOTE: discovered sidecars are NOT deduped against embedded streams of the same
|
||||||
|
// language. That's deliberate — a `Movie.en.srt` next to a video that also has an
|
||||||
|
// embedded English stream is usually a DIFFERENT track (full vs SDH, retimed, or
|
||||||
|
// a better translation), so silently dropping either would hide a choice the user
|
||||||
|
// may want. Both surface as separate, distinctly-labelled entries.
|
||||||
|
func DiscoverSidecarSubtitles(mediaPath string) []SubtitleTrack {
|
||||||
|
if mediaPath == "" || strings.Contains(mediaPath, "://") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
dir := filepath.Dir(mediaPath)
|
||||||
|
videoBase := strings.TrimSuffix(filepath.Base(mediaPath), filepath.Ext(mediaPath))
|
||||||
|
videoBaseLower := strings.ToLower(videoBase)
|
||||||
|
|
||||||
|
var out []SubtitleTrack
|
||||||
|
seen := make(map[string]bool) // absolute path dedupe
|
||||||
|
|
||||||
|
// 1. Siblings in the media's own directory whose name starts with the video
|
||||||
|
// base name: "Movie.srt", "Movie.en.srt", "Movie.en.forced.ass", …
|
||||||
|
addFromDir(dir, func(name string) bool {
|
||||||
|
return strings.HasPrefix(strings.ToLower(name), videoBaseLower)
|
||||||
|
}, videoBase, &out, seen)
|
||||||
|
|
||||||
|
// 2. A Subs/Subtitles subfolder: take EVERY subtitle file (the whole folder
|
||||||
|
// belongs to this release). Filenames there are usually language-named
|
||||||
|
// ("2_English.srt", "spa.ass") with no video-base prefix.
|
||||||
|
if entries, err := os.ReadDir(dir); err == nil {
|
||||||
|
for _, e := range entries {
|
||||||
|
if e.IsDir() && subFolderNames[strings.ToLower(e.Name())] {
|
||||||
|
addFromDir(filepath.Join(dir, e.Name()), func(string) bool { return true }, "", &out, seen)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// addFromDir scans one directory, emitting a SubtitleTrack for each text sidecar
|
||||||
|
// whose name passes `match`. stripPrefix (the video base, may be "") is removed
|
||||||
|
// before parsing language/role tokens so "Movie.en.forced.srt" parses as "en"+forced.
|
||||||
|
func addFromDir(dir string, match func(name string) bool, stripPrefix string, out *[]SubtitleTrack, seen map[string]bool) {
|
||||||
|
entries, err := os.ReadDir(dir)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Pre-index .idx files so a paired .sub is recognised as VobSub (bitmap) and skipped.
|
||||||
|
idxBases := make(map[string]bool)
|
||||||
|
for _, e := range entries {
|
||||||
|
if !e.IsDir() && strings.EqualFold(filepath.Ext(e.Name()), ".idx") {
|
||||||
|
idxBases[strings.ToLower(strings.TrimSuffix(e.Name(), filepath.Ext(e.Name())))] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, e := range entries {
|
||||||
|
if e.IsDir() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
name := e.Name()
|
||||||
|
ext := strings.ToLower(filepath.Ext(name))
|
||||||
|
codec, ok := sidecarSubExts[ext]
|
||||||
|
if !ok || !match(name) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// VobSub: a .sub paired with a same-named .idx is bitmap, not text. Skip.
|
||||||
|
if ext == ".sub" && idxBases[strings.ToLower(strings.TrimSuffix(name, ext))] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
abs := filepath.Join(dir, name)
|
||||||
|
if seen[abs] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[abs] = true
|
||||||
|
|
||||||
|
lang, forced, title := parseSidecarName(name, ext, stripPrefix)
|
||||||
|
*out = append(*out, SubtitleTrack{
|
||||||
|
Lang: lang,
|
||||||
|
Codec: codec,
|
||||||
|
Title: title,
|
||||||
|
Forced: forced,
|
||||||
|
External: true,
|
||||||
|
Path: abs,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseSidecarName extracts (lang, forced, title) from a subtitle filename.
|
||||||
|
// stripPrefix (the video base) is removed first; the remainder is tokenised on
|
||||||
|
// common separators and scanned for a language code + role markers. Unknown →
|
||||||
|
// lang "und". The title is a human hint ("Forced", "SDH") or "".
|
||||||
|
func parseSidecarName(name, ext, stripPrefix string) (lang string, forced bool, title string) {
|
||||||
|
stem := strings.TrimSuffix(name, filepath.Ext(name))
|
||||||
|
if stripPrefix != "" && len(stem) >= len(stripPrefix) &&
|
||||||
|
strings.EqualFold(stem[:len(stripPrefix)], stripPrefix) {
|
||||||
|
stem = stem[len(stripPrefix):]
|
||||||
|
}
|
||||||
|
lang = "und"
|
||||||
|
var roles []string
|
||||||
|
for _, tok := range strings.FieldsFunc(stem, func(r rune) bool {
|
||||||
|
return r == '.' || r == '_' || r == '-' || r == ' ' || r == '[' || r == ']' || r == '(' || r == ')'
|
||||||
|
}) {
|
||||||
|
low := strings.ToLower(strings.TrimSpace(tok))
|
||||||
|
if low == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if forcedTokens[low] {
|
||||||
|
forced = true
|
||||||
|
roles = append(roles, "Forced")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if v, isSDH := sdhTokens[low]; isSDH && v {
|
||||||
|
roles = append(roles, "SDH")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// First token that maps to a real language wins. Try release-naming
|
||||||
|
// aliases (chs/lat/…) first, then the standard ISO normaliser. NormalizeLang
|
||||||
|
// echoes unknown input back lowercased, so accept only a mapped result
|
||||||
|
// (different from the raw token, or already a known 2-letter code).
|
||||||
|
if lang == "und" {
|
||||||
|
if alias, ok := sidecarLangAliases[low]; ok {
|
||||||
|
lang = alias
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if norm := NormalizeLang(low); norm != "und" && (norm != low || len(low) == 2) && isKnownLang(norm) {
|
||||||
|
lang = norm
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
title = strings.Join(roles, " ")
|
||||||
|
return lang, forced, title
|
||||||
|
}
|
||||||
|
|
||||||
|
// isKnownLang reports whether code is a value present in langNormalize (i.e. a
|
||||||
|
// real ISO 639-1 we recognise) — guards against treating a random filename token
|
||||||
|
// ("web", "dl") as a language.
|
||||||
|
func isKnownLang(code string) bool {
|
||||||
|
for _, v := range langNormalize {
|
||||||
|
if v == code {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
113
internal/library/mediainfo/sidecar_subs_test.go
Normal file
113
internal/library/mediainfo/sidecar_subs_test.go
Normal file
|
|
@ -0,0 +1,113 @@
|
||||||
|
package mediainfo
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func writeFile(t *testing.T, path, content string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||||
|
t.Fatalf("write %s: %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func findTrack(tracks []SubtitleTrack, base string) *SubtitleTrack {
|
||||||
|
for i := range tracks {
|
||||||
|
if filepath.Base(tracks[i].Path) == base {
|
||||||
|
return &tracks[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiscoverSidecarSubtitles_Siblings(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
video := filepath.Join(dir, "Witch.Hat.Atelier.S01E10.mkv")
|
||||||
|
writeFile(t, video, "x")
|
||||||
|
writeFile(t, filepath.Join(dir, "Witch.Hat.Atelier.S01E10.srt"), "1\n00:00:01,000 --> 00:00:02,000\nhi\n")
|
||||||
|
writeFile(t, filepath.Join(dir, "Witch.Hat.Atelier.S01E10.es.ass"), "[Script Info]")
|
||||||
|
writeFile(t, filepath.Join(dir, "Witch.Hat.Atelier.S01E10.en.forced.srt"), "x")
|
||||||
|
// Unrelated file with a different base must NOT be matched as a sibling.
|
||||||
|
writeFile(t, filepath.Join(dir, "Other.Movie.srt"), "x")
|
||||||
|
|
||||||
|
tracks := DiscoverSidecarSubtitles(video)
|
||||||
|
if len(tracks) != 3 {
|
||||||
|
t.Fatalf("want 3 sibling tracks, got %d: %+v", len(tracks), tracks)
|
||||||
|
}
|
||||||
|
for _, tr := range tracks {
|
||||||
|
if !tr.External || tr.Path == "" {
|
||||||
|
t.Errorf("track not marked external w/ path: %+v", tr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if es := findTrack(tracks, "Witch.Hat.Atelier.S01E10.es.ass"); es == nil || es.Lang != "es" || es.Codec != "ass" {
|
||||||
|
t.Errorf("es.ass mis-parsed: %+v", es)
|
||||||
|
}
|
||||||
|
if fr := findTrack(tracks, "Witch.Hat.Atelier.S01E10.en.forced.srt"); fr == nil || fr.Lang != "en" || !fr.Forced {
|
||||||
|
t.Errorf("forced track mis-parsed: %+v", fr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiscoverSidecarSubtitles_SubsFolder(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
video := filepath.Join(dir, "Movie.2024.1080p.mkv")
|
||||||
|
writeFile(t, video, "x")
|
||||||
|
subs := filepath.Join(dir, "Subs")
|
||||||
|
if err := os.Mkdir(subs, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
writeFile(t, filepath.Join(subs, "2_English.srt"), "x")
|
||||||
|
writeFile(t, filepath.Join(subs, "spa.ass"), "x")
|
||||||
|
|
||||||
|
tracks := DiscoverSidecarSubtitles(video)
|
||||||
|
if len(tracks) != 2 {
|
||||||
|
t.Fatalf("want 2 Subs/ tracks, got %d: %+v", len(tracks), tracks)
|
||||||
|
}
|
||||||
|
if en := findTrack(tracks, "2_English.srt"); en == nil || en.Lang != "en" {
|
||||||
|
t.Errorf("English mis-parsed: %+v", en)
|
||||||
|
}
|
||||||
|
if es := findTrack(tracks, "spa.ass"); es == nil || es.Lang != "es" {
|
||||||
|
t.Errorf("spa mis-parsed: %+v", es)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseSidecarName_ReleaseAliases(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name, ext, prefix, wantLang string
|
||||||
|
}{
|
||||||
|
{"[DMG] Orange [01].chs.ass", ".ass", "", "zh"}, // Chinese Simplified fansub code → GBK
|
||||||
|
{"Show.cht.srt", ".srt", "Show", "zh-Hant"}, // Chinese Traditional → Big5
|
||||||
|
{"Movie.big5.srt", ".srt", "Movie", "zh-Hant"}, // Traditional via codepage token
|
||||||
|
{"Movie.lat.srt", ".srt", "Movie", "es"}, // Latin-American Spanish
|
||||||
|
{"Movie.latino.srt", ".srt", "Movie", "es"}, //
|
||||||
|
{"Pelicula.esp.srt", ".srt", "Pelicula", "es"}, //
|
||||||
|
{"Anime.VOSTFR.ass", ".ass", "Anime", "fr"}, // French fansub
|
||||||
|
{"X.kan.srt", ".srt", "X", "kn"}, // Kannada via langNormalize add
|
||||||
|
{"X.mal.srt", ".srt", "X", "ml"}, // Malayalam
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
lang, _, _ := parseSidecarName(c.name, c.ext, c.prefix)
|
||||||
|
if lang != c.wantLang {
|
||||||
|
t.Errorf("%s: got lang %q, want %q", c.name, lang, c.wantLang)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiscoverSidecarSubtitles_VobSubSkipped(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
video := filepath.Join(dir, "Film.mkv")
|
||||||
|
writeFile(t, video, "x")
|
||||||
|
writeFile(t, filepath.Join(dir, "Film.idx"), "x")
|
||||||
|
writeFile(t, filepath.Join(dir, "Film.sub"), "x") // VobSub bitmap → skip
|
||||||
|
tracks := DiscoverSidecarSubtitles(video)
|
||||||
|
if len(tracks) != 0 {
|
||||||
|
t.Fatalf("VobSub .sub+.idx must be skipped, got %d: %+v", len(tracks), tracks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiscoverSidecarSubtitles_RemoteURLNoop(t *testing.T) {
|
||||||
|
if tracks := DiscoverSidecarSubtitles("https://example.com/movie.mkv"); tracks != nil {
|
||||||
|
t.Fatalf("remote URL must yield no sidecars, got %+v", tracks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -42,10 +42,23 @@ type AudioTrack struct {
|
||||||
Default bool `json:"default"`
|
Default bool `json:"default"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// SubtitleTrack represents a single subtitle stream.
|
// SubtitleTrack represents a single subtitle source — either an EMBEDDED stream
|
||||||
|
// (the common case, identified by its ffmpeg `0:s:N` order in the slice) or an
|
||||||
|
// EXTERNAL sidecar file sitting next to the media (Path set, External true).
|
||||||
|
//
|
||||||
|
// External sidecars (a `.srt`/`.ass`/`.vtt` named after the video, or one in a
|
||||||
|
// `Subs/` subfolder) are appended AFTER all embedded tracks so the embedded
|
||||||
|
// tracks keep slice positions equal to their `0:s:N` index — the web's
|
||||||
|
// resolveSubtitleTracks relies on that for embedded, and switches to Path-based
|
||||||
|
// addressing for external (served via /sub?p=<file>&i=-1).
|
||||||
type SubtitleTrack struct {
|
type SubtitleTrack struct {
|
||||||
Lang string `json:"lang"`
|
Lang string `json:"lang"`
|
||||||
Codec string `json:"codec"`
|
Codec string `json:"codec"`
|
||||||
Title string `json:"title"`
|
Title string `json:"title"`
|
||||||
Forced bool `json:"forced"`
|
Forced bool `json:"forced"`
|
||||||
|
// External is true for a sidecar file; false (omitted) for an embedded stream.
|
||||||
|
External bool `json:"external,omitempty"`
|
||||||
|
// Path is the absolute filesystem path of the sidecar file (External only).
|
||||||
|
// Empty for embedded streams (those live inside the media container).
|
||||||
|
Path string `json:"path,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue