feat(transcode): dynamic H.264 level + HW probe + capability reporting

Three related fixes around 4K-source transcoding that left the web
player stuck on "preparing session" with no useful diagnostics:

1. Dynamic -level:v derived from output height (hls.go, transcoder.go).
   The previous fixed "4.0" silently rejected anything taller than 1080p
   inside libx264 — "frame MB size > level limit", "DPB size > level
   limit" — and emitted unplayable segments. Helper H264LevelForHeight()
   now picks 4.0 / 5.0 / 5.1 / 6.0 from the actual encode height.

2. New `unarr probe-hwaccel` diagnostic command. Lists the HW encoders
   compiled into ffmpeg, the device files / drivers present, and the
   backend the daemon would actually pick today. Surfaces the canonical
   gotcha: a host with an RTX 3090 + nvidia-smi but a Homebrew ffmpeg
   built without --enable-nvenc still falls back to libx264 software.

3. Register payload now includes hwAccel + maxTranscodeHeight so the web
   side can suggest a smaller alternate quality before the user even
   tries to play a 4K source on a software-only host. Software-only =
   1080p cap, any HW backend = 2160p cap.

internal/agent/daemon.go

@@ -23,6 +23,8 @@ type DaemonConfig struct {
 	TailscaleIP        string   // Tailscale IP (reported in sync for stream URL resolution)
 	CanDelete          bool     // library.allow_delete is enabled
 	ScanPaths          []string // configured scan paths for file deletion validation
+	HWAccel            string   // detected encoder backend ("nvenc"/"qsv"/"vaapi"/"videotoolbox"/"none")
+	MaxTranscodeHeight int      // resolution cap the agent can transcode comfortably (px)
 }
 
 // Daemon manages agent registration and the sync loop.
@@ -87,6 +89,8 @@ func (d *Daemon) Register(ctx context.Context) error {
 		StreamPort:         d.cfg.StreamPort,
 		LanIP:              d.cfg.LanIP,
 		TailscaleIP:        d.cfg.TailscaleIP,
+		HWAccel:            d.cfg.HWAccel,
+		MaxTranscodeHeight: d.cfg.MaxTranscodeHeight,
 	}
 	if free, total, err := DiskInfo(d.cfg.DownloadDir); err == nil {
 		req.DiskFreeBytes = free

internal/agent/types.go

@@ -18,6 +18,14 @@ type RegisterRequest struct {
 	StreamPort     int    `json:"streamPort,omitempty"`
 	LanIP          string `json:"lanIp,omitempty"`
 	TailscaleIP    string `json:"tailscaleIp,omitempty"`
+	// Transcode capabilities — let the web side suggest a smarter quality
+	// before the player even starts. HWAccel is the picked backend
+	// ("nvenc"/"qsv"/"vaapi"/"videotoolbox"/"none"). MaxTranscodeHeight is
+	// the largest output resolution the agent can encode comfortably; for
+	// software-only ffmpeg this is 1080p, with a real GPU encoder it goes
+	// up to 2160p.
+	HWAccel            string `json:"hwAccel,omitempty"`
+	MaxTranscodeHeight int    `json:"maxTranscodeHeight,omitempty"`
 }
 
 // RegisterResponse is returned by the server after registration.

internal/cmd/daemon.go

@@ -17,6 +17,7 @@ import (
 	"github.com/torrentclaw/unarr/internal/config"
 	"github.com/torrentclaw/unarr/internal/engine"
 	"github.com/torrentclaw/unarr/internal/library"
+	"github.com/torrentclaw/unarr/internal/library/mediainfo"
 	"github.com/torrentclaw/unarr/internal/usenet/download"
 )
 
@@ -135,6 +136,16 @@ func runDaemonStart() error {
 
 	userAgent := "unarr/" + Version
 
+	// Probe HW accel + derive a sensible transcode resolution cap. The cap
+	// is what the web side uses to decide whether the user should pre-empt
+	// transcoding by downloading a smaller version (4K source on a software
+	// libx264-only host is the canonical case where pre-download wins).
+	hwAccelPick := engine.DetectHWAccel(context.Background(), cfg.Library.FFmpegPath)
+	maxTranscodeHeight := 1080
+	if hwAccelPick != engine.HWAccelNone {
+		maxTranscodeHeight = 2160
+	}
+
 	// Create daemon config
 	daemonCfg := agent.DaemonConfig{
 		AgentID:            cfg.Agent.ID,
@@ -146,6 +157,8 @@ func runDaemonStart() error {
 		TailscaleIP:        engine.TailscaleIP(),
 		CanDelete:          cfg.Library.AllowDelete,
 		ScanPaths:          library.ResolveScanPaths(cfg.Download.Dir, cfg.Organize.MoviesDir, cfg.Organize.TVShowsDir, cfg.Library.ScanPath),
+		HWAccel:            string(hwAccelPick),
+		MaxTranscodeHeight: maxTranscodeHeight,
 	}
 
 	// Create HTTP client — single communication channel
@@ -237,6 +250,18 @@ func runDaemonStart() error {
 	}
 	d.UpdateStreamPort(streamSrv.Port())
 
+	// Warn at startup if transcode is enabled but ffmpeg/ffprobe are missing.
+	// HLS sessions get rejected at runtime (see daemon.go ~line 455), but
+	// surfacing it here gives the operator a chance to install ffmpeg before
+	// a user hits a confusing "rejected" line in the logs.
+	if cfg.Download.Transcode.Enabled {
+		if _, err := mediainfo.ResolveFFmpeg(cfg.Library.FFmpegPath); err != nil {
+			log.Printf("[hls] transcode enabled but ffmpeg/ffprobe not found — install ffmpeg to use HLS")
+		} else if _, err := mediainfo.ResolveFFprobe(cfg.Library.FFprobePath); err != nil {
+			log.Printf("[hls] transcode enabled but ffmpeg/ffprobe not found — install ffmpeg to use HLS")
+		}
+	}
+
 	// Wire sync client callbacks
 	sc := d.SyncClient()
 	sc.GetFreeSlots = manager.FreeSlots

internal/cmd/probe_hwaccel.go

@@ -0,0 +1,176 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/fatih/color"
+	"github.com/spf13/cobra"
+	"github.com/torrentclaw/unarr/internal/engine"
+)
+
+// newProbeHWAccelCmd reports the hardware-acceleration capabilities the daemon
+// would actually use for HLS/WebRTC transcoding. The motivation: a beefy host
+// (e.g. RTX 3090) can still fall back to software encoding when the installed
+// ffmpeg binary was built without nvenc/qsv/vaapi support — Homebrew ffmpeg
+// is a common offender. Without this command, users see slow / failing 4K
+// transcodes and no obvious way to diagnose where the regression sits.
+func newProbeHWAccelCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   "probe-hwaccel",
+		Short: "Diagnose hardware-acceleration availability",
+		Long: `Report the hardware-acceleration backends the daemon would pick for
+transcoding, plus exactly why each one was kept or rejected.
+
+Checks performed:
+  - ffmpeg / ffprobe paths
+  - which HW encoders the ffmpeg binary supports (h264_nvenc, h264_qsv, h264_vaapi…)
+  - whether the matching device files / drivers are actually present
+  - which backend the daemon would pick today (HWAccelNone means software)
+
+Use this when transcoding feels slow or fails on 4K — the most common cause
+is a software-only ffmpeg build, not a missing GPU.`,
+		Example: `  unarr probe-hwaccel`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runProbeHWAccel()
+		},
+	}
+}
+
+func runProbeHWAccel() error {
+	bold := color.New(color.Bold)
+	green := color.New(color.FgGreen)
+	yellow := color.New(color.FgYellow)
+	red := color.New(color.FgRed)
+
+	fmt.Println()
+	bold.Println("  Hardware acceleration probe")
+	fmt.Println()
+
+	// 1. Locate ffmpeg / ffprobe.
+	ffmpegPath, ffmpegErr := exec.LookPath("ffmpeg")
+	ffprobePath, ffprobeErr := exec.LookPath("ffprobe")
+
+	bold.Println("  Binaries")
+	if ffmpegErr != nil {
+		red.Printf("    x ffmpeg not on PATH\n")
+		fmt.Println()
+		yellow.Println("    HW probe needs ffmpeg. Install it:")
+		fmt.Println("      Ubuntu/Debian: sudo apt install ffmpeg")
+		fmt.Println("      macOS:         brew install ffmpeg")
+		fmt.Println()
+		return nil
+	}
+	green.Printf("    OK ffmpeg  %s\n", ffmpegPath)
+	if ffprobeErr != nil {
+		yellow.Printf("    !  ffprobe not on PATH (HLS still works, source probing falls back to ffmpeg)\n")
+	} else {
+		green.Printf("    OK ffprobe %s\n", ffprobePath)
+	}
+	fmt.Println()
+
+	// 2. List encoders the ffmpeg binary supports.
+	bold.Println("  HW encoders compiled in")
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	out, err := exec.CommandContext(ctx, ffmpegPath, "-hide_banner", "-encoders").CombinedOutput()
+	if err != nil {
+		red.Printf("    x ffmpeg -encoders failed: %v\n", err)
+		fmt.Println()
+		return nil
+	}
+	encoders := string(out)
+
+	hwEncoders := []struct {
+		name    string
+		family  string
+		family2 string
+	}{
+		{"h264_nvenc", "NVIDIA NVENC", "hevc_nvenc"},
+		{"h264_qsv", "Intel Quick Sync", "hevc_qsv"},
+		{"h264_vaapi", "Linux VA-API (Intel/AMD)", "hevc_vaapi"},
+		{"h264_videotoolbox", "macOS VideoToolbox", "hevc_videotoolbox"},
+	}
+	anyHWEncoder := false
+	for _, e := range hwEncoders {
+		hasH264 := strings.Contains(encoders, e.name)
+		hasHEVC := strings.Contains(encoders, e.family2)
+		if hasH264 || hasHEVC {
+			anyHWEncoder = true
+			green.Printf("    OK %s\n", e.family)
+			if hasH264 {
+				fmt.Printf("       %s\n", e.name)
+			}
+			if hasHEVC {
+				fmt.Printf("       %s\n", e.family2)
+			}
+		}
+	}
+	if !anyHWEncoder {
+		red.Printf("    x No HW encoders compiled in\n")
+		fmt.Println()
+		yellow.Println("    Most likely your ffmpeg was built without --enable-nvenc /")
+		yellow.Println("    --enable-libmfx / --enable-vaapi. Brew's default formula is one")
+		yellow.Println("    common offender. On Ubuntu, the system package ships with VAAPI")
+		yellow.Println("    by default and NVENC if you have CUDA installed.")
+	}
+	fmt.Println()
+
+	// 3. Device-file checks.
+	bold.Println("  Devices / drivers")
+	checks := []struct {
+		path string
+		desc string
+	}{
+		{"/dev/nvidia0", "NVIDIA GPU"},
+		{"/dev/dri/renderD128", "Linux DRM render node (used by VA-API + QSV)"},
+	}
+	for _, c := range checks {
+		if fileExistsLocal(c.path) {
+			green.Printf("    OK %s — %s\n", c.path, c.desc)
+		} else {
+			yellow.Printf("    -  %s — %s (not present)\n", c.path, c.desc)
+		}
+	}
+	if _, err := exec.LookPath("nvidia-smi"); err == nil {
+		green.Printf("    OK nvidia-smi on PATH\n")
+	} else {
+		yellow.Printf("    -  nvidia-smi not on PATH\n")
+	}
+	if runtime.GOOS == "darwin" {
+		fmt.Printf("    .  macOS host — VideoToolbox available if encoder was compiled in\n")
+	}
+	fmt.Println()
+
+	// 4. Daemon's actual decision.
+	engine.ResetHWAccelCache()
+	pick := engine.DetectHWAccel(ctx, ffmpegPath)
+	bold.Println("  Daemon would pick")
+	switch pick {
+	case engine.HWAccelNone:
+		red.Printf("    x %s — software libx264 only\n", pick)
+		fmt.Println()
+		yellow.Println("    On a slow CPU 1080p will lag and 4K is effectively unwatchable.")
+		yellow.Println("    Fix: rebuild / reinstall ffmpeg with HW encoder support, then:")
+		fmt.Println()
+		fmt.Println("      unarr daemon restart")
+	default:
+		green.Printf("    OK %s\n", pick)
+		fmt.Printf("    encoder: %s (h264) / %s (hevc)\n", pick.FFmpegVideoCodec("h264"), pick.FFmpegVideoCodec("hevc"))
+	}
+	fmt.Println()
+
+	return nil
+}
+
+// fileExistsLocal stats a path. Mirrors engine.fileExists without exporting it.
+func fileExistsLocal(path string) bool {
+	_, err := os.Stat(path)
+	return err == nil
+}

internal/cmd/root.go

@@ -104,6 +104,8 @@ Source:         https://github.com/torrentclaw/unarr`,
 	statsCmd.GroupID = "system"
 	doctorCmd := newDoctorCmd()
 	doctorCmd.GroupID = "system"
+	probeHWAccelCmd := newProbeHWAccelCmd()
+	probeHWAccelCmd.GroupID = "system"
 	cleanCmd := newCleanCmd()
 	cleanCmd.GroupID = "system"
 	selfUpdateCmd := newSelfUpdateCmd()
@@ -140,6 +142,7 @@ Source:         https://github.com/torrentclaw/unarr`,
 		// System & Diagnostics
 		statsCmd,
 		doctorCmd,
+		probeHWAccelCmd,
 		cleanCmd,
 		selfUpdateCmd,
 		versionCmd,

internal/engine/hls.go

@@ -845,9 +845,21 @@ func buildHLSFFmpegArgsAt(cfg HLSSessionConfig, probe *StreamProbe, tmpDir strin
 	case "h264_qsv":
 		args = append(args, "-preset", "medium", "-look_ahead", "0")
 	}
-	args = append(args, "-profile:v", "main", "-level:v", "4.0")
-
+	// Derive H.264 level from the actual output height. A fixed "4.0" caps the
+	// encoder at 1080p — anything taller (1440p, 4K source on quality=original)
+	// fails libx264 with "frame MB size > level limit" and emits unplayable
+	// segments. The output height matches qcap.MaxHeight when the source is
+	// downscaled, otherwise probe.Height (already populated by ffprobe).
 	qcap := resolveQualityCap(cfg.Quality)
+	outputHeight := qcap.MaxHeight
+	if outputHeight == 0 {
+		outputHeight = cfg.Transcode.MaxHeight
+	}
+	if outputHeight == 0 || (probe.Height > 0 && probe.Height < outputHeight) {
+		outputHeight = probe.Height
+	}
+	args = append(args, "-profile:v", "main", "-level:v", H264LevelForHeight(outputHeight))
+
 	bitrate := qcap.VideoBitrate
 	if bitrate == "" {
 		bitrate = cfg.Transcode.VideoBitrate

internal/engine/hwaccel.go

@@ -128,3 +128,31 @@ func (h HWAccel) FFmpegVideoCodec(target string) string {
 		return "libx264"
 	}
 }
+
+// H264LevelForHeight returns the lowest H.264 profile level capable of encoding
+// a stream at the given output pixel height (assumes ~16:9, ≤30 fps). The
+// previous code used a fixed "4.0" which silently rejects anything above 1080p
+// — libx264 logs "frame MB size > level limit" and emits a corrupt stream.
+// Returning a tighter level on smaller outputs keeps player compatibility on
+// older devices where the encoder can't auto-pick.
+func H264LevelForHeight(height int) string {
+	switch {
+	case height <= 0:
+		// Unknown source — pick a level that covers up to 4K so we never
+		// re-introduce the silent-failure mode that motivated this helper.
+		return "5.1"
+	case height <= 480:
+		return "3.0"
+	case height <= 720:
+		return "3.1"
+	case height <= 1080:
+		return "4.0"
+	case height <= 1440:
+		return "5.0"
+	case height <= 2160:
+		return "5.1"
+	default:
+		// 4K @ 60 fps and 8K all fall under 6.x.
+		return "6.0"
+	}
+}

internal/engine/transcoder.go

@@ -25,6 +25,7 @@ type TranscodeOpts struct {
 	VideoBitrate string // e.g. "5M"
 	AudioBitrate string // e.g. "192k"
 	MaxHeight    int    // optional downscale cap (e.g. 720)
+	SourceHeight int    // probed source height — used to derive a sane H.264 level
 	StartSeconds float64
 	FFmpegPath   string
 }
@@ -235,7 +236,16 @@ func buildFFmpegArgs(filePath string, opts TranscodeOpts) []string {
 		// can fail with "VaapiWrapper: failed initializing" on Linux boxes
 		// where VA-API isn't fully wired up. `main` keeps a clean software
 		// decode fallback on every desktop + mobile platform.
-		args = append(args, "-profile:v", "main", "-level:v", "4.0")
+		//
+		// Level is derived from the actual output height — a fixed "4.0"
+		// silently rejects 4K and 1440p sources at the libx264 macroblock
+		// limits and produces unplayable streams. opts.MaxHeight is the
+		// downscale cap when set; falling through means "encode at source".
+		levelHeight := opts.MaxHeight
+		if levelHeight == 0 || (opts.SourceHeight > 0 && opts.SourceHeight < levelHeight) {
+			levelHeight = opts.SourceHeight
+		}
+		args = append(args, "-profile:v", "main", "-level:v", H264LevelForHeight(levelHeight))
 		args = append(args, "-b:v", coalesce(opts.VideoBitrate, "5M"))
 		// Filter chain:
 		//   1. scale (optional) — cap height + force even width.

internal/engine/webrtc_stream.go

@@ -190,6 +190,7 @@ func buildStreamSource(
 		VideoBitrate: videoBitrate,
 		AudioBitrate: tc.AudioBitrate,
 		MaxHeight:    maxHeight,
+		SourceHeight: probe.Height,
 		FFmpegPath:   tc.FFmpegPath,
 	}
 	return newTranscodeSource(ctx, abs, probe, action, opts, displayName)