feat(transcode): dynamic H.264 level + HW probe + capability reporting

Three related fixes around 4K-source transcoding that left the web
player stuck on "preparing session" with no useful diagnostics:

1. Dynamic -level:v derived from output height (hls.go, transcoder.go).
   The previous fixed "4.0" silently rejected anything taller than 1080p
   inside libx264 — "frame MB size > level limit", "DPB size > level
   limit" — and emitted unplayable segments. Helper H264LevelForHeight()
   now picks 4.0 / 5.0 / 5.1 / 6.0 from the actual encode height.

2. New `unarr probe-hwaccel` diagnostic command. Lists the HW encoders
   compiled into ffmpeg, the device files / drivers present, and the
   backend the daemon would actually pick today. Surfaces the canonical
   gotcha: a host with an RTX 3090 + nvidia-smi but a Homebrew ffmpeg
   built without --enable-nvenc still falls back to libx264 software.

3. Register payload now includes hwAccel + maxTranscodeHeight so the web
   side can suggest a smaller alternate quality before the user even
   tries to play a 4K source on a software-only host. Software-only =
   1080p cap, any HW backend = 2160p cap.

internal/engine/hls.go

@@ -845,9 +845,21 @@ func buildHLSFFmpegArgsAt(cfg HLSSessionConfig, probe *StreamProbe, tmpDir strin
 	case "h264_qsv":
 		args = append(args, "-preset", "medium", "-look_ahead", "0")
 	}
-	args = append(args, "-profile:v", "main", "-level:v", "4.0")
-
+	// Derive H.264 level from the actual output height. A fixed "4.0" caps the
+	// encoder at 1080p — anything taller (1440p, 4K source on quality=original)
+	// fails libx264 with "frame MB size > level limit" and emits unplayable
+	// segments. The output height matches qcap.MaxHeight when the source is
+	// downscaled, otherwise probe.Height (already populated by ffprobe).
 	qcap := resolveQualityCap(cfg.Quality)
+	outputHeight := qcap.MaxHeight
+	if outputHeight == 0 {
+		outputHeight = cfg.Transcode.MaxHeight
+	}
+	if outputHeight == 0 || (probe.Height > 0 && probe.Height < outputHeight) {
+		outputHeight = probe.Height
+	}
+	args = append(args, "-profile:v", "main", "-level:v", H264LevelForHeight(outputHeight))
+
 	bitrate := qcap.VideoBitrate
 	if bitrate == "" {
 		bitrate = cfg.Transcode.VideoBitrate

internal/engine/hwaccel.go

@@ -128,3 +128,31 @@ func (h HWAccel) FFmpegVideoCodec(target string) string {
 		return "libx264"
 	}
 }
+
+// H264LevelForHeight returns the lowest H.264 profile level capable of encoding
+// a stream at the given output pixel height (assumes ~16:9, ≤30 fps). The
+// previous code used a fixed "4.0" which silently rejects anything above 1080p
+// — libx264 logs "frame MB size > level limit" and emits a corrupt stream.
+// Returning a tighter level on smaller outputs keeps player compatibility on
+// older devices where the encoder can't auto-pick.
+func H264LevelForHeight(height int) string {
+	switch {
+	case height <= 0:
+		// Unknown source — pick a level that covers up to 4K so we never
+		// re-introduce the silent-failure mode that motivated this helper.
+		return "5.1"
+	case height <= 480:
+		return "3.0"
+	case height <= 720:
+		return "3.1"
+	case height <= 1080:
+		return "4.0"
+	case height <= 1440:
+		return "5.0"
+	case height <= 2160:
+		return "5.1"
+	default:
+		// 4K @ 60 fps and 8K all fall under 6.x.
+		return "6.0"
+	}
+}

internal/engine/transcoder.go

@@ -25,6 +25,7 @@ type TranscodeOpts struct {
 	VideoBitrate string // e.g. "5M"
 	AudioBitrate string // e.g. "192k"
 	MaxHeight    int    // optional downscale cap (e.g. 720)
+	SourceHeight int    // probed source height — used to derive a sane H.264 level
 	StartSeconds float64
 	FFmpegPath   string
 }
@@ -235,7 +236,16 @@ func buildFFmpegArgs(filePath string, opts TranscodeOpts) []string {
 		// can fail with "VaapiWrapper: failed initializing" on Linux boxes
 		// where VA-API isn't fully wired up. `main` keeps a clean software
 		// decode fallback on every desktop + mobile platform.
-		args = append(args, "-profile:v", "main", "-level:v", "4.0")
+		//
+		// Level is derived from the actual output height — a fixed "4.0"
+		// silently rejects 4K and 1440p sources at the libx264 macroblock
+		// limits and produces unplayable streams. opts.MaxHeight is the
+		// downscale cap when set; falling through means "encode at source".
+		levelHeight := opts.MaxHeight
+		if levelHeight == 0 || (opts.SourceHeight > 0 && opts.SourceHeight < levelHeight) {
+			levelHeight = opts.SourceHeight
+		}
+		args = append(args, "-profile:v", "main", "-level:v", H264LevelForHeight(levelHeight))
 		args = append(args, "-b:v", coalesce(opts.VideoBitrate, "5M"))
 		// Filter chain:
 		//   1. scale (optional) — cap height + force even width.

internal/engine/webrtc_stream.go

@@ -190,6 +190,7 @@ func buildStreamSource(
 		VideoBitrate: videoBitrate,
 		AudioBitrate: tc.AudioBitrate,
 		MaxHeight:    maxHeight,
+		SourceHeight: probe.Height,
 		FFmpegPath:   tc.FFmpegPath,
 	}
 	return newTranscodeSource(ctx, abs, probe, action, opts, displayName)