feat(usenet): implement full NNTP download pipeline
Complete usenet download support for unarr CLI: - NZB XML parser with password extraction from <head> meta - yEnc decoder with CRC32 verification - NNTP client with TLS, auth, and connection pool (up to 10 conns) - Segment downloader with parallel workers and progress reporting - Post-processing: par2 verify/repair, unrar/7z extraction with password support - Agent client methods: SearchNzbs, DownloadNzb, GetUsenetCredentials - UsenetDownloader implementing full Downloader interface - Daemon wiring: UsenetDownloader passed to Manager E2E tested: Oppenheimer 1080p (2.94 GB) downloaded via NNTP in 77.6s.
This commit is contained in:
Deivid Soto
parent
5f337eebd7
commit
e332c0a6e4
15 changed files with 3016 additions and 23 deletions
210
internal/usenet/yenc/decode.go
Normal file
210
internal/usenet/yenc/decode.go
Normal file
|
|
@ -0,0 +1,210 @@
|
|||
package yenc
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Part represents a decoded yEnc part (one NNTP article body).
|
||||
type Part struct {
|
||||
Name string // filename from =ybegin
|
||||
Number int // part number (1-based)
|
||||
Total int // total parts (from =ybegin total=N)
|
||||
Begin int64 // byte offset start (from =ypart begin=N, 1-based)
|
||||
End int64 // byte offset end (from =ypart end=N, inclusive)
|
||||
Size int64 // total file size (from =ybegin size=N)
|
||||
CRC32 uint32 // CRC32 of this part's data (from =yend pcrc32)
|
||||
Data []byte // decoded binary data
|
||||
}
|
||||
|
||||
// Decode reads a yEnc encoded article body and returns the decoded part.
|
||||
// The reader should contain the raw article body (after NNTP BODY response).
|
||||
func Decode(r io.Reader) (*Part, error) {
|
||||
scanner := bufio.NewScanner(r)
|
||||
scanner.Buffer(make([]byte, 0, 1024*1024), 10*1024*1024) // up to 10MB per article
|
||||
|
||||
part := &Part{}
|
||||
|
||||
// Phase 1: Find and parse =ybegin header
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if strings.HasPrefix(line, "=ybegin ") {
|
||||
parseYBegin(part, line)
|
||||
break
|
||||
}
|
||||
}
|
||||
if part.Name == "" && part.Size == 0 {
|
||||
return nil, fmt.Errorf("yenc: no =ybegin header found")
|
||||
}
|
||||
|
||||
// Phase 2: Find optional =ypart header (for multipart)
|
||||
// Peek at next line
|
||||
if scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if strings.HasPrefix(line, "=ypart ") {
|
||||
parseYPart(part, line)
|
||||
} else {
|
||||
// Not a ypart line, decode it as data
|
||||
part.Data = append(part.Data, decodeLine(line)...)
|
||||
}
|
||||
}
|
||||
|
||||
// Phase 3: Decode data lines until =yend
|
||||
hasher := crc32.NewIEEE()
|
||||
// Hash data we already decoded (if any from non-ypart line)
|
||||
if len(part.Data) > 0 {
|
||||
hasher.Write(part.Data)
|
||||
}
|
||||
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
|
||||
if strings.HasPrefix(line, "=yend") {
|
||||
parseYEnd(part, line)
|
||||
break
|
||||
}
|
||||
|
||||
decoded := decodeLine(line)
|
||||
hasher.Write(decoded)
|
||||
part.Data = append(part.Data, decoded...)
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, fmt.Errorf("yenc: read error: %w", err)
|
||||
}
|
||||
|
||||
// Verify CRC32 if provided
|
||||
if part.CRC32 != 0 {
|
||||
computed := hasher.Sum32()
|
||||
if computed != part.CRC32 {
|
||||
return nil, fmt.Errorf("yenc: CRC32 mismatch: expected %08x, got %08x", part.CRC32, computed)
|
||||
}
|
||||
}
|
||||
|
||||
return part, nil
|
||||
}
|
||||
|
||||
// DecodeBytes decodes a yEnc encoded byte slice.
|
||||
func DecodeBytes(data []byte) (*Part, error) {
|
||||
return Decode(bytes.NewReader(data))
|
||||
}
|
||||
|
||||
// decodeLine decodes a single line of yEnc data.
|
||||
// yEnc encoding: each byte = (original + 42) % 256
|
||||
// Escape character '=' followed by next byte: (escapedByte - 64 - 42) % 256
|
||||
func decodeLine(line string) []byte {
|
||||
out := make([]byte, 0, len(line))
|
||||
escaped := false
|
||||
|
||||
for i := 0; i < len(line); i++ {
|
||||
b := line[i]
|
||||
|
||||
if escaped {
|
||||
// Escaped byte: subtract 106 (42 + 64)
|
||||
out = append(out, b-106)
|
||||
escaped = false
|
||||
continue
|
||||
}
|
||||
|
||||
if b == '=' {
|
||||
escaped = true
|
||||
continue
|
||||
}
|
||||
|
||||
// Normal byte: subtract 42
|
||||
out = append(out, b-42)
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// parseYBegin parses "=ybegin part=1 total=50 line=128 size=768000 name=file.mkv"
|
||||
func parseYBegin(p *Part, line string) {
|
||||
p.Number = getIntParam(line, "part")
|
||||
p.Total = getIntParam(line, "total")
|
||||
p.Size = int64(getIntParam(line, "size"))
|
||||
|
||||
// Name is special: it's everything after "name=" to end of line
|
||||
if idx := strings.Index(line, "name="); idx >= 0 {
|
||||
p.Name = strings.TrimSpace(line[idx+5:])
|
||||
}
|
||||
}
|
||||
|
||||
// parseYPart parses "=ypart begin=1 end=768000"
|
||||
func parseYPart(p *Part, line string) {
|
||||
p.Begin = int64(getIntParam(line, "begin"))
|
||||
p.End = int64(getIntParam(line, "end"))
|
||||
}
|
||||
|
||||
// parseYEnd parses "=yend size=768000 part=1 pcrc32=ABCD1234 crc32=ABCD1234"
|
||||
func parseYEnd(p *Part, line string) {
|
||||
// pcrc32 is the CRC of this part; crc32 is the CRC of the whole file (only on last part)
|
||||
if hex := getHexParam(line, "pcrc32"); hex != 0 {
|
||||
p.CRC32 = hex
|
||||
} else if hex := getHexParam(line, "crc32"); hex != 0 && p.Total <= 1 {
|
||||
// For single-part files, crc32 is the only CRC
|
||||
p.CRC32 = hex
|
||||
}
|
||||
}
|
||||
|
||||
// getIntParam extracts an integer parameter from a yEnc header line.
|
||||
func getIntParam(line, key string) int {
|
||||
prefix := key + "="
|
||||
idx := strings.Index(line, prefix)
|
||||
if idx < 0 {
|
||||
return 0
|
||||
}
|
||||
start := idx + len(prefix)
|
||||
end := start
|
||||
for end < len(line) && line[end] >= '0' && line[end] <= '9' {
|
||||
end++
|
||||
}
|
||||
if end == start {
|
||||
return 0
|
||||
}
|
||||
v, _ := strconv.Atoi(line[start:end])
|
||||
return v
|
||||
}
|
||||
|
||||
// getHexParam extracts a hex parameter (like CRC32) from a yEnc header line.
|
||||
// Uses word-boundary matching to avoid "pcrc32" matching "crc32".
|
||||
func getHexParam(line, key string) uint32 {
|
||||
prefix := key + "="
|
||||
idx := strings.Index(line, prefix)
|
||||
if idx < 0 {
|
||||
return 0
|
||||
}
|
||||
// Ensure we're matching the exact key, not a suffix (e.g., "crc32" should not match "pcrc32")
|
||||
if idx > 0 && line[idx-1] != ' ' && line[idx-1] != '\t' {
|
||||
// Try finding another occurrence after this one
|
||||
rest := line[idx+1:]
|
||||
nextIdx := strings.Index(rest, prefix)
|
||||
if nextIdx < 0 {
|
||||
return 0
|
||||
}
|
||||
idx = idx + 1 + nextIdx
|
||||
if idx > 0 && line[idx-1] != ' ' && line[idx-1] != '\t' {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
start := idx + len(prefix)
|
||||
end := start
|
||||
for end < len(line) && ((line[end] >= '0' && line[end] <= '9') ||
|
||||
(line[end] >= 'a' && line[end] <= 'f') ||
|
||||
(line[end] >= 'A' && line[end] <= 'F')) {
|
||||
end++
|
||||
}
|
||||
if end == start {
|
||||
return 0
|
||||
}
|
||||
v, err := strconv.ParseUint(line[start:end], 16, 32)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return uint32(v)
|
||||
}
|
||||
208
internal/usenet/yenc/decode_test.go
Normal file
208
internal/usenet/yenc/decode_test.go
Normal file
|
|
@ -0,0 +1,208 @@
|
|||
package yenc
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDecodeLine(t *testing.T) {
|
||||
// yEnc: each byte = (original + 42) % 256
|
||||
// So to encode byte 0x00, we store 0x2A (42)
|
||||
// To encode byte 0x01, we store 0x2B (43)
|
||||
|
||||
// Encode "Hello" manually:
|
||||
// H=72 → 72+42=114='r'
|
||||
// e=101 → 101+42=143='\x8f'
|
||||
// l=108 → 108+42=150='\x96'
|
||||
// l=108 → same
|
||||
// o=111 → 111+42=153='\x99'
|
||||
input := string([]byte{114, 143, 150, 150, 153})
|
||||
decoded := decodeLine(input)
|
||||
if string(decoded) != "Hello" {
|
||||
t.Errorf("decodeLine: got %q, want %q", string(decoded), "Hello")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeLineWithEscape(t *testing.T) {
|
||||
// Escaped characters: =\x00 (NUL), =\n (LF), =\r (CR), == (=)
|
||||
// Escape: '=' followed by byte, decoded as (byte - 64 - 42) = (byte - 106)
|
||||
|
||||
// To encode byte 0x00 (NUL): escape it → '=' + (0 + 42 + 64) = '=' + 106 = '=' + 'j'
|
||||
input := "=j" // should decode to 0x00
|
||||
decoded := decodeLine(input)
|
||||
if len(decoded) != 1 || decoded[0] != 0x00 {
|
||||
t.Errorf("escape decode: got %v, want [0x00]", decoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeSimpleArticle(t *testing.T) {
|
||||
// Create a simple yEnc encoded article
|
||||
original := []byte("Hello, World! This is a test of yEnc encoding.")
|
||||
encoded := encodeForTest(original)
|
||||
|
||||
crc := crc32.ChecksumIEEE(original)
|
||||
|
||||
article := fmt.Sprintf("=ybegin line=128 size=%d name=test.txt\r\n%s\r\n=yend size=%d crc32=%08x\r\n",
|
||||
len(original), encoded, len(original), crc)
|
||||
|
||||
part, err := Decode(strings.NewReader(article))
|
||||
if err != nil {
|
||||
t.Fatalf("Decode failed: %v", err)
|
||||
}
|
||||
|
||||
if part.Name != "test.txt" {
|
||||
t.Errorf("Name: got %q, want %q", part.Name, "test.txt")
|
||||
}
|
||||
if part.Size != int64(len(original)) {
|
||||
t.Errorf("Size: got %d, want %d", part.Size, len(original))
|
||||
}
|
||||
if !bytes.Equal(part.Data, original) {
|
||||
t.Errorf("Data mismatch:\n got: %s\n want: %s", hex.EncodeToString(part.Data), hex.EncodeToString(original))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeMultipart(t *testing.T) {
|
||||
original := []byte("Part one data here")
|
||||
encoded := encodeForTest(original)
|
||||
|
||||
crc := crc32.ChecksumIEEE(original)
|
||||
|
||||
article := fmt.Sprintf("=ybegin part=1 total=3 line=128 size=1000 name=movie.mkv\r\n"+
|
||||
"=ypart begin=1 end=%d\r\n"+
|
||||
"%s\r\n"+
|
||||
"=yend size=%d part=1 pcrc32=%08x\r\n",
|
||||
len(original), encoded, len(original), crc)
|
||||
|
||||
part, err := Decode(strings.NewReader(article))
|
||||
if err != nil {
|
||||
t.Fatalf("Decode failed: %v", err)
|
||||
}
|
||||
|
||||
if part.Number != 1 {
|
||||
t.Errorf("Number: got %d, want 1", part.Number)
|
||||
}
|
||||
if part.Total != 3 {
|
||||
t.Errorf("Total: got %d, want 3", part.Total)
|
||||
}
|
||||
if part.Begin != 1 {
|
||||
t.Errorf("Begin: got %d, want 1", part.Begin)
|
||||
}
|
||||
if part.End != int64(len(original)) {
|
||||
t.Errorf("End: got %d, want %d", part.End, len(original))
|
||||
}
|
||||
if part.Name != "movie.mkv" {
|
||||
t.Errorf("Name: got %q", part.Name)
|
||||
}
|
||||
if !bytes.Equal(part.Data, original) {
|
||||
t.Error("Data mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeCRC32Mismatch(t *testing.T) {
|
||||
original := []byte("test data")
|
||||
encoded := encodeForTest(original)
|
||||
|
||||
article := fmt.Sprintf("=ybegin line=128 size=%d name=test.bin\r\n%s\r\n=yend size=%d crc32=deadbeef\r\n",
|
||||
len(original), encoded, len(original))
|
||||
|
||||
_, err := Decode(strings.NewReader(article))
|
||||
if err == nil {
|
||||
t.Error("expected CRC32 mismatch error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "CRC32 mismatch") {
|
||||
t.Errorf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeNoHeader(t *testing.T) {
|
||||
_, err := Decode(strings.NewReader("just some random data\r\n"))
|
||||
if err == nil {
|
||||
t.Error("expected error for missing header")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeBytes(t *testing.T) {
|
||||
original := []byte("quick test")
|
||||
encoded := encodeForTest(original)
|
||||
crc := crc32.ChecksumIEEE(original)
|
||||
|
||||
article := fmt.Sprintf("=ybegin line=128 size=%d name=q.bin\r\n%s\r\n=yend size=%d crc32=%08x\r\n",
|
||||
len(original), encoded, len(original), crc)
|
||||
|
||||
part, err := DecodeBytes([]byte(article))
|
||||
if err != nil {
|
||||
t.Fatalf("DecodeBytes failed: %v", err)
|
||||
}
|
||||
if !bytes.Equal(part.Data, original) {
|
||||
t.Error("Data mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeBinaryData(t *testing.T) {
|
||||
// Test with all byte values 0-255
|
||||
original := make([]byte, 256)
|
||||
for i := range original {
|
||||
original[i] = byte(i)
|
||||
}
|
||||
|
||||
encoded := encodeForTest(original)
|
||||
crc := crc32.ChecksumIEEE(original)
|
||||
|
||||
article := fmt.Sprintf("=ybegin line=128 size=%d name=binary.bin\r\n%s\r\n=yend size=%d crc32=%08x\r\n",
|
||||
len(original), encoded, len(original), crc)
|
||||
|
||||
part, err := Decode(strings.NewReader(article))
|
||||
if err != nil {
|
||||
t.Fatalf("Decode failed: %v", err)
|
||||
}
|
||||
if !bytes.Equal(part.Data, original) {
|
||||
t.Errorf("Binary data mismatch: got %d bytes, want %d", len(part.Data), len(original))
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetIntParam(t *testing.T) {
|
||||
line := "=ybegin part=5 total=100 line=128 size=768000 name=file.mkv"
|
||||
if v := getIntParam(line, "part"); v != 5 {
|
||||
t.Errorf("part: got %d", v)
|
||||
}
|
||||
if v := getIntParam(line, "total"); v != 100 {
|
||||
t.Errorf("total: got %d", v)
|
||||
}
|
||||
if v := getIntParam(line, "size"); v != 768000 {
|
||||
t.Errorf("size: got %d", v)
|
||||
}
|
||||
if v := getIntParam(line, "missing"); v != 0 {
|
||||
t.Errorf("missing: got %d", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetHexParam(t *testing.T) {
|
||||
line := "=yend size=1000 pcrc32=ABCD1234 crc32=deadbeef"
|
||||
if v := getHexParam(line, "pcrc32"); v != 0xABCD1234 {
|
||||
t.Errorf("pcrc32: got %08x, want ABCD1234", v)
|
||||
}
|
||||
if v := getHexParam(line, "crc32"); v != 0xdeadbeef {
|
||||
t.Errorf("crc32: got %08x, want deadbeef", v)
|
||||
}
|
||||
}
|
||||
|
||||
// encodeForTest encodes data using yEnc for testing purposes.
|
||||
func encodeForTest(data []byte) string {
|
||||
var buf bytes.Buffer
|
||||
for _, b := range data {
|
||||
encoded := byte((int(b) + 42) % 256)
|
||||
// Escape special bytes: NUL, LF, CR, '=', '.'
|
||||
switch encoded {
|
||||
case 0x00, 0x0A, 0x0D, 0x3D, 0x2E:
|
||||
buf.WriteByte('=')
|
||||
buf.WriteByte(byte((int(encoded) + 64) % 256))
|
||||
default:
|
||||
buf.WriteByte(encoded)
|
||||
}
|
||||
}
|
||||
return buf.String()
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue