feat(usenet): implement full NNTP download pipeline
Complete usenet download support for unarr CLI: - NZB XML parser with password extraction from <head> meta - yEnc decoder with CRC32 verification - NNTP client with TLS, auth, and connection pool (up to 10 conns) - Segment downloader with parallel workers and progress reporting - Post-processing: par2 verify/repair, unrar/7z extraction with password support - Agent client methods: SearchNzbs, DownloadNzb, GetUsenetCredentials - UsenetDownloader implementing full Downloader interface - Daemon wiring: UsenetDownloader passed to Manager E2E tested: Oppenheimer 1080p (2.94 GB) downloaded via NNTP in 77.6s.
This commit is contained in:
Deivid Soto
parent
5f337eebd7
commit
e332c0a6e4
15 changed files with 3016 additions and 23 deletions
308
internal/usenet/download/downloader.go
Normal file
308
internal/usenet/download/downloader.go
Normal file
|
|
@ -0,0 +1,308 @@
|
|||
package download
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/torrentclaw/torrentclaw-cli/internal/ui"
|
||||
"github.com/torrentclaw/torrentclaw-cli/internal/usenet/nntp"
|
||||
"github.com/torrentclaw/torrentclaw-cli/internal/usenet/nzb"
|
||||
"github.com/torrentclaw/torrentclaw-cli/internal/usenet/yenc"
|
||||
)
|
||||
|
||||
// Progress is emitted during download.
|
||||
type Progress struct {
|
||||
FileName string
|
||||
SegmentsDone int
|
||||
SegmentsTotal int
|
||||
BytesDownloaded int64
|
||||
BytesTotal int64
|
||||
SpeedBps int64
|
||||
}
|
||||
|
||||
// Downloader orchestrates downloading all segments of NZB files via NNTP.
|
||||
type Downloader struct {
|
||||
nntp *nntp.Client
|
||||
}
|
||||
|
||||
// NewDownloader creates a usenet segment downloader.
|
||||
func NewDownloader(nntpClient *nntp.Client) *Downloader {
|
||||
return &Downloader{nntp: nntpClient}
|
||||
}
|
||||
|
||||
// DownloadFile downloads all segments of a single NZB file and assembles them.
|
||||
// Returns the path to the assembled file.
|
||||
func (d *Downloader) DownloadFile(ctx context.Context, file nzb.File, outputDir string, progressCh chan<- Progress) (string, error) {
|
||||
fileName := file.Filename()
|
||||
if fileName == "" {
|
||||
fileName = fmt.Sprintf("usenet_%d", time.Now().UnixNano())
|
||||
}
|
||||
|
||||
destPath := filepath.Join(outputDir, fileName)
|
||||
|
||||
// Ensure output directory exists
|
||||
if err := os.MkdirAll(outputDir, 0o755); err != nil {
|
||||
return "", fmt.Errorf("mkdir: %w", err)
|
||||
}
|
||||
|
||||
totalBytes := file.TotalBytes()
|
||||
totalSegs := len(file.Segments)
|
||||
|
||||
// Sort segments by number
|
||||
segments := make([]nzb.Segment, len(file.Segments))
|
||||
copy(segments, file.Segments)
|
||||
sort.Slice(segments, func(i, j int) bool {
|
||||
return segments[i].Number < segments[j].Number
|
||||
})
|
||||
|
||||
// Create/open output file
|
||||
outFile, err := os.Create(destPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create file: %w", err)
|
||||
}
|
||||
defer outFile.Close()
|
||||
|
||||
// Pre-allocate file if we know the size
|
||||
if totalBytes > 0 {
|
||||
outFile.Truncate(totalBytes)
|
||||
}
|
||||
|
||||
// Download segments using worker pool
|
||||
var downloaded atomic.Int64
|
||||
var segsDone atomic.Int32
|
||||
startTime := time.Now()
|
||||
|
||||
// Create work channel
|
||||
type segWork struct {
|
||||
seg nzb.Segment
|
||||
index int
|
||||
}
|
||||
workCh := make(chan segWork, len(segments))
|
||||
for i, seg := range segments {
|
||||
workCh <- segWork{seg: seg, index: i}
|
||||
}
|
||||
close(workCh)
|
||||
|
||||
// Track file offsets for each segment (sequential assembly)
|
||||
offsets := make([]int64, len(segments))
|
||||
var offset int64
|
||||
for i, seg := range segments {
|
||||
offsets[i] = offset
|
||||
offset += seg.Bytes
|
||||
}
|
||||
|
||||
// Progress reporter goroutine
|
||||
stopProgress := make(chan struct{})
|
||||
go func() {
|
||||
ticker := time.NewTicker(500 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
dl := downloaded.Load()
|
||||
elapsed := time.Since(startTime).Seconds()
|
||||
var speed int64
|
||||
if elapsed > 0 {
|
||||
speed = int64(float64(dl) / elapsed)
|
||||
}
|
||||
if progressCh != nil {
|
||||
select {
|
||||
case progressCh <- Progress{
|
||||
FileName: fileName,
|
||||
SegmentsDone: int(segsDone.Load()),
|
||||
SegmentsTotal: totalSegs,
|
||||
BytesDownloaded: dl,
|
||||
BytesTotal: totalBytes,
|
||||
SpeedBps: speed,
|
||||
}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
case <-stopProgress:
|
||||
return
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Workers — one per NNTP connection
|
||||
numWorkers := d.nntp.ActiveConnections()
|
||||
if numWorkers <= 0 {
|
||||
numWorkers = 1
|
||||
}
|
||||
|
||||
errCh := make(chan error, 1)
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for w := 0; w < numWorkers; w++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for work := range workCh {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
data, err := d.downloadSegment(ctx, work.seg)
|
||||
if err != nil {
|
||||
select {
|
||||
case errCh <- fmt.Errorf("segment %d (%s): %w", work.seg.Number, work.seg.MessageID, err):
|
||||
default:
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Write decoded data at the correct offset
|
||||
// WriteAt is safe for concurrent non-overlapping writes
|
||||
_, writeErr := outFile.WriteAt(data, offsets[work.index])
|
||||
|
||||
if writeErr != nil {
|
||||
select {
|
||||
case errCh <- fmt.Errorf("write segment %d: %w", work.seg.Number, writeErr):
|
||||
default:
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
downloaded.Add(int64(len(data)))
|
||||
segsDone.Add(1)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Wait for all workers
|
||||
wg.Wait()
|
||||
|
||||
// Stop progress reporter before sending final progress
|
||||
close(stopProgress)
|
||||
|
||||
// Check for errors
|
||||
select {
|
||||
case err := <-errCh:
|
||||
os.Remove(destPath)
|
||||
return "", err
|
||||
default:
|
||||
}
|
||||
|
||||
// Check context cancellation
|
||||
if ctx.Err() != nil {
|
||||
os.Remove(destPath)
|
||||
return "", ctx.Err()
|
||||
}
|
||||
|
||||
// Final progress report
|
||||
dl := downloaded.Load()
|
||||
elapsed := time.Since(startTime).Seconds()
|
||||
var speed int64
|
||||
if elapsed > 0 {
|
||||
speed = int64(float64(dl) / elapsed)
|
||||
}
|
||||
if progressCh != nil {
|
||||
select {
|
||||
case progressCh <- Progress{
|
||||
FileName: fileName,
|
||||
SegmentsDone: totalSegs,
|
||||
SegmentsTotal: totalSegs,
|
||||
BytesDownloaded: dl,
|
||||
BytesTotal: totalBytes,
|
||||
SpeedBps: speed,
|
||||
}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// Truncate to actual size (in case pre-allocation was larger)
|
||||
actualSize := downloaded.Load()
|
||||
if actualSize > 0 {
|
||||
outFile.Truncate(actualSize)
|
||||
}
|
||||
|
||||
log.Printf("[usenet] downloaded %s (%d segments, %s)", fileName, totalSegs, ui.FormatBytes(actualSize))
|
||||
return destPath, nil
|
||||
}
|
||||
|
||||
// DownloadNZB downloads content files from an NZB (rars or direct content).
|
||||
// Par2 files are NOT downloaded initially — they're only fetched on demand
|
||||
// if extraction fails (via DownloadPar2).
|
||||
// Returns a map of filename → filepath for all downloaded files.
|
||||
func (d *Downloader) DownloadNZB(ctx context.Context, n *nzb.NZB, outputDir string, progressCh chan<- Progress) (map[string]string, error) {
|
||||
// Determine which files to download (NO par2 initially)
|
||||
var filesToDownload []nzb.File
|
||||
|
||||
if n.HasRars() {
|
||||
filesToDownload = n.RarFiles()
|
||||
} else {
|
||||
filesToDownload = n.ContentFiles()
|
||||
}
|
||||
|
||||
if len(filesToDownload) == 0 {
|
||||
return nil, fmt.Errorf("no downloadable files found in NZB")
|
||||
}
|
||||
|
||||
results := make(map[string]string)
|
||||
|
||||
for _, file := range filesToDownload {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return results, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
path, err := d.DownloadFile(ctx, file, outputDir, progressCh)
|
||||
if err != nil {
|
||||
return results, fmt.Errorf("download %s: %w", file.Filename(), err)
|
||||
}
|
||||
results[file.Filename()] = path
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// DownloadPar2 downloads par2 parity files from the NZB.
|
||||
// Called on-demand when extraction/verification fails.
|
||||
func (d *Downloader) DownloadPar2(ctx context.Context, n *nzb.NZB, outputDir string, progressCh chan<- Progress) (map[string]string, error) {
|
||||
par2Files := n.Par2Files()
|
||||
if len(par2Files) == 0 {
|
||||
return nil, fmt.Errorf("no par2 files in NZB")
|
||||
}
|
||||
|
||||
results := make(map[string]string)
|
||||
for _, file := range par2Files {
|
||||
path, err := d.DownloadFile(ctx, file, outputDir, progressCh)
|
||||
if err != nil {
|
||||
log.Printf("[usenet] par2 download failed (non-fatal): %v", err)
|
||||
continue
|
||||
}
|
||||
results[file.Filename()] = path
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// downloadSegment downloads and decodes a single segment.
|
||||
func (d *Downloader) downloadSegment(ctx context.Context, seg nzb.Segment) ([]byte, error) {
|
||||
// Download article body via NNTP
|
||||
body, err := d.nntp.Body(ctx, seg.MessageID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("nntp body: %w", err)
|
||||
}
|
||||
|
||||
// Decode yEnc
|
||||
part, err := yenc.Decode(bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("yenc decode: %w", err)
|
||||
}
|
||||
|
||||
return part.Data, nil
|
||||
}
|
||||
|
||||
177
internal/usenet/download/e2e_test.go
Normal file
177
internal/usenet/download/e2e_test.go
Normal file
|
|
@ -0,0 +1,177 @@
|
|||
package download_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/torrentclaw/torrentclaw-cli/internal/usenet/download"
|
||||
"github.com/torrentclaw/torrentclaw-cli/internal/usenet/nntp"
|
||||
"github.com/torrentclaw/torrentclaw-cli/internal/usenet/nzb"
|
||||
"github.com/torrentclaw/torrentclaw-cli/internal/usenet/postprocess"
|
||||
)
|
||||
|
||||
// TestE2EDownload is a real end-to-end test that downloads from Usenet.
|
||||
// It requires:
|
||||
// - NZB file at /tmp/oppenheimer-test.nzb
|
||||
// - NNTP credentials in env: NNTP_USER, NNTP_PASS
|
||||
// - Network access to reader.torrentclaw.com:563
|
||||
//
|
||||
// Run with: go test -v -run TestE2EDownload -tags e2e -timeout 30m ./internal/usenet/download/
|
||||
func TestE2EDownload(t *testing.T) {
|
||||
if os.Getenv("NNTP_USER") == "" || os.Getenv("NNTP_PASS") == "" {
|
||||
t.Skip("NNTP_USER and NNTP_PASS not set — skipping e2e test")
|
||||
}
|
||||
|
||||
nzbPath := os.Getenv("NZB_FILE")
|
||||
if nzbPath == "" {
|
||||
nzbPath = "/tmp/oppenheimer-test.nzb"
|
||||
}
|
||||
|
||||
// 1. Parse NZB
|
||||
f, err := os.Open(nzbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("open NZB: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
nzbFile, err := nzb.Parse(f)
|
||||
if err != nil {
|
||||
t.Fatalf("parse NZB: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("NZB: %d files, %d total segments, %.2f GB",
|
||||
len(nzbFile.Files), nzbFile.TotalSegments(),
|
||||
float64(nzbFile.TotalBytes())/1024/1024/1024)
|
||||
|
||||
if nzbFile.Password != "" {
|
||||
t.Logf("NZB password: %s", nzbFile.Password)
|
||||
}
|
||||
|
||||
t.Logf("Has rars: %v, Has par2: %v", nzbFile.HasRars(), nzbFile.HasPar2())
|
||||
|
||||
for _, file := range nzbFile.Files {
|
||||
t.Logf(" %s — %d segments, %.1f MB",
|
||||
file.Filename(), len(file.Segments),
|
||||
float64(file.TotalBytes())/1024/1024)
|
||||
}
|
||||
|
||||
// 2. Connect NNTP
|
||||
client := nntp.NewClient(nntp.Config{
|
||||
Host: "reader.torrentclaw.com",
|
||||
Port: 563,
|
||||
SSL: true,
|
||||
TLSServerName: "xsnews.nl",
|
||||
Username: os.Getenv("NNTP_USER"),
|
||||
Password: os.Getenv("NNTP_PASS"),
|
||||
MaxConnections: 10,
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
t.Log("Connecting NNTP (10 connections)...")
|
||||
if err := client.Connect(ctx); err != nil {
|
||||
t.Fatalf("NNTP connect: %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
t.Logf("NNTP connected: %s", client.Status())
|
||||
|
||||
// 3. Download all files
|
||||
outputDir, err := os.MkdirTemp("", "usenet-e2e-*")
|
||||
if err != nil {
|
||||
t.Fatalf("tmpdir: %v", err)
|
||||
}
|
||||
t.Logf("Output dir: %s", outputDir)
|
||||
// Don't cleanup automatically — let user inspect
|
||||
// defer os.RemoveAll(outputDir)
|
||||
|
||||
dl := download.NewDownloader(client)
|
||||
|
||||
progressCh := make(chan download.Progress, 64)
|
||||
go func() {
|
||||
for p := range progressCh {
|
||||
pct := 0
|
||||
if p.BytesTotal > 0 {
|
||||
pct = int(float64(p.BytesDownloaded) / float64(p.BytesTotal) * 100)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "\r [%s] %d%% — %s/%s @ %s/s (%d/%d segs) ",
|
||||
p.FileName,
|
||||
pct,
|
||||
formatSize(p.BytesDownloaded),
|
||||
formatSize(p.BytesTotal),
|
||||
formatSize(p.SpeedBps),
|
||||
p.SegmentsDone, p.SegmentsTotal)
|
||||
}
|
||||
fmt.Fprintln(os.Stderr)
|
||||
}()
|
||||
|
||||
downloadedFiles, err := dl.DownloadNZB(ctx, nzbFile, outputDir, progressCh)
|
||||
close(progressCh)
|
||||
if err != nil {
|
||||
t.Fatalf("download: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("Downloaded %d files:", len(downloadedFiles))
|
||||
for name, path := range downloadedFiles {
|
||||
fi, _ := os.Stat(path)
|
||||
size := int64(0)
|
||||
if fi != nil {
|
||||
size = fi.Size()
|
||||
}
|
||||
t.Logf(" %s → %s (%.1f MB)", name, path, float64(size)/1024/1024)
|
||||
}
|
||||
|
||||
// 4. Post-process
|
||||
t.Log("Post-processing...")
|
||||
result, err := postprocess.Process(outputDir, downloadedFiles, postprocess.Options{
|
||||
Password: nzbFile.Password,
|
||||
Cleanup: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("post-process: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("Post-process result:")
|
||||
t.Logf(" Final path: %s", result.FinalPath)
|
||||
t.Logf(" Repaired: %v", result.Repaired)
|
||||
t.Logf(" Extracted: %v", result.Extracted)
|
||||
t.Logf(" Files: %v", result.Files)
|
||||
|
||||
// Verify final file exists and has size
|
||||
if result.FinalPath != "" {
|
||||
fi, err := os.Stat(result.FinalPath)
|
||||
if err != nil {
|
||||
t.Errorf("final file stat: %v", err)
|
||||
} else {
|
||||
t.Logf(" Final size: %.2f GB", float64(fi.Size())/1024/1024/1024)
|
||||
}
|
||||
}
|
||||
|
||||
// List all files in output dir
|
||||
t.Log("Final directory contents:")
|
||||
filepath.Walk(outputDir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil || info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
rel, _ := filepath.Rel(outputDir, path)
|
||||
t.Logf(" %s (%.1f MB)", rel, float64(info.Size())/1024/1024)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func formatSize(b int64) string {
|
||||
const unit = 1024
|
||||
if b < unit {
|
||||
return fmt.Sprintf("%d B", b)
|
||||
}
|
||||
div, exp := int64(unit), 0
|
||||
for n := b / unit; n >= unit; n /= unit {
|
||||
div *= unit
|
||||
exp++
|
||||
}
|
||||
return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp])
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue