feat: improve daemon resilience, streaming, and usenet downloads
- Add daemon state persistence and stale resume file cleanup - Add TriggerPoll for WebSocket resume actions - Improve stream server with graceful shutdown and connection tracking - Add desktop notifications for download completion - Add media file organization with Movies/TV Shows detection - Improve usenet downloader with progress tracking and resume support - Add self-update package with GitHub release verification - Downgrade tablewriter to v0.0.5 (v1.x API breaking change)
This commit is contained in:
Deivid Soto
parent
e332c0a6e4
commit
197e33956a
24 changed files with 2310 additions and 84 deletions
|
|
@ -12,7 +12,6 @@ import (
|
|||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/torrentclaw/torrentclaw-cli/internal/ui"
|
||||
"github.com/torrentclaw/torrentclaw-cli/internal/usenet/nntp"
|
||||
"github.com/torrentclaw/torrentclaw-cli/internal/usenet/nzb"
|
||||
"github.com/torrentclaw/torrentclaw-cli/internal/usenet/yenc"
|
||||
|
|
@ -39,8 +38,11 @@ func NewDownloader(nntpClient *nntp.Client) *Downloader {
|
|||
}
|
||||
|
||||
// DownloadFile downloads all segments of a single NZB file and assembles them.
|
||||
// If tracker is non-nil, it is used for resume support: completed segments are
|
||||
// skipped, and progress is persisted to disk on pause/error.
|
||||
// fileIndex is the index of this file within the NZB (for the tracker).
|
||||
// Returns the path to the assembled file.
|
||||
func (d *Downloader) DownloadFile(ctx context.Context, file nzb.File, outputDir string, progressCh chan<- Progress) (string, error) {
|
||||
func (d *Downloader) DownloadFile(ctx context.Context, file nzb.File, fileIndex int, outputDir string, tracker *ProgressTracker, progressCh chan<- Progress) (string, error) {
|
||||
fileName := file.Filename()
|
||||
if fileName == "" {
|
||||
fileName = fmt.Sprintf("usenet_%d", time.Now().UnixNano())
|
||||
|
|
@ -53,6 +55,15 @@ func (d *Downloader) DownloadFile(ctx context.Context, file nzb.File, outputDir
|
|||
return "", fmt.Errorf("mkdir: %w", err)
|
||||
}
|
||||
|
||||
// If tracker says this file is fully done, skip entirely
|
||||
if tracker != nil && tracker.IsFileDone(fileIndex) {
|
||||
if _, err := os.Stat(destPath); err == nil {
|
||||
log.Printf("[usenet] skipping %s (fully downloaded in previous run)", fileName)
|
||||
return destPath, nil
|
||||
}
|
||||
// File was marked done but doesn't exist on disk — re-download
|
||||
}
|
||||
|
||||
totalBytes := file.TotalBytes()
|
||||
totalSegs := len(file.Segments)
|
||||
|
||||
|
|
@ -63,34 +74,6 @@ func (d *Downloader) DownloadFile(ctx context.Context, file nzb.File, outputDir
|
|||
return segments[i].Number < segments[j].Number
|
||||
})
|
||||
|
||||
// Create/open output file
|
||||
outFile, err := os.Create(destPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create file: %w", err)
|
||||
}
|
||||
defer outFile.Close()
|
||||
|
||||
// Pre-allocate file if we know the size
|
||||
if totalBytes > 0 {
|
||||
outFile.Truncate(totalBytes)
|
||||
}
|
||||
|
||||
// Download segments using worker pool
|
||||
var downloaded atomic.Int64
|
||||
var segsDone atomic.Int32
|
||||
startTime := time.Now()
|
||||
|
||||
// Create work channel
|
||||
type segWork struct {
|
||||
seg nzb.Segment
|
||||
index int
|
||||
}
|
||||
workCh := make(chan segWork, len(segments))
|
||||
for i, seg := range segments {
|
||||
workCh <- segWork{seg: seg, index: i}
|
||||
}
|
||||
close(workCh)
|
||||
|
||||
// Track file offsets for each segment (sequential assembly)
|
||||
offsets := make([]int64, len(segments))
|
||||
var offset int64
|
||||
|
|
@ -99,6 +82,76 @@ func (d *Downloader) DownloadFile(ctx context.Context, file nzb.File, outputDir
|
|||
offset += seg.Bytes
|
||||
}
|
||||
|
||||
// Open output file — resume-aware
|
||||
var outFile *os.File
|
||||
var err error
|
||||
resuming := false
|
||||
|
||||
if tracker != nil {
|
||||
if _, statErr := os.Stat(destPath); statErr == nil && tracker.CompletedSegments(fileIndex) > 0 {
|
||||
// Partial file exists and we have progress — open for read-write (no truncate)
|
||||
outFile, err = os.OpenFile(destPath, os.O_RDWR, 0o644)
|
||||
resuming = true
|
||||
}
|
||||
}
|
||||
|
||||
if outFile == nil {
|
||||
// Fresh start
|
||||
outFile, err = os.Create(destPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create file: %w", err)
|
||||
}
|
||||
// Pre-allocate file if we know the size
|
||||
if totalBytes > 0 {
|
||||
outFile.Truncate(totalBytes)
|
||||
}
|
||||
} else if err != nil {
|
||||
return "", fmt.Errorf("open file for resume: %w", err)
|
||||
}
|
||||
defer outFile.Close()
|
||||
|
||||
// Download segments using worker pool
|
||||
var downloaded atomic.Int64
|
||||
var segsDone atomic.Int32
|
||||
startTime := time.Now()
|
||||
|
||||
// Create work channel — skip already-completed segments
|
||||
type segWork struct {
|
||||
seg nzb.Segment
|
||||
index int
|
||||
}
|
||||
|
||||
pendingCount := 0
|
||||
for i := range segments {
|
||||
if tracker != nil && tracker.IsDone(fileIndex, i) {
|
||||
// Already downloaded — count towards progress
|
||||
downloaded.Add(segments[i].Bytes)
|
||||
segsDone.Add(1)
|
||||
} else {
|
||||
pendingCount++
|
||||
}
|
||||
}
|
||||
|
||||
if resuming {
|
||||
log.Printf("[usenet] resuming %s (%d/%d segments, %s/%s)",
|
||||
fileName, totalSegs-pendingCount, totalSegs,
|
||||
formatBytes(downloaded.Load()), formatBytes(totalBytes))
|
||||
}
|
||||
|
||||
if pendingCount == 0 {
|
||||
// All segments already done
|
||||
log.Printf("[usenet] %s already complete (%d segments)", fileName, totalSegs)
|
||||
return destPath, nil
|
||||
}
|
||||
|
||||
workCh := make(chan segWork, pendingCount)
|
||||
for i, seg := range segments {
|
||||
if tracker == nil || !tracker.IsDone(fileIndex, i) {
|
||||
workCh <- segWork{seg: seg, index: i}
|
||||
}
|
||||
}
|
||||
close(workCh)
|
||||
|
||||
// Progress reporter goroutine
|
||||
stopProgress := make(chan struct{})
|
||||
go func() {
|
||||
|
|
@ -177,6 +230,11 @@ func (d *Downloader) DownloadFile(ctx context.Context, file nzb.File, outputDir
|
|||
|
||||
downloaded.Add(int64(len(data)))
|
||||
segsDone.Add(1)
|
||||
|
||||
// Mark segment as completed in tracker
|
||||
if tracker != nil {
|
||||
tracker.MarkDone(fileIndex, work.index)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
|
@ -187,17 +245,21 @@ func (d *Downloader) DownloadFile(ctx context.Context, file nzb.File, outputDir
|
|||
// Stop progress reporter before sending final progress
|
||||
close(stopProgress)
|
||||
|
||||
// Check for errors
|
||||
// Check for errors — keep partial file for resume (don't delete)
|
||||
select {
|
||||
case err := <-errCh:
|
||||
os.Remove(destPath)
|
||||
if tracker != nil {
|
||||
tracker.Flush()
|
||||
}
|
||||
return "", err
|
||||
default:
|
||||
}
|
||||
|
||||
// Check context cancellation
|
||||
// Check context cancellation — keep partial file for resume (don't delete)
|
||||
if ctx.Err() != nil {
|
||||
os.Remove(destPath)
|
||||
if tracker != nil {
|
||||
tracker.Flush()
|
||||
}
|
||||
return "", ctx.Err()
|
||||
}
|
||||
|
||||
|
|
@ -228,15 +290,16 @@ func (d *Downloader) DownloadFile(ctx context.Context, file nzb.File, outputDir
|
|||
outFile.Truncate(actualSize)
|
||||
}
|
||||
|
||||
log.Printf("[usenet] downloaded %s (%d segments, %s)", fileName, totalSegs, ui.FormatBytes(actualSize))
|
||||
log.Printf("[usenet] downloaded %s (%d segments, %s)", fileName, totalSegs, formatBytes(actualSize))
|
||||
return destPath, nil
|
||||
}
|
||||
|
||||
// DownloadNZB downloads content files from an NZB (rars or direct content).
|
||||
// Par2 files are NOT downloaded initially — they're only fetched on demand
|
||||
// if extraction fails (via DownloadPar2).
|
||||
// If tracker is non-nil, completed files are skipped and progress is tracked per-segment.
|
||||
// Returns a map of filename → filepath for all downloaded files.
|
||||
func (d *Downloader) DownloadNZB(ctx context.Context, n *nzb.NZB, outputDir string, progressCh chan<- Progress) (map[string]string, error) {
|
||||
func (d *Downloader) DownloadNZB(ctx context.Context, n *nzb.NZB, outputDir string, tracker *ProgressTracker, progressCh chan<- Progress) (map[string]string, error) {
|
||||
// Determine which files to download (NO par2 initially)
|
||||
var filesToDownload []nzb.File
|
||||
|
||||
|
|
@ -250,6 +313,13 @@ func (d *Downloader) DownloadNZB(ctx context.Context, n *nzb.NZB, outputDir stri
|
|||
return nil, fmt.Errorf("no downloadable files found in NZB")
|
||||
}
|
||||
|
||||
// Build NZB file index mapping: Subject → index in n.Files
|
||||
// This maps each file to its position in the ProgressTracker
|
||||
nzbFileIndex := make(map[string]int)
|
||||
for i, f := range n.Files {
|
||||
nzbFileIndex[f.Subject] = i
|
||||
}
|
||||
|
||||
results := make(map[string]string)
|
||||
|
||||
for _, file := range filesToDownload {
|
||||
|
|
@ -259,7 +329,19 @@ func (d *Downloader) DownloadNZB(ctx context.Context, n *nzb.NZB, outputDir stri
|
|||
default:
|
||||
}
|
||||
|
||||
path, err := d.DownloadFile(ctx, file, outputDir, progressCh)
|
||||
fileIdx := nzbFileIndex[file.Subject]
|
||||
|
||||
// Skip fully completed files
|
||||
if tracker != nil && tracker.IsFileDone(fileIdx) {
|
||||
destPath := filepath.Join(outputDir, file.Filename())
|
||||
if _, err := os.Stat(destPath); err == nil {
|
||||
results[file.Filename()] = destPath
|
||||
log.Printf("[usenet] skipping %s (complete)", file.Filename())
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
path, err := d.DownloadFile(ctx, file, fileIdx, outputDir, tracker, progressCh)
|
||||
if err != nil {
|
||||
return results, fmt.Errorf("download %s: %w", file.Filename(), err)
|
||||
}
|
||||
|
|
@ -271,6 +353,7 @@ func (d *Downloader) DownloadNZB(ctx context.Context, n *nzb.NZB, outputDir stri
|
|||
|
||||
// DownloadPar2 downloads par2 parity files from the NZB.
|
||||
// Called on-demand when extraction/verification fails.
|
||||
// No resume tracking — par2 files are small and downloaded fresh.
|
||||
func (d *Downloader) DownloadPar2(ctx context.Context, n *nzb.NZB, outputDir string, progressCh chan<- Progress) (map[string]string, error) {
|
||||
par2Files := n.Par2Files()
|
||||
if len(par2Files) == 0 {
|
||||
|
|
@ -279,7 +362,7 @@ func (d *Downloader) DownloadPar2(ctx context.Context, n *nzb.NZB, outputDir str
|
|||
|
||||
results := make(map[string]string)
|
||||
for _, file := range par2Files {
|
||||
path, err := d.DownloadFile(ctx, file, outputDir, progressCh)
|
||||
path, err := d.DownloadFile(ctx, file, -1, outputDir, nil, progressCh)
|
||||
if err != nil {
|
||||
log.Printf("[usenet] par2 download failed (non-fatal): %v", err)
|
||||
continue
|
||||
|
|
@ -306,3 +389,15 @@ func (d *Downloader) downloadSegment(ctx context.Context, seg nzb.Segment) ([]by
|
|||
return part.Data, nil
|
||||
}
|
||||
|
||||
func formatBytes(b int64) string {
|
||||
const unit = 1024
|
||||
if b < unit {
|
||||
return fmt.Sprintf("%d B", b)
|
||||
}
|
||||
div, exp := int64(unit), 0
|
||||
for n := b / unit; n >= unit; n /= unit {
|
||||
div *= unit
|
||||
exp++
|
||||
}
|
||||
return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp])
|
||||
}
|
||||
|
|
|
|||
|
|
@ -109,7 +109,7 @@ func TestE2EDownload(t *testing.T) {
|
|||
fmt.Fprintln(os.Stderr)
|
||||
}()
|
||||
|
||||
downloadedFiles, err := dl.DownloadNZB(ctx, nzbFile, outputDir, progressCh)
|
||||
downloadedFiles, err := dl.DownloadNZB(ctx, nzbFile, outputDir, nil, progressCh)
|
||||
close(progressCh)
|
||||
if err != nil {
|
||||
t.Fatalf("download: %v", err)
|
||||
|
|
|
|||
345
internal/usenet/download/progress.go
Normal file
345
internal/usenet/download/progress.go
Normal file
|
|
@ -0,0 +1,345 @@
|
|||
package download
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/torrentclaw/torrentclaw-cli/internal/usenet/nzb"
|
||||
)
|
||||
|
||||
// Binary progress file format:
|
||||
// [4B magic "UNRP"] [1B version=1] [1B reserved] [2B fileCount]
|
||||
// [32B SHA-256 fingerprint]
|
||||
// Per file: [4B segCount] [ceil(segCount/8) bytes bitset]
|
||||
|
||||
var progressMagic = [4]byte{'U', 'N', 'R', 'P'}
|
||||
|
||||
const (
|
||||
progressVersion = 1
|
||||
headerSize = 4 + 1 + 1 + 2 + 32 // 40 bytes
|
||||
flushInterval = 2 * time.Second
|
||||
flushSegmentFreq = 100 // flush every N segment completions
|
||||
)
|
||||
|
||||
// fileProgress tracks completed segments for a single NZB file.
|
||||
type fileProgress struct {
|
||||
segCount int
|
||||
completed []byte // bitset: ceil(segCount/8) bytes
|
||||
doneCount atomic.Int32
|
||||
}
|
||||
|
||||
// ProgressTracker tracks segment-level download progress for resumable usenet downloads.
|
||||
// Thread-safe for concurrent use by multiple download workers.
|
||||
type ProgressTracker struct {
|
||||
taskID string
|
||||
fingerprint [32]byte
|
||||
dir string // directory where progress files are stored
|
||||
files []fileProgress
|
||||
|
||||
mu sync.Mutex
|
||||
dirty bool
|
||||
lastFlush time.Time
|
||||
markCount int // marks since last flush
|
||||
}
|
||||
|
||||
// Fingerprint computes a SHA-256 hash from all message-IDs in the NZB.
|
||||
// Used to validate that a progress file matches the same NZB content.
|
||||
func Fingerprint(n *nzb.NZB) [32]byte {
|
||||
var ids []string
|
||||
for _, f := range n.Files {
|
||||
for _, s := range f.Segments {
|
||||
ids = append(ids, s.MessageID)
|
||||
}
|
||||
}
|
||||
sort.Strings(ids)
|
||||
|
||||
h := sha256.New()
|
||||
for _, id := range ids {
|
||||
h.Write([]byte(id))
|
||||
h.Write([]byte{'\n'})
|
||||
}
|
||||
|
||||
var fp [32]byte
|
||||
copy(fp[:], h.Sum(nil))
|
||||
return fp
|
||||
}
|
||||
|
||||
// NewProgressTracker creates a tracker for the given NZB.
|
||||
// The dir parameter is the base directory for resume files (e.g. DataDir()/resume).
|
||||
func NewProgressTracker(taskID string, n *nzb.NZB, dir string) *ProgressTracker {
|
||||
files := make([]fileProgress, len(n.Files))
|
||||
for i, f := range n.Files {
|
||||
segCount := len(f.Segments)
|
||||
files[i] = fileProgress{
|
||||
segCount: segCount,
|
||||
completed: make([]byte, (segCount+7)/8),
|
||||
}
|
||||
}
|
||||
|
||||
return &ProgressTracker{
|
||||
taskID: taskID,
|
||||
fingerprint: Fingerprint(n),
|
||||
dir: dir,
|
||||
files: files,
|
||||
lastFlush: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
// progressPath returns the path to the binary progress file.
|
||||
func (p *ProgressTracker) progressPath() string {
|
||||
return filepath.Join(p.dir, p.taskID+".progress")
|
||||
}
|
||||
|
||||
// nzbPath returns the path to the cached NZB file.
|
||||
func (p *ProgressTracker) nzbPath() string {
|
||||
return filepath.Join(p.dir, p.taskID+".nzb")
|
||||
}
|
||||
|
||||
// Load reads a progress file from disk and validates its fingerprint.
|
||||
// Returns true if the file was loaded successfully and matches the current NZB.
|
||||
// Returns false if the file doesn't exist, is invalid, or has a different fingerprint.
|
||||
func (p *ProgressTracker) Load() (bool, error) {
|
||||
data, err := os.ReadFile(p.progressPath())
|
||||
if err != nil {
|
||||
return false, nil // file doesn't exist = fresh start
|
||||
}
|
||||
|
||||
if len(data) < headerSize {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Validate magic
|
||||
if data[0] != progressMagic[0] || data[1] != progressMagic[1] ||
|
||||
data[2] != progressMagic[2] || data[3] != progressMagic[3] {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Validate version
|
||||
if data[4] != progressVersion {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Validate file count
|
||||
fileCount := int(binary.LittleEndian.Uint16(data[6:8]))
|
||||
if fileCount != len(p.files) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Validate fingerprint
|
||||
var storedFP [32]byte
|
||||
copy(storedFP[:], data[8:40])
|
||||
if storedFP != p.fingerprint {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Read per-file bitsets
|
||||
offset := headerSize
|
||||
for i := range p.files {
|
||||
if offset+4 > len(data) {
|
||||
return false, nil
|
||||
}
|
||||
segCount := int(binary.LittleEndian.Uint32(data[offset : offset+4]))
|
||||
offset += 4
|
||||
|
||||
if segCount != p.files[i].segCount {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
bitsetLen := (segCount + 7) / 8
|
||||
if offset+bitsetLen > len(data) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
copy(p.files[i].completed, data[offset:offset+bitsetLen])
|
||||
offset += bitsetLen
|
||||
|
||||
// Count completed segments
|
||||
var count int32
|
||||
for seg := 0; seg < segCount; seg++ {
|
||||
if p.files[i].completed[seg/8]&(1<<uint(seg%8)) != 0 {
|
||||
count++
|
||||
}
|
||||
}
|
||||
p.files[i].doneCount.Store(count)
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// MarkDone marks a segment as completed. Thread-safe.
|
||||
// Automatically flushes to disk periodically.
|
||||
func (p *ProgressTracker) MarkDone(fileIndex, segIndex int) {
|
||||
if fileIndex < 0 || fileIndex >= len(p.files) {
|
||||
return
|
||||
}
|
||||
fp := &p.files[fileIndex]
|
||||
if segIndex < 0 || segIndex >= fp.segCount {
|
||||
return
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
fp.completed[segIndex/8] |= 1 << uint(segIndex%8)
|
||||
fp.doneCount.Add(1)
|
||||
p.dirty = true
|
||||
p.markCount++
|
||||
|
||||
shouldFlush := p.markCount >= flushSegmentFreq || time.Since(p.lastFlush) >= flushInterval
|
||||
p.mu.Unlock()
|
||||
|
||||
if shouldFlush {
|
||||
p.Flush()
|
||||
}
|
||||
}
|
||||
|
||||
// IsDone returns whether a specific segment has been completed.
|
||||
func (p *ProgressTracker) IsDone(fileIndex, segIndex int) bool {
|
||||
if fileIndex < 0 || fileIndex >= len(p.files) {
|
||||
return false
|
||||
}
|
||||
fp := &p.files[fileIndex]
|
||||
if segIndex < 0 || segIndex >= fp.segCount {
|
||||
return false
|
||||
}
|
||||
// Read without lock — single-byte read is atomic on aligned data,
|
||||
// and we only ever set bits (never clear), so a stale read just means
|
||||
// we might re-download a segment (harmless, WriteAt is idempotent).
|
||||
return fp.completed[segIndex/8]&(1<<uint(segIndex%8)) != 0
|
||||
}
|
||||
|
||||
// IsFileDone returns true if all segments of a file are completed.
|
||||
func (p *ProgressTracker) IsFileDone(fileIndex int) bool {
|
||||
if fileIndex < 0 || fileIndex >= len(p.files) {
|
||||
return false
|
||||
}
|
||||
fp := &p.files[fileIndex]
|
||||
return int(fp.doneCount.Load()) >= fp.segCount
|
||||
}
|
||||
|
||||
// CompletedSegments returns the number of completed segments for a file.
|
||||
func (p *ProgressTracker) CompletedSegments(fileIndex int) int {
|
||||
if fileIndex < 0 || fileIndex >= len(p.files) {
|
||||
return 0
|
||||
}
|
||||
return int(p.files[fileIndex].doneCount.Load())
|
||||
}
|
||||
|
||||
// CompletedBytes returns the total bytes of completed segments for a file.
|
||||
func (p *ProgressTracker) CompletedBytes(fileIndex int, segments []nzb.Segment) int64 {
|
||||
if fileIndex < 0 || fileIndex >= len(p.files) {
|
||||
return 0
|
||||
}
|
||||
var total int64
|
||||
for i, seg := range segments {
|
||||
if p.IsDone(fileIndex, i) {
|
||||
total += seg.Bytes
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// TotalCompleted returns total completed segments across all files.
|
||||
func (p *ProgressTracker) TotalCompleted() int {
|
||||
var total int
|
||||
for i := range p.files {
|
||||
total += int(p.files[i].doneCount.Load())
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// Flush writes the current progress state to disk atomically (tmp + rename).
|
||||
func (p *ProgressTracker) Flush() error {
|
||||
p.mu.Lock()
|
||||
if !p.dirty {
|
||||
p.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Calculate total size
|
||||
size := headerSize
|
||||
for i := range p.files {
|
||||
size += 4 + (p.files[i].segCount+7)/8
|
||||
}
|
||||
|
||||
buf := make([]byte, size)
|
||||
|
||||
// Header
|
||||
copy(buf[0:4], progressMagic[:])
|
||||
buf[4] = progressVersion
|
||||
buf[5] = 0 // reserved
|
||||
binary.LittleEndian.PutUint16(buf[6:8], uint16(len(p.files)))
|
||||
copy(buf[8:40], p.fingerprint[:])
|
||||
|
||||
// Per-file bitsets
|
||||
offset := headerSize
|
||||
for i := range p.files {
|
||||
fp := &p.files[i]
|
||||
binary.LittleEndian.PutUint32(buf[offset:offset+4], uint32(fp.segCount))
|
||||
offset += 4
|
||||
bitsetLen := (fp.segCount + 7) / 8
|
||||
copy(buf[offset:offset+bitsetLen], fp.completed[:bitsetLen])
|
||||
offset += bitsetLen
|
||||
}
|
||||
|
||||
p.dirty = false
|
||||
p.markCount = 0
|
||||
p.lastFlush = time.Now()
|
||||
p.mu.Unlock()
|
||||
|
||||
// Atomic write: tmp file + rename
|
||||
if err := os.MkdirAll(p.dir, 0o755); err != nil {
|
||||
return fmt.Errorf("create resume dir: %w", err)
|
||||
}
|
||||
|
||||
tmpPath := p.progressPath() + ".tmp"
|
||||
if err := os.WriteFile(tmpPath, buf, 0o644); err != nil {
|
||||
return fmt.Errorf("write progress tmp: %w", err)
|
||||
}
|
||||
|
||||
if err := os.Rename(tmpPath, p.progressPath()); err != nil {
|
||||
os.Remove(tmpPath)
|
||||
return fmt.Errorf("rename progress: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Remove deletes both the progress file and cached NZB file.
|
||||
func (p *ProgressTracker) Remove() error {
|
||||
os.Remove(p.progressPath())
|
||||
os.Remove(p.nzbPath())
|
||||
// Also remove tmp file if it exists
|
||||
os.Remove(p.progressPath() + ".tmp")
|
||||
return nil
|
||||
}
|
||||
|
||||
// CleanStaleFiles removes resume files older than maxAge from the given directory.
|
||||
func CleanStaleFiles(dir string, maxAge time.Duration) int {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
removed := 0
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
info, err := e.Info()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if time.Since(info.ModTime()) > maxAge {
|
||||
if err := os.Remove(filepath.Join(dir, e.Name())); err == nil {
|
||||
removed++
|
||||
}
|
||||
}
|
||||
}
|
||||
return removed
|
||||
}
|
||||
398
internal/usenet/download/progress_test.go
Normal file
398
internal/usenet/download/progress_test.go
Normal file
|
|
@ -0,0 +1,398 @@
|
|||
package download
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"time"
|
||||
|
||||
"github.com/torrentclaw/torrentclaw-cli/internal/usenet/nzb"
|
||||
)
|
||||
|
||||
var fixedPast = time.Now().Add(-30 * 24 * time.Hour)
|
||||
|
||||
func makeTestNZB(fileCount, segsPerFile int) *nzb.NZB {
|
||||
n := &nzb.NZB{
|
||||
Files: make([]nzb.File, fileCount),
|
||||
}
|
||||
for i := 0; i < fileCount; i++ {
|
||||
segs := make([]nzb.Segment, segsPerFile)
|
||||
for j := 0; j < segsPerFile; j++ {
|
||||
segs[j] = nzb.Segment{
|
||||
Bytes: 750 * 1024,
|
||||
Number: j + 1,
|
||||
MessageID: segMsgID(i, j),
|
||||
}
|
||||
}
|
||||
n.Files[i] = nzb.File{
|
||||
Subject: `"testfile_` + string(rune('a'+i)) + `.rar" yEnc (1/` + string(rune('0'+segsPerFile)) + `)`,
|
||||
Segments: segs,
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func segMsgID(file, seg int) string {
|
||||
return "part" + itoa(seg) + ".file" + itoa(file) + "@example.com"
|
||||
}
|
||||
|
||||
func itoa(n int) string {
|
||||
if n == 0 {
|
||||
return "0"
|
||||
}
|
||||
s := ""
|
||||
for n > 0 {
|
||||
s = string(rune('0'+n%10)) + s
|
||||
n /= 10
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func TestFingerprint_Deterministic(t *testing.T) {
|
||||
n := makeTestNZB(3, 10)
|
||||
fp1 := Fingerprint(n)
|
||||
fp2 := Fingerprint(n)
|
||||
if fp1 != fp2 {
|
||||
t.Fatal("fingerprint should be deterministic")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFingerprint_DifferentNZB(t *testing.T) {
|
||||
n1 := makeTestNZB(3, 10)
|
||||
n2 := makeTestNZB(3, 11)
|
||||
if Fingerprint(n1) == Fingerprint(n2) {
|
||||
t.Fatal("different NZBs should have different fingerprints")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProgressTracker_NewAndFlush(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
n := makeTestNZB(2, 5)
|
||||
tracker := NewProgressTracker("test-task-1", n, dir)
|
||||
|
||||
// Mark some segments
|
||||
tracker.MarkDone(0, 0)
|
||||
tracker.MarkDone(0, 2)
|
||||
tracker.MarkDone(1, 4)
|
||||
|
||||
if err := tracker.Flush(); err != nil {
|
||||
t.Fatalf("flush: %v", err)
|
||||
}
|
||||
|
||||
// Verify file exists
|
||||
path := filepath.Join(dir, "test-task-1.progress")
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
t.Fatalf("progress file should exist: %v", err)
|
||||
}
|
||||
|
||||
// Verify state
|
||||
if !tracker.IsDone(0, 0) {
|
||||
t.Error("segment 0,0 should be done")
|
||||
}
|
||||
if tracker.IsDone(0, 1) {
|
||||
t.Error("segment 0,1 should NOT be done")
|
||||
}
|
||||
if !tracker.IsDone(0, 2) {
|
||||
t.Error("segment 0,2 should be done")
|
||||
}
|
||||
if !tracker.IsDone(1, 4) {
|
||||
t.Error("segment 1,4 should be done")
|
||||
}
|
||||
if tracker.CompletedSegments(0) != 2 {
|
||||
t.Errorf("file 0: expected 2 completed, got %d", tracker.CompletedSegments(0))
|
||||
}
|
||||
if tracker.CompletedSegments(1) != 1 {
|
||||
t.Errorf("file 1: expected 1 completed, got %d", tracker.CompletedSegments(1))
|
||||
}
|
||||
}
|
||||
|
||||
func TestProgressTracker_LoadRoundTrip(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
n := makeTestNZB(2, 8)
|
||||
|
||||
// Create and populate
|
||||
tracker1 := NewProgressTracker("test-task-2", n, dir)
|
||||
tracker1.MarkDone(0, 0)
|
||||
tracker1.MarkDone(0, 3)
|
||||
tracker1.MarkDone(0, 7)
|
||||
tracker1.MarkDone(1, 1)
|
||||
tracker1.MarkDone(1, 5)
|
||||
if err := tracker1.Flush(); err != nil {
|
||||
t.Fatalf("flush: %v", err)
|
||||
}
|
||||
|
||||
// Load into new tracker
|
||||
tracker2 := NewProgressTracker("test-task-2", n, dir)
|
||||
loaded, err := tracker2.Load()
|
||||
if err != nil {
|
||||
t.Fatalf("load: %v", err)
|
||||
}
|
||||
if !loaded {
|
||||
t.Fatal("should have loaded successfully")
|
||||
}
|
||||
|
||||
// Verify all bits match
|
||||
for _, tc := range []struct {
|
||||
file, seg int
|
||||
want bool
|
||||
}{
|
||||
{0, 0, true}, {0, 1, false}, {0, 2, false}, {0, 3, true},
|
||||
{0, 4, false}, {0, 5, false}, {0, 6, false}, {0, 7, true},
|
||||
{1, 0, false}, {1, 1, true}, {1, 2, false}, {1, 3, false},
|
||||
{1, 4, false}, {1, 5, true}, {1, 6, false}, {1, 7, false},
|
||||
} {
|
||||
got := tracker2.IsDone(tc.file, tc.seg)
|
||||
if got != tc.want {
|
||||
t.Errorf("file %d seg %d: got %v, want %v", tc.file, tc.seg, got, tc.want)
|
||||
}
|
||||
}
|
||||
|
||||
if tracker2.CompletedSegments(0) != 3 {
|
||||
t.Errorf("file 0: expected 3 completed, got %d", tracker2.CompletedSegments(0))
|
||||
}
|
||||
if tracker2.CompletedSegments(1) != 2 {
|
||||
t.Errorf("file 1: expected 2 completed, got %d", tracker2.CompletedSegments(1))
|
||||
}
|
||||
}
|
||||
|
||||
func TestProgressTracker_FingerprintMismatch(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
n1 := makeTestNZB(2, 5)
|
||||
n2 := makeTestNZB(2, 6) // different segment count = different fingerprint
|
||||
|
||||
// Write with n1
|
||||
tracker1 := NewProgressTracker("test-task-3", n1, dir)
|
||||
tracker1.MarkDone(0, 0)
|
||||
if err := tracker1.Flush(); err != nil {
|
||||
t.Fatalf("flush: %v", err)
|
||||
}
|
||||
|
||||
// Try to load with n2
|
||||
tracker2 := NewProgressTracker("test-task-3", n2, dir)
|
||||
loaded, err := tracker2.Load()
|
||||
if err != nil {
|
||||
t.Fatalf("load: %v", err)
|
||||
}
|
||||
if loaded {
|
||||
t.Fatal("should NOT load — fingerprint mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProgressTracker_IsFileDone(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
n := makeTestNZB(1, 4)
|
||||
tracker := NewProgressTracker("test-task-4", n, dir)
|
||||
|
||||
if tracker.IsFileDone(0) {
|
||||
t.Error("file should not be done yet")
|
||||
}
|
||||
|
||||
tracker.MarkDone(0, 0)
|
||||
tracker.MarkDone(0, 1)
|
||||
tracker.MarkDone(0, 2)
|
||||
if tracker.IsFileDone(0) {
|
||||
t.Error("file should not be done (3/4)")
|
||||
}
|
||||
|
||||
tracker.MarkDone(0, 3)
|
||||
if !tracker.IsFileDone(0) {
|
||||
t.Error("file should be done (4/4)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProgressTracker_ConcurrentMark(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
segCount := 1000
|
||||
n := makeTestNZB(1, segCount)
|
||||
tracker := NewProgressTracker("test-task-5", n, dir)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < segCount; i++ {
|
||||
wg.Add(1)
|
||||
go func(idx int) {
|
||||
defer wg.Done()
|
||||
tracker.MarkDone(0, idx)
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
if !tracker.IsFileDone(0) {
|
||||
t.Errorf("all segments should be done, got %d/%d", tracker.CompletedSegments(0), segCount)
|
||||
}
|
||||
|
||||
// Flush and reload
|
||||
if err := tracker.Flush(); err != nil {
|
||||
t.Fatalf("flush: %v", err)
|
||||
}
|
||||
|
||||
tracker2 := NewProgressTracker("test-task-5", n, dir)
|
||||
loaded, _ := tracker2.Load()
|
||||
if !loaded {
|
||||
t.Fatal("should load")
|
||||
}
|
||||
if !tracker2.IsFileDone(0) {
|
||||
t.Errorf("after reload: expected all done, got %d/%d", tracker2.CompletedSegments(0), segCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProgressTracker_Remove(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
n := makeTestNZB(1, 3)
|
||||
tracker := NewProgressTracker("test-task-6", n, dir)
|
||||
tracker.MarkDone(0, 0)
|
||||
if err := tracker.Flush(); err != nil {
|
||||
t.Fatalf("flush: %v", err)
|
||||
}
|
||||
|
||||
// Write a fake NZB cache file
|
||||
nzbPath := filepath.Join(dir, "test-task-6.nzb")
|
||||
os.WriteFile(nzbPath, []byte("<nzb/>"), 0o644)
|
||||
|
||||
// Both should exist
|
||||
if _, err := os.Stat(tracker.progressPath()); err != nil {
|
||||
t.Fatal("progress file should exist")
|
||||
}
|
||||
if _, err := os.Stat(nzbPath); err != nil {
|
||||
t.Fatal("nzb cache should exist")
|
||||
}
|
||||
|
||||
tracker.Remove()
|
||||
|
||||
if _, err := os.Stat(tracker.progressPath()); !os.IsNotExist(err) {
|
||||
t.Error("progress file should be removed")
|
||||
}
|
||||
if _, err := os.Stat(nzbPath); !os.IsNotExist(err) {
|
||||
t.Error("nzb cache should be removed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProgressTracker_LargeNZB(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
segCount := 30000
|
||||
n := makeTestNZB(1, segCount)
|
||||
tracker := NewProgressTracker("test-task-7", n, dir)
|
||||
|
||||
// Mark every other segment
|
||||
for i := 0; i < segCount; i += 2 {
|
||||
tracker.MarkDone(0, i)
|
||||
}
|
||||
|
||||
if err := tracker.Flush(); err != nil {
|
||||
t.Fatalf("flush: %v", err)
|
||||
}
|
||||
|
||||
// Check file size is compact
|
||||
info, err := os.Stat(tracker.progressPath())
|
||||
if err != nil {
|
||||
t.Fatalf("stat: %v", err)
|
||||
}
|
||||
// Header (40) + file header (4) + bitset (30000/8 = 3750) = 3794 bytes
|
||||
expectedMax := int64(4000)
|
||||
if info.Size() > expectedMax {
|
||||
t.Errorf("progress file too large: %d bytes (expected < %d)", info.Size(), expectedMax)
|
||||
}
|
||||
|
||||
// Reload and verify
|
||||
tracker2 := NewProgressTracker("test-task-7", n, dir)
|
||||
loaded, _ := tracker2.Load()
|
||||
if !loaded {
|
||||
t.Fatal("should load")
|
||||
}
|
||||
if tracker2.CompletedSegments(0) != segCount/2 {
|
||||
t.Errorf("expected %d completed, got %d", segCount/2, tracker2.CompletedSegments(0))
|
||||
}
|
||||
// Spot check
|
||||
if !tracker2.IsDone(0, 0) {
|
||||
t.Error("seg 0 should be done")
|
||||
}
|
||||
if tracker2.IsDone(0, 1) {
|
||||
t.Error("seg 1 should NOT be done")
|
||||
}
|
||||
if !tracker2.IsDone(0, 100) {
|
||||
t.Error("seg 100 should be done")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProgressTracker_CompletedBytes(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
n := makeTestNZB(1, 4)
|
||||
tracker := NewProgressTracker("test-task-8", n, dir)
|
||||
|
||||
tracker.MarkDone(0, 0)
|
||||
tracker.MarkDone(0, 2)
|
||||
|
||||
bytes := tracker.CompletedBytes(0, n.Files[0].Segments)
|
||||
expected := int64(2 * 750 * 1024) // 2 segments * 750KB
|
||||
if bytes != expected {
|
||||
t.Errorf("expected %d bytes, got %d", expected, bytes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProgressTracker_BoundsCheck(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
n := makeTestNZB(1, 3)
|
||||
tracker := NewProgressTracker("test-task-9", n, dir)
|
||||
|
||||
// Out-of-bounds should not panic
|
||||
tracker.MarkDone(-1, 0)
|
||||
tracker.MarkDone(0, -1)
|
||||
tracker.MarkDone(5, 0)
|
||||
tracker.MarkDone(0, 100)
|
||||
|
||||
if tracker.IsDone(-1, 0) {
|
||||
t.Error("out of bounds should return false")
|
||||
}
|
||||
if tracker.IsDone(5, 0) {
|
||||
t.Error("out of bounds should return false")
|
||||
}
|
||||
if tracker.IsFileDone(-1) {
|
||||
t.Error("out of bounds should return false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanStaleFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
// Create a "stale" file
|
||||
stalePath := filepath.Join(dir, "old-task.progress")
|
||||
os.WriteFile(stalePath, []byte("data"), 0o644)
|
||||
// Backdate modification time
|
||||
staleTime := os.Chtimes(stalePath, fixedPast, fixedPast)
|
||||
if staleTime != nil {
|
||||
t.Fatalf("chtimes: %v", staleTime)
|
||||
}
|
||||
|
||||
// Create a "fresh" file
|
||||
freshPath := filepath.Join(dir, "new-task.progress")
|
||||
os.WriteFile(freshPath, []byte("data"), 0o644)
|
||||
|
||||
removed := CleanStaleFiles(dir, 14*24*time.Hour) // 2 weeks — stale file is 30 days old
|
||||
if removed != 1 {
|
||||
t.Errorf("expected 1 removed, got %d", removed)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(stalePath); !os.IsNotExist(err) {
|
||||
t.Error("stale file should be removed")
|
||||
}
|
||||
if _, err := os.Stat(freshPath); err != nil {
|
||||
t.Error("fresh file should still exist")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProgressTracker_FlushNoOp(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
n := makeTestNZB(1, 3)
|
||||
tracker := NewProgressTracker("test-task-10", n, dir)
|
||||
|
||||
// Flush without any marks should be no-op
|
||||
if err := tracker.Flush(); err != nil {
|
||||
t.Fatalf("flush: %v", err)
|
||||
}
|
||||
|
||||
// File should not be created
|
||||
if _, err := os.Stat(tracker.progressPath()); !os.IsNotExist(err) {
|
||||
t.Error("no file should be created for empty flush")
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue