feat(usenet): implement full NNTP download pipeline
Complete usenet download support for unarr CLI: - NZB XML parser with password extraction from <head> meta - yEnc decoder with CRC32 verification - NNTP client with TLS, auth, and connection pool (up to 10 conns) - Segment downloader with parallel workers and progress reporting - Post-processing: par2 verify/repair, unrar/7z extraction with password support - Agent client methods: SearchNzbs, DownloadNzb, GetUsenetCredentials - UsenetDownloader implementing full Downloader interface - Daemon wiring: UsenetDownloader passed to Manager E2E tested: Oppenheimer 1080p (2.94 GB) downloaded via NNTP in 77.6s.
This commit is contained in:
Deivid Soto
parent
5f337eebd7
commit
e332c0a6e4
15 changed files with 3016 additions and 23 deletions
|
|
@ -40,26 +40,50 @@ func (c *Client) Register(ctx context.Context, req RegisterRequest) (*RegisterRe
|
|||
return &resp, nil
|
||||
}
|
||||
|
||||
// Heartbeat sends a periodic keep-alive signal.
|
||||
func (c *Client) Heartbeat(ctx context.Context, req HeartbeatRequest) error {
|
||||
var resp StatusResponse
|
||||
// Heartbeat sends a periodic keep-alive signal and returns server directives.
|
||||
func (c *Client) Heartbeat(ctx context.Context, req HeartbeatRequest) (*HeartbeatResponse, error) {
|
||||
var resp HeartbeatResponse
|
||||
if err := c.doPost(ctx, "/api/internal/agent/heartbeat", req, &resp); err != nil {
|
||||
return fmt.Errorf("heartbeat: %w", err)
|
||||
return nil, fmt.Errorf("heartbeat: %w", err)
|
||||
}
|
||||
return nil
|
||||
return &resp, nil
|
||||
}
|
||||
|
||||
// ClaimTasks polls for pending download tasks and claims them atomically.
|
||||
func (c *Client) ClaimTasks(ctx context.Context, agentID string) ([]Task, error) {
|
||||
// Also returns any stream requests for completed downloads.
|
||||
func (c *Client) ClaimTasks(ctx context.Context, agentID string) (*TasksResponse, error) {
|
||||
url := fmt.Sprintf("/api/internal/agent/tasks?agentId=%s", agentID)
|
||||
var resp TasksResponse
|
||||
if err := c.doGet(ctx, url, &resp); err != nil {
|
||||
return nil, fmt.Errorf("claim tasks: %w", err)
|
||||
}
|
||||
return resp.Tasks, nil
|
||||
return &resp, nil
|
||||
}
|
||||
|
||||
// ReportStatus reports download progress or completion for a task.
|
||||
// Deregister notifies the server that the agent is shutting down.
|
||||
func (c *Client) Deregister(ctx context.Context, agentID string) error {
|
||||
req := struct {
|
||||
AgentID string `json:"agentId"`
|
||||
}{AgentID: agentID}
|
||||
var resp StatusResponse
|
||||
if err := c.doPost(ctx, "/api/internal/agent/deregister", req, &resp); err != nil {
|
||||
return fmt.Errorf("deregister: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReportUpgradeResult reports the outcome of a self-upgrade attempt.
|
||||
func (c *Client) ReportUpgradeResult(ctx context.Context, result UpgradeResult) error {
|
||||
var resp struct {
|
||||
Success bool `json:"success"`
|
||||
}
|
||||
if err := c.doPost(ctx, "/api/internal/agent/upgrade-result", result, &resp); err != nil {
|
||||
return fmt.Errorf("report upgrade: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReportStatus reports download progress. Returns server-side flags the CLI must act on.
|
||||
func (c *Client) ReportStatus(ctx context.Context, update StatusUpdate) (*StatusResponse, error) {
|
||||
var resp StatusResponse
|
||||
|
|
@ -69,6 +93,66 @@ func (c *Client) ReportStatus(ctx context.Context, update StatusUpdate) (*Status
|
|||
return &resp, nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Usenet endpoints
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// SearchNzbs searches NZB indexers for matching content.
|
||||
func (c *Client) SearchNzbs(ctx context.Context, params NzbSearchParams) (*NzbSearchResponse, error) {
|
||||
var resp NzbSearchResponse
|
||||
if err := c.doPost(ctx, "/api/internal/agent/nzb-search", params, &resp); err != nil {
|
||||
return nil, fmt.Errorf("nzb search: %w", err)
|
||||
}
|
||||
return &resp, nil
|
||||
}
|
||||
|
||||
// DownloadNzb downloads the NZB file for the given nzbId.
|
||||
// Returns the raw NZB XML bytes.
|
||||
func (c *Client) DownloadNzb(ctx context.Context, nzbID string) ([]byte, error) {
|
||||
url := fmt.Sprintf("/api/internal/agent/nzb-download?nzbId=%s", nzbID)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
c.setHeaders(req)
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<16))
|
||||
return nil, fmt.Errorf("nzb download error %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(io.LimitReader(resp.Body, 100<<20)) // 100MB limit
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read nzb: %w", err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// GetUsenetCredentials fetches NNTP connection credentials.
|
||||
func (c *Client) GetUsenetCredentials(ctx context.Context) (*UsenetCredentials, error) {
|
||||
var resp UsenetCredentials
|
||||
if err := c.doGet(ctx, "/api/internal/agent/usenet-credentials", &resp); err != nil {
|
||||
return nil, fmt.Errorf("usenet credentials: %w", err)
|
||||
}
|
||||
return &resp, nil
|
||||
}
|
||||
|
||||
// GetUsenetUsage fetches current month's usenet quota usage.
|
||||
func (c *Client) GetUsenetUsage(ctx context.Context) (*UsenetUsageResponse, error) {
|
||||
var resp UsenetUsageResponse
|
||||
if err := c.doGet(ctx, "/api/internal/agent/usenet-usage", &resp); err != nil {
|
||||
return nil, fmt.Errorf("usenet usage: %w", err)
|
||||
}
|
||||
return &resp, nil
|
||||
}
|
||||
|
||||
// doPost sends a JSON POST request and decodes the response.
|
||||
func (c *Client) doPost(ctx context.Context, path string, body any, dst any) error {
|
||||
jsonBody, err := json.Marshal(body)
|
||||
|
|
|
|||
|
|
@ -66,6 +66,8 @@ type Task struct {
|
|||
Mode string `json:"mode,omitempty"` // download | stream
|
||||
DirectURL string `json:"directUrl,omitempty"` // HTTPS download URL (debrid, etc.)
|
||||
DirectFileName string `json:"directFileName,omitempty"` // Original filename from direct URL
|
||||
NzbID string `json:"nzbId,omitempty"` // Pre-resolved NZB ID from server
|
||||
NzbPassword string `json:"nzbPassword,omitempty"` // Password for encrypted NZB archives
|
||||
}
|
||||
|
||||
// TasksResponse wraps the array of tasks returned by the server.
|
||||
|
|
@ -141,3 +143,57 @@ type AgentInfo struct {
|
|||
LastPollAt time.Time
|
||||
ActiveTasks int
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Usenet types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// UsenetCredentials holds NNTP connection details for the CLI.
|
||||
type UsenetCredentials struct {
|
||||
Host string `json:"host"`
|
||||
Port int `json:"port"`
|
||||
SSL bool `json:"ssl"`
|
||||
TLSServerName string `json:"tlsServerName,omitempty"` // override for cert validation (e.g., "xsnews.nl")
|
||||
Username string `json:"username"`
|
||||
Password string `json:"password"`
|
||||
MaxConnections int `json:"maxConnections"`
|
||||
}
|
||||
|
||||
// NzbSearchParams defines search criteria for NZB indexers.
|
||||
type NzbSearchParams struct {
|
||||
Query string `json:"query,omitempty"`
|
||||
IMDbID string `json:"imdbId,omitempty"`
|
||||
TVDbID string `json:"tvdbId,omitempty"`
|
||||
Season *int `json:"season,omitempty"`
|
||||
Episode *int `json:"episode,omitempty"`
|
||||
Limit int `json:"limit,omitempty"`
|
||||
}
|
||||
|
||||
// NzbSearchResult represents a single NZB found by the indexer.
|
||||
type NzbSearchResult struct {
|
||||
Title string `json:"title"`
|
||||
NzbID string `json:"nzbId"`
|
||||
Category string `json:"category"`
|
||||
Size int64 `json:"size"`
|
||||
PublishedAt string `json:"publishedAt"`
|
||||
Grabs int `json:"grabs"`
|
||||
Group string `json:"group"`
|
||||
Poster string `json:"poster"`
|
||||
Attributes map[string]string `json:"attributes"`
|
||||
}
|
||||
|
||||
// NzbSearchResponse wraps search results.
|
||||
type NzbSearchResponse struct {
|
||||
Results []NzbSearchResult `json:"results"`
|
||||
Total int `json:"total"`
|
||||
Offset int `json:"offset"`
|
||||
}
|
||||
|
||||
// UsenetUsageResponse holds quota information.
|
||||
type UsenetUsageResponse struct {
|
||||
UsedBytes int64 `json:"usedBytes"`
|
||||
QuotaBytes int64 `json:"quotaBytes"`
|
||||
PercentUsed float64 `json:"percentUsed"`
|
||||
RemainingBytes int64 `json:"remainingBytes"`
|
||||
QuotaResetDate string `json:"quotaResetDate"`
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue