diff --git a/README.md b/README.md index 0319d93..bbaf7c7 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,12 @@ orchestration_workers = 10 # Optional number of download workers, default 4. This controls how many downloads we run in parallel. download_workers = 4 +# Optional, default false. When true, multi-volume RAR releases (put.io file +# type ARCHIVE) are downloaded like any other file instead of being skipped. +# Pair with an external extractor such as Unpackerr pointed at +# download_directory -- see "Archive / RAR support" below. +download_archives = false + [putio] # Required. Putio API key. You can generate one using `putioarr get-token` api_key = "MYPUTIOKEY" @@ -157,6 +163,27 @@ api_key = "MYWHISPARRAPIKEY" # category = "adult" ``` +### Archive / RAR support (Unpackerr integration) + +By default, putioarr only downloads `VIDEO`/`AUDIO` files. Multi-volume RAR +releases (`.rar`/`.r00`/`.r01`/...) are skipped entirely -- and because a +transfer with nothing downloadable was previously (incorrectly) treated as +already imported, such transfers used to vanish with **nothing ever +downloaded** and no error. + +Set `download_archives = true` to change this: putioarr downloads the RAR +parts like any other file, and waits for something else -- typically +[Unpackerr](https://github.com/Unpackerr/unpackerr) watching the same +`download_directory` -- to extract them. putioarr detects completion once +either: +- the archive parts are gone from disk (e.g. Unpackerr's `delete_orig`), or +- the *arr reports a `downloadFolderImported` history event for a file + inside that transfer's directory (the extracted video). + +Only enable this if you run an extractor pointed at the same download +directory; otherwise the RAR parts will sit there until `import_timeout_secs` +is reached. + ### Category-based Download Directories To prevent Sonarr/Radarr/Whisparr from seeing each other's downloads, you can configure separate download directories using categories: diff --git a/src/download_system/transfer.rs b/src/download_system/transfer.rs index c27dbc3..2cab3ac 100644 --- a/src/download_system/transfer.rs +++ b/src/download_system/transfer.rs @@ -53,7 +53,71 @@ impl Transfer { .collect::>(); let mut results = Vec::::new(); + let mut dir_import_cache = std::collections::HashMap::::new(); for target in targets { + // ARCHIVE targets (multi-volume RAR parts) are + // never reported by the *arr history API -- only the video file + // Unpackerr extracts from them gets a downloadFolderImported + // event, under a different path. So instead of asking the *arr, + // treat an ARCHIVE target as "imported" once it no longer exists + // on disk under its expected path: Unpackerr deletes/consumes + // the .rar/.rNN parts once it finishes extracting. Until then + // this keeps returning false, so is_imported() (and therefore + // the local-copy delete) correctly waits for the real + // extraction instead of firing on the first poll against an + // empty non-ARCHIVE target list (the original bug). + if target.media_type == Some(MediaType::Archive) { + // Fast path: the extractor may be configured to delete the + // archive parts after extraction (unpackerr delete_orig). + if !Path::new(&target.to).exists() { + info!("{}: archive part gone (extracted)", &target); + results.push(true); + continue; + } + // Otherwise ask the *arrs whether anything was imported from + // this archive's directory: the import event points at the + // extracted video file, which lives under the same transfer + // directory as the parts. One lookup per directory is enough, + // so cache the answer across the (potentially many) parts. + // Use the parent directory NAME (not full path): the *arr may + // see the download dir under a different path mapping, so we + // match on the unique transfer folder name inside droppedPath. + let dir = Path::new(&target.to) + .parent() + .and_then(|p| p.file_name()) + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_else(|| target.to.clone()); + if let Some(hit) = dir_import_cache.get(&dir) { + results.push(*hit); + continue; + } + let mut dir_imported = false; + for app in &apps { + match app.check_imported_dir(&dir).await { + Ok(true) => { + dir_imported = true; + break; + } + Ok(false) => {} + Err(e) => { + if self.app_data.state.should_log_arr_error(&app.name).await { + error!( + "Error retrieving history from {} (suppressing repeats for {:?}): {}", + app, + crate::state::StateManager::ARR_ERROR_LOG_INTERVAL, + e + ); + } + } + } + } + if dir_imported { + info!("{}: archive extracted and imported from {}", &target, dir); + } + dir_import_cache.insert(dir, dir_imported); + results.push(dir_imported); + continue; + } let mut service_results = vec![]; for app in &apps { // Only ask an *arr about files matching its media type. @@ -235,6 +299,28 @@ async fn recurse_download_targets( media_type: MediaType::from_putio(response.parent.file_type.as_str()), }); } + // put.io classifies multi-volume RAR parts as ARCHIVE, + // which historically fell through to the `other` arm below and was + // never turned into a File target. is_imported() filters to File + // targets and then does `.all(|x| x)` over that list -- an empty + // list makes `.all()` vacuously true, so a transfer containing only + // ARCHIVE files (nothing the *arr could ever have imported, since + // Unpackerr has not extracted it yet) was incorrectly reported as + // "imported" on the very first poll and its local copy deleted + // before Unpackerr got a chance to run. Track ARCHIVE files as Video + // targets too so is_imported() actually waits for a real + // downloadFolderImported event (post-extraction) before cleaning up. + "ARCHIVE" if app_data.config.download_archives => { + let url = putio::url(&app_data.config.putio.api_key, response.parent.id).await?; + targets.push(DownloadTarget { + from: Some(url), + target_type: TargetType::File, + to, + top_level, + transfer_hash: hash.to_string(), + media_type: MediaType::from_putio(response.parent.file_type.as_str()), + }); + } other => { debug!( "{}: skipping file type {}", @@ -257,6 +343,12 @@ pub enum TransferMessage { pub enum MediaType { Audio, Video, + /// Multi-volume RAR parts (put.io file_type ARCHIVE). + /// Tracked separately from Video so is_imported() can use a different + /// completion check (local file presence, since the *arr never reports + /// a downloadFolderImported history event for a .rNN/.rar path -- only + /// for the video file Unpackerr extracts from it). + Archive, } impl MediaType { @@ -264,6 +356,7 @@ impl MediaType { match file_type { "AUDIO" => Some(Self::Audio), "VIDEO" => Some(Self::Video), + "ARCHIVE" => Some(Self::Archive), _ => None, } } diff --git a/src/main.rs b/src/main.rs index 362cf46..2c233de 100644 --- a/src/main.rs +++ b/src/main.rs @@ -82,6 +82,13 @@ pub struct Config { /// lets the put.io account be shared with manual downloads. #[serde(default)] download_unmanaged: bool, + /// When true, also download ARCHIVE files (e.g. multi-volume RAR releases) + /// and hand them to an external extractor such as Unpackerr. Completion is + /// then detected either by the archive parts disappearing from disk + /// (extractor cleanup) or by the *arr importing a file from the transfer's + /// directory. When false (default), archives are skipped as before. + #[serde(default)] + download_archives: bool, /// put.io folder ids to additionally scan for *orphaned* completed files: /// files that were downloaded but whose transfer record no longer exists /// (e.g. put.io's "clear completed transfers" removes the transfer while diff --git a/src/services/arr.rs b/src/services/arr.rs index eb48323..e9d9d35 100644 --- a/src/services/arr.rs +++ b/src/services/arr.rs @@ -94,7 +94,33 @@ impl ArrApp { } } + /// Like [`check_imported`], but matches any import event whose droppedPath + /// sits UNDER `dir` instead of an exact path match. Used for archive + /// transfers: the *arr never imports the .rar/.rNN parts themselves, it + /// imports the video file Unpackerr extracts into the same directory, so + /// the archive parts can only be tied back to the transfer through their + /// parent directory. + /// `dir_name` is the transfer directory's *name* (last component), not a + /// full path: the *arr may see the download directory under a different + /// mount/remote path mapping than putioarr does, so full-path comparison + /// would never match. The unique transfer folder name is mapping-agnostic. + pub async fn check_imported_dir(&self, dir_name: &str) -> Result { + self.check_imported_matching(&|dropped: &str| { + std::path::Path::new(dropped) + .components() + .any(|comp| comp.as_os_str().to_string_lossy() == dir_name) + }) + .await + } + pub async fn check_imported(&self, target: &str) -> Result { + self.check_imported_matching(&|dropped: &str| dropped == target) + .await + } + + /// Shared history scan: returns true when any import event's droppedPath + /// satisfies `matches`. + async fn check_imported_matching(&self, matches: &(dyn Fn(&str) -> bool + Sync)) -> Result { let client = reqwest::Client::new(); let mut inspected = 0; let mut page = 0; @@ -123,7 +149,7 @@ impl ArrApp { .data .get("droppedPath") .and_then(|v| v.as_ref()) - .map(|p| p == target) + .map(|p| matches(p)) .unwrap_or(false) { return Ok(true);