Add archive support (Unpackerr integration): fix silent no-op on archive-only transfers, opt-in RAR downloading

README.md

@@ -130,6 +130,12 @@ orchestration_workers = 10
 # Optional number of download workers, default 4. This controls how many downloads we run in parallel.
 download_workers = 4
 
+# Optional, default false. When true, multi-volume RAR releases (put.io file
+# type ARCHIVE) are downloaded like any other file instead of being skipped.
+# Pair with an external extractor such as Unpackerr pointed at
+# download_directory -- see "Archive / RAR support" below.
+download_archives = false
+
 [putio]
 # Required. Putio API key. You can generate one using `putioarr get-token`
 api_key =  "MYPUTIOKEY"
@@ -157,6 +163,27 @@ api_key = "MYWHISPARRAPIKEY"
 # category = "adult"
 ```
 
+### Archive / RAR support (Unpackerr integration)
+
+By default, putioarr only downloads `VIDEO`/`AUDIO` files. Multi-volume RAR
+releases (`.rar`/`.r00`/`.r01`/...) are skipped entirely -- and because a
+transfer with nothing downloadable was previously (incorrectly) treated as
+already imported, such transfers used to vanish with **nothing ever
+downloaded** and no error.
+
+Set `download_archives = true` to change this: putioarr downloads the RAR
+parts like any other file, and waits for something else -- typically
+[Unpackerr](https://github.com/Unpackerr/unpackerr) watching the same
+`download_directory` -- to extract them. putioarr detects completion once
+either:
+- the archive parts are gone from disk (e.g. Unpackerr's `delete_orig`), or
+- the *arr reports a `downloadFolderImported` history event for a file
+  inside that transfer's directory (the extracted video).
+
+Only enable this if you run an extractor pointed at the same download
+directory; otherwise the RAR parts will sit there until `import_timeout_secs`
+is reached.
+
 ### Category-based Download Directories
 
 To prevent Sonarr/Radarr/Whisparr from seeing each other's downloads, you can configure separate download directories using categories:

src/download_system/transfer.rs

@@ -53,7 +53,71 @@ impl Transfer {
             .collect::<Vec<DownloadTarget>>();
 
         let mut results = Vec::<bool>::new();
+        let mut dir_import_cache = std::collections::HashMap::<String, bool>::new();
         for target in targets {
+            // ARCHIVE targets (multi-volume RAR parts) are
+            // never reported by the *arr history API -- only the video file
+            // Unpackerr extracts from them gets a downloadFolderImported
+            // event, under a different path. So instead of asking the *arr,
+            // treat an ARCHIVE target as "imported" once it no longer exists
+            // on disk under its expected path: Unpackerr deletes/consumes
+            // the .rar/.rNN parts once it finishes extracting. Until then
+            // this keeps returning false, so is_imported() (and therefore
+            // the local-copy delete) correctly waits for the real
+            // extraction instead of firing on the first poll against an
+            // empty non-ARCHIVE target list (the original bug).
+            if target.media_type == Some(MediaType::Archive) {
+                // Fast path: the extractor may be configured to delete the
+                // archive parts after extraction (unpackerr delete_orig).
+                if !Path::new(&target.to).exists() {
+                    info!("{}: archive part gone (extracted)", &target);
+                    results.push(true);
+                    continue;
+                }
+                // Otherwise ask the *arrs whether anything was imported from
+                // this archive's directory: the import event points at the
+                // extracted video file, which lives under the same transfer
+                // directory as the parts. One lookup per directory is enough,
+                // so cache the answer across the (potentially many) parts.
+                // Use the parent directory NAME (not full path): the *arr may
+                // see the download dir under a different path mapping, so we
+                // match on the unique transfer folder name inside droppedPath.
+                let dir = Path::new(&target.to)
+                    .parent()
+                    .and_then(|p| p.file_name())
+                    .map(|n| n.to_string_lossy().to_string())
+                    .unwrap_or_else(|| target.to.clone());
+                if let Some(hit) = dir_import_cache.get(&dir) {
+                    results.push(*hit);
+                    continue;
+                }
+                let mut dir_imported = false;
+                for app in &apps {
+                    match app.check_imported_dir(&dir).await {
+                        Ok(true) => {
+                            dir_imported = true;
+                            break;
+                        }
+                        Ok(false) => {}
+                        Err(e) => {
+                            if self.app_data.state.should_log_arr_error(&app.name).await {
+                                error!(
+                                    "Error retrieving history from {} (suppressing repeats for {:?}): {}",
+                                    app,
+                                    crate::state::StateManager::ARR_ERROR_LOG_INTERVAL,
+                                    e
+                                );
+                            }
+                        }
+                    }
+                }
+                if dir_imported {
+                    info!("{}: archive extracted and imported from {}", &target, dir);
+                }
+                dir_import_cache.insert(dir, dir_imported);
+                results.push(dir_imported);
+                continue;
+            }
             let mut service_results = vec![];
             for app in &apps {
                 // Only ask an *arr about files matching its media type.
@@ -235,6 +299,28 @@ async fn recurse_download_targets(
                 media_type: MediaType::from_putio(response.parent.file_type.as_str()),
             });
         }
+        // put.io classifies multi-volume RAR parts as ARCHIVE,
+        // which historically fell through to the `other` arm below and was
+        // never turned into a File target. is_imported() filters to File
+        // targets and then does `.all(|x| x)` over that list -- an empty
+        // list makes `.all()` vacuously true, so a transfer containing only
+        // ARCHIVE files (nothing the *arr could ever have imported, since
+        // Unpackerr has not extracted it yet) was incorrectly reported as
+        // "imported" on the very first poll and its local copy deleted
+        // before Unpackerr got a chance to run. Track ARCHIVE files as Video
+        // targets too so is_imported() actually waits for a real
+        // downloadFolderImported event (post-extraction) before cleaning up.
+        "ARCHIVE" if app_data.config.download_archives => {
+            let url = putio::url(&app_data.config.putio.api_key, response.parent.id).await?;
+            targets.push(DownloadTarget {
+                from: Some(url),
+                target_type: TargetType::File,
+                to,
+                top_level,
+                transfer_hash: hash.to_string(),
+                media_type: MediaType::from_putio(response.parent.file_type.as_str()),
+            });
+        }
         other => {
             debug!(
                 "{}: skipping file type {}",
@@ -257,13 +343,20 @@ pub enum TransferMessage {
 pub enum MediaType {
     Audio,
     Video,
+    /// Multi-volume RAR parts (put.io file_type ARCHIVE).
+    /// Tracked separately from Video so is_imported() can use a different
+    /// completion check (local file presence, since the *arr never reports
+    /// a downloadFolderImported history event for a .rNN/.rar path -- only
+    /// for the video file Unpackerr extracts from it).
+    Archive,
 }
 
 impl MediaType {
     pub fn from_putio(file_type: &str) -> Option<Self> {
         match file_type {
             "AUDIO" => Some(Self::Audio),
             "VIDEO" => Some(Self::Video),
+            "ARCHIVE" => Some(Self::Archive),
             _ => None,
         }
     }

src/main.rs

@@ -82,6 +82,13 @@ pub struct Config {
     /// lets the put.io account be shared with manual downloads.
     #[serde(default)]
     download_unmanaged: bool,
+    /// When true, also download ARCHIVE files (e.g. multi-volume RAR releases)
+    /// and hand them to an external extractor such as Unpackerr. Completion is
+    /// then detected either by the archive parts disappearing from disk
+    /// (extractor cleanup) or by the *arr importing a file from the transfer's
+    /// directory. When false (default), archives are skipped as before.
+    #[serde(default)]
+    download_archives: bool,
     /// put.io folder ids to additionally scan for *orphaned* completed files:
     /// files that were downloaded but whose transfer record no longer exists
     /// (e.g. put.io's "clear completed transfers" removes the transfer while

src/services/arr.rs

@@ -94,7 +94,33 @@ impl ArrApp {
         }
     }
 
+    /// Like [`check_imported`], but matches any import event whose droppedPath
+    /// sits UNDER `dir` instead of an exact path match. Used for archive
+    /// transfers: the *arr never imports the .rar/.rNN parts themselves, it
+    /// imports the video file Unpackerr extracts into the same directory, so
+    /// the archive parts can only be tied back to the transfer through their
+    /// parent directory.
+    /// `dir_name` is the transfer directory's *name* (last component), not a
+    /// full path: the *arr may see the download directory under a different
+    /// mount/remote path mapping than putioarr does, so full-path comparison
+    /// would never match. The unique transfer folder name is mapping-agnostic.
+    pub async fn check_imported_dir(&self, dir_name: &str) -> Result<bool> {
+        self.check_imported_matching(&|dropped: &str| {
+            std::path::Path::new(dropped)
+                .components()
+                .any(|comp| comp.as_os_str().to_string_lossy() == dir_name)
+        })
+        .await
+    }
+
     pub async fn check_imported(&self, target: &str) -> Result<bool> {
+        self.check_imported_matching(&|dropped: &str| dropped == target)
+            .await
+    }
+
+    /// Shared history scan: returns true when any import event's droppedPath
+    /// satisfies `matches`.
+    async fn check_imported_matching(&self, matches: &(dyn Fn(&str) -> bool + Sync)) -> Result<bool> {
         let client = reqwest::Client::new();
         let mut inspected = 0;
         let mut page = 0;
@@ -123,7 +149,7 @@ impl ArrApp {
                         .data
                         .get("droppedPath")
                         .and_then(|v| v.as_ref())
-                        .map(|p| p == target)
+                        .map(|p| matches(p))
                         .unwrap_or(false)
                 {
                     return Ok(true);