diff --git a/.gitignore b/.gitignore index b146056..559a492 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ # SPDX-License-Identifier: BSD-3-Clause target + +# Local test PDF fixtures (not committed to repo; place plain PDFs here for offline tests) +anonymizer_data/*.pdf diff --git a/Cargo.lock b/Cargo.lock index 503c1e0..87e72c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -84,9 +84,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.21" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ "anstyle", "anstyle-parse", @@ -99,15 +99,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "anstyle-parse" -version = "0.2.7" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" dependencies = [ "utf8parse", ] @@ -132,6 +132,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + [[package]] name = "argminmax" version = "0.6.3" @@ -192,9 +198,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.10.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" [[package]] name = "block-buffer" @@ -223,15 +229,15 @@ checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae" [[package]] name = "bumpalo" -version = "3.19.0" +version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" [[package]] name = "bytemuck" -version = "1.24.0" +version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" dependencies = [ "bytemuck_derive", ] @@ -244,7 +250,7 @@ checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -255,9 +261,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "calamine" @@ -276,9 +282,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.49" +version = "1.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90583009037521a116abf44494efecd645ba48b6622457080f080b85544e2215" +checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" dependencies = [ "find-msvc-tools", "jobserver", @@ -294,9 +300,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "chrono" -version = "0.4.42" +version = "0.4.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" dependencies = [ "iana-time-zone", "js-sys", @@ -316,9 +322,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.53" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", "clap_derive", @@ -326,9 +332,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.53" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ "anstream", "anstyle", @@ -338,27 +344,27 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.49" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] name = "clap_lex" -version = "0.7.6" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "cmake" -version = "0.1.56" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b042e5d8a74ae91bb0961acd039822472ec99f8ab0948cbf6d1369588f8be586" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" dependencies = [ "cc", ] @@ -374,9 +380,9 @@ dependencies = [ [[package]] name = "colorchoice" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "colored" @@ -390,9 +396,9 @@ dependencies = [ [[package]] name = "comfy-table" -version = "7.2.1" +version = "7.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b03b7db8e0b4b2fdad6c551e634134e99ec000e5c8c3b6856c65e8bbaded7a3b" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" dependencies = [ "crossterm", "unicode-segmentation", @@ -409,6 +415,16 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -482,7 +498,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.1", "crossterm_winapi", "document-features", "parking_lot", @@ -531,9 +547,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.5.5" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ "powerfmt", ] @@ -555,7 +571,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -603,7 +619,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -624,9 +640,9 @@ dependencies = [ [[package]] name = "ethnum" -version = "1.5.2" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca81e6b4777c89fd810c25a4be2b1bd93ea034fbe58e6a75216a34c6b82c539b" +checksum = "40404c3f5f511ec4da6fe866ddf6a717c309fdbb69fbbad7b0f3edab8f2e835f" [[package]] name = "etradeTaxReturnHelper" @@ -636,6 +652,7 @@ dependencies = [ "chrono", "clap", "csv", + "flate2", "fltk", "holidays", "log", @@ -659,9 +676,9 @@ checksum = "95765f67b4b18863968b4a1bd5bb576f732b29a4a28c7cd84c09fa3e2875f33c" [[package]] name = "fastrand" -version = "2.3.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] name = "fax" @@ -685,15 +702,15 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.5" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "flate2" -version = "1.1.5" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", "miniz_oxide", @@ -727,6 +744,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "foreign-types" version = "0.3.2" @@ -759,57 +782,56 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", ] [[package]] name = "futures-core" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" [[package]] name = "futures-io" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" [[package]] name = "futures-sink" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" [[package]] name = "futures-task" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] name = "futures-util" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-core", "futures-io", "futures-task", "memchr", "pin-project-lite", - "pin-utils", "slab", ] [[package]] name = "generic-array" -version = "0.14.9" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -817,9 +839,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", "js-sys", @@ -836,8 +858,21 @@ checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", "wasip2", + "wasip3", ] [[package]] @@ -878,9 +913,18 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.16.1" +version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" [[package]] name = "heck" @@ -987,9 +1031,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.64" +version = "0.1.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1011,12 +1055,13 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ "displaydoc", "potential_utf", + "utf8_iter", "yoke", "zerofrom", "zerovec", @@ -1024,9 +1069,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ "displaydoc", "litemap", @@ -1037,9 +1082,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -1051,15 +1096,15 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] name = "icu_properties" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ "icu_collections", "icu_locale_core", @@ -1071,15 +1116,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] name = "icu_provider" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ "displaydoc", "icu_locale_core", @@ -1090,6 +1135,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + [[package]] name = "idna" version = "1.1.0" @@ -1113,12 +1164,14 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.12.1" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.0", + "serde", + "serde_core", ] [[package]] @@ -1132,9 +1185,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.11.0" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] name = "is_terminal_polyfill" @@ -1153,9 +1206,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.15" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jobserver" @@ -1175,10 +1228,12 @@ checksum = "229d53d58899083193af11e15917b5640cd40b29ff475a1fe4ef725deb02d0f2" [[package]] name = "js-sys" -version = "0.3.83" +version = "0.3.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" +checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] @@ -1189,29 +1244,35 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + [[package]] name = "libc" -version = "0.2.178" +version = "0.2.185" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" +checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" [[package]] name = "libm" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "linux-raw-sys" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "litrs" @@ -1261,9 +1322,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "memchr" -version = "2.7.6" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "memmap2" @@ -1298,9 +1359,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", "wasi", @@ -1331,9 +1392,9 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.14" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" dependencies = [ "libc", "log", @@ -1367,18 +1428,18 @@ dependencies = [ [[package]] name = "ntapi" -version = "0.4.1" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" dependencies = [ "winapi", ] [[package]] name = "num-conv" -version = "0.1.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" [[package]] name = "num-traits" @@ -1401,9 +1462,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "once_cell_polyfill" @@ -1419,11 +1480,11 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "openssl" -version = "0.10.75" +version = "0.10.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +checksum = "bfe4646e360ec77dff7dde40ed3d6c5fee52d156ef4a62f53973d38294dad87f" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.1", "cfg-if", "foreign-types", "libc", @@ -1440,20 +1501,20 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] name = "openssl-probe" -version = "0.1.6" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-sys" -version = "0.9.111" +version = "0.9.113" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +checksum = "ad2f2c0eba47118757e4c6d2bff2838f3e0523380021356e7875e858372ce644" dependencies = [ "cc", "libc", @@ -1543,21 +1604,15 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pin-project-lite" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" - -[[package]] -name = "pin-utils" -version = "0.1.0" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pkg-config" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "planus" @@ -1574,7 +1629,7 @@ version = "0.35.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df8e52f9236eb722da0990a70bbb1216dcc7a77bcb00c63439d2d982823e90d5" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", "polars-core", "polars-io", "polars-lazy", @@ -1600,7 +1655,7 @@ dependencies = [ "ethnum", "fast-float", "foreign_vec", - "getrandom 0.2.16", + "getrandom 0.2.17", "hashbrown 0.14.5", "itoa", "lz4", @@ -1623,7 +1678,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae73d5b8e55decde670caba1cc82b61f14bfb9a72503198f0997d657a98dcfd6" dependencies = [ "ahash", - "bitflags 2.10.0", + "bitflags 2.11.1", "bytemuck", "chrono", "comfy-table", @@ -1695,7 +1750,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3555f759705be6dd0d3762d16a0b8787b2dc4da73b57465f3b2bf1a070ba8f20" dependencies = [ "ahash", - "bitflags 2.10.0", + "bitflags 2.11.1", "glob", "once_cell", "polars-arrow", @@ -1849,9 +1904,9 @@ dependencies = [ [[package]] name = "potential_utf" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ "zerovec", ] @@ -1871,11 +1926,21 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.117", +] + [[package]] name = "proc-macro2" -version = "1.0.103" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ "unicode-ident", ] @@ -1892,9 +1957,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.42" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] @@ -1905,11 +1970,17 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "rand" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", "rand_chacha", @@ -1932,7 +2003,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", ] [[package]] @@ -1947,9 +2018,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" dependencies = [ "either", "rayon-core", @@ -1971,14 +2042,14 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.1", ] [[package]] name = "regex" -version = "1.12.2" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" dependencies = [ "aho-corasick", "memchr", @@ -1988,9 +2059,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" dependencies = [ "aho-corasick", "memchr", @@ -1999,9 +2070,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.8" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" [[package]] name = "reqwest" @@ -2060,11 +2131,11 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.2" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys", @@ -2088,15 +2159,15 @@ checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "ryu" -version = "1.0.20" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "schannel" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" dependencies = [ "windows-sys 0.61.2", ] @@ -2109,12 +2180,12 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "security-framework" -version = "2.11.1" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.10.0", - "core-foundation", + "bitflags 2.11.1", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -2122,9 +2193,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.15.0" +version = "2.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" dependencies = [ "core-foundation-sys", "libc", @@ -2132,9 +2203,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" @@ -2163,7 +2234,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -2211,9 +2282,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "simd-adler32" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" [[package]] name = "simdutf8" @@ -2235,9 +2306,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" @@ -2289,12 +2360,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -2357,7 +2428,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -2373,9 +2444,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.111" +version = "2.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" dependencies = [ "proc-macro2", "quote", @@ -2396,7 +2467,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -2420,7 +2491,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", - "core-foundation", + "core-foundation 0.9.4", "system-configuration-sys", ] @@ -2442,12 +2513,12 @@ checksum = "c1bbb9f3c5c463a01705937a24fdabc5047929ac764b2d5b9cf681c1f5041ed5" [[package]] name = "tempfile" -version = "3.23.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", "rustix", "windows-sys 0.61.2", @@ -2470,14 +2541,14 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] name = "time" -version = "0.3.44" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", "itoa", @@ -2485,22 +2556,22 @@ dependencies = [ "num-conv", "num_threads", "powerfmt", - "serde", + "serde_core", "time-core", "time-macros", ] [[package]] name = "time-core" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" [[package]] name = "time-macros" -version = "0.2.24" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" dependencies = [ "num-conv", "time-core", @@ -2508,9 +2579,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", "zerovec", @@ -2518,9 +2589,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" dependencies = [ "tinyvec_macros", ] @@ -2533,15 +2604,15 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.48.0" +version = "1.52.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" +checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" dependencies = [ "bytes", "libc", "mio", "pin-project-lite", - "socket2 0.6.1", + "socket2 0.6.3", "windows-sys 0.61.2", ] @@ -2557,9 +2628,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.17" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" dependencies = [ "bytes", "futures-core", @@ -2576,9 +2647,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ "pin-project-lite", "tracing-core", @@ -2586,9 +2657,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.35" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a04e24fab5c89c6a36eb8558c9656f30d81de51dfa4d3b45f26b21d61fa0a6c" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", ] @@ -2619,9 +2690,9 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.22" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-normalization" @@ -2640,9 +2711,9 @@ checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" [[package]] name = "unicode-segmentation" -version = "1.12.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" [[package]] name = "unicode-width" @@ -2650,11 +2721,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "url" -version = "2.5.7" +version = "2.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" dependencies = [ "form_urlencoded", "idna", @@ -2703,18 +2780,27 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.1+wasi-0.2.4" +version = "1.0.3+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", ] [[package]] name = "wasm-bindgen" -version = "0.2.106" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" +checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" dependencies = [ "cfg-if", "once_cell", @@ -2725,22 +2811,19 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.56" +version = "0.4.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" +checksum = "f371d383f2fb139252e0bfac3b81b265689bf45b6874af544ffa4c975ac1ebf8" dependencies = [ - "cfg-if", "js-sys", - "once_cell", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.106" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" +checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2748,31 +2831,65 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.106" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" +checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.106" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags 2.11.1", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + [[package]] name = "web-sys" -version = "0.3.83" +version = "0.3.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" +checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" dependencies = [ "js-sys", "wasm-bindgen", @@ -2836,7 +2953,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -2847,7 +2964,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -2901,15 +3018,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -2943,30 +3051,13 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", + "windows_i686_gnullvm", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -2979,12 +3070,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -2997,12 +3082,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -3015,24 +3094,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -3045,12 +3112,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -3063,12 +3124,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -3081,12 +3136,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -3099,12 +3148,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winreg" version = "0.50.0" @@ -3117,15 +3160,103 @@ dependencies = [ [[package]] name = "wit-bindgen" -version = "0.46.0" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck 0.5.0", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck 0.5.0", + "indexmap", + "prettyplease", + "syn 2.0.117", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.117", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags 2.11.1", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] [[package]] name = "writeable" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "xxhash-rust" @@ -3135,9 +3266,9 @@ checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" [[package]] name = "yoke" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -3146,62 +3277,62 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.31" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.31" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] name = "zerofrom" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", "synstructure", ] [[package]] name = "zerotrie" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" dependencies = [ "displaydoc", "yoke", @@ -3210,9 +3341,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ "yoke", "zerofrom", @@ -3221,13 +3352,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 70309e8..737f934 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,9 @@ exclude = [ [[bin]] name = "etradeTaxReturnHelper" path = "src/main.rs" +[[bin]] +name = "etradeAnonymizer" +path = "src/anonymizer/anonymizer.rs" [[bin]] name = "gen_exchange_rates" @@ -49,5 +52,4 @@ polars = "0.35.4" csv = "1.3.0" serde_json = { version = "=1.0.133", optional = true } holidays = { version = "0.1.0", default-features = false, features = ["PL"] } - - +flate2 = "1.1.5" diff --git a/README.md b/README.md index 1997c30..bac3181 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ diff --git a/src/anonymizer/README.md b/src/anonymizer/README.md new file mode 100644 index 0000000..7d7f6dd --- /dev/null +++ b/src/anonymizer/README.md @@ -0,0 +1,115 @@ + + +# etradeAnonymizer + +Minimal Rust tool for anonymizing E*TRADE / Morgan Stanley PDF statements by replacing personally identifiable information while preserving calculation-relevant data. + +## Usage + +Anonymize a PDF (creates `anonymous_statement.pdf` by default): +``` +cargo run --bin etradeAnonymizer -- anonymize statement.pdf +``` + +Specify output file: +``` +cargo run --bin etradeAnonymizer -- anonymize input.pdf output.pdf +``` + +List all text tokens (for debugging): +``` +cargo run --bin etradeAnonymizer -- list statement.pdf +``` + +## Build & Test +``` +cargo build --release --bin etradeAnonymizer +# Tests live in the library target (there are no tests in the binary itself): +cargo test --lib anonymizer +``` + +Resulting binary: `target/release/etradeAnonymizer`. + +## Anonymization Strategy + +The tool processes PDF FlateDecode streams and applies smart text replacement: + +1. **Preserve calculation-relevant strings**: + - Strings containing `CLIENT STATEMENT` or `For the Period` + - Everything between `CASH FLOW ACTIVITY BY DATE` and `NET CREDITS/(DEBITS)` (inclusive) + +2. **Replace all other text**: + - The current implementation replaces non-preserved strings with stable numeric tokens (`0`, `1`, `2`, ...). + - This removes PII while keeping structure and relative token identity for verification and testing. + +## Technical Specification (YAGNI Scope) + +This tool follows the **YAGNI (You Ain't Gonna Need It)** principle, focusing only on the subset of the PDF standard actually used in the target documents. + +### Supported Features +- **PDF Standard:** Basic PDF 1.3 structure. +- **Streams:** `stream` objects compressed with `/FlateDecode` or raw (uncompressed) with an explicit `/Length`. +- **Text Blocks:** Data contained between `BT` (Begin Text) and `ET` (End Text) operators. +- **Text Operators:** Extraction from `(...) Tj` and `[...] TJ` arrays. +- **Unescaping:** Support for standard PDF escape sequences: `\n`, `\r`, `\t`, `\b`, `\f`, `\(`, `\)`, `\\`, and octal sequences `\ddd`. +- **Encoding:** Text treated as ASCII/Latin-1 (internally handled as UTF-8). + + +## Design Notes +- **Regex-based Discovery:** The tool uses optimized regular expressions to scan the PDF binary for stream object headers. This allows for fast location of data without full PDF structure parsing. +- Strict PDF header (`%PDF-1.3`) enforcement; files with any other header are rejected. +- FlateDecode and uncompressed streams with an explicit `/Length` are processed. +- Replacement recompresses; if no level fits original size, original compressed stream is kept. + +## Testing & Development + +### Running Tests +Most unit tests run automatically with `cargo test`. Integration tests that require real PDF files are marked `#[ignore]` and will not run on CI. + +To run the integration tests locally: +1. Place the plain PDF fixtures (`sample_statement.pdf`, `sample_statement_anonymized.pdf`) in the `anonymizer_data/` directory (git-ignored). +2. Run: + ```bash + cargo test --lib anonymizer -- --ignored + # or without GUI dependencies: + cargo test --lib anonymizer --no-default-features -- --ignored + ``` + +### Known Limitations (YAGNI Scope) +- **Indirect Length Objects:** The scanner currently only supports streams with an explicit numeric `/Length` in the dictionary. It will skip streams where the length is a reference (e.g., `/Length 12 0 R`). +- **Standard PDF Filters:** Only `/FlateDecode` is supported for text extraction. Image streams (e.g., `/DCTDecode`) and font files are intentionally ignored as they do not contain PII in target documents. + +### Why Padding? (Architecture Note) +This tool avoids full PDF parsing and rebuilding. Instead, it modifies streams **in-place**. +- PDF files rely on a Cross-Reference (XREF) table that stores the byte offset of every object. +- If we changed the length of a stream object, all subsequent object offsets would shift, invalidating the XREF table. +- To avoid rebuilding the XREF table, we ensure the modified stream is **exactly the same length** as the original. +- If the new compressed data is smaller, we **pad** the remainder with null bytes (`0x00`). +- If the new compressed data is larger than the original, we fall back to keeping the original stream to avoid file corruption. + +### Exact PDF object pattern searched +The tool searches for PDF objects that exactly match the following pattern: + +``` + obj +<< +/Length +/Filter [/FlateDecode] +>> +stream + +endstream +endobj +``` + +Note about `list` showing "0 tokens": the command always prints a stream dump header, but the token extractor only reports tokens when it finds PDF text-showing operators (e.g. `Tj`, `TJ`, `\'`, `"`). Non-text streams (images, fonts, etc.) will naturally show "0 tokens". + +## License +See `BSD-3-Clause` in `LICENSES/` directory. + +## Disclaimer + +Please note: this tool attempts to detect and replace everything except the information required for calculation by analyzing tokens in PDF streams that are strictly defined, but there is no guarantee that all PII will be detected or removed. You must manually review the resulting file before sharing it. diff --git a/src/anonymizer/anonymizer.rs b/src/anonymizer/anonymizer.rs new file mode 100644 index 0000000..4df0e0f --- /dev/null +++ b/src/anonymizer/anonymizer.rs @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: 2025 RustInFinance +// SPDX-License-Identifier: BSD-3-Clause + +//! etradeAnonymizer - PDF anonymization tool for E*TRADE / Morgan Stanley statements. +//! +//! This tool provides two subcommands: +//! - `list`: List all text tokens from FlateDecode streams in a PDF +//! - `anonymize`: Anonymize PDF by replacing all PII except calculation-relevant data +//! +//! The tool operates on tightly structured PDF FlateDecode streams and preserves +//! the original file structure by performing in-place replacements with exact-size matching. + +// Submodules are exported by `src/anonymizer/mod.rs` via the library crate. +// Since this file is a binary entry point, reference them via `etradeTaxReturnHelper::anonymizer::...`. + +use clap::{Parser, Subcommand}; +use etradeTaxReturnHelper::anonymizer; +use std::error::Error; +use std::path::PathBuf; + +/// Tool for anonymizing PDF files by replacing specific strings in FlateDecode streams +#[derive(Parser)] +#[command(name = "etradeAnonymizer")] +#[command(version, about, long_about = None)] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Subcommand)] +enum Commands { + /// List all text tokens from FlateDecode streams in the PDF + List { + /// Path to the input PDF file + input_file: PathBuf, + }, + /// Anonymize PDF by replacing all text except calculation-relevant data + Anonymize { + /// Path to the input PDF file + input_file: PathBuf, + /// Path to the output PDF file (optional, defaults to anonymous_.pdf) + output_file: Option, + }, +} + +fn main() -> Result<(), Box> { + // Default to `warn` level; RUST_LOG env var overrides this if set. + simple_logger::SimpleLogger::new() + .with_level(log::LevelFilter::Warn) + .env() + .init() + .unwrap(); + + let cli = Cli::parse(); + + match cli.command { + Commands::List { input_file } => anonymizer::list::list_texts(&input_file), + Commands::Anonymize { + input_file, + output_file, + } => { + let output = + output_file.unwrap_or_else(|| anonymizer::path::anonymous_output_path(&input_file)); + anonymizer::replace::replace_pii_smart(&input_file, &output) + } + } +} diff --git a/src/anonymizer/list.rs b/src/anonymizer/list.rs new file mode 100644 index 0000000..c861ff9 --- /dev/null +++ b/src/anonymizer/list.rs @@ -0,0 +1,63 @@ +// SPDX-FileCopyrightText: 2025 RustInFinance +// SPDX-License-Identifier: BSD-3-Clause + +//! Text listing module for anonymizer. +//! +//! This module provides functionality to extract and list all text tokens from +//! FlateDecode and uncompressed streams (with an explicit /Length) in a PDF. Each +//! token is printed with a global index, useful for understanding the structure and +//! content of the PDF before anonymization. + +use super::pdf::{extract_texts_from_stream, read_pdf, stream_scanner}; +use log::warn; +use std::error::Error; +use std::path::Path; + +/// List all text tokens from FlateDecode and uncompressed streams in the PDF at `input_path`. +/// +/// Prints each extracted token with a global index to stdout. +/// Logs warnings for streams that fail to decompress or have invalid markers. +/// +/// # Arguments +/// * `input_path` - Path to the input PDF file. +/// +/// # Returns +/// `Ok(())` on success, or an error if the PDF cannot be read. +pub fn list_texts(input_path: &Path) -> Result<(), Box> { + let pdf_data = read_pdf(input_path)?; + + let mut global_text_id = 0; + for (stream_id, stream) in stream_scanner(&pdf_data).enumerate() { + if !stream.valid_end_marker { + warn!( + "Skipping stream due to end-marker mismatch for object at {}", + stream.object_start + ); + continue; + } + + let extraction = extract_texts_from_stream(stream.compressed, stream.is_compressed); + + match extraction { + Ok(extracted_texts) => { + println!("---------------------------------------------------------------"); + println!("STREAM #{} ({} tokens)", stream_id, extracted_texts.len()); + println!("---------------------------------------------------------------"); + for (txt, _start, _end) in extracted_texts.iter() { + // Sanitize token for console: escape non-printable bytes to avoid + // terminal mojibake when streams contain binary data. + let safe: String = txt.chars().flat_map(|c| c.escape_default()).collect(); + println!(" [{}] {}", global_text_id, safe); + global_text_id += 1; + } + } + Err(e) => { + warn!( + "Failed to extract texts from stream at {}: {}", + stream.object_start, e + ); + } + } + } + Ok(()) +} diff --git a/src/anonymizer/mod.rs b/src/anonymizer/mod.rs new file mode 100644 index 0000000..430e978 --- /dev/null +++ b/src/anonymizer/mod.rs @@ -0,0 +1,9 @@ +// SPDX-FileCopyrightText: 2025 RustInFinance +// SPDX-License-Identifier: BSD-3-Clause + +//! Anonymizer module re-exports. + +pub mod list; +pub mod path; +pub mod pdf; +pub mod replace; diff --git a/src/anonymizer/path.rs b/src/anonymizer/path.rs new file mode 100644 index 0000000..96707ab --- /dev/null +++ b/src/anonymizer/path.rs @@ -0,0 +1,64 @@ +// SPDX-FileCopyrightText: 2025 RustInFinance +// SPDX-License-Identifier: BSD-3-Clause + +//! Path utility module for anonymizer. +//! +//! Provides helper functions for generating anonymized output file paths +//! by prefixing filenames with `anonymous_` while preserving directory structure. + +use std::path::PathBuf; + +/// Build an output path by prefixing the input filename with `anonymous_`. +/// +/// Preserves the parent directory if present and returns a `PathBuf`. +/// +/// # Examples +/// ```ignore +/// use std::path::Path; +/// let input = Path::new("data/statement.pdf"); +/// let output = anonymous_output_path(input); +/// assert_eq!(output, Path::new("data/anonymous_statement.pdf")); +/// ``` +pub fn anonymous_output_path(input_path: &std::path::Path) -> PathBuf { + let file_name = input_path + .file_name() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_else(|| input_path.to_string_lossy().into_owned()); + + if let Some(parent) = input_path.parent() { + let mut pb = PathBuf::from(parent); + pb.push(format!("anonymous_{}", file_name)); + pb + } else { + PathBuf::from(format!("anonymous_{}", file_name)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_anonymous_output_path_no_parent() { + let in_path = std::path::Path::new("statement.pdf"); + let out = anonymous_output_path(in_path); + assert_eq!(out, std::path::PathBuf::from("anonymous_statement.pdf")); + } + + #[test] + fn test_anonymous_output_path_with_parent() { + let in_path = std::path::Path::new("some/dir/statement.pdf"); + let out = anonymous_output_path(in_path); + assert_eq!( + out, + std::path::PathBuf::from("some/dir/anonymous_statement.pdf") + ); + } + + #[test] + fn test_anonymous_output_path_unicode_filename() { + let in_path = std::path::Path::new("résumé.pdf"); + let out = anonymous_output_path(in_path); + assert_eq!(out, std::path::PathBuf::from("anonymous_résumé.pdf")); + } +} diff --git a/src/anonymizer/pdf.rs b/src/anonymizer/pdf.rs new file mode 100644 index 0000000..657a6b3 --- /dev/null +++ b/src/anonymizer/pdf.rs @@ -0,0 +1,1115 @@ +// SPDX-FileCopyrightText: 2025 RustInFinance +// SPDX-License-Identifier: BSD-3-Clause + +//! PDF parsing utilities: header validation, stream extraction, text token parsing. +//! This module is intentionally strict and only supports a narrow subset of PDF +//! objects used by the target documents: FlateDecode streams with explicit /Length. + +use flate2::read::ZlibDecoder; +use log::{error, warn}; +use regex::bytes::Regex; +use std::error::Error; +use std::fs::File; +use std::io::Read; + +// Centralized constants and helpers for PDF parsing to reduce duplication between list/replace. +/// Expected PDF header (strictly enforced). +pub(crate) const PDF_HEADER: &[u8] = b"%PDF-1.3"; +/// Regex matching any stream object with an explicit `/Length`. +/// Uses (?s) DOTALL so the dictionary may span newlines; `[^>]` keeps a match +/// within a single dictionary (no nested `>`), matching the narrow object shapes +/// these documents use. The scanner then classifies each match by inspecting the +/// dictionary text only (never the binary payload), in this order: +/// - contains `/Length1` -> embedded font program (FontFile/FontFile2), skipped +/// regardless of compression: binary font data has no PDF text tokens and must +/// never be modified, +/// - otherwise contains `/FlateDecode` -> compressed (zlib-decoded and scanned), +/// - otherwise contains `/Filter` -> unsupported filter (e.g. `/DCTDecode` +/// image), skipped, +/// - otherwise -> genuine uncompressed content stream, scanned verbatim. +pub(crate) const OBJ_STREAM_RE: &str = + r"(?s)\d+\s+\d+\s+obj\s*<<[^>]*?/Length\s+(\d+)[^>]*?>>\s*stream\r?\n"; + +/// Read entire PDF file and validate strict header. +pub fn read_pdf(path: &std::path::Path) -> Result, Box> { + let mut file = File::open(path)?; + let mut pdf_data = Vec::new(); + file.read_to_end(&mut pdf_data)?; + if pdf_data.len() < PDF_HEADER.len() || &pdf_data[0..PDF_HEADER.len()] != PDF_HEADER { + error!( + "Unsupported PDF version or invalid PDF header at '{}'.", + path.display() + ); + return Err("Invalid PDF header".into()); + } + Ok(pdf_data) +} + +/// Lightweight representation of a stream slice inside a PDF (compressed or uncompressed). +pub struct StreamData<'a> { + pub object_start: usize, + pub data_start: usize, + pub compressed: &'a [u8], + pub valid_end_marker: bool, + pub is_compressed: bool, +} + +/// Iterator over stream objects, avoiding allocating a full Vec upfront. +pub struct StreamScanner<'a> { + re: Regex, + data: &'a [u8], + search_from: usize, +} + +/// Create a new streaming iterator over PDF stream objects (compressed and uncompressed). +pub fn stream_scanner<'a>(pdf_data: &'a [u8]) -> StreamScanner<'a> { + StreamScanner { + re: Regex::new(OBJ_STREAM_RE).unwrap(), + data: pdf_data, + search_from: 0, + } +} + +/// True if the byte slice `haystack` contains the contiguous subslice `needle`. +fn bytes_contain(haystack: &[u8], needle: &[u8]) -> bool { + needle.len() <= haystack.len() && haystack.windows(needle.len()).any(|w| w == needle) +} + +impl<'a> Iterator for StreamScanner<'a> { + type Item = StreamData<'a>; + fn next(&mut self) -> Option { + while self.search_from < self.data.len() { + let caps = match self.re.captures_at(self.data, self.search_from) { + Some(c) => c, + None => { + self.search_from = self.data.len(); + return None; + } + }; + + let whole = caps.get(0)?; + self.search_from = whole.end(); + + // Classify by inspecting only the matched object header/dictionary + // (never the binary payload, which begins after `stream`). + let dict = whole.as_bytes(); + let is_compressed = if bytes_contain(dict, b"/Length1") { + // Embedded font program (FontFile/FontFile2): binary font data + // with no PDF text tokens. Checked first so that even a + // Flate-compressed font is skipped, never scanned/modified. + continue; + } else if bytes_contain(dict, b"/FlateDecode") { + true + } else if bytes_contain(dict, b"/Filter") { + // Unsupported filter (e.g. /DCTDecode image): don't decode/scan. + continue; + } else { + false + }; + + if let Some((data, data_start, valid)) = extract_stream_bytes(self.data, &caps) { + return Some(StreamData { + object_start: whole.start(), + data_start, + compressed: data, + valid_end_marker: valid, + is_compressed, + }); + } else { + continue; // skip invalid capture + } + } + None + } +} + +/// Given a capture for a stream object, validate the end marker and return the raw stream bytes plus a validity flag. +pub(crate) fn extract_stream_bytes<'a>( + pdf_data: &'a [u8], + caps: ®ex::bytes::Captures<'a>, +) -> Option<(&'a [u8], usize, bool)> { + // Strict project rule: expected end marker is fixed here + const EXPECTED_END: &[u8] = b"\nendstream\nendobj"; + // Validate capture groups + let whole = match caps.get(0) { + Some(m) => m, + None => { + error!("PDF object capture missing whole-match"); + return None; + } + }; + let length_bytes = match caps.get(1) { + Some(m) => m.as_bytes(), + None => { + error!( + "PDF object capture missing /Length group at object starting {}", + whole.start() + ); + return None; + } + }; + + // Parse length strictly; if it fails, we consider this object invalid + let length = match std::str::from_utf8(length_bytes) + .ok() + .and_then(|s| s.parse::().ok()) + { + Some(v) => v, + None => { + error!( + "Invalid /Length value '{}' in object starting at {}", + String::from_utf8_lossy(length_bytes), + whole.start() + ); + return None; + } + }; + + let data_start = whole.end(); + let stream_end = match data_start.checked_add(length) { + Some(v) => v, + None => { + error!( + "Stream end overflow for object at {} (length={})", + data_start, length + ); + return None; + } + }; + + // strict bounds checks: must be entirely within pdf_data + if stream_end > pdf_data.len() { + error!( + "Stream end out of bounds for object starting at {}: stream_end={} pdf_len={}", + data_start, + stream_end, + pdf_data.len() + ); + return None; + } + if stream_end + EXPECTED_END.len() > pdf_data.len() { + error!( + "End marker out of bounds after stream_end {} for object starting at {} (pdf_len={})", + stream_end, + data_start, + pdf_data.len() + ); + return None; + } + + // Validate exact end marker (requirements are strict) + let debug_slice = &pdf_data[stream_end..stream_end + EXPECTED_END.len()]; + if debug_slice != EXPECTED_END { + warn!( + "End marker mismatch for object starting at {}: found {:?}, expected {:?}", + data_start, debug_slice, EXPECTED_END + ); + // Return decompressed candidate but indicate end marker mismatch for caller decision + return Some((&pdf_data[data_start..stream_end], data_start, false)); + } + + Some((&pdf_data[data_start..stream_end], data_start, true)) +} + +/// Extracted text token: the decoded string plus its start/end byte offsets +/// within the decompressed stream. +pub type TextToken = (String, usize, usize); + +/// Extract texts with positions from a compressed or uncompressed stream. +pub fn extract_texts_from_stream( + stream_data: &[u8], + is_compressed: bool, +) -> Result, Box> { + if is_compressed { + let mut decoder = ZlibDecoder::new(stream_data); + let mut decompressed = Vec::new(); + decoder.read_to_end(&mut decompressed)?; + Ok(scan_decompressed_text(&decompressed)) + } else { + Ok(scan_decompressed_text(stream_data)) + } +} + +/// Extract text tokens (with positions) from an already-decompressed content stream. +/// +/// Use this when the caller has already decompressed the stream to avoid a second +/// decompression pass. +pub fn extract_texts_from_decompressed(decompressed: &[u8]) -> Vec { + scan_decompressed_text(decompressed) +} + +/// association of text literal bytes and their positions in the decompressed buffer. +fn scan_decompressed_text(decompressed: &[u8]) -> Vec<(String, usize, usize)> { + // Linear scanner to find parenthesized literal strings reliably + // and associate them with text operators. This avoids brittle regexes + // and correctly handles escapes, octal sequences and nested parentheses. + fn skip_whitespace(buf: &[u8], mut idx: usize) -> usize { + while idx < buf.len() { + match buf[idx] { + b' ' | b'\t' | b'\n' | b'\r' | 0x0C => idx += 1, + _ => break, + } + } + idx + } + + fn parse_literal(buf: &[u8], open_idx: usize) -> Option<(usize, usize)> { + // open_idx points to '(' + let mut i = open_idx + 1; + let mut depth: i32 = 1; + while i < buf.len() { + match buf[i] { + b'\\' => { + // escape: skip next byte if present (octal handled by unescape) + i += 1; + if i < buf.len() { + // skip the escaped character + i += 1; + } + } + b'(' => { + depth += 1; + i += 1; + } + b')' => { + depth -= 1; + i += 1; + if depth == 0 { + // return indexes of inner content (exclude parentheses) + return Some((open_idx + 1, i - 1)); + } + } + _ => i += 1, + } + } + None + } + + let mut extracted: Vec<(String, usize, usize)> = Vec::new(); + // Tokens found since the most recent `BT`. They are committed to `extracted` + // only when a matching `ET` closes the text object. A stray `BT` that never + // closes (typical when non-content binary slips through) leaves its pending + // tokens uncommitted, so we never rewrite coincidental byte patterns. + let mut pending: Vec<(String, usize, usize)> = Vec::new(); + let mut i = 0usize; + let mut in_text_object = false; + + while i < decompressed.len() { + // Look for BT/ET operators to track text object state + if i + 1 < decompressed.len() { + if &decompressed[i..i + 2] == b"BT" { + in_text_object = true; + i += 2; + continue; + } else if &decompressed[i..i + 2] == b"ET" { + in_text_object = false; + // Commit only fully-closed text objects. + extracted.append(&mut pending); + i += 2; + continue; + } + } + + match decompressed[i] { + b'(' if in_text_object => { + if let Some((s, e)) = parse_literal(decompressed, i) { + // after the closing paren at index e+1, check for text operators + let after = skip_whitespace(decompressed, e + 1); + let mut is_text = false; + if after < decompressed.len() { + let op1 = decompressed[after]; + if op1 == b'\'' || op1 == b'"' { + is_text = true; + } else if after + 1 < decompressed.len() { + let op2 = &decompressed[after..after + 2]; + if op2 == b"Tj" || op2 == b"TJ" { + is_text = true; + } + } + } + + if is_text { + let raw = &decompressed[s..e]; + let unescaped = unescape_pdf_string(raw); + pending.push((String::from_utf8_lossy(&unescaped).to_string(), s, e)); + } + i = e + 1; // continue after closing paren + } else { + break; // malformed string — stop scanning + } + } + b'[' if in_text_object => { + // parse array until matching ']', collecting inner literal items + let mut arr_i = i + 1; + let mut found_end = false; + let mut inner_literals: Vec<(usize, usize)> = Vec::new(); + while arr_i < decompressed.len() { + match decompressed[arr_i] { + b'(' => { + if let Some((s, e)) = parse_literal(decompressed, arr_i) { + inner_literals.push((s, e)); + arr_i = e + 1; + } else { + break; // malformed + } + } + b']' => { + // check for TJ operator after the array + let after = skip_whitespace(decompressed, arr_i + 1); + if after + 1 < decompressed.len() + && &decompressed[after..after + 2] == b"TJ" + { + for (s, e) in inner_literals.iter() { + let raw = &decompressed[*s..*e]; + let unescaped = unescape_pdf_string(raw); + pending.push(( + String::from_utf8_lossy(&unescaped).to_string(), + *s, + *e, + )); + } + } + i = arr_i; + found_end = true; + break; + } + b'\\' => { + // skip escaped char inside array (not inside literal) + arr_i += 1; + if arr_i < decompressed.len() { + arr_i += 1; + } + } + _ => arr_i += 1, + } + } + if !found_end { + break; // malformed array — stop scanning to avoid infinite loop + } + } + _ => i += 1, + } + } + + // sort and dedup by start index + extracted.sort_by_key(|t| t.1); + extracted.dedup_by(|a, b| a.1 == b.1); + extracted +} + +/// Unescape PDF string literal escape sequences per PDF 1.3 spec (Table 3.2). +/// Handles: \n \r \t \b \f \( \) \\ and \ddd (octal, 1-3 digits). +/// Per spec: "If the character following the backslash is not one of those shown +/// in the table, the backslash is ignored." +fn unescape_pdf_string(data: &[u8]) -> Vec { + let mut result = Vec::with_capacity(data.len()); + let mut i = 0; + + while i < data.len() { + if data[i] == b'\\' && i + 1 < data.len() { + let (output, bytes_consumed) = handle_pdf_escape(&data[i + 1..]); + if let Some(byte) = output { + result.push(byte); + } + i += bytes_consumed; + } else { + result.push(data[i]); + i += 1; + } + } + result +} + +/// Handle a single PDF escape sequence starting after the backslash. +/// Returns (output byte if any, number of bytes to advance including the backslash). +fn handle_pdf_escape(data: &[u8]) -> (Option, usize) { + if data.is_empty() { + return (None, 1); // Lone backslash at end + } + + match data[0] { + b'n' => (Some(b'\n'), 2), + b'r' => (Some(b'\r'), 2), + b't' => (Some(b'\t'), 2), + b'b' => (Some(b'\x08'), 2), // backspace + b'f' => (Some(b'\x0C'), 2), // form feed + b'(' => (Some(b'('), 2), + b')' => (Some(b')'), 2), + b'\\' => (Some(b'\\'), 2), + b'0'..=b'7' => parse_pdf_octal_escape(data), + // Per spec: ignore backslash for unrecognized escapes + _ => (Some(data[0]), 2), + } +} + +/// Parse octal escape sequence \ddd (1-3 octal digits). +/// Returns (parsed byte, bytes consumed including backslash). +fn parse_pdf_octal_escape(data: &[u8]) -> (Option, usize) { + let mut end = 0; + // Consume up to 3 octal digits + while end < data.len() && end < 3 && data[end].is_ascii_digit() && data[end] <= b'7' { + end += 1; + } + + if let Ok(octal_str) = std::str::from_utf8(&data[..end]) { + if let Ok(value) = u8::from_str_radix(octal_str, 8) { + return (Some(value), end + 1); // +1 for the backslash + } + } + + // Fallback: ignore backslash if parsing fails + (Some(data[0]), 2) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_unescape_simple_escapes() { + // Test all simple escape sequences + assert_eq!(unescape_pdf_string(br"\n"), b"\n"); + assert_eq!(unescape_pdf_string(br"\r"), b"\r"); + assert_eq!(unescape_pdf_string(br"\t"), b"\t"); + assert_eq!(unescape_pdf_string(br"\b"), b"\x08"); // backspace + assert_eq!(unescape_pdf_string(br"\f"), b"\x0C"); // form feed + assert_eq!(unescape_pdf_string(br"\("), b"("); + assert_eq!(unescape_pdf_string(br"\)"), b")"); + assert_eq!(unescape_pdf_string(br"\\"), b"\\"); + } + + #[test] + fn test_unescape_octal_sequences() { + // Single digit octal + assert_eq!(unescape_pdf_string(br"\0"), b"\x00"); + assert_eq!(unescape_pdf_string(br"\7"), b"\x07"); + + // Two digit octal + assert_eq!(unescape_pdf_string(br"\53"), b"+"); // \053 = 43 decimal = '+' + + // Three digit octal + assert_eq!(unescape_pdf_string(br"\053"), b"+"); + assert_eq!(unescape_pdf_string(br"\245"), b"\xA5"); // 165 decimal + assert_eq!(unescape_pdf_string(br"\307"), b"\xC7"); // 199 decimal + + // Octal followed by non-digit (from PDF spec example) + assert_eq!(unescape_pdf_string(br"\0053"), b"\x053"); // \005 + '3' + } + + #[test] + fn test_unescape_real_world_case() { + // The actual case from the PDF that was failing + assert_eq!( + unescape_pdf_string(br"NET CREDITS/\(DEBITS\)"), + b"NET CREDITS/(DEBITS)" + ); + + // Dollar amount with parentheses + assert_eq!(unescape_pdf_string(br"\(6,085.80\)"), b"(6,085.80)"); + + // Date range + assert_eq!( + unescape_pdf_string(br"\(9/1/25-9/30/25\)"), + b"(9/1/25-9/30/25)" + ); + } + + #[test] + fn test_unescape_unrecognized_escape() { + // Per spec: "If the character following the backslash is not one of those + // shown in the table, the backslash is ignored." + assert_eq!(unescape_pdf_string(br"\x"), b"x"); + assert_eq!(unescape_pdf_string(br"\q"), b"q"); + assert_eq!(unescape_pdf_string(br"\Z"), b"Z"); + } + + #[test] + fn test_unescape_mixed_content() { + // Mix of regular text, escapes, and parentheses + assert_eq!( + unescape_pdf_string(br"Hello\nWorld\t\(test\)"), + b"Hello\nWorld\t(test)" + ); + + // \\ becomes \, then 053 is literal text (not preceded by backslash after unescape) + // Then \245 becomes byte 0xA5 + assert_eq!( + unescape_pdf_string(br"Price: \(\\053\245\)"), + b"Price: (\\053\xA5)" + ); + } + + #[test] + fn test_unescape_edge_cases() { + // Empty string + assert_eq!(unescape_pdf_string(b""), b""); + + // No escapes + assert_eq!(unescape_pdf_string(b"plain text"), b"plain text"); + + // Backslash at end (no following character) + assert_eq!(unescape_pdf_string(b"text\\"), b"text\\"); + } + + #[test] + fn test_scan_array_operator_boundaries() { + // Simulation: [ (text) ] TJ + // Ensures the parser stops exactly at ']' and doesn't skip the next byte. + let buf = b"BT [(text)] TJ ET"; + let extracted = scan_decompressed_text(buf); + + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].0, "text"); + } + + #[test] + fn test_nested_parentheses() { + // PDF spec allows nested parens if balanced: (Text (inner) more) + let buf = b"BT (Outer (inner) text) Tj ET"; + let extracted = scan_decompressed_text(buf); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].0, "Outer (inner) text"); + } + + #[test] + fn test_binary_junk_resilience() { + // Stream containing binary junk that might look like operators + let mut buf = Vec::new(); + buf.extend_from_slice(b"BT (Valid) Tj "); + buf.push(0x01); + buf.push(0x28); // binary '(' + buf.extend_from_slice(b"JUNK"); + buf.push(0x29); // binary ')' + buf.extend_from_slice(b" ET"); + + let extracted = scan_decompressed_text(&buf); + // Should find "Valid" and the junk string, but not crash + assert!(extracted.iter().any(|(s, _, _)| s == "Valid")); + assert!(!extracted.is_empty()); + } + + #[test] + fn test_array_operator_at_et_boundary() { + // Test: [(text)]TJ ET + // This checks if the parser correctly handles the end of the text block immediately after an array. + let buf = b"BT [(text)]TJ ET"; + let extracted = scan_decompressed_text(buf); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].0, "text"); + } + + #[test] + fn test_empty_and_max_length_literals() { + // Test edge cases for literal lengths + let buf = b"BT () Tj (A) Tj (BC) Tj ET"; + let extracted = scan_decompressed_text(buf); + assert_eq!(extracted.len(), 3); + assert_eq!(extracted[0].0, ""); + assert_eq!(extracted[1].0, "A"); + assert_eq!(extracted[2].0, "BC"); + } + + // --- Synthetic stream_scanner classification tests --- + // These build tiny in-memory PDF snippets (no real fixtures) to exercise how + // stream_scanner distinguishes FlateDecode, plain uncompressed, and non-Flate + // filtered (e.g. /DCTDecode image) streams. + + /// Zlib-compress `data` the same way real FlateDecode streams are stored. + fn flate_compress(data: &[u8]) -> Vec { + use flate2::write::ZlibEncoder; + use flate2::Compression; + use std::io::Write; + let mut enc = ZlibEncoder::new(Vec::new(), Compression::default()); + enc.write_all(data).unwrap(); + enc.finish().unwrap() + } + + /// Wrap raw stream bytes in a minimal `N 0 obj << /Length L{extra} >> stream ... endstream endobj`. + /// `extra` holds any additional dictionary entries (e.g. ` /Filter [/FlateDecode]`). + fn build_stream_object(obj_num: u32, extra_dict: &str, body: &[u8]) -> Vec { + let mut buf = Vec::new(); + buf.extend_from_slice( + format!( + "{} 0 obj\n<< /Length {}{} >>\nstream\n", + obj_num, + body.len(), + extra_dict + ) + .as_bytes(), + ); + buf.extend_from_slice(body); + buf.extend_from_slice(b"\nendstream\nendobj\n"); + buf + } + + #[test] + fn test_scanner_classifies_flate_stream_as_compressed() { + let compressed = flate_compress(b"BT (SECRET) Tj ET"); + let pdf = build_stream_object(1, " /Filter [/FlateDecode]", &compressed); + + let streams: Vec<_> = stream_scanner(&pdf).collect(); + assert_eq!(streams.len(), 1); + assert!(streams[0].is_compressed); + assert!(streams[0].valid_end_marker); + + let texts = + extract_texts_from_stream(streams[0].compressed, streams[0].is_compressed).unwrap(); + assert_eq!(texts.len(), 1); + assert_eq!(texts[0].0, "SECRET"); + } + + #[test] + fn test_scanner_classifies_plain_length_stream_as_uncompressed() { + // A dictionary with an explicit /Length but no /Filter is treated as + // uncompressed and read verbatim (no zlib decode). + let body = b"BT (HELLO) Tj ET"; + let pdf = build_stream_object(2, "", body); + + let streams: Vec<_> = stream_scanner(&pdf).collect(); + assert_eq!(streams.len(), 1); + assert!(!streams[0].is_compressed); + assert_eq!(streams[0].compressed, body); + + let texts = + extract_texts_from_stream(streams[0].compressed, streams[0].is_compressed).unwrap(); + assert_eq!(texts.len(), 1); + assert_eq!(texts[0].0, "HELLO"); + } + + #[test] + fn test_scanner_skips_non_flate_filter_stream() { + // A /DCTDecode (JPEG) image stream declares a /Filter we don't support. + // The scanner skips it entirely rather than mislabeling it uncompressed, + // so it is never decoded, scanned, or modified. + let body: &[u8] = &[0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10]; // JPEG SOI marker bytes + let pdf = build_stream_object(3, " /Filter /DCTDecode /Subtype /Image", body); + + let streams: Vec<_> = stream_scanner(&pdf).collect(); + assert!( + streams.is_empty(), + "image stream should be skipped, got {}", + streams.len() + ); + } + + #[test] + fn test_scanner_iterates_multiple_mixed_streams() { + // Two consecutive stream objects (one FlateDecode, one plain) are both + // yielded, in order, with the correct compression classification. + let compressed = flate_compress(b"BT (A) Tj ET"); + let mut pdf = Vec::new(); + pdf.extend_from_slice(b"%PDF-1.3\n"); + pdf.extend_from_slice(&build_stream_object( + 1, + " /Filter [/FlateDecode]", + &compressed, + )); + pdf.extend_from_slice(&build_stream_object(2, "", b"BT (B) Tj ET")); + + let streams: Vec<_> = stream_scanner(&pdf).collect(); + assert_eq!(streams.len(), 2); + assert!(streams[0].is_compressed); + assert!(!streams[1].is_compressed); + } + + #[test] + fn test_scanner_skips_font_program_stream() { + // An embedded font program is identified by /Length1 (FontFile/FontFile2) + // and must never be scanned: its binary body carries no PDF text tokens. + let body: &[u8] = b"%!PS-AdobeFont binary font data"; + let pdf = build_stream_object(3, " /Length1 12 /Length2 8 /Length3 0", body); + + let streams: Vec<_> = stream_scanner(&pdf).collect(); + assert!( + streams.is_empty(), + "font program should be skipped, got {}", + streams.len() + ); + } + + #[test] + fn test_scanner_skips_compressed_font_program_stream() { + // A Flate-compressed font program (/Length1 + /FlateDecode) must also be + // skipped: /Length1 is checked before the FlateDecode branch. + let compressed = flate_compress(b"binary font program bytes"); + let pdf = build_stream_object(6, " /Length1 25 /Filter /FlateDecode", &compressed); + + let streams: Vec<_> = stream_scanner(&pdf).collect(); + assert!( + streams.is_empty(), + "compressed font program should be skipped, got {}", + streams.len() + ); + } + + #[test] + fn test_scanner_classifies_name_form_flate_as_compressed() { + // FlateDecode written as a name (/Filter /FlateDecode) rather than + // an array is recognized by the unified classifier and decoded. + let compressed = flate_compress(b"BT (SECRET) Tj ET"); + let pdf = build_stream_object(4, " /Filter /FlateDecode", &compressed); + + let streams: Vec<_> = stream_scanner(&pdf).collect(); + assert_eq!(streams.len(), 1); + assert!(streams[0].is_compressed); + + let texts = + extract_texts_from_stream(streams[0].compressed, streams[0].is_compressed).unwrap(); + assert_eq!(texts.len(), 1); + assert_eq!(texts[0].0, "SECRET"); + } + + #[test] + fn test_scanner_discards_unclosed_text_object() { + // A `BT` never closed by `ET` (typical of non-content binary) must yield + // no tokens, even when it contains (...)Tj-like patterns. + let pdf = build_stream_object(5, "", b"BT (SHOULD NOT APPEAR) Tj (NOR THIS) Tj"); + let streams: Vec<_> = stream_scanner(&pdf).collect(); + assert_eq!(streams.len(), 1); + let texts = + extract_texts_from_stream(streams[0].compressed, streams[0].is_compressed).unwrap(); + assert!( + texts.is_empty(), + "unclosed BT must yield no tokens, got {:?}", + texts + ); + } + + /// Returns path to `anonymizer_data/` for offline integration tests. + /// Place plain PDF fixtures in that directory locally; they are git-ignored. + fn test_pdf_path(name: &str) -> std::path::PathBuf { + let manifest = env!("CARGO_MANIFEST_DIR"); + let path = std::path::Path::new(manifest) + .join("anonymizer_data") + .join(name); + assert!( + path.exists(), + "Missing test fixture (place the PDF in anonymizer_data/): {}", + path.display() + ); + path + } + + #[test] + #[ignore = "requires local PDF fixtures in anonymizer_data/"] + fn test_end_to_end_cash_flow_preserved() { + let orig_path = test_pdf_path("sample_statement.pdf"); + + // Create anonymized output via the library replace function + let out_path = std::env::temp_dir().join("sample_statement_anonymized_test.pdf"); + crate::anonymizer::replace::replace_pii_smart(&orig_path, &out_path) + .expect("anonymize failed"); + + let orig_bytes = read_pdf(&orig_path).expect("read orig"); + let out_bytes = read_pdf(&out_path).expect("read out"); + + fn extract_section(pdf_bytes: &[u8]) -> Option> { + for stream in stream_scanner(pdf_bytes) { + if let Ok(caps) = extract_texts_from_stream(stream.compressed, stream.is_compressed) + { + if let Some(pos) = caps.iter().position(|(s, _st, _en)| { + s.trim().eq_ignore_ascii_case("CASH FLOW ACTIVITY BY DATE") + }) { + let mut sec = Vec::new(); + for cap in caps.iter().skip(pos) { + sec.push(cap.0.clone()); + if cap.0.to_uppercase().contains("NET CREDITS") { + return Some(sec); + } + } + return Some(sec); + } + } + } + None + } + + let orig_sec = extract_section(&orig_bytes).expect("orig sec"); + let out_sec = extract_section(&out_bytes).expect("out sec"); + + assert!(orig_sec + .iter() + .any(|s| s.trim().eq_ignore_ascii_case("CASH FLOW ACTIVITY BY DATE"))); + assert!(orig_sec + .iter() + .any(|s| s.to_uppercase().contains("NET CREDITS"))); + assert_eq!( + orig_sec, out_sec, + "CASH FLOW section changed after anonymize" + ); + + let _ = std::fs::remove_file(&out_path); + } + + #[test] + #[ignore = "requires local PDF fixtures in anonymizer_data/"] + fn test_extract_integration_investments() { + // Integration-style test: open the sample PDF from the repo and + // assert that our extractor finds the visible "INVESTMENTS" string + // in at least one stream. This makes the regression explicit. + let pdf_path = test_pdf_path("sample_statement.pdf"); + let pdf_bytes = match read_pdf(&pdf_path) { + Ok(b) => b, + Err(e) => panic!("Failed to read sample PDF: {}", e), + }; + + let mut found = false; + for stream in stream_scanner(&pdf_bytes) { + let compressed = stream.compressed; + if let Ok(texts) = extract_texts_from_stream(compressed, stream.is_compressed) { + for (s, _st, _en) in texts.iter() { + if s.to_uppercase().contains("INVESTMENTS") { + found = true; + break; + } + } + } + if found { + break; + } + } + + assert!( + found, + "Extractor did not find 'INVESTMENTS' in sample PDF streams" + ); + } + + #[test] + #[ignore = "requires local PDF fixtures in anonymizer_data/"] + fn test_indexed_list_output_consistency() { + use std::collections::HashMap; + + let pdf_path = test_pdf_path("sample_statement.pdf"); + let pdf_bytes = match read_pdf(&pdf_path) { + Ok(b) => b, + Err(e) => panic!("Failed to read sample PDF: {}", e), + }; + + // preserved tokens that should remain literal in the output + let preserved = ["CLIENT STATEMENT", "For the Period"]; + + let mut global_map: HashMap = HashMap::new(); + let mut next_idx: usize = 0; + + // First pass: build global mapping of non-preserved strings to indices + for stream in stream_scanner(&pdf_bytes) { + let caps = match extract_texts_from_stream(stream.compressed, stream.is_compressed) { + Ok(c) => c, + Err(_) => continue, + }; + for (s, _st, _en) in caps { + let key = s.clone(); + if preserved.iter().any(|p| key.starts_with(p)) { + continue; // preserve + } + if !global_map.contains_key(&key) { + global_map.insert(key.clone(), next_idx); + next_idx += 1; + } + } + } + + // We expect the INVESTMENTS phrase to be present in the map and thus indexed + let investments_key = global_map + .keys() + .find(|k| k.to_uppercase().contains("INVESTMENTS")); + assert!( + investments_key.is_some(), + "INVESTMENTS was not indexed in the global map" + ); + + // Second pass: ensure consistent replacement (same string -> same index) + for stream in stream_scanner(&pdf_bytes) { + let caps = match extract_texts_from_stream(stream.compressed, stream.is_compressed) { + Ok(c) => c, + Err(_) => continue, + }; + for (s, _st, _en) in caps { + let key = s.clone(); + if preserved.iter().any(|p| key.starts_with(p)) { + // preserved tokens must remain textual + assert!(preserved.iter().any(|p| key.starts_with(p))); + } else { + let idx = global_map.get(&key).expect("Missing mapping for token"); + // index must be stable and within range + assert!(*idx < next_idx); + } + } + } + } + + #[test] + #[ignore = "requires local PDF fixtures in anonymizer_data/"] + fn test_anonymized_output_matches_expected() { + // Run the anonymizer on the input PDF and compare extracted text + // tokens from the result with those from the expected anonymized PDF. + let input_path = test_pdf_path("sample_statement.pdf"); + let expected_path = test_pdf_path("sample_statement_anonymized.pdf"); + + // Run anonymizer + let actual_path = std::env::temp_dir().join("test_anonymized_output_cmp.pdf"); + crate::anonymizer::replace::replace_pii_smart(&input_path, &actual_path) + .expect("anonymize failed"); + + // Extract all text tokens from both PDFs + fn all_tokens(pdf_path: &std::path::Path) -> Vec> { + let data = read_pdf(pdf_path).expect("read pdf"); + let mut result = Vec::new(); + for stream in stream_scanner(&data) { + if let Ok(caps) = extract_texts_from_stream(stream.compressed, stream.is_compressed) + { + let texts: Vec = caps + .into_iter() + .map(|(s, _, _)| s) + // Filter out non-printable binary junk to allow stable comparison + .filter(|s| { + s.chars() + .all(|c| c.is_ascii_graphic() || c.is_ascii_whitespace()) + }) + .collect(); + if !texts.is_empty() { + result.push(texts); + } + } + } + result + } + + let expected_tokens = all_tokens(&expected_path); + let actual_tokens = all_tokens(&actual_path); + + // We compare as many streams as we have in the older baseline + for (i, (exp, act)) in expected_tokens.iter().zip(actual_tokens.iter()).enumerate() { + assert_eq!( + exp, act, + "Stream {} tokens differ.\nExpected: {:?}\nActual: {:?}", + i, exp, act + ); + } + + // Ensure we didn't lose any data (new engine should find AT LEAST as many as old) + assert!( + actual_tokens.len() >= expected_tokens.len(), + "Regression: new engine found fewer text streams ({}) than the old one ({})", + actual_tokens.len(), + expected_tokens.len() + ); + + let _ = std::fs::remove_file(&actual_path); + } + + #[test] + #[ignore = "requires local PDF fixtures in anonymizer_data/"] + fn test_pdf_structure_skeleton_integrity() { + // Ultimate integrity test: verify that outside of the text literals (...), + // every single byte of the PDF stream (operators, coordinates, etc.) + // remains exactly the same. + let input_path = test_pdf_path("sample_statement.pdf"); + let actual_path = std::env::temp_dir().join("integrity_skeleton_test.pdf"); + crate::anonymizer::replace::replace_pii_smart(&input_path, &actual_path) + .expect("anonymize failed"); + + let input_data = read_pdf(&input_path).expect("read input"); + let actual_data = read_pdf(&actual_path).expect("read actual"); + + fn get_stream_skeleton(stream_data: &[u8]) -> Vec { + let mut decoder = ZlibDecoder::new(stream_data); + let mut decompressed = Vec::new(); + let _ = decoder.read_to_end(&mut decompressed); + + let mut skeleton = Vec::with_capacity(decompressed.len()); + let mut i = 0; + while i < decompressed.len() { + if decompressed[i] == b'(' { + // Skip everything inside literal until matching ) + skeleton.push(b'('); + skeleton.push(b'*'); // Placeholder for any text + skeleton.push(b')'); + + let mut depth = 1; + i += 1; + while i < decompressed.len() && depth > 0 { + if decompressed[i] == b'\\' && i + 1 < decompressed.len() { + i += 2; + continue; + } + if decompressed[i] == b'(' { + depth += 1; + } else if decompressed[i] == b')' { + depth -= 1; + } + i += 1; + } + } else { + skeleton.push(decompressed[i]); + i += 1; + } + } + skeleton + } + + let input_streams: Vec<_> = stream_scanner(&input_data).collect(); + let actual_streams: Vec<_> = stream_scanner(&actual_data).collect(); + + assert_eq!( + input_streams.len(), + actual_streams.len(), + "Stream count changed!" + ); + + for (i, (in_s, act_s)) in input_streams.iter().zip(actual_streams.iter()).enumerate() { + let skel_in = get_stream_skeleton(in_s.compressed); + let skel_act = get_stream_skeleton(act_s.compressed); + + assert_eq!( + skel_in, skel_act, + "Integrity violation in stream {}: non-textual bytes were modified!", + i + ); + } + + let _ = std::fs::remove_file(&actual_path); + } + + #[test] + #[ignore = "requires local PDF fixtures in anonymizer_data/"] + fn test_font_program_streams_not_modified() { + // Embedded font programs (/Length1) must be byte-for-byte identical after + // anonymization. Guards against the scanner ever treating coincidental + // byte patterns in binary font data as text tokens. + let input_path = test_pdf_path("sample_statement.pdf"); + let out_path = std::env::temp_dir().join("font_integrity_test.pdf"); + crate::anonymizer::replace::replace_pii_smart(&input_path, &out_path) + .expect("anonymize failed"); + + let inp = read_pdf(&input_path).expect("read input"); + let out = read_pdf(&out_path).expect("read output"); + assert_eq!(inp.len(), out.len(), "file length changed"); + + let re = regex::bytes::Regex::new( + r"(?s)\d+\s+\d+\s+obj\s*<<[^>]*?/Length\s+(\d+)[^>]*?/Length1\s+\d+[^>]*?>>\s*stream\r?\n", + ) + .unwrap(); + let mut checked = 0; + for caps in re.captures_iter(&inp) { + let whole = caps.get(0).unwrap(); + let len: usize = std::str::from_utf8(caps.get(1).unwrap().as_bytes()) + .unwrap() + .parse() + .unwrap(); + let start = whole.end(); + assert_eq!( + &inp[start..start + len], + &out[start..start + len], + "embedded font program bytes changed after anonymize" + ); + checked += 1; + } + assert!( + checked > 0, + "expected at least one /Length1 font program in fixture" + ); + let _ = std::fs::remove_file(&out_path); + } +} diff --git a/src/anonymizer/replace.rs b/src/anonymizer/replace.rs new file mode 100644 index 0000000..fb97558 --- /dev/null +++ b/src/anonymizer/replace.rs @@ -0,0 +1,436 @@ +// SPDX-FileCopyrightText: 2025 RustInFinance +// SPDX-License-Identifier: BSD-3-Clause + +//! String replacement module for anonymizer. +//! +//! This module applies specified text replacements to all FlateDecode streams in a PDF. +//! For each stream, the module: +//! 1. Decompresses the stream data +//! 2. Applies all specified string replacements +//! 3. Recompresses the modified text to the exact original size (with padding if necessary) +//! 4. Writes the modified PDF to the output file +//! +//! The in-place replacement strategy avoids rebuilding the PDF's XREF table, +//! ensuring the output PDF remains valid without full PDF structure parsing. + +use super::pdf::{extract_texts_from_decompressed, read_pdf, stream_scanner}; +use flate2::write::ZlibEncoder; +use flate2::Compression; +use log::{debug, info, warn}; +use std::fs::File; +use std::io::Write; +use std::path::Path; + +/// Replace PII using smart anonymization. +/// +/// 1. Preserve strings containing 'CLIENT STATEMENT' or 'For the Period' +/// 2. Replace all other strings with stable numeric tokens (0, 1, 2, ...) +/// 3. When 'CASH FLOW ACTIVITY BY DATE' is encountered, preserve all strings until +/// 'NET CREDITS/(DEBITS)' is found (inclusive) +/// +/// # Arguments +/// * `input_path` - Path to the input PDF file +/// * `output_path` - Path where the anonymized PDF will be written +/// +/// # Returns +/// * `Ok(())` on success +/// * `Err` if PDF cannot be read or processed +pub fn replace_pii_smart( + input_path: &Path, + output_path: &Path, +) -> Result<(), Box> { + info!("Smart anonymization mode"); + info!("Loading: {}", input_path.display()); + + let pdf_data = read_pdf(input_path)?; + debug!("PDF Size: {} bytes", pdf_data.len()); + + let mut output_data = pdf_data.clone(); + let mut streams_modified = 0; + let mut streams_total = 0; + let mut streams_skipped = 0; + let mut letter_counter = 0; + let mut in_cash_flow_section = false; + + for stream in stream_scanner(&pdf_data) { + if !stream.valid_end_marker { + warn!( + "Skipping stream due to end-marker mismatch for object at {}", + stream.object_start + ); + streams_skipped += 1; + continue; + } + streams_total += 1; + + let stream_data = stream.compressed; + let data_start = stream.data_start; + let is_compressed = stream.is_compressed; + + debug!( + "═══ Stream #{} (compressed={}) ═══", + streams_total, is_compressed + ); + debug!( + "Position: {}-{} ({} B)", + data_start, + data_start + stream_data.len(), + stream_data.len() + ); + + match process_stream_smart( + stream_data, + is_compressed, + &mut letter_counter, + &mut in_cash_flow_section, + ) { + Ok((new_data, modified)) => { + // Check size and warn if too large + if new_data.len() > stream_data.len() { + warn!( + "Cannot fit modified data: {} > {} bytes", + new_data.len(), + stream_data.len() + ); + warn!("Skipping modifications for this stream"); + streams_skipped += 1; + continue; + } + + // Count only streams whose modifications are actually written out. + if modified { + streams_modified += 1; + } + + // Write new data + for (idx, &byte) in new_data.iter().enumerate() { + output_data[data_start + idx] = byte; + } + + // Pad remaining space with NULL bytes to maintain the exact original stream + // length. Preserving the byte length keeps every subsequent object offset (and + // therefore the XREF table) valid, so we never need to rebuild it. This applies + // to both compressed and uncompressed streams; the trailing padding sits after + // the logical end of the stream content and is ignored by PDF readers. + let padding_len = stream_data.len() - new_data.len(); + for idx in new_data.len()..stream_data.len() { + output_data[data_start + idx] = 0x00; + } + + if padding_len > 0 { + debug!("Applied padding of {} bytes to stream", padding_len); + } + + debug!("Data size: {} → {} B", stream_data.len(), new_data.len()); + } + Err(e) => { + warn!("Failed to process stream: {}", e); + streams_skipped += 1; + } + } + } + + // Warn loudly (output is still produced): a cash-flow section that opened but + // never closed means every token after it was preserved and may still contain PII. + if in_cash_flow_section { + warn!( + "CASH FLOW section opened but its closing marker was never found; \ + tokens after it were preserved and may not be anonymized" + ); + } + + info!("Saving: {}", output_path.display()); + File::create(output_path)?.write_all(&output_data)?; + + info!("DONE!"); + info!( + "Streams: total={} modified={}", + streams_total, streams_modified + ); + if streams_skipped > 0 { + warn!( + "Skipped {} stream(s); their bytes were left unmodified and may still contain PII", + streams_skipped + ); + } + info!("File: {}", output_path.display()); + + Ok(()) +} + +/// Check if a text should be preserved based on content rules. +fn should_preserve_text(text: &str) -> bool { + text.contains("CLIENT STATEMENT") || text.contains("For the Period") +} + +/// Process a single stream with smart anonymization. +/// Returns (output_data, was_modified). +/// For compressed streams, returns compressed data. For uncompressed, returns raw data. +fn process_stream_smart( + stream_data: &[u8], + is_compressed: bool, + letter_counter: &mut usize, + in_cash_flow_section: &mut bool, +) -> Result<(Vec, bool), Box> { + use flate2::read::ZlibDecoder; + use std::io::Read; + + let original_len = stream_data.len(); + + info!( + "Processing stream len={} compressed={}", + original_len, is_compressed + ); + + // Decompress if needed + let mut decompressed = Vec::new(); + if is_compressed { + let mut decoder = ZlibDecoder::new(stream_data); + decoder.read_to_end(&mut decompressed)?; + } else { + decompressed = stream_data.to_vec(); + } + + debug!("Decompressed: {} B", decompressed.len()); + + // Extract texts along with their byte positions in the decompressed stream. + // Reuse the buffer we just decompressed above to avoid a second decompression. + let texts = extract_texts_from_decompressed(&decompressed); + + // Build replacement list based on rules + let mut replacements: Vec<(usize, usize, Vec)> = Vec::new(); + for (text, start, end) in texts.iter() { + let current_token_index = *letter_counter; + *letter_counter += 1; + + // Log token detection for debugging + debug!( + "Token[{}] '{}' at {}..{} (in_cash_flow={})", + current_token_index, text, start, end, in_cash_flow_section + ); + + // Preserve start/end markers of the CASH FLOW section explicitly. + if text.contains("CASH FLOW ACTIVITY BY DATE") { + *in_cash_flow_section = true; + debug!("Entering CASH FLOW section"); + debug!("Preserving token[{}] '{}'", current_token_index, text); + continue; + } + + // Preserve the closing marker and then exit the cash flow section. + if text.contains("NET CREDITS/(DEBITS)") { + debug!("Preserving token[{}] '{}'", current_token_index, text); + *in_cash_flow_section = false; + debug!("Exiting CASH FLOW section"); + continue; + } + + // Check if we should preserve this text by general rules + if *in_cash_flow_section || should_preserve_text(text) { + debug!("Preserving token[{}] '{}'", current_token_index, text); + continue; + } else { + // Replace entire token with its global index number + let replacement_str = current_token_index.to_string(); + + // PDF string literals are stored escaped in the decompressed stream, + // but our replacement uses only ASCII digits, so converting + // to bytes is safe here. + let replacement_bytes = replacement_str.as_bytes().to_vec(); + debug!( + "Scheduling replacement for token[{}] '{}' -> '{}' ({} bytes)", + current_token_index, + text, + replacement_str, + replacement_bytes.len() + ); + replacements.push((*start, *end, replacement_bytes)); + } + } + + // Apply replacements to decompressed data. Perform in reverse order of + // positions so earlier replacements do not affect subsequent indices. + let mut modified_data = decompressed.clone(); + let mut was_modified = false; + + if !replacements.is_empty() { + // Sort by start descending + replacements.sort_by_key(|t| std::cmp::Reverse(t.0)); + for (start, end, repl_bytes) in replacements.iter() { + let s = *start; + let e = *end; + // Build new buffer with replacement applied + let mut new_buf = Vec::with_capacity(modified_data.len() - (e - s) + repl_bytes.len()); + new_buf.extend_from_slice(&modified_data[..s]); + new_buf.extend_from_slice(repl_bytes); + new_buf.extend_from_slice(&modified_data[e..]); + modified_data = new_buf; + was_modified = true; + debug!( + "Applied replacement at {}..{} ({} B)", + s, + e, + repl_bytes.len() + ); + } + } + + // Recompress to fit original size (only for compressed streams) + if was_modified { + if is_compressed { + if let Some((compressed, level)) = + find_fitting_compression(&modified_data, original_len) + { + debug!( + "Compressed with level {} ({} B <= {} B)", + level, + compressed.len(), + original_len + ); + return Ok((compressed, true)); + } else { + warn!( + "Cannot fit modified data into original size ({}B); keeping original", + original_len + ); + return Ok((stream_data.to_vec(), false)); + } + } else { + // Uncompressed stream - return modified data directly + return Ok((modified_data, true)); + } + } + + Ok((stream_data.to_vec(), false)) +} + +/// Try progressive zlib compression levels (0..=9) returning the first compressed form whose length is <= `max_size`. +fn find_fitting_compression(data: &[u8], max_size: usize) -> Option<(Vec, u32)> { + for level in 0..=9 { + let mut encoder = ZlibEncoder::new(Vec::new(), Compression::new(level)); + if encoder.write_all(data).is_err() { + continue; + } + let compressed = encoder.finish().ok()?; + if compressed.len() <= max_size { + return Some((compressed, level)); + } + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + use flate2::read::ZlibDecoder; + use std::io::Read; + + /// Helper: run `process_stream_smart` on uncompressed content with fresh state. + fn process_uncompressed(content: &[u8]) -> (Vec, bool, usize) { + let mut counter = 0usize; + let mut in_cash_flow = false; + let (out, modified) = + process_stream_smart(content, false, &mut counter, &mut in_cash_flow).unwrap(); + (out, modified, counter) + } + + #[test] + fn test_replaces_pii_and_preserves_marker_uncompressed() { + // "JAN KOWALSKI" is PII -> replaced with its token index (0). + // "CLIENT STATEMENT" is a preserved marker -> kept verbatim. + let content = b"BT (JAN KOWALSKI) Tj (CLIENT STATEMENT) Tj ET"; + let (out, modified, counter) = process_uncompressed(content); + + assert!(modified, "stream with PII should be reported as modified"); + assert_eq!(out, b"BT (0) Tj (CLIENT STATEMENT) Tj ET"); + // Two tokens were scanned, so the global counter advanced by two. + assert_eq!(counter, 2); + } + + #[test] + fn test_preserves_cash_flow_section_then_resumes_replacement() { + // Everything from "CASH FLOW ACTIVITY BY DATE" to "NET CREDITS/(DEBITS)" + // (inclusive) must be preserved; tokens after the section are replaced again. + let content = + b"BT (CASH FLOW ACTIVITY BY DATE) Tj (9/1/25) Tj (100.00) Tj (NET CREDITS/(DEBITS)) Tj (SECRET) Tj ET"; + let (out, modified, counter) = process_uncompressed(content); + + assert!(modified); + let out_str = String::from_utf8(out).unwrap(); + // In-section tokens are untouched. + assert!(out_str.contains("(CASH FLOW ACTIVITY BY DATE)")); + assert!(out_str.contains("(9/1/25)")); + assert!(out_str.contains("(100.00)")); + assert!(out_str.contains("(NET CREDITS/(DEBITS))")); + // The token after the section ("SECRET") is the 5th token (index 4). + assert!(out_str.contains("(4) Tj ET")); + assert!(!out_str.contains("SECRET")); + assert_eq!(counter, 5); + } + + #[test] + fn test_no_change_when_all_tokens_preserved_uncompressed() { + let content = b"BT (CLIENT STATEMENT) Tj (For the Period) Tj ET"; + let (out, modified, counter) = process_uncompressed(content); + + assert!(!modified, "fully preserved stream must not be modified"); + assert_eq!(out, content, "unmodified stream must be returned verbatim"); + assert_eq!(counter, 2); + } + + #[test] + fn test_compressed_stream_roundtrip_replacement() { + // A real FlateDecode stream: compress a content stream, run it through the + // compressed path, and verify the replacement survives a decompress round-trip. + // The original is stored uncompressed (`Compression::none()`) so that the + // (shorter) modified stream is guaranteed to fit the original size without + // depending on exact compressor byte counts. + let content = b"BT (SECRET) Tj (CLIENT STATEMENT) Tj ET"; + + let mut encoder = ZlibEncoder::new(Vec::new(), Compression::none()); + encoder.write_all(content).unwrap(); + let compressed = encoder.finish().unwrap(); + let original_len = compressed.len(); + + let mut counter = 0usize; + let mut in_cash_flow = false; + let (out, modified) = + process_stream_smart(&compressed, true, &mut counter, &mut in_cash_flow).unwrap(); + + assert!(modified); + // Output must still fit the original compressed size (XREF-preserving invariant). + assert!( + out.len() <= original_len, + "recompressed stream ({}) must fit original size ({})", + out.len(), + original_len + ); + + // Decompress and verify: "SECRET" -> token index 0, marker preserved. + let mut decoder = ZlibDecoder::new(&out[..]); + let mut decompressed = Vec::new(); + decoder.read_to_end(&mut decompressed).unwrap(); + assert_eq!(decompressed, b"BT (0) Tj (CLIENT STATEMENT) Tj ET"); + assert_eq!(counter, 2); + } + + #[test] + fn test_cash_flow_section_stays_open_without_closing_marker() { + // Opening marker with no closing "NET CREDITS/(DEBITS)": the section stays + // open, so later tokens are preserved (not anonymized). This is the state + // that replace_pii_smart surfaces via a warning. + let content = b"BT (CASH FLOW ACTIVITY BY DATE) Tj (STILL SECRET) Tj ET"; + let mut counter = 0usize; + let mut in_cash_flow = false; + let (out, _modified) = + process_stream_smart(content, false, &mut counter, &mut in_cash_flow).unwrap(); + + assert!( + in_cash_flow, + "section must remain open when the closing marker is absent" + ); + // The post-marker token was preserved verbatim (the potential leak the warning flags). + let out_str = String::from_utf8(out).unwrap(); + assert!(out_str.contains("(STILL SECRET)")); + } +} diff --git a/src/lib.rs b/src/lib.rs index e5cdd9a..e93dc07 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,6 +8,9 @@ mod pdfparser; mod transactions; mod xlsxparser; +// Public because the `etradeAnonymizer` binary references it as `etradeTaxReturnHelper::anonymizer::...`. +pub mod anonymizer; + type ReqwestClient = reqwest::blocking::Client; pub use logging::ResultExt;