Repository navigation
fix(artifacts): inspect text behind printable binary prefixes #794
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
6a1b0d6
bb81156
2529288
bd46d38
6460651
72cf6ea
4a0cbd8
666a758
781475c
8f8d5b8
32c722b
0e3992b
93e06fe
6d685d4
5f88b16
7183371
a0b5caa
58b0273
5bba1e7
c808261
fe4fac2
b5a1c04
a2209b6
4ba349f
c3325d0
eaebfab
8112ce3
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -64,6 +64,7 @@ class ArtifactRecord(TypedDict): | |
| contains_nul: bool | ||
| misleading_extension: bool | ||
| referenced: bool | ||
| readable_binary: NotRequired[bool] | ||
| reason: NotRequired[str] | ||
| inherited_exclusion_reason: NotRequired[str] | ||
|
|
||
|
|
@@ -367,6 +368,11 @@ def classify_artifact(path: str, data: bytes, *, referenced: bool = False) -> Ar | |
| decoded = data.decode("utf-8", errors="replace") | ||
| decodable = False | ||
|
|
||
| # These printable prefixes are also valid instruction text. Keep the | ||
| # claimed binary format's coverage limits, but inspect the readable bytes. | ||
| # Lossy decoding must not let one invalid byte or a binary comment hide text. | ||
| readable_binary = data.startswith((b"MZ", b"GIF87a", b"GIF89a", b"%PDF-")) | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This gate is now the prefix alone, so every real GIF, PDF and PE file gets its whole lossy Measured on a transcription of the head (main 5edc347 plus this PR's src diff, byte-identical to the head), with an unreferenced real file beside a benign SKILL.md and
Please keep the evasion fix but scan a bounded printable-run projection instead of the whole decode:
See finding 1. |
||
|
|
||
| if has_binary_magic: | ||
| kind = ContentKind.BINARY | ||
| elif decodable: | ||
|
|
@@ -388,6 +394,8 @@ def classify_artifact(path: str, data: bytes, *, referenced: bool = False) -> Ar | |
| disposition = ( | ||
| ArtifactDisposition.PARTIAL | ||
| if referenced and kind is not ContentKind.TEXT | ||
| else ArtifactDisposition.ANALYZED | ||
| if readable_binary | ||
| else ArtifactDisposition.OUT_OF_SCOPE | ||
| if kind is ContentKind.BINARY | ||
| else ArtifactDisposition.ANALYZED | ||
|
|
@@ -401,9 +409,39 @@ def classify_artifact(path: str, data: bytes, *, referenced: bool = False) -> Ar | |
| "contains_nul": contains_nul, | ||
| "misleading_extension": misleading, | ||
| "referenced": referenced, | ||
| "readable_binary": readable_binary, | ||
| } | ||
|
|
||
|
|
||
| def pdf_xmp_format_spans(content: str) -> tuple[int, int, int]: | ||
| """Recognize one standard XMP packet's BOM and trailing ASCII padding only.""" | ||
| header = '<?xpacket begin="\ufeff" id="W5M0MpCehiHzreSzNTczkc9d"?>' | ||
| if not content.startswith("%PDF-"): | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This helper only recognizes the packet when it receives text that starts with Transcription results:
Please compute the spans once on the whole artifact, for example in |
||
| return -1, -1, -1 | ||
| start = content.find(header) | ||
| if start < 0: | ||
| # Unicode normalization removes the format-only BOM in derived views. | ||
| header = header.replace("\ufeff", "") | ||
| start = content.find(header) | ||
| if start < 0 or content.find(header, start + len(header)) >= 0: | ||
| return -1, -1, -1 | ||
| metadata = content.find("<x:xmpmeta", start + len(header)) | ||
| close = content.find("</x:xmpmeta>", metadata) if metadata >= 0 else -1 | ||
| if close < 0: | ||
| return -1, -1, -1 | ||
| padding_start = close + len("</x:xmpmeta>") | ||
| padding_end = padding_start | ||
| while padding_end < len(content) and content[padding_end] in " \t\r\n": | ||
| padding_end += 1 | ||
| if not content.startswith(('<?xpacket end="w"?>', '<?xpacket end="r"?>'), padding_end): | ||
| return -1, -1, -1 | ||
| return ( | ||
| (start + header.index("\ufeff") if "\ufeff" in header else -1), | ||
| padding_start, | ||
| padding_end, | ||
| ) | ||
|
|
||
|
|
||
| def promote_artifact_to_decoded_text(artifact: ArtifactRecord) -> None: | ||
| """Apply a successful format-aware text decode without erasing prior limits.""" | ||
| generic_binary_scope = artifact["content_kind"] is ContentKind.BINARY and ( | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This row holds for the synthetic ASCII fixture but not for real files. Transcription with a real
icon.gif(125 B, LZW) andmanual.pdf(1,479 B, Flate):binary_contentinscope_exclusions, SAFE 0, exits 0/0/0.analyzed,scope_exclusionsempty, AE4 on both, SAFE 12, exits 0/0/1.In the local sample, 47 of 49 real GIF/PDF files got AE4. Row 121 is also no longer accurate for GIF images, and the compressed data is never decoded.
After the finding 1 fix, please:
scope_exclusionsunder a distinct reason, for examplebinary_text_projection;test-unitcurrently fails.See finding 6.