Repository navigation
Expand file tree
/
Copy pathqueue.py
More file actions
308 lines (238 loc) · 9.23 KB
/
Copy pathqueue.py
File metadata and controls
308 lines (238 loc) · 9.23 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
#!/usr/bin/env python3
"""
Durable collection queue, derived from the append-only logs.
Queue state is a pure function of (url-inventory.jsonl, seeds.txt,
manifest.jsonl): nothing is ever "in memory only", a crashed worker
leaves no stuck FETCHING rows (its unattempted URLs simply remain
pending), and rebuilding the queue is always safe and idempotent.
States per URL:
DISCOVERED known, never attempted -> pending
ARCHIVED latest attempt succeeded (bytes hashed+stored)
NON_CONTENT_TECHNICAL_RESPONSE
latest attempt "succeeded" but the bytes are a
bot challenge / interstitial / application
shell (per recovery/challenge-observations.jsonl)
-> owned by the recovery pipeline, never counted
as archived content, never re-hammered here
NOT_FOUND latest attempt 404/410 -> observation, no retry
ACCESS_RESTRICTED latest attempt 401/403 -> recorded gap, never
evaded, no retry
RATE_LIMITED latest attempt 429 -> retry-eligible
TEMPORARY_ERROR latest attempt 5xx/timeout/network -> retry-
eligible
ROBOTS_DISALLOWED refused by robots.txt -> recorded, no retry
MANUAL_REVIEW host not allowlisted or unclassifiable -> human
Retry-eligible states stop being emitted as pending once a URL has
been attempted --max-attempts times (they stay in the queue file as a
recorded gap).
Outputs:
queue.jsonl full queue state (atomic replace)
--pending-out seed-format file of pending work, capped at
--limit (DISCOVERED first, then retries)
"""
from __future__ import annotations
import argparse
import json
import os
import sys
from pathlib import Path
from archive import parse_seed, sanitize_url
from discover import seed_label, seed_provenance
from manifest_store import iter_jsonl_records
RETRYABLE = {"RATE_LIMITED", "TEMPORARY_ERROR"}
NO_RETRY = {
"ARCHIVED",
"NON_CONTENT_TECHNICAL_RESPONSE",
"NOT_FOUND",
"ACCESS_RESTRICTED",
"ROBOTS_DISALLOWED",
"MANUAL_REVIEW",
}
# Challenge-detector states whose "successful" capture is not content.
NON_CONTENT_CLASSIFICATIONS = {
"BOT_CHALLENGE",
"ACCESS_INTERSTITIAL",
"APPLICATION_SHELL",
"REDIRECT_STUB",
}
def load_challenged_urls(path: Path) -> set[str]:
"""URLs whose latest capture is a non-content technical response."""
challenged: set[str] = set()
if not path.exists():
return challenged
with path.open("r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
try:
row = json.loads(line)
except json.JSONDecodeError:
continue
if row.get("classification") in NON_CONTENT_CLASSIFICATIONS:
challenged.add(sanitize_url(row.get("url", "")))
return challenged
def classify(record: dict, challenged: set[str] = frozenset()) -> str:
if record.get("sha256"):
if sanitize_url(record.get("url", "")) in challenged:
# 200-with-bytes that the challenge detector classified as
# a bot challenge / interstitial / app shell. The recovery
# pipeline owns these on a low-frequency cadence; the
# ordinary queue neither counts them as archived nor
# re-hammers them.
return "NON_CONTENT_TECHNICAL_RESPONSE"
return "ARCHIVED"
error = (record.get("error") or "").lower()
status = record.get("http_status")
if error == "robots_disallowed":
return "ROBOTS_DISALLOWED"
if error == "host_not_allowlisted":
return "MANUAL_REVIEW"
if status in (401, 403):
return "ACCESS_RESTRICTED"
if status in (404, 410):
return "NOT_FOUND"
if status == 429:
return "RATE_LIMITED"
return "TEMPORARY_ERROR"
def load_known_urls(inventory: Path, seeds: Path) -> dict[str, dict]:
known: dict[str, dict] = {}
if seeds.exists():
for raw in seeds.read_text(encoding="utf-8").splitlines():
line = raw.strip()
if not line or line.startswith("#"):
continue
url, provenance, source = parse_seed(line)
known[sanitize_url(url)] = {
"provenance": provenance,
"source": source,
"origin": "seeds",
}
if inventory.exists():
with inventory.open("r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
record = json.loads(line)
url = sanitize_url(record.get("url", ""))
if url and url not in known:
known[url] = {
"provenance": seed_provenance(record),
"source": seed_label(record),
"origin": "inventory",
}
return known
def load_attempts(manifest: Path) -> dict[str, dict]:
"""Last record + attempt count per URL, in file (=time) order."""
attempts: dict[str, dict] = {}
for record in iter_jsonl_records(manifest):
url = sanitize_url(record.get("url", ""))
if not url:
continue
entry = attempts.setdefault(
url, {"count": 0, "last": None, "ever_archived": False}
)
entry["count"] += 1
entry["last"] = record
if record.get("sha256"):
entry["ever_archived"] = True
return attempts
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--inventory", default="url-inventory.jsonl")
parser.add_argument("--seeds", default="seeds.txt")
parser.add_argument("--manifest", default="manifest.jsonl")
parser.add_argument("--queue-out", default="queue.jsonl")
parser.add_argument("--pending-out", default=None)
parser.add_argument("--limit", type=int, default=4000)
parser.add_argument("--max-attempts", type=int, default=5)
parser.add_argument(
"--challenge-observations",
default="recovery/challenge-observations.jsonl",
help=(
"Challenge-detector observations; captures classified as "
"non-content there are reported as "
"NON_CONTENT_TECHNICAL_RESPONSE instead of ARCHIVED"
),
)
args = parser.parse_args()
known = load_known_urls(Path(args.inventory), Path(args.seeds))
attempts = load_attempts(Path(args.manifest))
challenged = load_challenged_urls(Path(args.challenge_observations))
# URLs seen only in the manifest (e.g. link-following) are still
# part of the queue's world view.
for url, entry in attempts.items():
if url not in known:
last = entry["last"]
known[url] = {
"provenance": last.get("provenance", "GOV-PUBLIC"),
"source": last.get("source"),
"origin": "crawl",
}
rows = []
counts: dict[str, int] = {}
pending_new = []
pending_retry = []
for url in sorted(known):
meta = known[url]
entry = attempts.get(url)
if entry is None:
state = "DISCOVERED"
count = 0
last_status = None
last_at = None
else:
state = classify(entry["last"], challenged)
count = entry["count"]
last_status = entry["last"].get("http_status")
last_at = entry["last"].get("retrieved_at")
counts[state] = counts.get(state, 0) + 1
retry_eligible = (
state in RETRYABLE and count < args.max_attempts
)
rows.append({
"url": url,
"state": state,
"attempts": count,
"retry_eligible": retry_eligible,
"last_http_status": last_status,
"last_attempt_at": last_at,
"provenance": meta["provenance"],
"source": meta["source"],
"origin": meta["origin"],
})
seed_line = (
f"{url}|{meta['provenance']}|{meta['source'] or ''}"
)
if state == "DISCOVERED":
pending_new.append(seed_line)
elif retry_eligible:
pending_retry.append(seed_line)
# Atomic queue write: never leave a half-written queue behind.
queue_path = Path(args.queue_out)
tmp = queue_path.with_suffix(".tmp")
with tmp.open("w", encoding="utf-8") as f:
for row in rows:
f.write(json.dumps(row, sort_keys=True) + "\n")
os.replace(tmp, queue_path)
pending = (pending_new + pending_retry)[: args.limit]
if args.pending_out:
pending_path = Path(args.pending_out)
tmp = pending_path.with_suffix(".tmp")
tmp.write_text(
"\n".join(pending) + ("\n" if pending else ""),
encoding="utf-8",
)
os.replace(tmp, pending_path)
print(f"Queue: {len(rows)} URLs")
for state in sorted(counts):
print(f" {state}: {counts[state]}")
print(
f"Pending emitted: {len(pending)} "
f"({len(pending_new)} new, {len(pending_retry)} retry-eligible, "
f"limit {args.limit})"
)
return 0
if __name__ == "__main__":
sys.exit(main())