@@ -341,8 +341,8 @@ class DagFileProcessorManager(LoggingMixin):
341341 _deferred_api_callbacks : list [CallbackRequest ] = attrs .field (factory = list , init = False )
342342 _deferred_callback_retries : dict [int , DeferredCallback ] = attrs .field (factory = dict , init = False )
343343 _callback_attempts : dict [UUID , list [ProcessorWorkItem ]] = attrs .field (factory = dict , init = False )
344- _priority_claims : dict [str , PriorityParseClaim ] = attrs .field (factory = dict , init = False )
345- _pending_work_acks : list [tuple [Literal ["callbacks" , "priority" ], ProcessorWorkItem , bool ]] = attrs .field (
344+ _priority_claims : dict [tuple [ str , UUID ] , PriorityParseClaim ] = attrs .field (factory = dict , init = False )
345+ _pending_work_acks : list [tuple [Literal ["callbacks" , "priority" ], ProcessorWorkItem ]] = attrs .field (
346346 factory = list , init = False
347347 )
348348 _num_run : int = attrs .field (default = 0 , init = False )
@@ -810,7 +810,7 @@ def claim_priority_files(self) -> list[DagFileInfo]:
810810 available = (
811811 self ._parallelism
812812 - len (self ._priority_claims )
813- - sum (kind == "priority" for kind , _ , _ in self ._pending_work_acks )
813+ - sum (kind == "priority" for kind , _ in self ._pending_work_acks )
814814 )
815815 if available <= 0 :
816816 return []
@@ -822,7 +822,7 @@ def claim_priority_files(self) -> list[DagFileInfo]:
822822 return []
823823 files = []
824824 for item in work :
825- self ._priority_claims [item .id ] = PriorityParseClaim (
825+ self ._priority_claims [item .id , item . claim_id ] = PriorityParseClaim (
826826 work = item , refresh_generation = self ._bundle_refresh_generations [item .bundle_name ] + 1
827827 )
828828 bundle = bundles [item .bundle_name ]
@@ -897,7 +897,7 @@ def fetch_callbacks(self) -> list[CallbackRequest]:
897897 self .max_callbacks_per_loop
898898 - len (self ._callback_claims )
899899 - sum (len (work ) for work in self ._callback_attempts .values ())
900- - sum (kind == "callbacks" for kind , _ , _ in self ._pending_work_acks )
900+ - sum (kind == "callbacks" for kind , _ in self ._pending_work_acks )
901901 )
902902 if available <= 0 :
903903 return list (self ._deferred_api_callbacks )
@@ -922,9 +922,9 @@ def _acknowledge_requested_work(self, *, deadline: float | None = None) -> None:
922922 if deadline is None :
923923 deadline = time .monotonic () + _API_SEND_BUDGET_SECONDS
924924 while self ._pending_work_acks and time .monotonic () < deadline :
925- kind , item , failed = self ._pending_work_acks [0 ]
925+ kind , item = self ._pending_work_acks [0 ]
926926 try :
927- self .api_client .acknowledge_work (kind , item , failed = failed )
927+ self .api_client .acknowledge_work (kind , item )
928928 except httpx2 .HTTPStatusError as error :
929929 if error .response .status_code >= 500 :
930930 self .log .warning ("Unable to acknowledge requested work; retaining the claim" )
@@ -944,13 +944,13 @@ def _acknowledge_requested_work(self, *, deadline: float | None = None) -> None:
944944
945945 def _finish_callback_attempt (self , proc : BaseDagFileProcessorProcess ) -> None :
946946 for item in self ._callback_attempts .pop (proc .id , []):
947- self ._pending_work_acks .append (("callbacks" , item , proc . _exit_code != 0 ))
947+ self ._pending_work_acks .append (("callbacks" , item ))
948948
949- def _finish_priority_claims (self , attempt_id : UUID , * , failed : bool ) -> None :
949+ def _finish_priority_claims (self , attempt_id : UUID ) -> None :
950950 """Acknowledge only requests assigned to this import, never newer requests for its file."""
951951 for key , claim in list (self ._priority_claims .items ()):
952952 if claim .attempt_id == attempt_id :
953- self ._pending_work_acks .append (("priority" , claim .work , failed ))
953+ self ._pending_work_acks .append (("priority" , claim .work ))
954954 del self ._priority_claims [key ]
955955
956956 def _get_priority_claims (self , file : DagFileInfo ) -> list [PriorityParseClaim ]:
@@ -1065,7 +1065,7 @@ def _add_callback_to_queue(self, request: CallbackRequest) -> None:
10651065 if get_claiming_importer (request .filepath , request .bundle_name ) is not None :
10661066 self ._log_dropped_lang_sdk_callback (request )
10671067 if item := self ._callback_claims .pop (id (request ), None ):
1068- self ._pending_work_acks .append (("callbacks" , item , True ))
1068+ self ._pending_work_acks .append (("callbacks" , item ))
10691069 self ._remove_deferred_callback (request )
10701070 return
10711071 retry = self ._deferred_callback_retries .get (id (request ))
@@ -1098,7 +1098,7 @@ def _remove_deferred_callback(self, request: CallbackRequest) -> None:
10981098 self ._deferred_callback_retries .pop (id (request ), None )
10991099
11001100 def _defer_claimed_callback (self , request : CallbackRequest ) -> None :
1101- """Retry preparing a claimed callback's bundle with backoff, then acknowledge it as failed ."""
1101+ """Retry preparing a claimed callback's bundle with backoff, then drop the unpreparable request ."""
11021102 now = time .monotonic ()
11031103 timeout = conf .getint ("dag_processor" , "job_heartbeat_timeout" )
11041104 retry = self ._deferred_callback_retries .setdefault (id (request ), DeferredCallback (now + timeout ))
@@ -1111,7 +1111,7 @@ def _defer_claimed_callback(self, request: CallbackRequest) -> None:
11111111 request .bundle_version ,
11121112 timeout ,
11131113 )
1114- self ._pending_work_acks .append (("callbacks" , self ._callback_claims .pop (id (request )), True ))
1114+ self ._pending_work_acks .append (("callbacks" , self ._callback_claims .pop (id (request ))))
11151115 self ._remove_deferred_callback (request )
11161116 return
11171117 retry .attempts += 1
@@ -1214,7 +1214,7 @@ def _publish_bundle_inventory(self, bundle_name: str, known_files: dict[str, set
12141214 and item .relative_fileloc not in body .files
12151215 and not is_source_discovery_failed (item .relative_fileloc , failed_sources )
12161216 ):
1217- self ._pending_work_acks .append (("priority" , item , True ))
1217+ self ._pending_work_acks .append (("priority" , item ))
12181218 del self ._priority_claims [key ]
12191219
12201220 def _discard_unusable_bundle_source (self , bundle_name : str ) -> None :
@@ -1232,7 +1232,7 @@ def _discard_unusable_bundle_source(self, bundle_name: str) -> None:
12321232 )
12331233 for key , claim in list (self ._priority_claims .items ()):
12341234 if claim .work .bundle_name == bundle_name and claim .attempt_id not in active_attempts :
1235- self ._pending_work_acks .append (("priority" , claim .work , True ))
1235+ self ._pending_work_acks .append (("priority" , claim .work ))
12361236 del self ._priority_claims [key ]
12371237
12381238 def _invalidate_bundle_parse_source (self , bundle_name : str ) -> None :
@@ -1882,11 +1882,11 @@ def handle_parsing_result(
18821882 file .bundle_name ,
18831883 )
18841884 self ._record_failed_publication (file , next_stat )
1885- self ._finish_priority_claims (proc .id , failed = True )
1885+ self ._finish_priority_claims (proc .id )
18861886 return
18871887
18881888 if proc .parsing_result is None and not is_callback_only :
1889- self ._finish_priority_claims (proc .id , failed = True )
1889+ self ._finish_priority_claims (proc .id )
18901890 self ._file_stats [file ] = next_stat
18911891
18921892 def _record_failed_publication (self , file : DagFileInfo , stat : DagFileStat ) -> None :
@@ -1964,15 +1964,14 @@ def _publish_pending_results(self, *, deadline: float | None = None) -> None:
19641964 continue
19651965 self .log .exception ("Failed to publish parsing result for %s" , file )
19661966 self ._record_failed_publication (file , pending .stat )
1967- self ._finish_priority_claims (pending .body .attempt_id , failed = True )
19681967 else :
19691968 self ._file_stats [file ] = pending .stat
1970- self ._finish_priority_claims (pending .body .attempt_id , failed = False )
1969+ self ._finish_priority_claims (pending .body .attempt_id )
19711970 del self ._pending_publications [file ]
19721971
19731972 def _build_rejection_report (self , pending : PendingDagPublication , reason : str ) -> PendingDagPublication :
19741973 """Replace a result that cannot be published with an import error users can see."""
1975- self ._finish_priority_claims (pending .body .attempt_id , failed = True )
1974+ self ._finish_priority_claims (pending .body .attempt_id )
19761975 self ._dispatch_sequence += 1
19771976 if len (reason ) > _MAX_REJECTION_REASON_LENGTH :
19781977 reason = reason [:_MAX_REJECTION_REASON_LENGTH ] + "..."
@@ -2505,7 +2504,7 @@ def _kill_timed_out_processors(self):
25052504 )
25062505 self ._file_stats [file ] = stat
25072506 if not processor .had_callbacks :
2508- self ._finish_priority_claims (processor .id , failed = True )
2507+ self ._finish_priority_claims (processor .id )
25092508
25102509 # Clean up `self._processors` after iterating over it
25112510 for proc in processors_to_remove :
0 commit comments