From 39b4c2689cd1deb209b0e1f9c81137006ec9c45d Mon Sep 17 00:00:00 2001 From: Dejan Zele Pejchev Date: Tue, 19 May 2026 13:59:34 +0200 Subject: [PATCH] Emit JobRunTerminated event when executor observes pod terminal phase Signed-off-by: Dejan Zele Pejchev --- internal/common/ingest/testfixtures/event.go | 10 + internal/executor/domain/pod_metadata.go | 1 + internal/executor/reporter/event.go | 22 + .../executor/service/job_state_reporter.go | 76 +- .../service/job_state_reporter_test.go | 164 +++- internal/executor/util/pod_util.go | 36 + internal/executor/util/pod_util_test.go | 78 ++ .../instructions/instructions.go | 3 +- .../039_add_runs_failed_timestamp.sql | 1 + internal/scheduler/database/models.go | 1 + internal/scheduler/database/query.sql.go | 9 +- internal/scheduler/jobdb/job.go | 1 + internal/scheduler/jobdb/job_run.go | 16 + internal/scheduler/jobdb/job_run_test.go | 3 + internal/scheduler/jobdb/reconciliation.go | 3 +- internal/scheduler/metrics.go | 11 +- internal/scheduler/metrics/state_metrics.go | 14 +- .../scheduler/metrics/state_metrics_test.go | 36 + internal/scheduler/scheduler_test.go | 16 +- internal/scheduleringester/dbops.go | 13 + internal/scheduleringester/dbops_test.go | 9 + internal/scheduleringester/instructions.go | 7 + .../scheduleringester/instructions_test.go | 4 + internal/scheduleringester/schedulerdb.go | 34 +- .../scheduleringester/schedulerdb_test.go | 68 +- .../server/event/conversion/conversions.go | 1 + pkg/armadaevents/events.pb.go | 823 ++++++++++++------ pkg/armadaevents/events.proto | 11 + 28 files changed, 1176 insertions(+), 295 deletions(-) create mode 100644 internal/scheduler/database/migrations/039_add_runs_failed_timestamp.sql diff --git a/internal/common/ingest/testfixtures/event.go b/internal/common/ingest/testfixtures/event.go index d8645fd16b1..f647c9a39e7 100644 --- a/internal/common/ingest/testfixtures/event.go +++ b/internal/common/ingest/testfixtures/event.go @@ -467,6 +467,16 @@ var JobRunPreempted = &armadaevents.EventSequence_Event{ }, } +var JobRunTerminated = &armadaevents.EventSequence_Event{ + Created: testfixtures.BasetimeProto, + Event: &armadaevents.EventSequence_Event_JobRunTerminated{ + JobRunTerminated: &armadaevents.JobRunTerminated{ + JobId: JobId, + RunId: RunId, + }, + }, +} + var JobRunFailed = &armadaevents.EventSequence_Event{ Created: testfixtures.BasetimeProto, Event: &armadaevents.EventSequence_Event_JobRunErrors{ diff --git a/internal/executor/domain/pod_metadata.go b/internal/executor/domain/pod_metadata.go index f08c7649868..84bd9f2a1e6 100644 --- a/internal/executor/domain/pod_metadata.go +++ b/internal/executor/domain/pod_metadata.go @@ -14,4 +14,5 @@ const ( IngressReported = "ingress_reported" MarkedForDeletion = "deletion_requested" JobPreemptedAnnotation = "reported_preempted" + JobRunTerminatedReported = "reported_terminated" ) diff --git a/internal/executor/reporter/event.go b/internal/executor/reporter/event.go index 1119d781310..d60fb5c9f73 100644 --- a/internal/executor/reporter/event.go +++ b/internal/executor/reporter/event.go @@ -190,6 +190,28 @@ func CreateJobIngressInfoEvent(pod *v1.Pod, clusterId string, associatedServices return sequence, nil } +func CreateJobRunTerminatedEvent(pod *v1.Pod) (*armadaevents.EventSequence, error) { + sequence := createEmptySequence(pod) + jobId, runId, err := extractIds(pod) + if err != nil { + return nil, err + } + finishedAt, err := types.TimestampProto(util.LatestContainerFinishedAt(pod)) + if err != nil { + return nil, err + } + sequence.Events = append(sequence.Events, &armadaevents.EventSequence_Event{ + Created: finishedAt, + Event: &armadaevents.EventSequence_Event_JobRunTerminated{ + JobRunTerminated: &armadaevents.JobRunTerminated{ + JobId: jobId, + RunId: runId, + }, + }, + }) + return sequence, nil +} + func CreateSimpleJobPreemptedEvent(pod *v1.Pod) (*armadaevents.EventSequence, error) { sequence := createEmptySequence(pod) preemptedJobId, preemptedRunId, err := extractIds(pod) diff --git a/internal/executor/service/job_state_reporter.go b/internal/executor/service/job_state_reporter.go index 2bfd8c87155..e894603320e 100644 --- a/internal/executor/service/job_state_reporter.go +++ b/internal/executor/service/job_state_reporter.go @@ -51,7 +51,10 @@ func (stateReporter *JobStateReporter) podEventHandler() cache.ResourceEventHand log.Errorf("Failed to process pod event due to it being an unexpected type. Failed to process %+v", obj) return } - go stateReporter.reportCurrentStatus(pod) + go func() { + stateReporter.reportCurrentStatus(pod) + stateReporter.attemptToReportJobRunTerminatedEvent(pod) + }() }, UpdateFunc: func(oldObj, newObj interface{}) { oldPod, ok := oldObj.(*v1.Pod) @@ -64,7 +67,10 @@ func (stateReporter *JobStateReporter) podEventHandler() cache.ResourceEventHand log.Errorf("Failed to process pod event due to it being an unexpected type. Failed to process %+v", newObj) return } - go stateReporter.reportStatusUpdate(oldPod, newPod) + go func() { + stateReporter.reportStatusUpdate(oldPod, newPod) + stateReporter.attemptToReportJobRunTerminatedEvent(newPod) + }() }, } } @@ -142,6 +148,57 @@ func (stateReporter *JobStateReporter) reportCurrentStatus(pod *v1.Pod) { } } +// attemptToReportJobRunTerminatedEvent emits JobRunTerminated once per pod when the +// pod reaches a terminal phase (PodSucceeded or PodFailed). Called from two +// places: +// +// 1. Inline, from the pod-event handler, so we observe the terminal-phase +// transition the moment the informer reports it. This is required for +// pods armada itself deleted (cancel/preempt): once the containers exit, +// the Kubernetes API removes the pod object within a couple of seconds, +// and the executor's informer cache loses it. The reconciliation interval +// (default 15s) is wider than that window, so a reconcile-only design +// would race against pod deletion and never see the terminal phase - +// terminated_timestamp would stay NULL. +// +// 2. From the periodic reconciliation pass in ReportMissingJobEvents, as a +// safety net for inline emissions that did not happen (executor restart +// between phase transition and emission, informer watch hiccup). +// +// Idempotent: a successful emission annotates the pod with reported_terminated, +// and subsequent calls (inline or reconciliation) skip annotated pods. +func (stateReporter *JobStateReporter) attemptToReportJobRunTerminatedEvent(pod *v1.Pod) { + if !util.IsManagedPod(pod) { + return + } + if !util.IsInTerminalState(pod) { + return + } + if util.HasJobRunTerminatedBeenReported(pod) { + return + } + event, err := reporter.CreateJobRunTerminatedEvent(pod) + if err != nil { + log.Errorf("Failed to build JobRunTerminated event for pod %s: %v", pod.Name, err) + return + } + stateReporter.eventReporter.QueueEvent(reporter.EventMessage{Event: event, JobRunId: util.ExtractJobRunId(pod)}, func(err error) { + if err != nil { + log.Errorf("Failed to report JobRunTerminated for pod %s: %v", pod.Name, err) + return + } + if err := stateReporter.addAnnotationToMarkJobRunTerminatedReported(pod); err != nil { + log.Errorf("Failed to annotate pod %s as JobRunTerminated-reported: %v", pod.Name, err) + } + }) +} + +func (stateReporter *JobStateReporter) addAnnotationToMarkJobRunTerminatedReported(pod *v1.Pod) error { + return stateReporter.clusterContext.AddAnnotation(pod, map[string]string{ + domain2.JobRunTerminatedReported: time.Now().String(), + }) +} + func (stateReporter *JobStateReporter) addAnnotationToMarkStateReported(pod *v1.Pod) error { annotations := make(map[string]string) annotationName := string(pod.Status.Phase) @@ -183,6 +240,21 @@ func (stateReporter *JobStateReporter) ReportMissingJobEvents() { stateReporter.attemptToReportIngressInfoEvent(pod) } } + + // The pod-event handler normally emits JobRunTerminated inline the moment the + // informer reports terminal phase. This loop catches pods where that inline + // emission did not happen: the executor restarted after the pod went terminal + // but before the handler ran, or the informer dropped the watch event. + // Idempotent via the reported_terminated annotation set on successful emission. + podsWithTerminatedNotReported := util.FilterPods(allBatchPods, func(pod *v1.Pod) bool { + return util.IsInTerminalState(pod) && !util.HasJobRunTerminatedBeenReported(pod) + }) + + for _, pod := range podsWithTerminatedNotReported { + if !stateReporter.eventReporter.HasPendingEvents(pod) { + stateReporter.attemptToReportJobRunTerminatedEvent(pod) + } + } } func (stateReporter *JobStateReporter) attemptToReportIngressInfoEvent(pod *v1.Pod) { diff --git a/internal/executor/service/job_state_reporter_test.go b/internal/executor/service/job_state_reporter_test.go index 8191ae6afd1..54cc3bc10bf 100644 --- a/internal/executor/service/job_state_reporter_test.go +++ b/internal/executor/service/job_state_reporter_test.go @@ -73,13 +73,13 @@ func TestJobStateReporter_HandlesPodAddEvents(t *testing.T) { if test.expectEvent { assertExpectedEvents(t, test.pod, eventReporter.ReceivedEvents, test.expectedType) } else { - assert.Len(t, eventReporter.ReceivedEvents, 0) + assert.Len(t, filterOutJobRunTerminated(eventReporter.ReceivedEvents), 0) } if test.expectAnnotation { assertExpectedAnnotations(t, test.pod, fakeClusterContext) } else { - assert.Len(t, fakeClusterContext.AnnotationsAdded, 0) + assert.Len(t, annotationsExcludingTerminated(fakeClusterContext.AnnotationsAdded), 0) } } } @@ -120,8 +120,8 @@ func TestJobStateReporter_HandlesPodUpdateEvents(t *testing.T) { assertExpectedEvents(t, before, eventReporter.ReceivedEvents, test.expectedType) assertExpectedAnnotations(t, after, fakeClusterContext) } else { - assert.Len(t, eventReporter.ReceivedEvents, 0) - assert.Len(t, fakeClusterContext.AnnotationsAdded, 0) + assert.Len(t, filterOutJobRunTerminated(eventReporter.ReceivedEvents), 0) + assert.Len(t, annotationsExcludingTerminated(fakeClusterContext.AnnotationsAdded), 0) } } } @@ -135,7 +135,7 @@ func TestJobStateReporter_HandlesPodUpdateEvents_IgnoreUnmanagedPods(t *testing. fakeClusterContext.SimulateUpdateAddEvent(before, after) time.Sleep(time.Millisecond * 100) // Give time for async routine to process message - assert.Len(t, eventReporter.ReceivedEvents, 0) + assert.Len(t, filterOutJobRunTerminated(eventReporter.ReceivedEvents), 0) } func TestJobStateReporter_HandlesFailedPod_WithRetryableError(t *testing.T) { @@ -148,7 +148,7 @@ func TestJobStateReporter_HandlesFailedPod_WithRetryableError(t *testing.T) { jobStateReporter.podIssueHandler = &stubIssueHandler{detectAndRegisterFailedPodIssueResult: true, detectAndRegisterFailedPodIssueError: nil} fakeClusterContext.SimulateUpdateAddEvent(before, after) time.Sleep(time.Millisecond * 100) // Give time for async routine to process message - assert.Len(t, eventReporter.ReceivedEvents, 0) + assert.Len(t, filterOutJobRunTerminated(eventReporter.ReceivedEvents), 0) // Does not send event if issue handler already knows of issue for run jobStateReporter.podIssueHandler = &stubIssueHandler{ @@ -158,13 +158,13 @@ func TestJobStateReporter_HandlesFailedPod_WithRetryableError(t *testing.T) { } fakeClusterContext.SimulateUpdateAddEvent(before, after) time.Sleep(time.Millisecond * 100) // Give time for async routine to process message - assert.Len(t, eventReporter.ReceivedEvents, 0) + assert.Len(t, filterOutJobRunTerminated(eventReporter.ReceivedEvents), 0) // Does send event if issue handler errors jobStateReporter.podIssueHandler = &stubIssueHandler{detectAndRegisterFailedPodIssueResult: false, detectAndRegisterFailedPodIssueError: fmt.Errorf("error")} fakeClusterContext.SimulateUpdateAddEvent(before, after) time.Sleep(time.Millisecond * 100) // Give time for async routine to process message - assert.Len(t, eventReporter.ReceivedEvents, 1) + assert.Len(t, filterOutJobRunTerminated(eventReporter.ReceivedEvents), 1) assertExpectedEvents(t, before, eventReporter.ReceivedEvents, reflect.TypeOf(&armadaevents.EventSequence_Event_JobRunErrors{})) } @@ -235,7 +235,7 @@ func TestJobStateReporter_PodFailed_EmitsJobFailedEventWhenClassifierMatches(t * fakeClusterContext.SimulatePodAddEvent(pod) time.Sleep(time.Millisecond * 100) - assert.Len(t, eventReporter.ReceivedEvents, 1) + assert.Len(t, filterOutJobRunTerminated(eventReporter.ReceivedEvents), 1) } func TestJobStateReporter_PodFailed_SuppressesEmissionWhenRetryableIssueRegistered(t *testing.T) { @@ -250,7 +250,7 @@ func TestJobStateReporter_PodFailed_SuppressesEmissionWhenRetryableIssueRegister fakeClusterContext.SimulatePodAddEvent(pod) time.Sleep(time.Millisecond * 100) - assert.Len(t, eventReporter.ReceivedEvents, 0, "retryable issue path emits ReturnLease, not JobFailed") + assert.Len(t, filterOutJobRunTerminated(eventReporter.ReceivedEvents), 0, "retryable issue path emits ReturnLease, not JobFailed") } func TestJobStateReporter_PodFailed_SuppressesEmissionWhenIssueAlreadyExists(t *testing.T) { @@ -265,7 +265,7 @@ func TestJobStateReporter_PodFailed_SuppressesEmissionWhenIssueAlreadyExists(t * fakeClusterContext.SimulatePodAddEvent(pod) time.Sleep(time.Millisecond * 100) - assert.Len(t, eventReporter.ReceivedEvents, 0, "run already owned by issue handler - no direct emission from this path") + assert.Len(t, filterOutJobRunTerminated(eventReporter.ReceivedEvents), 0, "run already owned by issue handler - no direct emission from this path") } func TestJobStateReporter_PodFailed_DropsEventWhenReporterErrors(t *testing.T) { @@ -278,7 +278,7 @@ func TestJobStateReporter_PodFailed_DropsEventWhenReporterErrors(t *testing.T) { fakeClusterContext.SimulatePodAddEvent(pod) time.Sleep(time.Millisecond * 100) - assert.Len(t, eventReporter.ReceivedEvents, 0, "event is dropped when reporter errors - counter increment is gated behind successful emission") + assert.Len(t, filterOutJobRunTerminated(eventReporter.ReceivedEvents), 0, "event is dropped when reporter errors - counter increment is gated behind successful emission") } func TestJobStateReporter_PodFailed_EmitsEventWithEmptyCategoryWhenClassifierIsNil(t *testing.T) { @@ -289,29 +289,65 @@ func TestJobStateReporter_PodFailed_EmitsEventWithEmptyCategoryWhenClassifierIsN fakeClusterContext.SimulatePodAddEvent(pod) time.Sleep(time.Millisecond * 100) - assert.Len(t, eventReporter.ReceivedEvents, 1, "event still emitted with empty category/subcategory when classification is disabled") + assert.Len(t, filterOutJobRunTerminated(eventReporter.ReceivedEvents), 1, "event still emitted with empty category/subcategory when classification is disabled") } func assertExpectedEvents(t *testing.T, pod *v1.Pod, messages []reporter.EventMessage, expectedType reflect.Type) { - assert.Len(t, messages, 1) - assert.Len(t, messages[0].Event.Events, 1) + // Terminal-phase pods produce both the phase event and JobRunTerminated. Phase-event + // assertions filter out JobRunTerminated; tests that target JobRunTerminated keep it. + target := messages + if expectedType != reflect.TypeOf(&armadaevents.EventSequence_Event_JobRunTerminated{}) { + target = filterOutJobRunTerminated(messages) + } + assert.Len(t, target, 1) + assert.Len(t, target[0].Event.Events, 1) - assert.Equal(t, util.ExtractJobRunId(pod), messages[0].JobRunId) - assert.Equal(t, util.ExtractQueue(pod), messages[0].Event.Queue) - assert.Equal(t, util.ExtractJobSet(pod), messages[0].Event.JobSetName) + assert.Equal(t, util.ExtractJobRunId(pod), target[0].JobRunId) + assert.Equal(t, util.ExtractQueue(pod), target[0].Event.Queue) + assert.Equal(t, util.ExtractJobSet(pod), target[0].Event.JobSetName) - event := messages[0].Event.Events[0] + event := target[0].Event.Events[0] resultType := reflect.TypeOf(event.Event) assert.Equal(t, expectedType, resultType) } func assertExpectedAnnotations(t *testing.T, pod *v1.Pod, clusterContext *fakecontext.SyncFakeClusterContext) { - jobAnnotations := clusterContext.AnnotationsAdded[util.ExtractJobId(pod)] + jobAnnotations := annotationsExcludingTerminated(clusterContext.AnnotationsAdded)[util.ExtractJobId(pod)] assert.Len(t, jobAnnotations, 1) _, exists := jobAnnotations[string(pod.Status.Phase)] assert.True(t, exists) } +func filterOutJobRunTerminated(messages []reporter.EventMessage) []reporter.EventMessage { + out := make([]reporter.EventMessage, 0, len(messages)) + for _, m := range messages { + if len(m.Event.Events) == 1 { + if _, isTerminated := m.Event.Events[0].Event.(*armadaevents.EventSequence_Event_JobRunTerminated); isTerminated { + continue + } + } + out = append(out, m) + } + return out +} + +func annotationsExcludingTerminated(added map[string]map[string]string) map[string]map[string]string { + out := make(map[string]map[string]string, len(added)) + for jobId, anns := range added { + filtered := make(map[string]string, len(anns)) + for k, v := range anns { + if k == domain.JobRunTerminatedReported { + continue + } + filtered[k] = v + } + if len(filtered) > 0 { + out[jobId] = filtered + } + } + return out +} + type stubIssueHandler struct { runIdsWithIssues map[string]bool detectAndRegisterFailedPodIssueResult bool @@ -332,3 +368,91 @@ func copyWithUpdatedPhase(pod *v1.Pod, newPhase v1.PodPhase) *v1.Pod { result.Status.Phase = newPhase return result } + +func TestJobStateReporter_ReportJobRunTerminatedIfNeeded(t *testing.T) { + terminalManaged := makeTestPod(v1.PodStatus{Phase: v1.PodSucceeded}) + unmanaged := &v1.Pod{Status: v1.PodStatus{Phase: v1.PodFailed}} + nonTerminal := makeTestPod(v1.PodStatus{Phase: v1.PodRunning}) + alreadyReported := makeTestPod(v1.PodStatus{Phase: v1.PodSucceeded}) + alreadyReported.Annotations[domain.JobRunTerminatedReported] = time.Now().String() + markedForDeletion := makeTestPod(v1.PodStatus{Phase: v1.PodFailed}) + markedForDeletion.Annotations[domain.MarkedForDeletion] = time.Now().String() + + tests := map[string]struct { + pod *v1.Pod + addToCluster bool + expectEmit bool + expectAnnotation bool + }{ + "emits for terminal managed pod": { + pod: terminalManaged, addToCluster: true, expectEmit: true, expectAnnotation: true, + }, + "ignores unmanaged pod": { + pod: unmanaged, + }, + "ignores non-terminal pod": { + pod: nonTerminal, addToCluster: true, + }, + "no-op when already annotated": { + pod: alreadyReported, addToCluster: true, + }, + // The motivating case: armada-initiated cancel/preempt sets MarkedForDeletion. reportCurrentStatus + // skips those, but JobRunTerminated must still fire so terminated_timestamp can be filled. + "fires for marked-for-deletion pod": { + pod: markedForDeletion, addToCluster: true, expectEmit: true, expectAnnotation: true, + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + jobStateReporter, _, eventReporter, fakeClusterContext := setUpJobStateReporterTest(t) + if tc.addToCluster { + addPod(t, fakeClusterContext, tc.pod) + } + + jobStateReporter.attemptToReportJobRunTerminatedEvent(tc.pod) + + emitted := hasJobRunTerminated(eventReporter.ReceivedEvents) + assert.Equal(t, tc.expectEmit, emitted) + _, annotated := fakeClusterContext.AnnotationsAdded[util.ExtractJobId(tc.pod)][domain.JobRunTerminatedReported] + assert.Equal(t, tc.expectAnnotation, annotated) + }) + } +} + +func TestJobStateReporter_ReportMissingJobEvents_JobRunTerminatedReconciliation(t *testing.T) { + fresh := makeTestPod(v1.PodStatus{Phase: v1.PodFailed}) + alreadyReported := makeTestPod(v1.PodStatus{Phase: v1.PodSucceeded}) + alreadyReported.Annotations[domain.JobRunTerminatedReported] = time.Now().String() + alreadyReported.Annotations[string(v1.PodSucceeded)] = time.Now().String() // suppress phase reconcile too + + tests := map[string]struct { + pod *v1.Pod + expectEmit bool + }{ + // Pod reached terminal phase but JobRunTerminated was never emitted (e.g. informer watch hiccup). + // Reconciliation must catch it. + "reconciles missed event": {pod: fresh, expectEmit: true}, + "does not re-emit once annotated": {pod: alreadyReported, expectEmit: false}, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + jobStateReporter, _, eventReporter, fakeClusterContext := setUpJobStateReporterTest(t) + addPod(t, fakeClusterContext, tc.pod) + + jobStateReporter.ReportMissingJobEvents() + + assert.Equal(t, tc.expectEmit, hasJobRunTerminated(eventReporter.ReceivedEvents)) + }) + } +} + +func hasJobRunTerminated(messages []reporter.EventMessage) bool { + for _, m := range messages { + for _, e := range m.Event.Events { + if _, ok := e.Event.(*armadaevents.EventSequence_Event_JobRunTerminated); ok { + return true + } + } + } + return false +} diff --git a/internal/executor/util/pod_util.go b/internal/executor/util/pod_util.go index cc719728c7b..eaef2f77079 100644 --- a/internal/executor/util/pod_util.go +++ b/internal/executor/util/pod_util.go @@ -262,6 +262,42 @@ func IsReportedPreempted(pod *v1.Pod) bool { return exists } +func HasJobRunTerminatedBeenReported(pod *v1.Pod) bool { + _, exists := pod.Annotations[domain.JobRunTerminatedReported] + return exists +} + +// LatestContainerFinishedAt returns the latest FinishedAt across all container terminated states +// (init and main), which is when the kubelet observed the last container exit. Regular init +// containers always finish before main containers, so the max naturally lands on a main container +// when one ran; for init-container failures (main never started) it lands on the failing init +// container. Sidecar init containers (restartPolicy: Always) can outlive main containers, but +// the max still picks the correct latest exit. +// +// Falls back to time.Now() when the pod reached terminal phase without any container reporting a +// terminated state. The case this covers is the NodeController force-marking pods as Failed +// after node loss: the kubelet on the lost node never reported container exits. The actual stop +// time is unknowable here; time.Now() is the executor's observation moment, so the returned +// value is bounded above by reconciliation lag and never earlier than the real stop time. +// Callers that need stronger accuracy guarantees (e.g., a retry-gate deciding when to safely +// requeue) should combine this with a deadline. +func LatestContainerFinishedAt(pod *v1.Pod) time.Time { + var latest time.Time + consider := func(statuses []v1.ContainerStatus) { + for _, cs := range statuses { + if cs.State.Terminated != nil && cs.State.Terminated.FinishedAt.Time.After(latest) { + latest = cs.State.Terminated.FinishedAt.Time + } + } + } + consider(pod.Status.ContainerStatuses) + consider(pod.Status.InitContainerStatuses) + if latest.IsZero() { + return time.Now() + } + return latest +} + func IsPodFinishedAndReported(pod *v1.Pod) bool { if !IsInTerminalState(pod) || !HasCurrentStateBeenReported(pod) { diff --git a/internal/executor/util/pod_util_test.go b/internal/executor/util/pod_util_test.go index 5c40f808867..37e6a0f8e8f 100644 --- a/internal/executor/util/pod_util_test.go +++ b/internal/executor/util/pod_util_test.go @@ -799,3 +799,81 @@ func TestGroupByQueue_EmptyList(t *testing.T) { result := GroupByQueue([]*v1.Pod{}) assert.Len(t, result, 0) } + +func TestHasJobRunTerminatedBeenReported(t *testing.T) { + tests := map[string]struct { + annotations map[string]string + want bool + }{ + "annotation present": { + annotations: map[string]string{domain.JobRunTerminatedReported: time.Now().String()}, + want: true, + }, + "annotation missing": { + annotations: map[string]string{domain.IngressReported: time.Now().String()}, + want: false, + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{Annotations: tc.annotations}} + assert.Equal(t, tc.want, HasJobRunTerminatedBeenReported(pod)) + }) + } +} + +func TestLatestContainerFinishedAt(t *testing.T) { + earlier := metav1.NewTime(time.Date(2026, 5, 1, 10, 0, 0, 0, time.UTC)) + later := metav1.NewTime(time.Date(2026, 5, 1, 10, 0, 30, 0, time.UTC)) + terminated := func(at metav1.Time) v1.ContainerStatus { + return v1.ContainerStatus{State: v1.ContainerState{Terminated: &v1.ContainerStateTerminated{FinishedAt: at}}} + } + tests := map[string]struct { + containerStatuses []v1.ContainerStatus + initContainerStatuses []v1.ContainerStatus + want time.Time + wantFallback bool + }{ + "picks latest across multiple terminated containers": { + containerStatuses: []v1.ContainerStatus{terminated(earlier), terminated(later)}, + want: later.Time, + }, + "ignores running and waiting states": { + containerStatuses: []v1.ContainerStatus{ + {State: v1.ContainerState{Running: &v1.ContainerStateRunning{}}}, + terminated(earlier), + {State: v1.ContainerState{Waiting: &v1.ContainerStateWaiting{}}}, + }, + want: earlier.Time, + }, + "uses init container finish time when main never started": { + initContainerStatuses: []v1.ContainerStatus{terminated(earlier)}, + want: earlier.Time, + }, + "prefers main container over init container when both terminated": { + containerStatuses: []v1.ContainerStatus{terminated(later)}, + initContainerStatuses: []v1.ContainerStatus{terminated(earlier)}, + want: later.Time, + }, + "falls back to time.Now() when no container reported terminated": { + containerStatuses: []v1.ContainerStatus{ + {State: v1.ContainerState{Running: &v1.ContainerStateRunning{}}}, + }, + wantFallback: true, + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + pod := &v1.Pod{Status: v1.PodStatus{ + ContainerStatuses: tc.containerStatuses, + InitContainerStatuses: tc.initContainerStatuses, + }} + got := LatestContainerFinishedAt(pod) + if tc.wantFallback { + assert.WithinDuration(t, time.Now(), got, 5*time.Second) + } else { + assert.Equal(t, tc.want, got) + } + }) + } +} diff --git a/internal/lookoutingester/instructions/instructions.go b/internal/lookoutingester/instructions/instructions.go index fd90fa9720d..135f4449839 100644 --- a/internal/lookoutingester/instructions/instructions.go +++ b/internal/lookoutingester/instructions/instructions.go @@ -127,7 +127,8 @@ func (c *InstructionConverter) convertSequence( *armadaevents.EventSequence_Event_ResourceUtilisation, *armadaevents.EventSequence_Event_PartitionMarker, *armadaevents.EventSequence_Event_JobValidated, - *armadaevents.EventSequence_Event_JobRunPreemptionRequested: + *armadaevents.EventSequence_Event_JobRunPreemptionRequested, + *armadaevents.EventSequence_Event_JobRunTerminated: log.Debugf("Ignoring event type %T", event.GetEvent()) default: log.Warnf("Ignoring unknown event type %T", event.GetEvent()) diff --git a/internal/scheduler/database/migrations/039_add_runs_failed_timestamp.sql b/internal/scheduler/database/migrations/039_add_runs_failed_timestamp.sql new file mode 100644 index 00000000000..d514f9ced6f --- /dev/null +++ b/internal/scheduler/database/migrations/039_add_runs_failed_timestamp.sql @@ -0,0 +1 @@ +ALTER TABLE runs ADD COLUMN IF NOT EXISTS failed_timestamp timestamptz NULL; diff --git a/internal/scheduler/database/models.go b/internal/scheduler/database/models.go index a8cb88ae8a2..c6e4a01d75d 100644 --- a/internal/scheduler/database/models.go +++ b/internal/scheduler/database/models.go @@ -99,4 +99,5 @@ type Run struct { Pool string `db:"pool"` Terminated bool `db:"terminated"` PreemptReason *string `db:"preempt_reason"` + FailedTimestamp *time.Time `db:"failed_timestamp"` } diff --git a/internal/scheduler/database/query.sql.go b/internal/scheduler/database/query.sql.go index 7e261b53b72..16e3d132663 100644 --- a/internal/scheduler/database/query.sql.go +++ b/internal/scheduler/database/query.sql.go @@ -481,7 +481,7 @@ func (q *Queries) SelectInitialJobs(ctx context.Context, arg SelectInitialJobsPa } const selectInitialRuns = `-- name: SelectInitialRuns :many -SELECT run_id, job_id, created, job_set, executor, node, cancelled, running, succeeded, failed, returned, run_attempted, serial, last_modified, leased_timestamp, pending_timestamp, running_timestamp, terminated_timestamp, scheduled_at_priority, preempted, pending, preempted_timestamp, pod_requirements_overlay, preempt_requested, queue, pool, terminated, preempt_reason FROM runs WHERE serial > $1 AND job_id = ANY($3::text[]) ORDER BY serial LIMIT $2 +SELECT run_id, job_id, created, job_set, executor, node, cancelled, running, succeeded, failed, returned, run_attempted, serial, last_modified, leased_timestamp, pending_timestamp, running_timestamp, terminated_timestamp, scheduled_at_priority, preempted, pending, preempted_timestamp, pod_requirements_overlay, preempt_requested, queue, pool, terminated, preempt_reason, failed_timestamp FROM runs WHERE serial > $1 AND job_id = ANY($3::text[]) ORDER BY serial LIMIT $2 ` type SelectInitialRunsParams struct { @@ -528,6 +528,7 @@ func (q *Queries) SelectInitialRuns(ctx context.Context, arg SelectInitialRunsPa &i.Pool, &i.Terminated, &i.PreemptReason, + &i.FailedTimestamp, ); err != nil { return nil, err } @@ -831,7 +832,7 @@ func (q *Queries) SelectNewJobs(ctx context.Context, arg SelectNewJobsParams) ([ } const selectNewRuns = `-- name: SelectNewRuns :many -SELECT run_id, job_id, created, job_set, executor, node, cancelled, running, succeeded, failed, returned, run_attempted, serial, last_modified, leased_timestamp, pending_timestamp, running_timestamp, terminated_timestamp, scheduled_at_priority, preempted, pending, preempted_timestamp, pod_requirements_overlay, preempt_requested, queue, pool, terminated, preempt_reason FROM runs WHERE serial > $1 ORDER BY serial LIMIT $2 +SELECT run_id, job_id, created, job_set, executor, node, cancelled, running, succeeded, failed, returned, run_attempted, serial, last_modified, leased_timestamp, pending_timestamp, running_timestamp, terminated_timestamp, scheduled_at_priority, preempted, pending, preempted_timestamp, pod_requirements_overlay, preempt_requested, queue, pool, terminated, preempt_reason, failed_timestamp FROM runs WHERE serial > $1 ORDER BY serial LIMIT $2 ` type SelectNewRunsParams struct { @@ -877,6 +878,7 @@ func (q *Queries) SelectNewRuns(ctx context.Context, arg SelectNewRunsParams) ([ &i.Pool, &i.Terminated, &i.PreemptReason, + &i.FailedTimestamp, ); err != nil { return nil, err } @@ -889,7 +891,7 @@ func (q *Queries) SelectNewRuns(ctx context.Context, arg SelectNewRunsParams) ([ } const selectNewRunsForJobs = `-- name: SelectNewRunsForJobs :many -SELECT run_id, job_id, created, job_set, executor, node, cancelled, running, succeeded, failed, returned, run_attempted, serial, last_modified, leased_timestamp, pending_timestamp, running_timestamp, terminated_timestamp, scheduled_at_priority, preempted, pending, preempted_timestamp, pod_requirements_overlay, preempt_requested, queue, pool, terminated, preempt_reason FROM runs WHERE serial > $1 AND job_id = ANY($2::text[]) ORDER BY serial +SELECT run_id, job_id, created, job_set, executor, node, cancelled, running, succeeded, failed, returned, run_attempted, serial, last_modified, leased_timestamp, pending_timestamp, running_timestamp, terminated_timestamp, scheduled_at_priority, preempted, pending, preempted_timestamp, pod_requirements_overlay, preempt_requested, queue, pool, terminated, preempt_reason, failed_timestamp FROM runs WHERE serial > $1 AND job_id = ANY($2::text[]) ORDER BY serial ` type SelectNewRunsForJobsParams struct { @@ -935,6 +937,7 @@ func (q *Queries) SelectNewRunsForJobs(ctx context.Context, arg SelectNewRunsFor &i.Pool, &i.Terminated, &i.PreemptReason, + &i.FailedTimestamp, ); err != nil { return nil, err } diff --git a/internal/scheduler/jobdb/job.go b/internal/scheduler/jobdb/job.go index 9145b433613..3ac141b3f1d 100644 --- a/internal/scheduler/jobdb/job.go +++ b/internal/scheduler/jobdb/job.go @@ -794,6 +794,7 @@ func (job *Job) WithNewRun(executor, nodeId, nodeName, pool string, scheduledAtP nil, nil, nil, + nil, false, false, )) diff --git a/internal/scheduler/jobdb/job_run.go b/internal/scheduler/jobdb/job_run.go index c015b796e39..c4854252d6c 100644 --- a/internal/scheduler/jobdb/job_run.go +++ b/internal/scheduler/jobdb/job_run.go @@ -60,6 +60,10 @@ type JobRun struct { cancelled bool // The time at which the job was reported as cancelled, failed or succeeded by the executor. terminatedTime *time.Time + // The time at which Armada concluded the run failed. + // Set by MarkRunsFailed at JobRunErrors event time. Distinct from terminatedTime, + // which is the kubelet-observed pod stop time (NULL when no observation). + failedTime *time.Time // True if the job has been returned by the executor. returned bool // True if the job has been returned and the job was given a chance to run. @@ -236,6 +240,7 @@ func (jobDb *JobDb) CreateRun( runningTime *time.Time, preemptedTime *time.Time, terminatedTime *time.Time, + failedTime *time.Time, returned bool, runAttempted bool, ) *JobRun { @@ -262,6 +267,7 @@ func (jobDb *JobDb) CreateRun( runningTime: runningTime, preemptedTime: preemptedTime, terminatedTime: terminatedTime, + failedTime: failedTime, returned: returned, runAttempted: runAttempted, } @@ -364,6 +370,12 @@ func (run *JobRun) WithTerminatedTime(terminatedTime *time.Time) *JobRun { return run } +func (run *JobRun) WithFailedTime(failedTime *time.Time) *JobRun { + run = run.DeepCopy() + run.failedTime = failedTime + return run +} + func (run *JobRun) Pending() bool { return run.pending } @@ -487,6 +499,10 @@ func (run *JobRun) TerminatedTime() *time.Time { return run.terminatedTime } +func (run *JobRun) FailedTime() *time.Time { + return run.failedTime +} + // RunAttempted Returns true if the executor has attempted to run the job. func (run *JobRun) RunAttempted() bool { return run.runAttempted diff --git a/internal/scheduler/jobdb/job_run_test.go b/internal/scheduler/jobdb/job_run_test.go index 3307341e968..da7ca707d91 100644 --- a/internal/scheduler/jobdb/job_run_test.go +++ b/internal/scheduler/jobdb/job_run_test.go @@ -70,6 +70,7 @@ var baseJobRun = jobDb.CreateRun( nil, nil, nil, + nil, false, false, ) @@ -148,6 +149,7 @@ func TestDeepCopy(t *testing.T) { nil, nil, nil, + nil, true, true, ) @@ -174,6 +176,7 @@ func TestDeepCopy(t *testing.T) { nil, nil, nil, + nil, true, true, ) diff --git a/internal/scheduler/jobdb/reconciliation.go b/internal/scheduler/jobdb/reconciliation.go index 4ed69510a02..4b7bbb8f8ff 100644 --- a/internal/scheduler/jobdb/reconciliation.go +++ b/internal/scheduler/jobdb/reconciliation.go @@ -266,7 +266,7 @@ func (jobDb *JobDb) reconcileRunDifferences(jobRun *JobRun, jobRepoRun *database rst.Cancelled = true } if jobRepoRun.Failed && !jobRun.Failed() { - jobRun = jobRun.WithFailed(true).WithRunning(false).WithTerminatedTime(jobRepoRun.TerminatedTimestamp) + jobRun = jobRun.WithFailed(true).WithRunning(false).WithTerminatedTime(jobRepoRun.TerminatedTimestamp).WithFailedTime(jobRepoRun.FailedTimestamp) rst.Failed = true } if jobRepoRun.Succeeded && !jobRun.Succeeded() { @@ -380,6 +380,7 @@ func (jobDb *JobDb) schedulerRunFromDatabaseRun(dbRun *database.Run) *JobRun { dbRun.RunningTimestamp, dbRun.PreemptedTimestamp, dbRun.TerminatedTimestamp, + dbRun.FailedTimestamp, dbRun.Returned, dbRun.RunAttempted, ) diff --git a/internal/scheduler/metrics.go b/internal/scheduler/metrics.go index d4c281781f0..6cfe3de48c8 100644 --- a/internal/scheduler/metrics.go +++ b/internal/scheduler/metrics.go @@ -224,7 +224,16 @@ func (c *MetricsCollector) updateQueueMetrics(ctx *armadacontext.Context) ([]pro recorder = qs.queuedJobRecorder queuedTime := time.Unix(0, job.Created()) if job.HasRuns() { - terminationTimeOfLatestRun := job.LatestRun().TerminatedTime() + latestRun := job.LatestRun() + // TerminatedTime is NULL between MarkRunsFailed (event-emit time) and + // MarkRunsTerminated (kubelet-observed time via JobRunTerminated, up to + // one reconciliation interval later). Fall back to FailedTime in that + // window so a retried job's queued-time-in-state is not inflated back + // to job.Created(). + terminationTimeOfLatestRun := latestRun.TerminatedTime() + if terminationTimeOfLatestRun == nil { + terminationTimeOfLatestRun = latestRun.FailedTime() + } if terminationTimeOfLatestRun != nil && terminationTimeOfLatestRun.After(queuedTime) { queuedTime = *terminationTimeOfLatestRun } diff --git a/internal/scheduler/metrics/state_metrics.go b/internal/scheduler/metrics/state_metrics.go index 774b7a2cd89..0e3b563a845 100644 --- a/internal/scheduler/metrics/state_metrics.go +++ b/internal/scheduler/metrics/state_metrics.go @@ -230,7 +230,19 @@ func (m *jobStateMetrics) ReportStateTransitions( } } if jst.Failed { - duration, priorState := stateDuration(job, run, run.TerminatedTime()) + // FailedTime is the event-emit time of the failure (decision time for the + // preempt path, kubelet-observed time for organic failures). TerminatedTime + // shifts to kubelet-observed-after-grace for preempted runs once + // JobRunTerminated lands, which would distort this histogram. + // + // Fall back to TerminatedTime for pre-migration runs (failed_timestamp is + // NULL because column 039 did not exist yet); they still carry the failure + // time in terminated_timestamp under the pre-PR semantics. + failedAt := run.FailedTime() + if failedAt == nil { + failedAt = run.TerminatedTime() + } + duration, priorState := stateDuration(job, run, failedAt) m.updateStateDuration(job, failed, priorState, duration) m.completedRunDurations.WithLabelValues(job.Queue(), run.Pool()).Observe(duration) jobRunError := jobRunErrorsByRunId[run.Id()] diff --git a/internal/scheduler/metrics/state_metrics_test.go b/internal/scheduler/metrics/state_metrics_test.go index 8bf836ca717..13505ff25c2 100644 --- a/internal/scheduler/metrics/state_metrics_test.go +++ b/internal/scheduler/metrics/state_metrics_test.go @@ -185,6 +185,42 @@ func TestReportJobStateTransitions(t *testing.T) { }, }, "Failed": { + trackedResourceNames: []v1.ResourceName{"cpu"}, + jsts: []jobdb.JobStateTransitions{ + { + Job: baseJob. + WithUpdatedRun( + baseRun. + WithLeasedTime(baseTimePlusSeconds(60)). + WithPendingTime(baseTimePlusSeconds(62)). + WithRunningTime(baseTimePlusSeconds(72)). + WithFailedTime(baseTimePlusSeconds(80))), + Failed: true, + }, + }, + expectedJobStateCounterByQueue: map[[4]string]float64{ + {testQueue, testPool, "failed", "running"}: 1, + }, + expectedJobStateCounterByNode: map[[5]string]float64{ + {testNode, testPool, testCluster, "failed", "running"}: 1, + }, + expectedJobStateSecondsByQueue: map[[4]string]float64{ + {testQueue, testPool, "failed", "running"}: 8, + }, + expectedJobStateSecondsByNode: map[[5]string]float64{ + {testNode, testPool, testCluster, "failed", "running"}: 8, + }, + expectedJobStateResourceSecondsByQueue: map[[5]string]float64{ + {testQueue, testPool, "failed", "running", "cpu"}: 8 * 16, + }, + expectedJobStateResourceSecondsByNode: map[[6]string]float64{ + {testNode, testPool, testCluster, "failed", "running", "cpu"}: 8 * 16, + }, + }, + // Pre-migration failed run: failed_timestamp was added in migration 039, so any + // run that failed before that has it NULL but carries the failure time in + // terminated_timestamp under the pre-PR semantics. The metric branch falls back. + "Failed (pre-migration, FailedTime nil)": { trackedResourceNames: []v1.ResourceName{"cpu"}, jsts: []jobdb.JobStateTransitions{ { diff --git a/internal/scheduler/scheduler_test.go b/internal/scheduler/scheduler_test.go index eb80b73adac..71b12eb817c 100644 --- a/internal/scheduler/scheduler_test.go +++ b/internal/scheduler/scheduler_test.go @@ -268,9 +268,9 @@ var returnedOnceLeasedJob = testfixtures.NewJob( nil, nil, nil, + nil, true, - true, -)).WithNewRun("testExecutor", "test-node", "node", "pool", 5) + true)).WithNewRun("testExecutor", "test-node", "node", "pool", 5) var defaultJobError = &armadaevents.Error{ Terminal: true, @@ -354,9 +354,9 @@ var ( nil, nil, nil, + nil, true, - true, - )) + true)) ) type jobRunId struct { @@ -1572,9 +1572,9 @@ func TestScheduler_TestSyncInitialState(t *testing.T) { nil, nil, nil, + nil, false, - false, - )).WithQueued(false).WithQueuedVersion(1).WithPriceBand(bidstore.PriceBand_PRICE_BAND_C), + false)).WithQueued(false).WithQueuedVersion(1).WithPriceBand(bidstore.PriceBand_PRICE_BAND_C), }, expectedInitialJobDbIds: []string{queuedJob.Id()}, expectedJobsSerial: 1, @@ -1734,9 +1734,9 @@ func TestScheduler_TestSyncState(t *testing.T) { nil, nil, nil, + nil, false, - false, - )).WithQueued(false).WithQueuedVersion(2), + false)).WithQueued(false).WithQueuedVersion(2), }, expectedJobDbIds: []string{queuedJob.Id()}, }, diff --git a/internal/scheduleringester/dbops.go b/internal/scheduleringester/dbops.go index 5e879f3bd16..3dd6007f3ee 100644 --- a/internal/scheduleringester/dbops.go +++ b/internal/scheduleringester/dbops.go @@ -207,6 +207,7 @@ type ( MarkRunsRunning map[string]time.Time MarkRunsPending map[string]time.Time MarkRunsPreempted map[string]time.Time + MarkRunsTerminated map[string]time.Time InsertJobRunErrors map[string]*schedulerdb.JobRunError UpdateJobPriorities struct { key JobReprioritiseKey @@ -385,6 +386,10 @@ func (a MarkRunsPreempted) Merge(b DbOperation) bool { return mergeInMap(a, b) } +func (a MarkRunsTerminated) Merge(b DbOperation) bool { + return mergeInMap(a, b) +} + func (a InsertJobRunErrors) Merge(b DbOperation) bool { return mergeInMap(a, b) } @@ -547,6 +552,10 @@ func (a MarkRunsPreempted) CanBeAppliedBefore(b DbOperation) bool { return !definesRun(a, b) } +func (a MarkRunsTerminated) CanBeAppliedBefore(b DbOperation) bool { + return !definesRun(a, b) +} + func (a *InsertPartitionMarker) CanBeAppliedBefore(b DbOperation) bool { // Partition markers can never be brought forward return false @@ -780,6 +789,10 @@ func (a MarkRunsPreempted) GetOperation() Operation { return JobSetOperation } +func (a MarkRunsTerminated) GetOperation() Operation { + return JobSetOperation +} + func (a InsertJobRunErrors) GetOperation() Operation { return JobSetOperation } diff --git a/internal/scheduleringester/dbops_test.go b/internal/scheduleringester/dbops_test.go index 6ed18f0d597..e37c23c9b37 100644 --- a/internal/scheduleringester/dbops_test.go +++ b/internal/scheduleringester/dbops_test.go @@ -280,6 +280,15 @@ func TestDbOperationOptimisation(t *testing.T) { MarkRunsRunning{runIds[1]: time.Time{}}, // 3 InsertJobs{jobIds[2]: &schedulerdb.Job{JobID: jobIds[2]}}, // 3 }}, + "MarkRunsTerminated": {N: 3, Ops: []DbOperation{ + InsertJobs{jobIds[0]: &schedulerdb.Job{JobID: jobIds[0]}}, // 1 + InsertRuns{runIds[0]: &JobRunDetails{Queue: testQueueName, DbRun: &schedulerdb.Run{JobID: jobIds[0], RunID: runIds[0]}}}, // 2 + MarkRunsTerminated{runIds[0]: time.Time{}}, // 3 + InsertJobs{jobIds[1]: &schedulerdb.Job{JobID: jobIds[1]}}, // 3 + InsertRuns{runIds[1]: &JobRunDetails{Queue: testQueueName, DbRun: &schedulerdb.Run{JobID: jobIds[0], RunID: runIds[1]}}}, // 3 + MarkRunsTerminated{runIds[1]: time.Time{}}, // 3 + InsertJobs{jobIds[2]: &schedulerdb.Job{JobID: jobIds[2]}}, // 3 + }}, "InsertPartitionMarker": {N: 2, Ops: []DbOperation{ InsertJobs{jobIds[0]: &schedulerdb.Job{JobID: jobIds[0]}}, // 1 &InsertPartitionMarker{markers: []*schedulerdb.Marker{}}, // 2 diff --git a/internal/scheduleringester/instructions.go b/internal/scheduleringester/instructions.go index 57f249c53b9..9c4c6071bc3 100644 --- a/internal/scheduleringester/instructions.go +++ b/internal/scheduleringester/instructions.go @@ -112,6 +112,8 @@ func (c *JobSetEventsInstructionConverter) dbOperationsFromEventSequence(es *arm operationsFromEvent, err = c.handlePartitionMarker(event.GetPartitionMarker(), eventTime) case *armadaevents.EventSequence_Event_JobRunPreempted: operationsFromEvent, err = c.handleJobRunPreempted(event.GetJobRunPreempted(), eventTime) + case *armadaevents.EventSequence_Event_JobRunTerminated: + operationsFromEvent, err = c.handleJobRunTerminated(event.GetJobRunTerminated(), eventTime) case *armadaevents.EventSequence_Event_JobRunAssigned: operationsFromEvent, err = c.handleJobRunAssigned(event.GetJobRunAssigned(), eventTime) case *armadaevents.EventSequence_Event_JobValidated: @@ -260,6 +262,11 @@ func (c *JobSetEventsInstructionConverter) handleJobRunPreempted(jobRunPreempted return []DbOperation{MarkRunsPreempted{runId: preemptedTime}}, nil } +func (c *JobSetEventsInstructionConverter) handleJobRunTerminated(jobRunTerminated *armadaevents.JobRunTerminated, terminatedTime time.Time) ([]DbOperation, error) { + runId := jobRunTerminated.RunId + return []DbOperation{MarkRunsTerminated{runId: terminatedTime}}, nil +} + func (c *JobSetEventsInstructionConverter) handleJobRunAssigned(jobRunAssigned *armadaevents.JobRunAssigned, assignedTime time.Time) ([]DbOperation, error) { runId := jobRunAssigned.RunId return []DbOperation{MarkRunsPending{runId: assignedTime}}, nil diff --git a/internal/scheduleringester/instructions_test.go b/internal/scheduleringester/instructions_test.go index 14f70bedf95..3ae057f7cfd 100644 --- a/internal/scheduleringester/instructions_test.go +++ b/internal/scheduleringester/instructions_test.go @@ -123,6 +123,10 @@ func TestConvertEventSequence(t *testing.T) { events: []*armadaevents.EventSequence_Event{f.JobRunPreempted}, expected: []DbOperation{MarkRunsPreempted{f.RunId: f.BaseTime}}, }, + "job run terminated": { + events: []*armadaevents.EventSequence_Event{f.JobRunTerminated}, + expected: []DbOperation{MarkRunsTerminated{f.RunId: f.BaseTime}}, + }, "lease returned": { events: []*armadaevents.EventSequence_Event{f.LeaseReturned}, expected: []DbOperation{ diff --git a/internal/scheduleringester/schedulerdb.go b/internal/scheduleringester/schedulerdb.go index 78733872e4e..7023d9f5c9a 100644 --- a/internal/scheduleringester/schedulerdb.go +++ b/internal/scheduleringester/schedulerdb.go @@ -330,7 +330,7 @@ func (s *SchedulerDb) WriteDbOp(ctx *armadacontext.Context, tx pgx.Tx, op DbOper runAttempted = append(runAttempted, k) } } - sqlStmt := multiColumnRunsUpdateStmt("run_id", "failed", "terminated_timestamp") + sqlStmt := multiColumnRunsUpdateStmt("run_id", "failed", "failed_timestamp") // order of arguments is important. See multiColumnRunsUpdateStmt function for details if _, err := tx.Exec(ctx, sqlStmt, runIds, failed, failTimes); err != nil { return errors.WithStack(err) @@ -382,6 +382,23 @@ func (s *SchedulerDb) WriteDbOp(ctx *armadacontext.Context, tx pgx.Tx, op DbOper if _, err := tx.Exec(ctx, sqlStmt, runIds, preempted, preemptedTimes); err != nil { return errors.WithStack(err) } + case MarkRunsTerminated: + runIds := make([]string, 0, len(o)) + terminatedTimes := make([]interface{}, 0, len(o)) + for runId, terminatedTime := range o { + runIds = append(runIds, runId) + terminatedTimes = append(terminatedTimes, terminatedTime) + } + // terminated_timestamp may already be set by an earlier writer; the helper's + // IS NULL guard yields to whichever path got there first: + // - succeeded runs: MarkRunsSucceeded set it at JobRunSucceeded event time (kubelet observation) + // - cancelled runs: MarkJobsCancelled set it at the cancel-event time (armada decision) + // - failed runs (including preempted, which fail via JobRunErrors): MarkRunsFailed + // writes failed_timestamp, NOT terminated_timestamp - so this path fills it. + sqlStmt := runsSetTimestampIfNullStmt("run_id", "terminated_timestamp") + if _, err := tx.Exec(ctx, sqlStmt, runIds, terminatedTimes); err != nil { + return errors.WithStack(err) + } case InsertJobRunErrors: records := make([]any, len(o)) i := 0 @@ -727,6 +744,21 @@ func multiColumnRunsUpdateStmt(id, phaseColumn, timeStampColumn string) string { id, phaseColumn, timeStampColumn, "text") } +// runsSetTimestampIfNullStmt builds a batch UPDATE that fills a single timestamp +// column on runs only for rows where the column is currently NULL. Use for writers +// that must defer to any prior writer of the same column (e.g. MarkRunsTerminated +// yields to MarkRunsSucceeded and MarkJobsCancelled which already filled +// terminated_timestamp). +func runsSetTimestampIfNullStmt(idColumn, timestampColumn string) string { + return fmt.Sprintf(`update runs set + %[2]v = runs_temp.%[2]v + from (select * from unnest($1::text[], $2::timestamptz[])) + as runs_temp(%[1]v, %[2]v) + where runs.%[1]v = runs_temp.%[1]v + and runs.%[2]v is null;`, + idColumn, timestampColumn) +} + func filterJobsByPriorityClasses(jobs []schedulerdb.Job, priorityClasses []string) ([]schedulerdb.Job, error) { if len(priorityClasses) == 0 { return jobs, nil diff --git a/internal/scheduleringester/schedulerdb_test.go b/internal/scheduleringester/schedulerdb_test.go index 7611f139942..96e1496bb5c 100644 --- a/internal/scheduleringester/schedulerdb_test.go +++ b/internal/scheduleringester/schedulerdb_test.go @@ -887,7 +887,8 @@ func assertOpSuccess(t *testing.T, schedulerDb *SchedulerDb, serials map[string] assert.True(t, run.Failed) assert.Equal(t, expectedRun.LeaseReturned, run.Returned) assert.Equal(t, expectedRun.RunAttempted, run.RunAttempted) - assert.Equal(t, expectedRun.FailureTime, run.TerminatedTimestamp.UTC()) + require.NotNil(t, run.FailedTimestamp) + assert.Equal(t, expectedRun.FailureTime, run.FailedTimestamp.UTC()) numChanged++ } } @@ -1186,6 +1187,71 @@ func TestStore(t *testing.T) { require.NoError(t, err) } +func TestMarkRunsTerminated(t *testing.T) { + // Verifies the MarkRunsTerminated SQL guard `WHERE terminated_timestamp IS NULL` + // against each writer that may have already set the column. + jobId := util.ULID().String() + preemptedAt := time.Date(2026, 5, 14, 10, 0, 0, 0, time.UTC) + succeededAt := time.Date(2026, 5, 14, 10, 0, 5, 0, time.UTC) + failedAt := time.Date(2026, 5, 14, 10, 0, 5, 0, time.UTC) + jobRunTerminatedAt := time.Date(2026, 5, 14, 10, 0, 30, 0, time.UTC) + + tests := map[string]struct { + seed func(runId string) []DbOperation + want time.Time + }{ + // MarkRunsPreempted leaves terminated_timestamp NULL; the guard lets the write through. + "preempted run gets kubelet time": { + seed: func(runId string) []DbOperation { return []DbOperation{MarkRunsPreempted{runId: preemptedAt}} }, + want: jobRunTerminatedAt, + }, + // MarkRunsSucceeded filled terminated_timestamp; the guard blocks the overwrite. + "succeeded run keeps its event-time value": { + seed: func(runId string) []DbOperation { return []DbOperation{MarkRunsSucceeded{runId: succeededAt}} }, + want: succeededAt, + }, + // MarkRunsFailed writes failed_timestamp (not terminated_timestamp); column is NULL + // so the guard lets the kubelet observation through. + "failed run with no kubelet observation gets filled by JobRunTerminated": { + seed: func(runId string) []DbOperation { + return []DbOperation{MarkRunsFailed{runId: &JobRunFailed{FailureTime: failedAt}}} + }, + want: jobRunTerminatedAt, + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + runId := uuid.NewString() + ops := []DbOperation{ + InsertJobs{jobId: &schedulerdb.Job{ + JobID: jobId, JobSet: "set1", Groups: []byte{}, SubmitMessage: []byte{}, SchedulingInfo: []byte{}, + }}, + InsertRuns{runId: &JobRunDetails{Queue: testQueueName, DbRun: &schedulerdb.Run{ + JobID: jobId, RunID: runId, Queue: testQueueName, + }}}, + } + ops = append(ops, tc.seed(runId)...) + + ctx, cancel := armadacontext.WithTimeout(armadacontext.Background(), 10*time.Second) + defer cancel() + require.NoError(t, schedulerdb.WithTestDb(func(q *schedulerdb.Queries, db *pgxpool.Pool) error { + schedulerDb := NewSchedulerDb(db, metrics.NewMetrics("test_mark_runs_terminated_"+util.NewULID()), time.Second, time.Second, 10*time.Second, schedulerdb.JobMetadataMigrationPhaseLegacy) + require.NoError(t, schedulerDb.Store(ctx, &DbOperationsWithMessageIds{Ops: ops})) + require.NoError(t, schedulerDb.Store(ctx, &DbOperationsWithMessageIds{Ops: []DbOperation{ + MarkRunsTerminated{runId: jobRunTerminatedAt}, + }})) + + runs, err := q.SelectNewRunsForJobs(ctx, schedulerdb.SelectNewRunsForJobsParams{Serial: 0, JobIds: []string{jobId}}) + require.NoError(t, err) + require.Len(t, runs, 1) + require.NotNil(t, runs[0].TerminatedTimestamp) + assert.Equal(t, tc.want, runs[0].TerminatedTimestamp.UTC()) + return nil + })) + }) + } +} + func max[E constraints.Ordered](a, b E) E { if a > b { return a diff --git a/internal/server/event/conversion/conversions.go b/internal/server/event/conversion/conversions.go index 21e7c6ea8a8..e2caa1c7f4c 100644 --- a/internal/server/event/conversion/conversions.go +++ b/internal/server/event/conversion/conversions.go @@ -59,6 +59,7 @@ func FromEventSequence(es *armadaevents.EventSequence) ([]*api.EventMessage, err *armadaevents.EventSequence_Event_JobRunSucceeded, *armadaevents.EventSequence_Event_JobRequeued, *armadaevents.EventSequence_Event_JobValidated, + *armadaevents.EventSequence_Event_JobRunTerminated, *armadaevents.EventSequence_Event_PartitionMarker: // These events have no api analog right now, so we ignore log.Debugf("ignoring event type %T", esEvent) diff --git a/pkg/armadaevents/events.pb.go b/pkg/armadaevents/events.pb.go index ed7f800a3b7..ddfca2786e2 100644 --- a/pkg/armadaevents/events.pb.go +++ b/pkg/armadaevents/events.pb.go @@ -201,6 +201,7 @@ type EventSequence_Event struct { // *EventSequence_Event_JobPreemptionRequested // *EventSequence_Event_JobRunCancelled // *EventSequence_Event_JobValidated + // *EventSequence_Event_JobRunTerminated Event isEventSequence_Event_Event `protobuf_oneof:"event"` } @@ -312,6 +313,9 @@ type EventSequence_Event_JobRunCancelled struct { type EventSequence_Event_JobValidated struct { JobValidated *JobValidated `protobuf:"bytes,25,opt,name=jobValidated,proto3,oneof" json:"jobValidated,omitempty"` } +type EventSequence_Event_JobRunTerminated struct { + JobRunTerminated *JobRunTerminated `protobuf:"bytes,26,opt,name=jobRunTerminated,proto3,oneof" json:"jobRunTerminated,omitempty"` +} func (*EventSequence_Event_SubmitJob) isEventSequence_Event_Event() {} func (*EventSequence_Event_ReprioritiseJob) isEventSequence_Event_Event() {} @@ -336,6 +340,7 @@ func (*EventSequence_Event_JobRequeued) isEventSequence_Event_Event() func (*EventSequence_Event_JobPreemptionRequested) isEventSequence_Event_Event() {} func (*EventSequence_Event_JobRunCancelled) isEventSequence_Event_Event() {} func (*EventSequence_Event_JobValidated) isEventSequence_Event_Event() {} +func (*EventSequence_Event_JobRunTerminated) isEventSequence_Event_Event() {} func (m *EventSequence_Event) GetEvent() isEventSequence_Event_Event { if m != nil { @@ -512,6 +517,13 @@ func (m *EventSequence_Event) GetJobValidated() *JobValidated { return nil } +func (m *EventSequence_Event) GetJobRunTerminated() *JobRunTerminated { + if x, ok := m.GetEvent().(*EventSequence_Event_JobRunTerminated); ok { + return x.JobRunTerminated + } + return nil +} + // XXX_OneofWrappers is for the internal use of the proto package. func (*EventSequence_Event) XXX_OneofWrappers() []interface{} { return []interface{}{ @@ -538,6 +550,7 @@ func (*EventSequence_Event) XXX_OneofWrappers() []interface{} { (*EventSequence_Event_JobPreemptionRequested)(nil), (*EventSequence_Event_JobRunCancelled)(nil), (*EventSequence_Event_JobValidated)(nil), + (*EventSequence_Event_JobRunTerminated)(nil), } } @@ -3689,6 +3702,63 @@ func (m *JobRunCancelled) GetRunId() string { return "" } +// JobRunTerminated is emitted by the executor when it observes that the pod +// for a run has reached a terminal phase in Kubernetes. Only the executor +// produces this event. The timestamp on the enclosing sequence is the moment +// the executor saw the pod terminal, which is after the kubelet has stopped +// the containers. +type JobRunTerminated struct { + JobId string `protobuf:"bytes,1,opt,name=job_id,json=jobId,proto3" json:"jobId,omitempty"` + RunId string `protobuf:"bytes,2,opt,name=run_id,json=runId,proto3" json:"runId,omitempty"` +} + +func (m *JobRunTerminated) Reset() { *m = JobRunTerminated{} } +func (m *JobRunTerminated) String() string { return proto.CompactTextString(m) } +func (*JobRunTerminated) ProtoMessage() {} +func (*JobRunTerminated) Descriptor() ([]byte, []int) { + return fileDescriptor_6aab92ca59e015f8, []int{46} +} +func (m *JobRunTerminated) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *JobRunTerminated) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_JobRunTerminated.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *JobRunTerminated) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobRunTerminated.Merge(m, src) +} +func (m *JobRunTerminated) XXX_Size() int { + return m.Size() +} +func (m *JobRunTerminated) XXX_DiscardUnknown() { + xxx_messageInfo_JobRunTerminated.DiscardUnknown(m) +} + +var xxx_messageInfo_JobRunTerminated proto.InternalMessageInfo + +func (m *JobRunTerminated) GetJobId() string { + if m != nil { + return m.JobId + } + return "" +} + +func (m *JobRunTerminated) GetRunId() string { + if m != nil { + return m.RunId + } + return "" +} + func init() { proto.RegisterEnum("armadaevents.JobState", JobState_name, JobState_value) proto.RegisterEnum("armadaevents.KubernetesReason", KubernetesReason_name, KubernetesReason_value) @@ -3746,249 +3816,252 @@ func init() { proto.RegisterType((*JobPreemptionRequested)(nil), "armadaevents.JobPreemptionRequested") proto.RegisterType((*JobValidated)(nil), "armadaevents.JobValidated") proto.RegisterType((*JobRunCancelled)(nil), "armadaevents.JobRunCancelled") + proto.RegisterType((*JobRunTerminated)(nil), "armadaevents.JobRunTerminated") } func init() { proto.RegisterFile("pkg/armadaevents/events.proto", fileDescriptor_6aab92ca59e015f8) } var fileDescriptor_6aab92ca59e015f8 = []byte{ - // 3787 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x3b, 0x4d, 0x6f, 0x1b, 0xd7, - 0xb5, 0x1e, 0x7e, 0xf3, 0x50, 0xa2, 0xe8, 0xab, 0x0f, 0xd3, 0x4a, 0x2c, 0xca, 0x74, 0x5e, 0x62, - 0x07, 0x09, 0xe5, 0x38, 0x2f, 0x0f, 0xf9, 0x78, 0x48, 0x20, 0xda, 0xf2, 0x87, 0x62, 0xd9, 0x32, - 0x65, 0xe5, 0xf9, 0x3d, 0x04, 0x60, 0x86, 0x9c, 0x2b, 0x6a, 0x2c, 0x72, 0x86, 0x99, 0x0f, 0x45, - 0x02, 0x02, 0xbc, 0xa4, 0x48, 0xbb, 0xce, 0xa6, 0x40, 0x91, 0x4d, 0xb3, 0xe9, 0xa2, 0x05, 0xba, - 0x6c, 0x7f, 0x43, 0x17, 0x45, 0x91, 0x4d, 0x81, 0x6e, 0x4a, 0x14, 0x09, 0xba, 0xe1, 0xa2, 0x3f, - 0x20, 0xab, 0xe2, 0x7e, 0xcc, 0xcc, 0xbd, 0xc3, 0x4b, 0x8b, 0x72, 0x2c, 0xc3, 0x41, 0x56, 0xd2, - 0x9c, 0xcf, 0x3b, 0xe7, 0x9c, 0x7b, 0xe6, 0x9c, 0x73, 0x2f, 0xe1, 0x5c, 0x7f, 0xaf, 0xb3, 0xa2, - 0x3b, 0x3d, 0xdd, 0xd0, 0xf1, 0x3e, 0xb6, 0x3c, 0x77, 0x85, 0xfd, 0xa9, 0xf5, 0x1d, 0xdb, 0xb3, - 0xd1, 0x94, 0x88, 0x5a, 0xac, 0xee, 0xbd, 0xe9, 0xd6, 0x4c, 0x7b, 0x45, 0xef, 0x9b, 0x2b, 0x6d, - 0xdb, 0xc1, 0x2b, 0xfb, 0xaf, 0xad, 0x74, 0xb0, 0x85, 0x1d, 0xdd, 0xc3, 0x06, 0xe3, 0x58, 0xbc, - 0x28, 0xd0, 0x58, 0xd8, 0xfb, 0xc4, 0x76, 0xf6, 0x4c, 0xab, 0xa3, 0xa2, 0xac, 0x74, 0x6c, 0xbb, - 0xd3, 0xc5, 0x2b, 0xf4, 0xa9, 0xe5, 0xef, 0xac, 0x78, 0x66, 0x0f, 0xbb, 0x9e, 0xde, 0xeb, 0x73, - 0x82, 0xff, 0x8c, 0x44, 0xf5, 0xf4, 0xf6, 0xae, 0x69, 0x61, 0xe7, 0x70, 0x85, 0xae, 0xb7, 0x6f, - 0xae, 0x38, 0xd8, 0xb5, 0x7d, 0xa7, 0x8d, 0x47, 0xc4, 0xbe, 0x6d, 0x5a, 0x1e, 0x76, 0x2c, 0xbd, - 0xbb, 0xe2, 0xb6, 0x77, 0xb1, 0xe1, 0x77, 0xb1, 0x13, 0xfd, 0x67, 0xb7, 0x1e, 0xe2, 0xb6, 0xe7, - 0x8e, 0x00, 0x18, 0x6f, 0xf5, 0xef, 0xf3, 0x30, 0xbd, 0x46, 0xde, 0x75, 0x0b, 0x7f, 0xec, 0x63, - 0xab, 0x8d, 0xd1, 0x25, 0x48, 0x7f, 0xec, 0x63, 0x1f, 0x97, 0xb5, 0x65, 0xed, 0x62, 0xbe, 0x3e, - 0x3b, 0x1c, 0x54, 0x66, 0x28, 0xe0, 0x15, 0xbb, 0x67, 0x7a, 0xb8, 0xd7, 0xf7, 0x0e, 0x1b, 0x8c, - 0x02, 0xbd, 0x0d, 0x53, 0x0f, 0xed, 0x56, 0xd3, 0xc5, 0x5e, 0xd3, 0xd2, 0x7b, 0xb8, 0x9c, 0xa0, - 0x1c, 0xe5, 0xe1, 0xa0, 0x32, 0xf7, 0xd0, 0x6e, 0x6d, 0x61, 0xef, 0x8e, 0xde, 0x13, 0xd9, 0x20, - 0x82, 0xa2, 0x57, 0x21, 0xeb, 0xbb, 0xd8, 0x69, 0x9a, 0x46, 0x39, 0x49, 0xd9, 0xe6, 0x86, 0x83, - 0x4a, 0x89, 0x80, 0x6e, 0x19, 0x02, 0x4b, 0x86, 0x41, 0xd0, 0x2b, 0x90, 0xe9, 0x38, 0xb6, 0xdf, - 0x77, 0xcb, 0xa9, 0xe5, 0x64, 0x40, 0xcd, 0x20, 0x22, 0x35, 0x83, 0xa0, 0xbb, 0x90, 0x61, 0x0e, - 0x2c, 0xa7, 0x97, 0x93, 0x17, 0x0b, 0x57, 0xce, 0xd7, 0x44, 0xaf, 0xd6, 0xa4, 0x17, 0x66, 0x4f, - 0x4c, 0x20, 0xc3, 0x8b, 0x02, 0x79, 0x1c, 0xfc, 0x71, 0x16, 0xd2, 0x94, 0x0e, 0xbd, 0x0f, 0xd9, - 0xb6, 0x83, 0x89, 0xf5, 0xcb, 0x68, 0x59, 0xbb, 0x58, 0xb8, 0xb2, 0x58, 0x63, 0x5e, 0xad, 0x05, - 0x5e, 0xad, 0xdd, 0x0f, 0xbc, 0x5a, 0x9f, 0x1f, 0x0e, 0x2a, 0xa7, 0x39, 0xb9, 0x20, 0x35, 0x90, - 0x80, 0x36, 0x21, 0xef, 0xfa, 0xad, 0x9e, 0xe9, 0xad, 0xdb, 0x2d, 0x6a, 0xef, 0xc2, 0x95, 0x33, - 0xf2, 0x52, 0xb7, 0x02, 0x74, 0xfd, 0xcc, 0x70, 0x50, 0x99, 0x0d, 0xa9, 0x23, 0x69, 0x37, 0x4f, - 0x35, 0x22, 0x21, 0x68, 0x17, 0x66, 0x1c, 0xdc, 0x77, 0x4c, 0xdb, 0x31, 0x3d, 0xd3, 0xc5, 0x44, - 0x6e, 0x82, 0xca, 0x3d, 0x27, 0xcb, 0x6d, 0xc8, 0x44, 0xf5, 0x73, 0xc3, 0x41, 0xe5, 0x6c, 0x8c, - 0x53, 0xd2, 0x11, 0x17, 0x8b, 0x3c, 0x40, 0x31, 0xd0, 0x16, 0xf6, 0xa8, 0x2f, 0x0b, 0x57, 0x96, - 0x1f, 0xa9, 0x6c, 0x0b, 0x7b, 0xf5, 0xe5, 0xe1, 0xa0, 0xf2, 0xfc, 0x28, 0xbf, 0xa4, 0x52, 0x21, - 0x1f, 0x75, 0xa1, 0x24, 0x42, 0x0d, 0xf2, 0x82, 0x29, 0xaa, 0x73, 0x69, 0xbc, 0x4e, 0x42, 0x55, - 0x5f, 0x1a, 0x0e, 0x2a, 0x8b, 0x71, 0x5e, 0x49, 0xdf, 0x88, 0x64, 0xe2, 0x9f, 0xb6, 0x6e, 0xb5, - 0x71, 0x97, 0xa8, 0x49, 0xab, 0xfc, 0x73, 0x35, 0x40, 0x33, 0xff, 0x84, 0xd4, 0xb2, 0x7f, 0x42, - 0x30, 0xfa, 0x10, 0xa6, 0xc2, 0x07, 0x62, 0xaf, 0x0c, 0x8f, 0x21, 0xb5, 0x50, 0x62, 0xa9, 0xc5, - 0xe1, 0xa0, 0xb2, 0x20, 0xf2, 0x48, 0xa2, 0x25, 0x69, 0x91, 0xf4, 0x2e, 0xb3, 0x4c, 0x76, 0xbc, - 0x74, 0x46, 0x21, 0x4a, 0xef, 0x8e, 0x5a, 0x44, 0x92, 0x46, 0xa4, 0x93, 0x0d, 0xec, 0xb7, 0xdb, - 0x18, 0x1b, 0xd8, 0x28, 0xe7, 0x54, 0xd2, 0xd7, 0x05, 0x0a, 0x26, 0x5d, 0xe4, 0x91, 0xa5, 0x8b, - 0x18, 0x62, 0xeb, 0x87, 0x76, 0x6b, 0xcd, 0x71, 0x6c, 0xc7, 0x2d, 0xe7, 0x55, 0xb6, 0x5e, 0x0f, - 0xd0, 0xcc, 0xd6, 0x21, 0xb5, 0x6c, 0xeb, 0x10, 0xcc, 0xd7, 0xdb, 0xf0, 0xad, 0xdb, 0x58, 0x77, - 0xb1, 0x51, 0x86, 0x31, 0xeb, 0x0d, 0x29, 0xc2, 0xf5, 0x86, 0x90, 0x91, 0xf5, 0x86, 0x18, 0x64, - 0x40, 0x91, 0x3d, 0xaf, 0xba, 0xae, 0xd9, 0xb1, 0xb0, 0x51, 0x2e, 0x50, 0xf9, 0xcf, 0xab, 0xe4, - 0x07, 0x34, 0xf5, 0xe7, 0x87, 0x83, 0x4a, 0x59, 0xe6, 0x93, 0x74, 0xc4, 0x64, 0xa2, 0x8f, 0x60, - 0x9a, 0x41, 0x1a, 0xbe, 0x65, 0x99, 0x56, 0xa7, 0x3c, 0x45, 0x95, 0x3c, 0xa7, 0x52, 0xc2, 0x49, - 0xea, 0xcf, 0x0d, 0x07, 0x95, 0x33, 0x12, 0x97, 0xa4, 0x42, 0x16, 0x48, 0x32, 0x06, 0x03, 0x44, - 0x8e, 0x9d, 0x56, 0x65, 0x8c, 0x75, 0x99, 0x88, 0x65, 0x8c, 0x18, 0xa7, 0x9c, 0x31, 0x62, 0xc8, - 0xc8, 0x1f, 0xdc, 0xc9, 0xc5, 0xf1, 0xfe, 0xe0, 0x7e, 0x16, 0xfc, 0xa1, 0x70, 0xb5, 0x24, 0x0d, - 0x7d, 0xa6, 0xc1, 0xbc, 0xeb, 0xe9, 0x96, 0xa1, 0x77, 0x6d, 0x0b, 0xdf, 0xb2, 0x3a, 0x0e, 0x76, - 0xdd, 0x5b, 0xd6, 0x8e, 0x5d, 0x2e, 0x51, 0x3d, 0x17, 0x62, 0x89, 0x55, 0x45, 0x5a, 0xbf, 0x30, - 0x1c, 0x54, 0x2a, 0x4a, 0x29, 0x92, 0x66, 0xb5, 0x22, 0x74, 0x00, 0xb3, 0xc1, 0x47, 0x7a, 0xdb, - 0x33, 0xbb, 0xa6, 0xab, 0x7b, 0xa6, 0x6d, 0x95, 0x4f, 0x53, 0xfd, 0xe7, 0xe3, 0xf9, 0x69, 0x84, - 0xb0, 0x7e, 0x7e, 0x38, 0xa8, 0x9c, 0x53, 0x48, 0x90, 0x74, 0xab, 0x54, 0x44, 0x4e, 0xdc, 0x74, - 0x30, 0x21, 0xc4, 0x46, 0x79, 0x76, 0xbc, 0x13, 0x43, 0x22, 0xd1, 0x89, 0x21, 0x50, 0xe5, 0xc4, - 0x10, 0x49, 0x34, 0xf5, 0x75, 0xc7, 0x33, 0x89, 0xda, 0x0d, 0xdd, 0xd9, 0xc3, 0x4e, 0x79, 0x4e, - 0xa5, 0x69, 0x53, 0x26, 0x62, 0x9a, 0x62, 0x9c, 0xb2, 0xa6, 0x18, 0x12, 0x7d, 0xa9, 0x81, 0xbc, - 0x34, 0xd3, 0xb6, 0x1a, 0xe4, 0xa3, 0xed, 0x92, 0xd7, 0x9b, 0xa7, 0x4a, 0x5f, 0x7a, 0xc4, 0xeb, - 0x89, 0xe4, 0xf5, 0x97, 0x86, 0x83, 0xca, 0x85, 0xb1, 0xd2, 0xa4, 0x85, 0x8c, 0x57, 0x8a, 0x1e, - 0x40, 0x81, 0x20, 0x31, 0x2d, 0x7f, 0x8c, 0xf2, 0x02, 0x5d, 0xc3, 0xd9, 0xd1, 0x35, 0x70, 0x82, - 0xfa, 0xd9, 0xe1, 0xa0, 0x32, 0x2f, 0x70, 0x48, 0x7a, 0x44, 0x51, 0xe8, 0x0b, 0x0d, 0x48, 0xa0, - 0xab, 0xde, 0xf4, 0x0c, 0xd5, 0xf2, 0xc2, 0x88, 0x16, 0xd5, 0x6b, 0xbe, 0x30, 0x1c, 0x54, 0x96, - 0xd5, 0x72, 0x24, 0xdd, 0x63, 0x74, 0x45, 0x71, 0x14, 0x7e, 0x24, 0xca, 0xe5, 0xf1, 0x71, 0x14, - 0x12, 0x89, 0x71, 0x14, 0x02, 0x55, 0x71, 0x14, 0x22, 0x79, 0x32, 0xf8, 0x40, 0xef, 0x9a, 0x06, - 0x2d, 0xa6, 0xce, 0x8e, 0x49, 0x06, 0x21, 0x45, 0x98, 0x0c, 0x42, 0xc8, 0x48, 0x32, 0x88, 0x68, - 0xb3, 0x90, 0xa6, 0x22, 0xaa, 0x5f, 0xe5, 0x61, 0x56, 0xb1, 0xd5, 0x10, 0x86, 0xe9, 0x60, 0x1f, - 0x35, 0x4d, 0x92, 0x24, 0x92, 0x2a, 0x2b, 0xbf, 0xef, 0xb7, 0xb0, 0x63, 0x61, 0x0f, 0xbb, 0x81, - 0x0c, 0x9a, 0x25, 0xe8, 0x4a, 0x1c, 0x01, 0x22, 0xd4, 0x76, 0x53, 0x22, 0x1c, 0x7d, 0xa5, 0x41, - 0xb9, 0xa7, 0x1f, 0x34, 0x03, 0xa0, 0xdb, 0xdc, 0xb1, 0x9d, 0x66, 0x1f, 0x3b, 0xa6, 0x6d, 0xd0, - 0x4a, 0xb6, 0x70, 0xe5, 0xbf, 0x8f, 0xcc, 0x0b, 0xb5, 0x0d, 0xfd, 0x20, 0x00, 0xbb, 0xd7, 0x6d, - 0x67, 0x93, 0xb2, 0xaf, 0x59, 0x9e, 0x73, 0xc8, 0x12, 0x56, 0x4f, 0x85, 0x17, 0xd6, 0x34, 0xaf, - 0x24, 0x40, 0xbf, 0xd4, 0x60, 0xc1, 0xb3, 0x3d, 0xbd, 0xdb, 0x6c, 0xfb, 0x3d, 0xbf, 0xab, 0x7b, - 0xe6, 0x3e, 0x6e, 0xfa, 0xae, 0xde, 0xc1, 0xbc, 0x6c, 0x7e, 0xe7, 0xe8, 0xa5, 0xdd, 0x27, 0xfc, - 0x57, 0x43, 0xf6, 0x6d, 0xc2, 0xcd, 0x56, 0x56, 0x1d, 0x0e, 0x2a, 0x4b, 0x9e, 0x02, 0x2d, 0x2c, - 0x6c, 0x4e, 0x85, 0x47, 0x2f, 0x43, 0x86, 0xb4, 0x15, 0xa6, 0x41, 0xab, 0x23, 0xde, 0x82, 0x3c, - 0xb4, 0x5b, 0x52, 0x63, 0x90, 0xa6, 0x00, 0x42, 0xeb, 0xf8, 0x16, 0xa1, 0xcd, 0x46, 0xb4, 0x8e, - 0x6f, 0xc9, 0xb4, 0x14, 0x40, 0x9d, 0xa1, 0xef, 0x77, 0xd4, 0xce, 0xc8, 0x4d, 0xea, 0x8c, 0xd5, - 0xfd, 0xce, 0x23, 0x9d, 0xa1, 0xab, 0xf0, 0xa2, 0x33, 0x94, 0x04, 0x8b, 0x5f, 0x6b, 0xb0, 0x38, - 0xde, 0xcf, 0xe8, 0x02, 0x24, 0xf7, 0xf0, 0x21, 0xef, 0xc9, 0x4e, 0x0f, 0x07, 0x95, 0xe9, 0x3d, - 0x7c, 0x28, 0x48, 0x25, 0x58, 0xf4, 0xbf, 0x90, 0xde, 0xd7, 0xbb, 0x3e, 0xe6, 0x25, 0x7f, 0xad, - 0xc6, 0xda, 0xc9, 0x9a, 0xd8, 0x4e, 0xd6, 0xfa, 0x7b, 0x1d, 0x02, 0xa8, 0x05, 0x56, 0xa8, 0xdd, - 0xf3, 0x75, 0xcb, 0x33, 0xbd, 0x43, 0x66, 0x3b, 0x2a, 0x40, 0xb4, 0x1d, 0x05, 0xbc, 0x9d, 0x78, - 0x53, 0x5b, 0xfc, 0xb5, 0x06, 0x67, 0xc7, 0xfa, 0xfb, 0x99, 0x58, 0x21, 0x31, 0xe2, 0x78, 0xff, - 0x3c, 0x0b, 0x4b, 0x5c, 0x4f, 0xe5, 0xb4, 0x52, 0x62, 0x3d, 0x95, 0x4b, 0x94, 0x92, 0xd5, 0xef, - 0x33, 0x90, 0x0f, 0x1b, 0x3c, 0x74, 0x13, 0x4a, 0x06, 0x36, 0xfc, 0x7e, 0xd7, 0x6c, 0xd3, 0x48, - 0x23, 0x41, 0xcd, 0x3a, 0x6a, 0x9a, 0x5d, 0x25, 0x9c, 0x14, 0xde, 0x33, 0x31, 0x14, 0xba, 0x02, - 0x39, 0xde, 0xc8, 0x1c, 0xd2, 0xbc, 0x36, 0x5d, 0x5f, 0x18, 0x0e, 0x2a, 0x28, 0x80, 0x09, 0xac, - 0x21, 0x1d, 0x6a, 0x00, 0xb0, 0xc9, 0xc0, 0x06, 0xf6, 0x74, 0xde, 0x52, 0x95, 0xe5, 0xdd, 0x70, - 0x37, 0xc4, 0xb3, 0x1e, 0x3f, 0xa2, 0x17, 0x7b, 0xfc, 0x08, 0x8a, 0x3e, 0x04, 0xe8, 0xe9, 0xa6, - 0xc5, 0xf8, 0x78, 0xff, 0x54, 0x1d, 0x97, 0x61, 0x37, 0x42, 0x4a, 0x26, 0x3d, 0xe2, 0x14, 0xa5, - 0x47, 0x50, 0x74, 0x17, 0xb2, 0x7c, 0x96, 0x51, 0xce, 0xd0, 0xcd, 0xbb, 0x34, 0x4e, 0x34, 0x17, - 0x4b, 0xbb, 0x71, 0xce, 0x22, 0x76, 0xe3, 0x1c, 0x44, 0xcc, 0xd6, 0x35, 0x77, 0xb0, 0x67, 0xf6, - 0x30, 0xcd, 0x26, 0xdc, 0x6c, 0x01, 0x4c, 0x34, 0x5b, 0x00, 0x43, 0x6f, 0x02, 0xe8, 0xde, 0x86, - 0xed, 0x7a, 0x77, 0xad, 0x36, 0xa6, 0x1d, 0x51, 0x8e, 0x2d, 0x3f, 0x82, 0x8a, 0xcb, 0x8f, 0xa0, - 0xe8, 0x1d, 0x28, 0xf4, 0xf9, 0x17, 0xb8, 0xd5, 0xc5, 0xb4, 0xe3, 0xc9, 0xb1, 0x82, 0x41, 0x00, - 0x0b, 0xbc, 0x22, 0x35, 0xba, 0x01, 0x33, 0x6d, 0xdb, 0x6a, 0xfb, 0x8e, 0x83, 0xad, 0xf6, 0xe1, - 0x96, 0xbe, 0x83, 0x69, 0x77, 0x93, 0x63, 0xa1, 0x12, 0x43, 0x89, 0xa1, 0x12, 0x43, 0xa1, 0x37, - 0x20, 0x1f, 0x4e, 0x86, 0x68, 0x03, 0x93, 0xe7, 0x83, 0x86, 0x00, 0x28, 0x30, 0x47, 0x94, 0x64, - 0xf1, 0xa6, 0x7b, 0x8d, 0x07, 0x1d, 0xa6, 0x4d, 0x09, 0x5f, 0xbc, 0x00, 0x16, 0x17, 0x2f, 0x80, - 0x85, 0xfc, 0x5e, 0x3c, 0x32, 0xbf, 0x5f, 0x87, 0x12, 0x3e, 0x60, 0xd3, 0xad, 0x26, 0x61, 0xf2, - 0x1d, 0x93, 0xd6, 0xf3, 0x79, 0xd6, 0x49, 0x05, 0xb8, 0x75, 0xbb, 0xb5, 0xed, 0x98, 0x02, 0x7b, - 0x51, 0xc6, 0x84, 0xdb, 0x6e, 0xba, 0x54, 0x5c, 0x4f, 0xe5, 0x66, 0x4a, 0xa5, 0xea, 0x9f, 0x35, - 0x98, 0x53, 0x45, 0x5f, 0x6c, 0x27, 0x68, 0x4f, 0x64, 0x27, 0x7c, 0x00, 0xb9, 0xbe, 0x6d, 0x34, - 0xdd, 0x3e, 0x6e, 0xf3, 0xbc, 0x12, 0xdb, 0x07, 0x9b, 0xb6, 0xb1, 0xd5, 0xc7, 0xed, 0xff, 0x31, - 0xbd, 0xdd, 0xd5, 0x7d, 0xdb, 0x34, 0x6e, 0x9b, 0x2e, 0x0f, 0xd8, 0x3e, 0xc3, 0x48, 0xc5, 0x4e, - 0x96, 0x03, 0xeb, 0x39, 0xc8, 0x30, 0x2d, 0xd5, 0xbf, 0x24, 0xa1, 0x14, 0x8f, 0xf8, 0x1f, 0xd3, - 0xab, 0xa0, 0x07, 0x90, 0x35, 0x59, 0x2f, 0xc5, 0x6b, 0xb1, 0xff, 0x10, 0x32, 0x6f, 0x2d, 0x1a, - 0xac, 0xd6, 0xf6, 0x5f, 0xab, 0xf1, 0xa6, 0x8b, 0x9a, 0x80, 0x4a, 0xe6, 0x9c, 0xb2, 0x64, 0x0e, - 0x44, 0x0d, 0xc8, 0xba, 0xd8, 0xd9, 0x37, 0xdb, 0x98, 0xe7, 0xb5, 0x8a, 0x28, 0xb9, 0x6d, 0x3b, - 0x98, 0xc8, 0xdc, 0x62, 0x24, 0x91, 0x4c, 0xce, 0x23, 0xcb, 0xe4, 0x40, 0xf4, 0x01, 0xe4, 0xdb, - 0xb6, 0xb5, 0x63, 0x76, 0x36, 0xf4, 0x3e, 0xcf, 0x6c, 0xe7, 0x54, 0x52, 0xaf, 0x06, 0x44, 0x7c, - 0x3e, 0x14, 0x3c, 0xc6, 0xe6, 0x43, 0x21, 0x55, 0xe4, 0xd0, 0x7f, 0xa5, 0x00, 0x22, 0xe7, 0xa0, - 0xb7, 0xa0, 0x80, 0x0f, 0x70, 0xdb, 0xf7, 0x6c, 0x3a, 0x33, 0xd5, 0xa2, 0x51, 0x6b, 0x00, 0x96, - 0xb6, 0x0f, 0x44, 0x50, 0xb2, 0xc7, 0x2d, 0xbd, 0x87, 0xdd, 0xbe, 0xde, 0x0e, 0x66, 0xb4, 0x74, - 0x31, 0x21, 0x50, 0xdc, 0xe3, 0x21, 0x10, 0xbd, 0x08, 0x29, 0x3a, 0xd5, 0x65, 0xe3, 0x59, 0x34, - 0x1c, 0x54, 0x8a, 0x96, 0x3c, 0xcf, 0xa5, 0x78, 0xf4, 0x1e, 0x4c, 0xef, 0x85, 0x81, 0x47, 0xd6, - 0x96, 0xa2, 0x0c, 0xb4, 0x48, 0x8e, 0x10, 0xd2, 0xea, 0xa6, 0x44, 0x38, 0xda, 0x81, 0x82, 0x6e, - 0x59, 0xb6, 0x47, 0x3f, 0x5f, 0xc1, 0xc8, 0xf6, 0xd2, 0xb8, 0x30, 0xad, 0xad, 0x46, 0xb4, 0xac, - 0xec, 0xa2, 0x79, 0x47, 0x90, 0x20, 0xe6, 0x1d, 0x01, 0x8c, 0x1a, 0x90, 0xe9, 0xea, 0x2d, 0xdc, - 0x0d, 0xbe, 0x17, 0x2f, 0x8c, 0x55, 0x71, 0x9b, 0x92, 0x31, 0xe9, 0x74, 0x30, 0xcc, 0xf8, 0xc4, - 0xc1, 0x30, 0x83, 0x2c, 0xee, 0x40, 0x29, 0xbe, 0x9e, 0xc9, 0xca, 0x8c, 0x4b, 0x62, 0x99, 0x91, - 0x3f, 0xb2, 0xb2, 0xd1, 0xa1, 0x20, 0x2c, 0xea, 0x24, 0x54, 0x54, 0x7f, 0xab, 0xc1, 0x9c, 0x6a, - 0xef, 0xa2, 0x0d, 0x61, 0xc7, 0x6b, 0x7c, 0xfc, 0xa4, 0x08, 0x75, 0xce, 0x3b, 0x66, 0xab, 0x47, - 0x1b, 0xbd, 0x0e, 0x45, 0xcb, 0x36, 0x70, 0x53, 0x27, 0x0a, 0xba, 0xa6, 0xeb, 0x95, 0x13, 0x74, - 0xa4, 0x4f, 0xc7, 0x56, 0x04, 0xb3, 0x1a, 0x20, 0x04, 0xee, 0x69, 0x09, 0x51, 0xfd, 0x04, 0x66, - 0x62, 0x43, 0x65, 0xa9, 0xe8, 0x49, 0x4c, 0x58, 0xf4, 0x44, 0x5f, 0xa2, 0xe4, 0x51, 0x5f, 0x22, - 0xf6, 0x05, 0xa9, 0xfe, 0x3c, 0x01, 0x05, 0xa1, 0xc3, 0x47, 0x0f, 0x61, 0x86, 0x7f, 0x15, 0x4d, - 0xab, 0xc3, 0x3a, 0xc9, 0x04, 0x1f, 0x37, 0x8d, 0x9c, 0xb8, 0xac, 0xdb, 0xad, 0xad, 0x90, 0x96, - 0x36, 0x92, 0xf4, 0x1b, 0xe6, 0x4a, 0x30, 0xf1, 0x1b, 0x26, 0x63, 0xd0, 0x03, 0x58, 0xf0, 0xfb, - 0xa4, 0xbf, 0x6d, 0xba, 0xfc, 0xec, 0xa2, 0x69, 0xf9, 0xbd, 0x16, 0x76, 0xe8, 0xea, 0xd3, 0xac, - 0xe3, 0x62, 0x14, 0xc1, 0xe1, 0xc6, 0x1d, 0x8a, 0x17, 0x3b, 0x2e, 0x15, 0x5e, 0xb0, 0x43, 0x6a, - 0x42, 0x3b, 0xdc, 0x04, 0x34, 0x3a, 0xd5, 0x97, 0x7c, 0xa0, 0x4d, 0xe6, 0x83, 0xea, 0x01, 0x94, - 0xe2, 0xb3, 0xfa, 0xa7, 0xe4, 0xcb, 0x3d, 0xc8, 0x87, 0x93, 0x76, 0xf4, 0x0a, 0x64, 0x1c, 0xac, - 0xbb, 0xb6, 0xc5, 0x77, 0x0b, 0xdd, 0xf6, 0x0c, 0x22, 0x6e, 0x7b, 0x06, 0x79, 0x0c, 0x65, 0xf7, - 0x61, 0x8a, 0x19, 0xe9, 0xba, 0xd9, 0xf5, 0xb0, 0x83, 0xae, 0x41, 0xc6, 0xf5, 0x74, 0x0f, 0xbb, - 0x65, 0x6d, 0x39, 0x79, 0xb1, 0x78, 0x65, 0x61, 0x74, 0x8c, 0x4e, 0xd0, 0x6c, 0x1d, 0x8c, 0x52, - 0x5c, 0x07, 0x83, 0x54, 0x7f, 0xa6, 0xc1, 0x94, 0x78, 0x5a, 0xf0, 0x64, 0xc4, 0x1e, 0xcf, 0x18, - 0x24, 0x71, 0x4c, 0x89, 0x87, 0x0a, 0x27, 0x67, 0x4b, 0xf2, 0x15, 0x64, 0x47, 0x12, 0x4d, 0xdf, - 0xc5, 0x0e, 0x8f, 0x56, 0xfa, 0x15, 0x64, 0xe0, 0x6d, 0x57, 0x8a, 0x76, 0x88, 0xa0, 0xdc, 0x0d, - 0x64, 0xad, 0xe2, 0x11, 0x05, 0xea, 0x44, 0x83, 0x20, 0xb2, 0xc9, 0x5c, 0x9a, 0x8c, 0x26, 0x1d, - 0x04, 0xd1, 0x94, 0x25, 0xb1, 0x8b, 0x29, 0x4b, 0x42, 0x3c, 0x46, 0xc8, 0x7c, 0x9d, 0xa6, 0x6b, - 0x8d, 0x8e, 0x1c, 0x62, 0x35, 0x40, 0xf2, 0x18, 0x35, 0xc0, 0xab, 0x90, 0xa5, 0x49, 0x37, 0xdc, - 0xe2, 0xd4, 0x27, 0x04, 0x24, 0x1f, 0xb7, 0x32, 0xc8, 0x23, 0x52, 0x4d, 0xfa, 0x07, 0xa6, 0x9a, - 0x26, 0x9c, 0xdd, 0xd5, 0xdd, 0x66, 0x90, 0x1c, 0x8d, 0xa6, 0xee, 0x35, 0xc3, 0xbd, 0x9e, 0xa1, - 0x7d, 0x04, 0x1d, 0x62, 0xee, 0xea, 0xee, 0x56, 0x40, 0xb3, 0xea, 0x6d, 0x8e, 0xee, 0xfc, 0x05, - 0x35, 0x05, 0xda, 0x86, 0x79, 0xb5, 0xf0, 0x2c, 0x5d, 0x39, 0x9d, 0xb1, 0xbb, 0x8f, 0x94, 0x3c, - 0xab, 0x40, 0xa3, 0xcf, 0x35, 0x28, 0x93, 0xaf, 0xa0, 0x83, 0x3f, 0xf6, 0x4d, 0x07, 0xf7, 0x48, - 0x58, 0x34, 0xed, 0x7d, 0xec, 0x74, 0xf5, 0x43, 0x7e, 0x5c, 0x75, 0x7e, 0x34, 0xe5, 0x6f, 0xda, - 0x46, 0x43, 0x60, 0x60, 0xaf, 0xd6, 0x97, 0x81, 0x77, 0x99, 0x10, 0xf1, 0xd5, 0xd4, 0x14, 0x42, - 0x08, 0xc1, 0x31, 0x06, 0x63, 0x85, 0x23, 0x07, 0x63, 0x2f, 0x42, 0xaa, 0x6f, 0xdb, 0x5d, 0xda, - 0xc6, 0xf1, 0x4a, 0x8f, 0x3c, 0x8b, 0x95, 0x1e, 0x79, 0x16, 0x67, 0x17, 0xeb, 0xa9, 0x5c, 0xae, - 0x94, 0x27, 0x9f, 0xc3, 0xa2, 0x7c, 0xc2, 0x35, 0xba, 0xa1, 0x92, 0x27, 0xbe, 0xa1, 0x52, 0xc7, - 0xb0, 0x46, 0x7a, 0x62, 0x6b, 0x64, 0x26, 0xb7, 0x46, 0xf5, 0x8b, 0x04, 0x4c, 0x4b, 0x87, 0x70, - 0x3f, 0x4d, 0x33, 0xfc, 0x2a, 0x01, 0x0b, 0xea, 0x57, 0x3a, 0x91, 0x56, 0xf4, 0x26, 0x90, 0xa2, - 0xf2, 0x56, 0x54, 0x74, 0xcd, 0x8f, 0x74, 0xa2, 0xd4, 0x9c, 0x41, 0x45, 0x3a, 0x72, 0x8e, 0x17, - 0xb0, 0xa3, 0x07, 0x50, 0x30, 0x85, 0x13, 0xc3, 0xa4, 0xea, 0x60, 0x47, 0x3c, 0x27, 0x64, 0xa3, - 0x8e, 0x31, 0xa7, 0x83, 0xa2, 0xa8, 0x7a, 0x06, 0x52, 0xa4, 0x2a, 0xac, 0xee, 0x43, 0x96, 0x2f, - 0x07, 0xbd, 0x0e, 0x79, 0x9a, 0x8b, 0x69, 0x77, 0xc5, 0x4a, 0x78, 0x5a, 0xde, 0x10, 0x60, 0xec, - 0xc6, 0x4c, 0x2e, 0x80, 0xa1, 0xff, 0x02, 0x20, 0xe9, 0x87, 0x67, 0xe1, 0x04, 0xcd, 0x65, 0xb4, - 0x8b, 0xeb, 0xdb, 0xc6, 0x48, 0xea, 0xcd, 0x87, 0xc0, 0xea, 0xef, 0x13, 0x50, 0x10, 0xcf, 0x28, - 0x1f, 0x4b, 0xf9, 0xa7, 0x10, 0x74, 0xd8, 0x4d, 0xdd, 0x30, 0xc8, 0x5f, 0x1c, 0x7c, 0x28, 0x57, - 0xc6, 0x1a, 0x29, 0xf8, 0x7f, 0x35, 0xe0, 0x60, 0xfd, 0x14, 0xbd, 0x87, 0x61, 0xc6, 0x50, 0x82, - 0xd6, 0x52, 0x1c, 0xb7, 0xb8, 0x07, 0xf3, 0x4a, 0x51, 0x62, 0x17, 0x94, 0x7e, 0x52, 0x5d, 0xd0, - 0x6f, 0xd2, 0x30, 0xaf, 0x3c, 0x1b, 0x8e, 0x45, 0x70, 0xf2, 0x89, 0x44, 0xf0, 0x2f, 0x34, 0x95, - 0x65, 0xd9, 0xc1, 0xd0, 0x5b, 0x13, 0x1c, 0x58, 0x3f, 0x29, 0x1b, 0xcb, 0x61, 0x91, 0x7e, 0xac, - 0x98, 0xcc, 0x4c, 0x1a, 0x93, 0xe8, 0x32, 0x6b, 0x28, 0xa9, 0x2e, 0x76, 0x6c, 0x13, 0xec, 0xd0, - 0x98, 0xaa, 0x2c, 0x07, 0xa1, 0xf7, 0x60, 0x3a, 0xe0, 0x60, 0x63, 0x8c, 0x5c, 0x34, 0x63, 0xe0, - 0x34, 0xf1, 0x49, 0xc6, 0x94, 0x08, 0x17, 0xb2, 0x64, 0xfe, 0x18, 0x59, 0x12, 0x8e, 0xca, 0x92, - 0x4f, 0x35, 0x36, 0xa5, 0x54, 0x3b, 0xd0, 0x60, 0x26, 0x76, 0x25, 0xe3, 0x47, 0xff, 0xcd, 0x91, - 0x5e, 0xf0, 0x33, 0x0d, 0xf2, 0xe1, 0x8d, 0x1f, 0xb4, 0x0a, 0x19, 0xcc, 0x6e, 0x8d, 0xb0, 0xb4, - 0x33, 0x1b, 0xbb, 0xd1, 0x47, 0x70, 0xfc, 0x0e, 0x5f, 0xec, 0xa2, 0x48, 0x83, 0x33, 0x3e, 0x46, - 0x01, 0xfe, 0x07, 0x2d, 0x28, 0xc0, 0x47, 0x56, 0x91, 0xfc, 0xe1, 0xab, 0x38, 0x39, 0xd3, 0x7d, - 0x0f, 0x90, 0xa6, 0x6b, 0x21, 0x8d, 0xb4, 0x87, 0x9d, 0x9e, 0x69, 0xe9, 0x5d, 0x1a, 0x8a, 0x39, - 0xb6, 0xab, 0x03, 0x98, 0xb8, 0xab, 0x03, 0x18, 0xda, 0x85, 0x99, 0x68, 0x3c, 0x47, 0xc5, 0xa8, - 0xaf, 0x10, 0xbe, 0x2f, 0x13, 0xb1, 0xa3, 0x87, 0x18, 0xa7, 0x7c, 0x07, 0x20, 0x86, 0x44, 0x06, - 0x14, 0xdb, 0xb6, 0xe5, 0xe9, 0xa6, 0x85, 0x1d, 0xa6, 0x28, 0xa9, 0xba, 0x42, 0x75, 0x55, 0xa2, - 0x61, 0x43, 0x13, 0x99, 0x4f, 0xbe, 0x42, 0x25, 0xe3, 0xd0, 0x47, 0x30, 0x1d, 0x34, 0x42, 0x4c, - 0x49, 0x4a, 0x75, 0x85, 0x6a, 0x4d, 0x24, 0x61, 0x9b, 0x41, 0xe2, 0x92, 0xaf, 0x50, 0x49, 0x28, - 0xf4, 0x21, 0x4c, 0x75, 0x49, 0x87, 0xb6, 0x76, 0xd0, 0x37, 0x1d, 0x6c, 0xa8, 0x2f, 0xf5, 0xdd, - 0x16, 0x28, 0x58, 0xe2, 0x12, 0x79, 0xe4, 0xbb, 0x0c, 0x22, 0x86, 0xf8, 0xa3, 0xa7, 0x1f, 0x34, - 0x7c, 0xcb, 0x5d, 0x3b, 0xe0, 0x17, 0xb4, 0xb2, 0x2a, 0x7f, 0x6c, 0xc8, 0x44, 0xcc, 0x1f, 0x31, - 0x4e, 0xd9, 0x1f, 0x31, 0x24, 0xba, 0x4d, 0xf3, 0x32, 0x33, 0x12, 0xbb, 0xdc, 0xb7, 0x30, 0x52, - 0x50, 0x31, 0xfb, 0xb0, 0x71, 0x0c, 0x7f, 0x92, 0x84, 0x86, 0x12, 0x50, 0x17, 0x4a, 0x7d, 0xdb, - 0xa0, 0xaf, 0xdd, 0xc0, 0x9e, 0xef, 0x58, 0xd8, 0xe0, 0x8d, 0xd2, 0xd2, 0x88, 0x54, 0x89, 0x8a, - 0x7d, 0xbe, 0xe2, 0xbc, 0xf2, 0x55, 0xcd, 0x38, 0x16, 0x7d, 0x0a, 0x73, 0xb1, 0xab, 0x4a, 0xec, - 0x3d, 0x0a, 0xaa, 0x23, 0x8a, 0x75, 0x05, 0x25, 0xeb, 0x69, 0x55, 0x32, 0x24, 0xcd, 0x4a, 0x2d, - 0x44, 0x7b, 0x47, 0xb7, 0x3a, 0xeb, 0x76, 0x6b, 0xdb, 0xe2, 0x4d, 0xa0, 0xde, 0xea, 0x62, 0x7e, - 0x5b, 0x2f, 0xa6, 0xfd, 0x86, 0x82, 0x92, 0x69, 0x57, 0xc9, 0x90, 0xb5, 0xab, 0x28, 0xc2, 0x6b, - 0x49, 0xa4, 0xac, 0x08, 0xaf, 0xef, 0xa9, 0xae, 0x25, 0x31, 0x02, 0xe1, 0x5a, 0x12, 0x03, 0x28, - 0xae, 0x25, 0x31, 0x04, 0xbb, 0xd1, 0xd6, 0xb6, 0xad, 0xb6, 0xd9, 0x35, 0xe9, 0x88, 0x9b, 0x19, - 0xb5, 0xa8, 0xbe, 0xd1, 0x36, 0x42, 0x18, 0xdc, 0x68, 0x1b, 0x41, 0xc4, 0x6f, 0xb4, 0x8d, 0x10, - 0xa0, 0x9b, 0x50, 0xda, 0xd1, 0xcd, 0xae, 0xef, 0xe0, 0x66, 0x5b, 0xf7, 0x70, 0xc7, 0x76, 0x0e, - 0xf9, 0xc1, 0x1f, 0x8d, 0x6b, 0x8e, 0xbb, 0xca, 0x51, 0xe2, 0x11, 0x67, 0x0c, 0x85, 0xee, 0xc1, - 0x6c, 0x20, 0xc9, 0xf5, 0x5b, 0xa1, 0xb0, 0xd3, 0x54, 0x18, 0xbd, 0x87, 0xcc, 0xd1, 0x5b, 0x11, - 0x56, 0x90, 0x87, 0x46, 0xb1, 0xf5, 0x5c, 0x30, 0xe0, 0x5a, 0x4f, 0xe5, 0xd2, 0xa5, 0xcc, 0x7a, - 0x2a, 0x07, 0xa5, 0x02, 0x3f, 0x57, 0xbc, 0x07, 0x33, 0xb1, 0xcc, 0x88, 0xde, 0x85, 0xf0, 0x56, - 0xd0, 0xfd, 0xc3, 0x7e, 0x50, 0x76, 0x4b, 0xb7, 0x88, 0x08, 0x5c, 0x75, 0x8b, 0x88, 0xc0, 0xab, - 0x5f, 0xa6, 0x20, 0x17, 0x6c, 0xbd, 0x13, 0x69, 0xa4, 0x56, 0x20, 0xdb, 0xc3, 0x2e, 0xbd, 0xf9, - 0x93, 0x88, 0xea, 0x31, 0x0e, 0x12, 0xeb, 0x31, 0x0e, 0x92, 0xcb, 0xc5, 0xe4, 0x63, 0x95, 0x8b, - 0xa9, 0x89, 0xcb, 0x45, 0x4c, 0x0f, 0xbb, 0x85, 0x94, 0x1e, 0x9c, 0x11, 0x3d, 0xfa, 0x3b, 0x11, - 0x1c, 0x85, 0x8b, 0x8c, 0xb1, 0xa3, 0x70, 0x11, 0x85, 0xf6, 0xe0, 0xb4, 0x70, 0x8e, 0xc5, 0x07, - 0x98, 0x24, 0x95, 0x17, 0xc7, 0xdf, 0x2c, 0x68, 0x50, 0x2a, 0x96, 0xb0, 0xf6, 0x62, 0x50, 0xb1, - 0xde, 0x8e, 0xe3, 0x48, 0x48, 0x18, 0xb8, 0xe5, 0x77, 0x36, 0xb8, 0xd9, 0xb3, 0x51, 0x48, 0x88, - 0x70, 0x31, 0x24, 0x44, 0x78, 0xf5, 0x9f, 0x09, 0x28, 0xca, 0xef, 0x7b, 0x22, 0x81, 0xf1, 0x3a, - 0xe4, 0xf1, 0x81, 0xe9, 0x35, 0xdb, 0xb6, 0x81, 0x79, 0xd3, 0x49, 0xfd, 0x4c, 0x80, 0x57, 0x6d, - 0x43, 0xf2, 0x73, 0x00, 0x13, 0xa3, 0x29, 0x39, 0x51, 0x34, 0x45, 0xf3, 0xe2, 0xd4, 0x04, 0xf3, - 0x62, 0xa5, 0x9f, 0xf2, 0x27, 0xe3, 0xa7, 0xea, 0x37, 0x09, 0x28, 0xc5, 0xbf, 0x4f, 0xcf, 0xc6, - 0x16, 0x94, 0x77, 0x53, 0x72, 0xe2, 0xdd, 0xf4, 0x1e, 0x4c, 0x93, 0xa2, 0x52, 0xf7, 0x3c, 0x7e, - 0x51, 0x38, 0x45, 0xeb, 0x42, 0x96, 0x8d, 0x7c, 0x6b, 0x35, 0x80, 0x4b, 0xd9, 0x48, 0x80, 0x8f, - 0x84, 0x6e, 0xfa, 0x98, 0xa1, 0xfb, 0x79, 0x02, 0xa6, 0x37, 0x6d, 0xe3, 0x3e, 0xab, 0x37, 0xbd, - 0x67, 0xc5, 0x9e, 0x4f, 0x33, 0xa5, 0x55, 0x67, 0x60, 0x5a, 0x2a, 0x38, 0xab, 0x5f, 0xb0, 0x38, - 0x93, 0xbf, 0xeb, 0x3f, 0x3d, 0xbb, 0x14, 0x61, 0x4a, 0xac, 0x93, 0xab, 0x75, 0x98, 0x89, 0x95, - 0xb5, 0xe2, 0x0b, 0x68, 0x93, 0xbc, 0x40, 0xf5, 0x1a, 0xcc, 0xa9, 0xea, 0x3d, 0x21, 0xeb, 0x68, - 0x13, 0x1c, 0x72, 0xdd, 0x80, 0x39, 0x55, 0xdd, 0x76, 0xfc, 0xe5, 0xbc, 0xcb, 0x0f, 0x90, 0x79, - 0x85, 0x75, 0x6c, 0xfe, 0xeb, 0x30, 0xab, 0xa8, 0xb4, 0x8e, 0x2f, 0xe7, 0xaf, 0xe1, 0x00, 0x21, - 0xba, 0xdc, 0x7f, 0x1d, 0x4a, 0xfd, 0xe0, 0xa1, 0xc9, 0xdb, 0xd4, 0x74, 0x74, 0xdb, 0x2a, 0xc4, - 0xad, 0xc7, 0xfa, 0xd5, 0xa2, 0x8c, 0x91, 0xe5, 0xf0, 0x16, 0x36, 0xa3, 0x90, 0xd3, 0x88, 0xf5, - 0xb2, 0x45, 0x19, 0x23, 0xb8, 0x28, 0x7b, 0xb4, 0x8b, 0x68, 0x0b, 0x9c, 0xae, 0x7e, 0xa6, 0xc1, - 0x4c, 0xec, 0xc7, 0x07, 0xe8, 0x32, 0xe4, 0xe8, 0x2f, 0x03, 0xa3, 0xe6, 0x9f, 0x5a, 0x87, 0xc2, - 0xa4, 0x05, 0x64, 0x39, 0x08, 0xbd, 0x01, 0xf9, 0xf0, 0xf7, 0x08, 0xfc, 0x08, 0x9a, 0xc5, 0x6f, - 0x00, 0x94, 0xe2, 0x37, 0x00, 0xf2, 0xb9, 0xc1, 0xff, 0xc3, 0xd9, 0xb1, 0xbf, 0x44, 0x38, 0xd6, - 0x71, 0x67, 0x34, 0x00, 0x48, 0x1d, 0x6b, 0x00, 0x70, 0x00, 0x0b, 0xea, 0x1f, 0x08, 0x08, 0xda, - 0x13, 0x47, 0x6a, 0x8f, 0xac, 0x9f, 0x9c, 0xd0, 0xfa, 0x89, 0xea, 0x1e, 0x9d, 0x98, 0x84, 0x17, - 0xf1, 0xd1, 0x25, 0x48, 0xf7, 0x6d, 0xbb, 0xeb, 0xf2, 0x3b, 0x1e, 0x54, 0x1d, 0x05, 0x88, 0xea, - 0x28, 0xe0, 0x31, 0xe6, 0x33, 0x7e, 0x10, 0xc1, 0xd1, 0xcf, 0x0a, 0x9e, 0x82, 0x75, 0x5f, 0xbe, - 0x0c, 0xb9, 0xe0, 0x1c, 0x1d, 0x01, 0x64, 0xee, 0x6d, 0xaf, 0x6d, 0xaf, 0x5d, 0x2b, 0x9d, 0x42, - 0x05, 0xc8, 0x6e, 0xae, 0xdd, 0xb9, 0x76, 0xeb, 0xce, 0x8d, 0x92, 0x46, 0x1e, 0x1a, 0xdb, 0x77, - 0xee, 0x90, 0x87, 0xc4, 0xcb, 0xb7, 0xc5, 0xbb, 0x79, 0xbc, 0x02, 0x9c, 0x82, 0xdc, 0x6a, 0xbf, - 0x4f, 0x37, 0x2f, 0xe3, 0x5d, 0xdb, 0x37, 0x49, 0x46, 0x28, 0x69, 0x28, 0x0b, 0xc9, 0xbb, 0x77, - 0x37, 0x4a, 0x09, 0x34, 0x07, 0xa5, 0x6b, 0x58, 0x37, 0xba, 0xa6, 0x85, 0x83, 0xfc, 0x57, 0x4a, - 0xd6, 0x1f, 0xfe, 0xe9, 0xdb, 0x25, 0xed, 0x9b, 0x6f, 0x97, 0xb4, 0x7f, 0x7c, 0xbb, 0xa4, 0x7d, - 0xf9, 0xdd, 0xd2, 0xa9, 0x6f, 0xbe, 0x5b, 0x3a, 0xf5, 0xb7, 0xef, 0x96, 0x4e, 0xfd, 0xdf, 0xe5, - 0x8e, 0xe9, 0xed, 0xfa, 0xad, 0x5a, 0xdb, 0xee, 0xf1, 0x9f, 0x38, 0xf7, 0x1d, 0x9b, 0x24, 0x1a, - 0xfe, 0xb4, 0x12, 0xff, 0xed, 0xf3, 0xef, 0x12, 0xe7, 0x56, 0xe9, 0xe3, 0x26, 0xa3, 0xab, 0xdd, - 0xb2, 0x6b, 0x0c, 0x40, 0x7f, 0xed, 0xea, 0xb6, 0x32, 0xf4, 0x57, 0xad, 0xaf, 0xff, 0x3b, 0x00, - 0x00, 0xff, 0xff, 0xd3, 0x8d, 0xc0, 0xbf, 0x36, 0x3d, 0x00, 0x00, + // 3820 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x3b, 0x4b, 0x6f, 0x1b, 0xd7, + 0x7a, 0x1e, 0xbe, 0xf9, 0x51, 0xa2, 0xe8, 0xa3, 0x87, 0x69, 0x25, 0x16, 0x65, 0x3a, 0x4d, 0xec, + 0x20, 0xa1, 0x1c, 0xa7, 0x29, 0xf2, 0x28, 0x12, 0x88, 0xb6, 0xfc, 0x60, 0x2c, 0x5b, 0xa6, 0xac, + 0xd4, 0x2d, 0x02, 0x30, 0x43, 0xce, 0x11, 0x35, 0x12, 0x39, 0xc3, 0xcc, 0x43, 0x91, 0x80, 0x00, + 0x4d, 0x8a, 0xb4, 0xeb, 0x6c, 0x0a, 0x14, 0xd9, 0x34, 0x8b, 0x76, 0xd1, 0x02, 0x5d, 0xf6, 0x3f, + 0x14, 0x68, 0x51, 0x64, 0x73, 0x81, 0xbb, 0x22, 0x2e, 0x12, 0xdc, 0x0d, 0x17, 0xf7, 0x07, 0x64, + 0x75, 0x71, 0x1e, 0x33, 0x73, 0xce, 0xf0, 0xd0, 0xa2, 0x1c, 0x2b, 0x48, 0x90, 0x95, 0x34, 0xdf, + 0xf3, 0xcc, 0xf9, 0xbe, 0xf3, 0xcd, 0xf7, 0x38, 0x84, 0x4b, 0x83, 0x83, 0xee, 0x9a, 0xee, 0xf4, + 0x75, 0x43, 0xc7, 0x87, 0xd8, 0xf2, 0xdc, 0x35, 0xf6, 0xa7, 0x36, 0x70, 0x6c, 0xcf, 0x46, 0x33, + 0x22, 0x6a, 0xb9, 0x7a, 0xf0, 0xb6, 0x5b, 0x33, 0xed, 0x35, 0x7d, 0x60, 0xae, 0x75, 0x6c, 0x07, + 0xaf, 0x1d, 0xbe, 0xb1, 0xd6, 0xc5, 0x16, 0x76, 0x74, 0x0f, 0x1b, 0x8c, 0x63, 0xf9, 0xaa, 0x40, + 0x63, 0x61, 0xef, 0x33, 0xdb, 0x39, 0x30, 0xad, 0xae, 0x8a, 0xb2, 0xd2, 0xb5, 0xed, 0x6e, 0x0f, + 0xaf, 0xd1, 0xa7, 0xb6, 0xbf, 0xbb, 0xe6, 0x99, 0x7d, 0xec, 0x7a, 0x7a, 0x7f, 0xc0, 0x09, 0xfe, + 0x32, 0x12, 0xd5, 0xd7, 0x3b, 0x7b, 0xa6, 0x85, 0x9d, 0xe3, 0x35, 0xba, 0xde, 0x81, 0xb9, 0xe6, + 0x60, 0xd7, 0xf6, 0x9d, 0x0e, 0x1e, 0x13, 0xfb, 0xae, 0x69, 0x79, 0xd8, 0xb1, 0xf4, 0xde, 0x9a, + 0xdb, 0xd9, 0xc3, 0x86, 0xdf, 0xc3, 0x4e, 0xf4, 0x9f, 0xdd, 0xde, 0xc7, 0x1d, 0xcf, 0x1d, 0x03, + 0x30, 0xde, 0xea, 0xff, 0x2e, 0xc1, 0xec, 0x06, 0x79, 0xd7, 0x6d, 0xfc, 0xa9, 0x8f, 0xad, 0x0e, + 0x46, 0xd7, 0x20, 0xfd, 0xa9, 0x8f, 0x7d, 0x5c, 0xd6, 0x56, 0xb5, 0xab, 0xf9, 0xfa, 0xfc, 0x68, + 0x58, 0x99, 0xa3, 0x80, 0xd7, 0xec, 0xbe, 0xe9, 0xe1, 0xfe, 0xc0, 0x3b, 0x6e, 0x32, 0x0a, 0xf4, + 0x2e, 0xcc, 0xec, 0xdb, 0xed, 0x96, 0x8b, 0xbd, 0x96, 0xa5, 0xf7, 0x71, 0x39, 0x41, 0x39, 0xca, + 0xa3, 0x61, 0x65, 0x61, 0xdf, 0x6e, 0x6f, 0x63, 0xef, 0x81, 0xde, 0x17, 0xd9, 0x20, 0x82, 0xa2, + 0xd7, 0x21, 0xeb, 0xbb, 0xd8, 0x69, 0x99, 0x46, 0x39, 0x49, 0xd9, 0x16, 0x46, 0xc3, 0x4a, 0x89, + 0x80, 0xee, 0x19, 0x02, 0x4b, 0x86, 0x41, 0xd0, 0x6b, 0x90, 0xe9, 0x3a, 0xb6, 0x3f, 0x70, 0xcb, + 0xa9, 0xd5, 0x64, 0x40, 0xcd, 0x20, 0x22, 0x35, 0x83, 0xa0, 0x87, 0x90, 0x61, 0x06, 0x2c, 0xa7, + 0x57, 0x93, 0x57, 0x0b, 0x37, 0x2e, 0xd7, 0x44, 0xab, 0xd6, 0xa4, 0x17, 0x66, 0x4f, 0x4c, 0x20, + 0xc3, 0x8b, 0x02, 0xb9, 0x1f, 0xfc, 0xdb, 0x02, 0xa4, 0x29, 0x1d, 0xfa, 0x10, 0xb2, 0x1d, 0x07, + 0x93, 0xdd, 0x2f, 0xa3, 0x55, 0xed, 0x6a, 0xe1, 0xc6, 0x72, 0x8d, 0x59, 0xb5, 0x16, 0x58, 0xb5, + 0xf6, 0x38, 0xb0, 0x6a, 0x7d, 0x71, 0x34, 0xac, 0x9c, 0xe7, 0xe4, 0x82, 0xd4, 0x40, 0x02, 0xda, + 0x82, 0xbc, 0xeb, 0xb7, 0xfb, 0xa6, 0xd7, 0xb0, 0xdb, 0x74, 0xbf, 0x0b, 0x37, 0x2e, 0xc8, 0x4b, + 0xdd, 0x0e, 0xd0, 0xf5, 0x0b, 0xa3, 0x61, 0x65, 0x3e, 0xa4, 0x8e, 0xa4, 0xdd, 0x3d, 0xd7, 0x8c, + 0x84, 0xa0, 0x3d, 0x98, 0x73, 0xf0, 0xc0, 0x31, 0x6d, 0xc7, 0xf4, 0x4c, 0x17, 0x13, 0xb9, 0x09, + 0x2a, 0xf7, 0x92, 0x2c, 0xb7, 0x29, 0x13, 0xd5, 0x2f, 0x8d, 0x86, 0x95, 0x8b, 0x31, 0x4e, 0x49, + 0x47, 0x5c, 0x2c, 0xf2, 0x00, 0xc5, 0x40, 0xdb, 0xd8, 0xa3, 0xb6, 0x2c, 0xdc, 0x58, 0x7d, 0xaa, + 0xb2, 0x6d, 0xec, 0xd5, 0x57, 0x47, 0xc3, 0xca, 0x8b, 0xe3, 0xfc, 0x92, 0x4a, 0x85, 0x7c, 0xd4, + 0x83, 0x92, 0x08, 0x35, 0xc8, 0x0b, 0xa6, 0xa8, 0xce, 0x95, 0xc9, 0x3a, 0x09, 0x55, 0x7d, 0x65, + 0x34, 0xac, 0x2c, 0xc7, 0x79, 0x25, 0x7d, 0x63, 0x92, 0x89, 0x7d, 0x3a, 0xba, 0xd5, 0xc1, 0x3d, + 0xa2, 0x26, 0xad, 0xb2, 0xcf, 0xcd, 0x00, 0xcd, 0xec, 0x13, 0x52, 0xcb, 0xf6, 0x09, 0xc1, 0xe8, + 0x63, 0x98, 0x09, 0x1f, 0xc8, 0x7e, 0x65, 0xb8, 0x0f, 0xa9, 0x85, 0x92, 0x9d, 0x5a, 0x1e, 0x0d, + 0x2b, 0x4b, 0x22, 0x8f, 0x24, 0x5a, 0x92, 0x16, 0x49, 0xef, 0xb1, 0x9d, 0xc9, 0x4e, 0x96, 0xce, + 0x28, 0x44, 0xe9, 0xbd, 0xf1, 0x1d, 0x91, 0xa4, 0x11, 0xe9, 0xe4, 0x00, 0xfb, 0x9d, 0x0e, 0xc6, + 0x06, 0x36, 0xca, 0x39, 0x95, 0xf4, 0x86, 0x40, 0xc1, 0xa4, 0x8b, 0x3c, 0xb2, 0x74, 0x11, 0x43, + 0xf6, 0x7a, 0xdf, 0x6e, 0x6f, 0x38, 0x8e, 0xed, 0xb8, 0xe5, 0xbc, 0x6a, 0xaf, 0x1b, 0x01, 0x9a, + 0xed, 0x75, 0x48, 0x2d, 0xef, 0x75, 0x08, 0xe6, 0xeb, 0x6d, 0xfa, 0xd6, 0x7d, 0xac, 0xbb, 0xd8, + 0x28, 0xc3, 0x84, 0xf5, 0x86, 0x14, 0xe1, 0x7a, 0x43, 0xc8, 0xd8, 0x7a, 0x43, 0x0c, 0x32, 0xa0, + 0xc8, 0x9e, 0xd7, 0x5d, 0xd7, 0xec, 0x5a, 0xd8, 0x28, 0x17, 0xa8, 0xfc, 0x17, 0x55, 0xf2, 0x03, + 0x9a, 0xfa, 0x8b, 0xa3, 0x61, 0xa5, 0x2c, 0xf3, 0x49, 0x3a, 0x62, 0x32, 0xd1, 0x27, 0x30, 0xcb, + 0x20, 0x4d, 0xdf, 0xb2, 0x4c, 0xab, 0x5b, 0x9e, 0xa1, 0x4a, 0x5e, 0x50, 0x29, 0xe1, 0x24, 0xf5, + 0x17, 0x46, 0xc3, 0xca, 0x05, 0x89, 0x4b, 0x52, 0x21, 0x0b, 0x24, 0x11, 0x83, 0x01, 0x22, 0xc3, + 0xce, 0xaa, 0x22, 0x46, 0x43, 0x26, 0x62, 0x11, 0x23, 0xc6, 0x29, 0x47, 0x8c, 0x18, 0x32, 0xb2, + 0x07, 0x37, 0x72, 0x71, 0xb2, 0x3d, 0xb8, 0x9d, 0x05, 0x7b, 0x28, 0x4c, 0x2d, 0x49, 0x43, 0x5f, + 0x68, 0xb0, 0xe8, 0x7a, 0xba, 0x65, 0xe8, 0x3d, 0xdb, 0xc2, 0xf7, 0xac, 0xae, 0x83, 0x5d, 0xf7, + 0x9e, 0xb5, 0x6b, 0x97, 0x4b, 0x54, 0xcf, 0x95, 0x58, 0x60, 0x55, 0x91, 0xd6, 0xaf, 0x8c, 0x86, + 0x95, 0x8a, 0x52, 0x8a, 0xa4, 0x59, 0xad, 0x08, 0x1d, 0xc1, 0x7c, 0xf0, 0x91, 0xde, 0xf1, 0xcc, + 0x9e, 0xe9, 0xea, 0x9e, 0x69, 0x5b, 0xe5, 0xf3, 0x54, 0xff, 0xe5, 0x78, 0x7c, 0x1a, 0x23, 0xac, + 0x5f, 0x1e, 0x0d, 0x2b, 0x97, 0x14, 0x12, 0x24, 0xdd, 0x2a, 0x15, 0x91, 0x11, 0xb7, 0x1c, 0x4c, + 0x08, 0xb1, 0x51, 0x9e, 0x9f, 0x6c, 0xc4, 0x90, 0x48, 0x34, 0x62, 0x08, 0x54, 0x19, 0x31, 0x44, + 0x12, 0x4d, 0x03, 0xdd, 0xf1, 0x4c, 0xa2, 0x76, 0x53, 0x77, 0x0e, 0xb0, 0x53, 0x5e, 0x50, 0x69, + 0xda, 0x92, 0x89, 0x98, 0xa6, 0x18, 0xa7, 0xac, 0x29, 0x86, 0x44, 0x5f, 0x6b, 0x20, 0x2f, 0xcd, + 0xb4, 0xad, 0x26, 0xf9, 0x68, 0xbb, 0xe4, 0xf5, 0x16, 0xa9, 0xd2, 0x57, 0x9e, 0xf2, 0x7a, 0x22, + 0x79, 0xfd, 0x95, 0xd1, 0xb0, 0x72, 0x65, 0xa2, 0x34, 0x69, 0x21, 0x93, 0x95, 0xa2, 0x27, 0x50, + 0x20, 0x48, 0x4c, 0xd3, 0x1f, 0xa3, 0xbc, 0x44, 0xd7, 0x70, 0x71, 0x7c, 0x0d, 0x9c, 0xa0, 0x7e, + 0x71, 0x34, 0xac, 0x2c, 0x0a, 0x1c, 0x92, 0x1e, 0x51, 0x14, 0xfa, 0x4a, 0x03, 0xe2, 0xe8, 0xaa, + 0x37, 0xbd, 0x40, 0xb5, 0xbc, 0x34, 0xa6, 0x45, 0xf5, 0x9a, 0x2f, 0x8d, 0x86, 0x95, 0x55, 0xb5, + 0x1c, 0x49, 0xf7, 0x04, 0x5d, 0x91, 0x1f, 0x85, 0x1f, 0x89, 0x72, 0x79, 0xb2, 0x1f, 0x85, 0x44, + 0xa2, 0x1f, 0x85, 0x40, 0x95, 0x1f, 0x85, 0x48, 0x1e, 0x0c, 0x3e, 0xd2, 0x7b, 0xa6, 0x41, 0x93, + 0xa9, 0x8b, 0x13, 0x82, 0x41, 0x48, 0x11, 0x06, 0x83, 0x10, 0x32, 0x16, 0x0c, 0x42, 0x0c, 0x49, + 0x13, 0x98, 0xc2, 0xc7, 0xd8, 0xe9, 0x9b, 0x16, 0xd5, 0xb0, 0xac, 0x4a, 0x13, 0x1a, 0x31, 0x2a, + 0x96, 0x26, 0xc4, 0x79, 0xe5, 0x34, 0x21, 0x8e, 0xad, 0x67, 0x21, 0x4d, 0xc5, 0x55, 0xbf, 0xc9, + 0xc3, 0xbc, 0xe2, 0x60, 0x23, 0x0c, 0xb3, 0xc1, 0xa9, 0x6d, 0x99, 0x24, 0x24, 0x25, 0x55, 0x36, + 0xfd, 0xd0, 0x6f, 0x63, 0xc7, 0xc2, 0x1e, 0x76, 0x03, 0x19, 0x34, 0x26, 0xd1, 0xf7, 0x76, 0x04, + 0x88, 0x90, 0x49, 0xce, 0x88, 0x70, 0xf4, 0x8d, 0x06, 0xe5, 0xbe, 0x7e, 0xd4, 0x0a, 0x80, 0x6e, + 0x6b, 0xd7, 0x76, 0x5a, 0x03, 0xec, 0x98, 0xb6, 0x41, 0xf3, 0xe6, 0xc2, 0x8d, 0xbf, 0x3e, 0x31, + 0x0a, 0xd5, 0x36, 0xf5, 0xa3, 0x00, 0xec, 0xde, 0xb6, 0x9d, 0x2d, 0xca, 0xbe, 0x61, 0x79, 0xce, + 0x31, 0x0b, 0x8f, 0x7d, 0x15, 0x5e, 0x58, 0xd3, 0xa2, 0x92, 0x00, 0xfd, 0xb3, 0x06, 0x4b, 0x9e, + 0xed, 0xe9, 0xbd, 0x56, 0xc7, 0xef, 0xfb, 0x3d, 0xdd, 0x33, 0x0f, 0x71, 0xcb, 0x77, 0xf5, 0x2e, + 0xe6, 0x49, 0xfa, 0x7b, 0x27, 0x2f, 0xed, 0x31, 0xe1, 0xbf, 0x19, 0xb2, 0xef, 0x10, 0x6e, 0xb6, + 0xb2, 0xea, 0x68, 0x58, 0x59, 0xf1, 0x14, 0x68, 0x61, 0x61, 0x0b, 0x2a, 0x3c, 0x7a, 0x15, 0x32, + 0xa4, 0x88, 0x31, 0x0d, 0x9a, 0x8b, 0xf1, 0x82, 0x67, 0xdf, 0x6e, 0x4b, 0x65, 0x48, 0x9a, 0x02, + 0x08, 0xad, 0xe3, 0x5b, 0x84, 0x36, 0x1b, 0xd1, 0x3a, 0xbe, 0x25, 0xd3, 0x52, 0x00, 0x35, 0x86, + 0x7e, 0xd8, 0x55, 0x1b, 0x23, 0x37, 0xad, 0x31, 0xd6, 0x0f, 0xbb, 0x4f, 0x35, 0x86, 0xae, 0xc2, + 0x8b, 0xc6, 0x50, 0x12, 0x2c, 0x7f, 0xab, 0xc1, 0xf2, 0x64, 0x3b, 0xa3, 0x2b, 0x90, 0x3c, 0xc0, + 0xc7, 0xbc, 0x02, 0x3c, 0x3f, 0x1a, 0x56, 0x66, 0x0f, 0xf0, 0xb1, 0x20, 0x95, 0x60, 0xd1, 0xdf, + 0x42, 0xfa, 0x50, 0xef, 0xf9, 0x98, 0x17, 0x18, 0xb5, 0x1a, 0x2b, 0x5e, 0x6b, 0x62, 0xf1, 0x5a, + 0x1b, 0x1c, 0x74, 0x09, 0xa0, 0x16, 0xec, 0x42, 0xed, 0x91, 0xaf, 0x5b, 0x9e, 0xe9, 0x1d, 0xb3, + 0xbd, 0xa3, 0x02, 0xc4, 0xbd, 0xa3, 0x80, 0x77, 0x13, 0x6f, 0x6b, 0xcb, 0xff, 0xaa, 0xc1, 0xc5, + 0x89, 0xf6, 0xfe, 0x45, 0xac, 0x90, 0x6c, 0xe2, 0x64, 0xfb, 0xfc, 0x12, 0x96, 0xd8, 0x48, 0xe5, + 0xb4, 0x52, 0xa2, 0x91, 0xca, 0x25, 0x4a, 0xc9, 0xea, 0x8f, 0x19, 0xc8, 0x87, 0xe5, 0x24, 0xba, + 0x0b, 0x25, 0x03, 0x1b, 0xfe, 0xa0, 0x67, 0x76, 0xa8, 0xa7, 0x11, 0xa7, 0x66, 0xf5, 0x3b, 0x8d, + 0xe5, 0x12, 0x4e, 0x72, 0xef, 0xb9, 0x18, 0x0a, 0xdd, 0x80, 0x1c, 0x2f, 0x9b, 0x8e, 0x69, 0x5c, + 0x9b, 0xad, 0x2f, 0x8d, 0x86, 0x15, 0x14, 0xc0, 0x04, 0xd6, 0x90, 0x0e, 0x35, 0x01, 0x58, 0x1f, + 0x62, 0x13, 0x7b, 0x3a, 0x2f, 0xe0, 0xca, 0xf2, 0x69, 0x78, 0x18, 0xe2, 0x59, 0x47, 0x21, 0xa2, + 0x17, 0x3b, 0x0a, 0x11, 0x14, 0x7d, 0x0c, 0xd0, 0xd7, 0x4d, 0x8b, 0xf1, 0xf1, 0x6a, 0xad, 0x3a, + 0x29, 0xc2, 0x6e, 0x86, 0x94, 0x4c, 0x7a, 0xc4, 0x29, 0x4a, 0x8f, 0xa0, 0xe8, 0x21, 0x64, 0x79, + 0xe7, 0xa4, 0x9c, 0xa1, 0x87, 0x77, 0x65, 0x92, 0x68, 0x2e, 0x96, 0xd6, 0xfe, 0x9c, 0x45, 0xac, + 0xfd, 0x39, 0x88, 0x6c, 0x5b, 0xcf, 0xdc, 0xc5, 0x9e, 0xd9, 0xc7, 0x34, 0x9a, 0xf0, 0x6d, 0x0b, + 0x60, 0xe2, 0xb6, 0x05, 0x30, 0xf4, 0x36, 0x80, 0xee, 0x6d, 0xda, 0xae, 0xf7, 0xd0, 0xea, 0x60, + 0x5a, 0x7f, 0xe5, 0xd8, 0xf2, 0x23, 0xa8, 0xb8, 0xfc, 0x08, 0x8a, 0xde, 0x83, 0xc2, 0x80, 0x7f, + 0xef, 0xdb, 0x3d, 0x4c, 0xeb, 0xab, 0x1c, 0x4b, 0x4f, 0x04, 0xb0, 0xc0, 0x2b, 0x52, 0xa3, 0x3b, + 0x30, 0xd7, 0xb1, 0xad, 0x8e, 0xef, 0x38, 0xd8, 0xea, 0x1c, 0x6f, 0xeb, 0xbb, 0x98, 0xd6, 0x52, + 0x39, 0xe6, 0x2a, 0x31, 0x94, 0xe8, 0x2a, 0x31, 0x14, 0x7a, 0x0b, 0xf2, 0x61, 0x1f, 0x8a, 0x96, + 0x4b, 0x79, 0xde, 0xd6, 0x08, 0x80, 0x02, 0x73, 0x44, 0x49, 0x16, 0x6f, 0xba, 0xb7, 0xb8, 0xd3, + 0x61, 0x5a, 0x02, 0xf1, 0xc5, 0x0b, 0x60, 0x71, 0xf1, 0x02, 0x58, 0x88, 0xef, 0xc5, 0x13, 0xe3, + 0xfb, 0x6d, 0x28, 0xe1, 0x23, 0xd6, 0x4b, 0x6b, 0x11, 0x26, 0xdf, 0x31, 0x69, 0xf5, 0x90, 0x67, + 0x75, 0x5b, 0x80, 0x6b, 0xd8, 0xed, 0x1d, 0xc7, 0x14, 0xd8, 0x8b, 0x32, 0x26, 0x3c, 0x76, 0xb3, + 0xa5, 0x62, 0x23, 0x95, 0x9b, 0x2b, 0x95, 0xaa, 0xff, 0xa7, 0xc1, 0x82, 0xca, 0xfb, 0x62, 0x27, + 0x41, 0x7b, 0x2e, 0x27, 0xe1, 0x23, 0xc8, 0x0d, 0x6c, 0xa3, 0xe5, 0x0e, 0x70, 0x87, 0xc7, 0x95, + 0xd8, 0x39, 0xd8, 0xb2, 0x8d, 0xed, 0x01, 0xee, 0xfc, 0x8d, 0xe9, 0xed, 0xad, 0x1f, 0xda, 0xa6, + 0x71, 0xdf, 0x74, 0xb9, 0xc3, 0x0e, 0x18, 0x46, 0x4a, 0x78, 0xb2, 0x1c, 0x58, 0xcf, 0x41, 0x86, + 0x69, 0xa9, 0xfe, 0x7f, 0x12, 0x4a, 0x71, 0x8f, 0xff, 0x35, 0xbd, 0x0a, 0x7a, 0x02, 0x59, 0x93, + 0x55, 0x6e, 0x3c, 0x17, 0xfb, 0x0b, 0x21, 0xf2, 0xd6, 0xa2, 0x36, 0x6e, 0xed, 0xf0, 0x8d, 0x1a, + 0x2f, 0xf1, 0xe8, 0x16, 0x50, 0xc9, 0x9c, 0x53, 0x96, 0xcc, 0x81, 0xa8, 0x09, 0x59, 0x17, 0x3b, + 0x87, 0x66, 0x07, 0xf3, 0xb8, 0x56, 0x11, 0x25, 0x77, 0x6c, 0x07, 0x13, 0x99, 0xdb, 0x8c, 0x24, + 0x92, 0xc9, 0x79, 0x64, 0x99, 0x1c, 0x88, 0x3e, 0x82, 0x7c, 0xc7, 0xb6, 0x76, 0xcd, 0xee, 0xa6, + 0x3e, 0xe0, 0x91, 0xed, 0x92, 0x4a, 0xea, 0xcd, 0x80, 0x88, 0x77, 0xa3, 0x82, 0xc7, 0x58, 0x37, + 0x2a, 0xa4, 0x8a, 0x0c, 0xfa, 0xa7, 0x14, 0x40, 0x64, 0x1c, 0xf4, 0x0e, 0x14, 0xf0, 0x11, 0xee, + 0xf8, 0x9e, 0x4d, 0x3b, 0xb4, 0x5a, 0xd4, 0xd8, 0x0d, 0xc0, 0xd2, 0xf1, 0x81, 0x08, 0x4a, 0xce, + 0xb8, 0xa5, 0xf7, 0xb1, 0x3b, 0xd0, 0x3b, 0x41, 0x47, 0x98, 0x2e, 0x26, 0x04, 0x8a, 0x67, 0x3c, + 0x04, 0xa2, 0x97, 0x21, 0x45, 0x7b, 0xc8, 0xac, 0x19, 0x8c, 0x46, 0xc3, 0x4a, 0xd1, 0x92, 0xbb, + 0xc7, 0x14, 0x8f, 0x3e, 0x80, 0xd9, 0x83, 0xd0, 0xf1, 0xc8, 0xda, 0x52, 0x94, 0x81, 0x26, 0xc9, + 0x11, 0x42, 0x5a, 0xdd, 0x8c, 0x08, 0x47, 0xbb, 0x50, 0xd0, 0x2d, 0xcb, 0xf6, 0xe8, 0xe7, 0x2b, + 0x68, 0x10, 0x5f, 0x9b, 0xe4, 0xa6, 0xb5, 0xf5, 0x88, 0x96, 0xa5, 0x5d, 0x34, 0xee, 0x08, 0x12, + 0xc4, 0xb8, 0x23, 0x80, 0x51, 0x13, 0x32, 0x3d, 0xbd, 0x8d, 0x7b, 0xc1, 0xf7, 0xe2, 0xa5, 0x89, + 0x2a, 0xee, 0x53, 0x32, 0x26, 0x9d, 0xb6, 0xa1, 0x19, 0x9f, 0xd8, 0x86, 0x66, 0x90, 0xe5, 0x5d, + 0x28, 0xc5, 0xd7, 0x33, 0x5d, 0x9a, 0x71, 0x4d, 0x4c, 0x33, 0xf2, 0x27, 0x66, 0x36, 0x3a, 0x14, + 0x84, 0x45, 0x9d, 0x85, 0x8a, 0xea, 0x7f, 0x68, 0xb0, 0xa0, 0x3a, 0xbb, 0x68, 0x53, 0x38, 0xf1, + 0x1a, 0x6f, 0x76, 0x29, 0x5c, 0x9d, 0xf3, 0x4e, 0x38, 0xea, 0xd1, 0x41, 0xaf, 0x43, 0xd1, 0xb2, + 0x0d, 0xdc, 0xd2, 0x89, 0x82, 0x9e, 0xe9, 0x7a, 0xe5, 0x04, 0x1d, 0x20, 0xd0, 0x26, 0x19, 0xc1, + 0xac, 0x07, 0x08, 0x81, 0x7b, 0x56, 0x42, 0x54, 0x3f, 0x83, 0xb9, 0x58, 0x0b, 0x5b, 0x4a, 0x7a, + 0x12, 0x53, 0x26, 0x3d, 0xd1, 0x97, 0x28, 0x79, 0xd2, 0x97, 0x88, 0x7d, 0x41, 0xaa, 0xff, 0x98, + 0x80, 0x82, 0xd0, 0x4f, 0x40, 0xfb, 0x30, 0xc7, 0xbf, 0x8a, 0xa6, 0xd5, 0x65, 0x95, 0x64, 0x82, + 0x37, 0xb7, 0xc6, 0xe6, 0x3b, 0x0d, 0xbb, 0xbd, 0x1d, 0xd2, 0xd2, 0x42, 0x92, 0x7e, 0xc3, 0x5c, + 0x09, 0x26, 0x7e, 0xc3, 0x64, 0x0c, 0x7a, 0x02, 0x4b, 0xfe, 0x80, 0x54, 0xd3, 0x2d, 0x97, 0x4f, + 0x4a, 0x5a, 0x96, 0xdf, 0x6f, 0x63, 0x87, 0xae, 0x3e, 0xcd, 0x2a, 0x2e, 0x46, 0x11, 0x8c, 0x52, + 0x1e, 0x50, 0xbc, 0x58, 0x71, 0xa9, 0xf0, 0xc2, 0x3e, 0xa4, 0xa6, 0xdc, 0x87, 0xbb, 0x80, 0xc6, + 0x67, 0x08, 0x92, 0x0d, 0xb4, 0xe9, 0x6c, 0x50, 0x3d, 0x82, 0x52, 0x7c, 0x32, 0xf0, 0x33, 0xd9, + 0xf2, 0x00, 0xf2, 0x61, 0x5f, 0x1f, 0xbd, 0x06, 0x19, 0x07, 0xeb, 0xae, 0x6d, 0xf1, 0xd3, 0x42, + 0x8f, 0x3d, 0x83, 0x88, 0xc7, 0x9e, 0x41, 0x9e, 0x41, 0xd9, 0x63, 0x98, 0x61, 0x9b, 0x74, 0xdb, + 0xec, 0x79, 0xd8, 0x41, 0xb7, 0x20, 0xe3, 0x7a, 0xba, 0x87, 0xdd, 0xb2, 0xb6, 0x9a, 0xbc, 0x5a, + 0xbc, 0xb1, 0x34, 0xde, 0xb4, 0x27, 0x68, 0xb6, 0x0e, 0x46, 0x29, 0xae, 0x83, 0x41, 0xaa, 0xff, + 0xa0, 0xc1, 0x8c, 0x38, 0x9b, 0x78, 0x3e, 0x62, 0x4f, 0xb7, 0x19, 0x24, 0x70, 0xcc, 0x88, 0x23, + 0x8c, 0xb3, 0xdb, 0x4b, 0xf2, 0x15, 0x64, 0x03, 0x90, 0x96, 0xef, 0x62, 0x87, 0x7b, 0x2b, 0xfd, + 0x0a, 0x32, 0xf0, 0x8e, 0x2b, 0x79, 0x3b, 0x44, 0x50, 0x6e, 0x06, 0xb2, 0x56, 0x71, 0x20, 0x82, + 0xba, 0x51, 0x23, 0x88, 0x1c, 0x32, 0x97, 0x06, 0xa3, 0x69, 0x1b, 0x41, 0x34, 0x64, 0x49, 0xec, + 0x62, 0xc8, 0x92, 0x10, 0xcf, 0xe0, 0x32, 0xdf, 0xa6, 0xe9, 0x5a, 0xa3, 0x01, 0x47, 0x2c, 0x07, + 0x48, 0x9e, 0x22, 0x07, 0x78, 0x1d, 0xb2, 0x34, 0xe8, 0x86, 0x47, 0x9c, 0xda, 0x84, 0x80, 0xe4, + 0xe1, 0x2e, 0x83, 0x3c, 0x25, 0xd4, 0xa4, 0x7f, 0x62, 0xa8, 0x69, 0xc1, 0xc5, 0x3d, 0xdd, 0x6d, + 0x05, 0xc1, 0xd1, 0x68, 0xe9, 0x5e, 0x2b, 0x3c, 0xeb, 0x19, 0x5a, 0x47, 0xd0, 0x96, 0xe9, 0x9e, + 0xee, 0x6e, 0x07, 0x34, 0xeb, 0xde, 0xd6, 0xf8, 0xc9, 0x5f, 0x52, 0x53, 0xa0, 0x1d, 0x58, 0x54, + 0x0b, 0xcf, 0xd2, 0x95, 0xd3, 0x8e, 0xbe, 0xfb, 0x54, 0xc9, 0xf3, 0x0a, 0x34, 0xfa, 0x52, 0x83, + 0x32, 0xf9, 0x0a, 0x3a, 0xf8, 0x53, 0xdf, 0x74, 0x70, 0x9f, 0xb8, 0x45, 0xcb, 0x3e, 0xc4, 0x4e, + 0x4f, 0x3f, 0xe6, 0xc3, 0xb1, 0xcb, 0xe3, 0x21, 0x7f, 0xcb, 0x36, 0x9a, 0x02, 0x03, 0x7b, 0xb5, + 0x81, 0x0c, 0x7c, 0xc8, 0x84, 0x88, 0xaf, 0xa6, 0xa6, 0x10, 0x5c, 0x08, 0x4e, 0xd1, 0x18, 0x2b, + 0x9c, 0xd8, 0x18, 0x7b, 0x19, 0x52, 0x03, 0xdb, 0xee, 0xd1, 0x32, 0x8e, 0x67, 0x7a, 0xe4, 0x59, + 0xcc, 0xf4, 0xc8, 0xb3, 0xd8, 0xbb, 0x68, 0xa4, 0x72, 0xb9, 0x52, 0x9e, 0x7c, 0x0e, 0x8b, 0xf2, + 0x3c, 0x6d, 0xfc, 0x40, 0x25, 0xcf, 0xfc, 0x40, 0xa5, 0x4e, 0xb1, 0x1b, 0xe9, 0xa9, 0x77, 0x23, + 0x33, 0xfd, 0x6e, 0x54, 0xbf, 0x4a, 0xc0, 0xac, 0x34, 0xf2, 0xfb, 0x6d, 0x6e, 0xc3, 0xbf, 0x24, + 0x60, 0x49, 0xfd, 0x4a, 0x67, 0x52, 0x8a, 0xde, 0x05, 0x92, 0x54, 0xde, 0x8b, 0x92, 0xae, 0xc5, + 0xb1, 0x4a, 0x94, 0x6e, 0x67, 0x90, 0x91, 0x8e, 0x4d, 0x0d, 0x03, 0x76, 0xf4, 0x04, 0x0a, 0xa6, + 0x30, 0x9f, 0x4c, 0xaa, 0xc6, 0x48, 0xe2, 0x54, 0x92, 0xb5, 0x3a, 0x26, 0xcc, 0x22, 0x45, 0x51, + 0xf5, 0x0c, 0xa4, 0x48, 0x56, 0x58, 0x3d, 0x84, 0x2c, 0x5f, 0x0e, 0x7a, 0x13, 0xf2, 0x34, 0x16, + 0xd3, 0xea, 0x8a, 0xa5, 0xf0, 0x34, 0xbd, 0x21, 0xc0, 0xd8, 0xfd, 0x9c, 0x5c, 0x00, 0x43, 0x7f, + 0x05, 0x40, 0xc2, 0x0f, 0x8f, 0xc2, 0x09, 0x1a, 0xcb, 0x68, 0x15, 0x37, 0xb0, 0x8d, 0xb1, 0xd0, + 0x9b, 0x0f, 0x81, 0xd5, 0xff, 0x4a, 0x40, 0x41, 0x9c, 0x88, 0x3e, 0x93, 0xf2, 0xcf, 0x21, 0xa8, + 0xb0, 0x5b, 0xba, 0x61, 0x90, 0xbf, 0x38, 0xf8, 0x50, 0xae, 0x4d, 0xdc, 0xa4, 0xe0, 0xff, 0xf5, + 0x80, 0x83, 0xd5, 0x53, 0x74, 0x9c, 0x63, 0xc6, 0x50, 0x82, 0xd6, 0x52, 0x1c, 0xb7, 0x7c, 0x00, + 0x8b, 0x4a, 0x51, 0x62, 0x15, 0x94, 0x7e, 0x5e, 0x55, 0xd0, 0xbf, 0xa7, 0x61, 0x51, 0x39, 0x89, + 0x8e, 0x79, 0x70, 0xf2, 0xb9, 0x78, 0xf0, 0x3f, 0x69, 0xaa, 0x9d, 0x65, 0x83, 0xa1, 0x77, 0xa6, + 0x18, 0x8f, 0x3f, 0xaf, 0x3d, 0x96, 0xdd, 0x22, 0xfd, 0x4c, 0x3e, 0x99, 0x99, 0xd6, 0x27, 0xd1, + 0x75, 0x56, 0x50, 0x52, 0x5d, 0x6c, 0x6c, 0x13, 0x9c, 0xd0, 0x98, 0xaa, 0x2c, 0x07, 0xa1, 0x0f, + 0x60, 0x36, 0xe0, 0x60, 0x6d, 0x8c, 0x5c, 0xd4, 0x63, 0xe0, 0x34, 0xf1, 0x4e, 0xc6, 0x8c, 0x08, + 0x17, 0xa2, 0x64, 0xfe, 0x14, 0x51, 0x12, 0x4e, 0x8a, 0x92, 0x3f, 0xab, 0x6f, 0x4a, 0xa1, 0x76, + 0xa8, 0xc1, 0x5c, 0xec, 0x02, 0xc8, 0xaf, 0xfe, 0x9b, 0x23, 0xbd, 0xe0, 0x17, 0x1a, 0xe4, 0xc3, + 0xfb, 0x45, 0x68, 0x1d, 0x32, 0x98, 0xdd, 0x51, 0x61, 0x61, 0x67, 0x3e, 0x76, 0x7f, 0x90, 0xe0, + 0xf8, 0x8d, 0xc1, 0xd8, 0xb5, 0x94, 0x26, 0x67, 0x7c, 0x86, 0x04, 0xfc, 0xbf, 0xb5, 0x20, 0x01, + 0x1f, 0x5b, 0x45, 0xf2, 0xa7, 0xaf, 0xe2, 0xec, 0xb6, 0xee, 0x47, 0x80, 0x34, 0x5d, 0x0b, 0x29, + 0xa4, 0x3d, 0x36, 0x15, 0xef, 0x51, 0x57, 0xcc, 0xb1, 0x53, 0x1d, 0xc0, 0xc4, 0x53, 0x1d, 0xc0, + 0xd0, 0x1e, 0xcc, 0x45, 0xed, 0x39, 0x2a, 0x46, 0x7d, 0x61, 0xf1, 0x43, 0x99, 0x88, 0x8d, 0x1e, + 0x62, 0x9c, 0xf2, 0x8d, 0x83, 0x18, 0x12, 0x19, 0x50, 0xec, 0xd8, 0x96, 0xa7, 0x9b, 0x16, 0x76, + 0x98, 0xa2, 0xa4, 0xea, 0xc2, 0xd6, 0x4d, 0x89, 0x86, 0x35, 0x4d, 0x64, 0x3e, 0xf9, 0xc2, 0x96, + 0x8c, 0x43, 0x9f, 0xc0, 0x6c, 0x50, 0x08, 0x31, 0x25, 0x29, 0xd5, 0x85, 0xad, 0x0d, 0x91, 0x84, + 0x1d, 0x06, 0x89, 0x4b, 0xbe, 0xb0, 0x25, 0xa1, 0xd0, 0xc7, 0x30, 0xd3, 0x23, 0x15, 0xda, 0xc6, + 0xd1, 0xc0, 0x74, 0xb0, 0xa1, 0xbe, 0x42, 0x78, 0x5f, 0xa0, 0x60, 0x81, 0x4b, 0xe4, 0x91, 0x6f, + 0x4e, 0x88, 0x18, 0x62, 0x8f, 0xbe, 0x7e, 0xd4, 0xf4, 0x2d, 0x77, 0xe3, 0x88, 0x5f, 0x07, 0xcb, + 0xaa, 0xec, 0xb1, 0x29, 0x13, 0x31, 0x7b, 0xc4, 0x38, 0x65, 0x7b, 0xc4, 0x90, 0xe8, 0x3e, 0x8d, + 0xcb, 0x6c, 0x93, 0xd8, 0x55, 0xc2, 0xa5, 0xb1, 0x84, 0x8a, 0xed, 0x0f, 0x6b, 0xc7, 0xf0, 0x27, + 0x49, 0x68, 0x28, 0x01, 0xf5, 0xa0, 0x34, 0xb0, 0x0d, 0xfa, 0xda, 0x4d, 0xec, 0xf9, 0x8e, 0x85, + 0x0d, 0x5e, 0x28, 0xad, 0x8c, 0x49, 0x95, 0xa8, 0xd8, 0xe7, 0x2b, 0xce, 0x2b, 0xdf, 0xf8, 0x88, + 0x63, 0xd1, 0xe7, 0xb0, 0x10, 0xbb, 0x18, 0xc5, 0xde, 0xa3, 0xa0, 0x1a, 0x51, 0x34, 0x14, 0x94, + 0xac, 0xa6, 0x55, 0xc9, 0x90, 0x34, 0x2b, 0xb5, 0x10, 0xed, 0x5d, 0xdd, 0xea, 0x36, 0xec, 0xf6, + 0x8e, 0xc5, 0x8b, 0x40, 0xbd, 0xdd, 0xc3, 0xfc, 0x6e, 0x60, 0x4c, 0xfb, 0x1d, 0x05, 0x25, 0xd3, + 0xae, 0x92, 0x21, 0x6b, 0x57, 0x51, 0x84, 0x97, 0xa0, 0x48, 0x5a, 0x11, 0x5e, 0x16, 0x54, 0x5d, + 0x82, 0x62, 0x04, 0xc2, 0x25, 0x28, 0x06, 0x50, 0x5c, 0x82, 0x62, 0x08, 0x76, 0x7f, 0xae, 0x63, + 0x5b, 0x1d, 0xb3, 0x67, 0xd2, 0x16, 0x37, 0xdb, 0xd4, 0xa2, 0xfa, 0xfe, 0xdc, 0x18, 0x61, 0x70, + 0x7f, 0x6e, 0x0c, 0x11, 0xbf, 0x3f, 0x37, 0x46, 0x80, 0xee, 0x42, 0x69, 0x57, 0x37, 0x7b, 0xbe, + 0x83, 0x5b, 0x1d, 0xdd, 0xc3, 0x5d, 0xdb, 0x39, 0xe6, 0x83, 0x3f, 0xea, 0xd7, 0x1c, 0x77, 0x93, + 0xa3, 0xc4, 0x11, 0x67, 0x0c, 0x85, 0x1e, 0xc1, 0x7c, 0x20, 0xc9, 0xf5, 0xdb, 0xa1, 0xb0, 0xf3, + 0x54, 0x18, 0xbd, 0xf5, 0xcc, 0xd1, 0xdb, 0x11, 0x56, 0x90, 0x87, 0xc6, 0xb1, 0xf5, 0x5c, 0xd0, + 0xe0, 0x6a, 0xa4, 0x72, 0xe9, 0x52, 0xa6, 0x91, 0xca, 0x41, 0xa9, 0xc0, 0xe7, 0x8a, 0x8f, 0x60, + 0x2e, 0x16, 0x19, 0xd1, 0xfb, 0x10, 0xde, 0x0a, 0x7a, 0x7c, 0x3c, 0x08, 0xd2, 0x6e, 0xe9, 0x16, + 0x11, 0x81, 0xab, 0x6e, 0x11, 0x11, 0x78, 0xf5, 0xeb, 0x14, 0xe4, 0x82, 0xa3, 0x77, 0x26, 0x85, + 0xd4, 0x1a, 0x64, 0xfb, 0xd8, 0xa5, 0x37, 0x7f, 0x12, 0x51, 0x3e, 0xc6, 0x41, 0x62, 0x3e, 0xc6, + 0x41, 0x72, 0xba, 0x98, 0x7c, 0xa6, 0x74, 0x31, 0x35, 0x75, 0xba, 0x88, 0xe9, 0xb0, 0x5b, 0x08, + 0xe9, 0xc1, 0x8c, 0xe8, 0xe9, 0xdf, 0x89, 0x60, 0x14, 0x2e, 0x32, 0xc6, 0x46, 0xe1, 0x22, 0x0a, + 0x1d, 0xc0, 0x79, 0x61, 0x8e, 0xc5, 0x1b, 0x98, 0x24, 0x94, 0x17, 0x27, 0xdf, 0x2c, 0x68, 0x52, + 0x2a, 0x16, 0xb0, 0x0e, 0x62, 0x50, 0x31, 0xdf, 0x8e, 0xe3, 0x88, 0x4b, 0x18, 0xb8, 0xed, 0x77, + 0x37, 0xf9, 0xb6, 0x67, 0x23, 0x97, 0x10, 0xe1, 0xa2, 0x4b, 0x88, 0xf0, 0xea, 0x1f, 0x13, 0x50, + 0x94, 0xdf, 0xf7, 0x4c, 0x1c, 0xe3, 0x4d, 0xc8, 0xe3, 0x23, 0xd3, 0x6b, 0x75, 0x6c, 0x03, 0xf3, + 0xa2, 0x93, 0xda, 0x99, 0x00, 0x6f, 0xda, 0x86, 0x64, 0xe7, 0x00, 0x26, 0x7a, 0x53, 0x72, 0x2a, + 0x6f, 0x8a, 0xfa, 0xc5, 0xa9, 0x29, 0xfa, 0xc5, 0x4a, 0x3b, 0xe5, 0xcf, 0xc6, 0x4e, 0xd5, 0xef, + 0x12, 0x50, 0x8a, 0x7f, 0x9f, 0x7e, 0x19, 0x47, 0x50, 0x3e, 0x4d, 0xc9, 0xa9, 0x4f, 0xd3, 0x07, + 0x30, 0x4b, 0x92, 0x4a, 0xdd, 0xf3, 0xf8, 0xb5, 0xe4, 0x14, 0xcd, 0x0b, 0x59, 0x34, 0xf2, 0xad, + 0xf5, 0x00, 0x2e, 0x45, 0x23, 0x01, 0x3e, 0xe6, 0xba, 0xe9, 0x53, 0xba, 0xee, 0x97, 0x09, 0x98, + 0xdd, 0xb2, 0x8d, 0xe8, 0xb6, 0xe6, 0x6f, 0x2f, 0xa4, 0x55, 0xe7, 0x60, 0x56, 0x4a, 0x38, 0xab, + 0x5f, 0x31, 0x3f, 0x93, 0xbf, 0xeb, 0xbf, 0xbd, 0x7d, 0x29, 0xc2, 0x8c, 0x98, 0x27, 0x57, 0xeb, + 0x30, 0x17, 0x4b, 0x6b, 0xc5, 0x17, 0xd0, 0xa6, 0x79, 0x81, 0xea, 0x2d, 0x58, 0x50, 0xe5, 0x7b, + 0x42, 0xd4, 0xd1, 0xa6, 0x18, 0x72, 0xdd, 0x81, 0x05, 0x55, 0xde, 0x76, 0xfa, 0xe5, 0xbc, 0xcf, + 0x07, 0xc8, 0x3c, 0xc3, 0x3a, 0x35, 0xff, 0x6d, 0x98, 0x57, 0x64, 0x5a, 0xa7, 0x97, 0xf3, 0xbb, + 0xb0, 0x81, 0x10, 0xfd, 0x94, 0xe0, 0x36, 0x94, 0x06, 0xc1, 0x43, 0x8b, 0x97, 0xa9, 0xe9, 0xe8, + 0xb6, 0x55, 0x88, 0x6b, 0xc4, 0xea, 0xd5, 0xa2, 0x8c, 0x91, 0xe5, 0xf0, 0x12, 0x36, 0xa3, 0x90, + 0xd3, 0x8c, 0xd5, 0xb2, 0x45, 0x19, 0x23, 0x98, 0x28, 0x7b, 0xb2, 0x89, 0x68, 0x09, 0x9c, 0xae, + 0x7e, 0xa1, 0xc1, 0x5c, 0xec, 0xa7, 0x0e, 0xe8, 0x3a, 0xe4, 0xe8, 0xef, 0x10, 0xa3, 0xe2, 0x9f, + 0xee, 0x0e, 0x85, 0x49, 0x0b, 0xc8, 0x72, 0x10, 0x7a, 0x0b, 0xf2, 0xe1, 0xaf, 0x1f, 0xf8, 0x08, + 0x9a, 0xf9, 0x6f, 0x00, 0x94, 0xfc, 0x37, 0x00, 0xf2, 0xbe, 0xc1, 0xdf, 0xc3, 0xc5, 0x89, 0xbf, + 0x7b, 0x38, 0xd5, 0xb8, 0x33, 0x6a, 0x00, 0xa4, 0x4e, 0xd5, 0x00, 0x38, 0x82, 0x25, 0xf5, 0xcf, + 0x11, 0x04, 0xed, 0x89, 0x13, 0xb5, 0x47, 0xbb, 0x9f, 0x9c, 0x72, 0xf7, 0x13, 0xd5, 0x03, 0xda, + 0x31, 0x89, 0xae, 0xfd, 0x5f, 0x83, 0xf4, 0xc0, 0xb6, 0x7b, 0x2e, 0xbf, 0xe3, 0x41, 0xd5, 0x51, + 0x80, 0xa8, 0x8e, 0x02, 0x9e, 0xa1, 0x3f, 0xe3, 0x07, 0x1e, 0x1c, 0xfd, 0x88, 0xe1, 0xe7, 0xd8, + 0xdd, 0x7d, 0x28, 0xc5, 0x7f, 0xa4, 0x20, 0xe8, 0xd5, 0x4e, 0xa1, 0x37, 0x71, 0x92, 0xde, 0x57, + 0xaf, 0x43, 0x2e, 0x98, 0xd9, 0x23, 0x80, 0xcc, 0xa3, 0x9d, 0x8d, 0x9d, 0x8d, 0x5b, 0xa5, 0x73, + 0xa8, 0x00, 0xd9, 0xad, 0x8d, 0x07, 0xb7, 0xee, 0x3d, 0xb8, 0x53, 0xd2, 0xc8, 0x43, 0x73, 0xe7, + 0xc1, 0x03, 0xf2, 0x90, 0x78, 0xf5, 0xbe, 0x78, 0x0f, 0x90, 0x67, 0x9b, 0x33, 0x90, 0x5b, 0x1f, + 0x0c, 0x68, 0xa0, 0x60, 0xbc, 0x1b, 0x87, 0x26, 0x89, 0x3e, 0x25, 0x0d, 0x65, 0x21, 0xf9, 0xf0, + 0xe1, 0x66, 0x29, 0x81, 0x16, 0xa0, 0x74, 0x0b, 0xeb, 0x46, 0xcf, 0xb4, 0x70, 0x10, 0x6b, 0x4b, + 0xc9, 0xfa, 0xfe, 0xff, 0x7c, 0xbf, 0xa2, 0x7d, 0xf7, 0xfd, 0x8a, 0xf6, 0x87, 0xef, 0x57, 0xb4, + 0xaf, 0x7f, 0x58, 0x39, 0xf7, 0xdd, 0x0f, 0x2b, 0xe7, 0x7e, 0xff, 0xc3, 0xca, 0xb9, 0xbf, 0xbb, + 0xde, 0x35, 0xbd, 0x3d, 0xbf, 0x5d, 0xeb, 0xd8, 0x7d, 0xfe, 0xe3, 0xed, 0x81, 0x63, 0x93, 0xa0, + 0xc6, 0x9f, 0xd6, 0xe2, 0xbf, 0xea, 0xfe, 0xcf, 0xc4, 0xa5, 0x75, 0xfa, 0xb8, 0xc5, 0xe8, 0x6a, + 0xf7, 0xec, 0x1a, 0x03, 0xd0, 0xdf, 0xf1, 0xba, 0xed, 0x0c, 0xfd, 0xbd, 0xee, 0x9b, 0x7f, 0x0e, + 0x00, 0x00, 0xff, 0xff, 0x1d, 0x1c, 0x58, 0xc5, 0x10, 0x3e, 0x00, 0x00, } func (m *EventSequence) Marshal() (dAtA []byte, err error) { @@ -4605,6 +4678,29 @@ func (m *EventSequence_Event_JobValidated) MarshalToSizedBuffer(dAtA []byte) (in } return len(dAtA) - i, nil } +func (m *EventSequence_Event_JobRunTerminated) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *EventSequence_Event_JobRunTerminated) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + if m.JobRunTerminated != nil { + { + size, err := m.JobRunTerminated.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintEvents(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xd2 + } + return len(dAtA) - i, nil +} func (m *ResourceUtilisation) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -5402,20 +5498,20 @@ func (m *JobSetFilter) MarshalToSizedBuffer(dAtA []byte) (int, error) { var l int _ = l if len(m.States) > 0 { - dAtA41 := make([]byte, len(m.States)*10) - var j40 int + dAtA42 := make([]byte, len(m.States)*10) + var j41 int for _, num := range m.States { for num >= 1<<7 { - dAtA41[j40] = uint8(uint64(num)&0x7f | 0x80) + dAtA42[j41] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j40++ + j41++ } - dAtA41[j40] = uint8(num) - j40++ + dAtA42[j41] = uint8(num) + j41++ } - i -= j40 - copy(dAtA[i:], dAtA41[:j40]) - i = encodeVarintEvents(dAtA, i, uint64(j40)) + i -= j41 + copy(dAtA[i:], dAtA42[:j41]) + i = encodeVarintEvents(dAtA, i, uint64(j41)) i-- dAtA[i] = 0xa } @@ -5450,20 +5546,20 @@ func (m *CancelJobSet) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA[i] = 0x12 } if len(m.States) > 0 { - dAtA43 := make([]byte, len(m.States)*10) - var j42 int + dAtA44 := make([]byte, len(m.States)*10) + var j43 int for _, num := range m.States { for num >= 1<<7 { - dAtA43[j42] = uint8(uint64(num)&0x7f | 0x80) + dAtA44[j43] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j42++ + j43++ } - dAtA43[j42] = uint8(num) - j42++ + dAtA44[j43] = uint8(num) + j43++ } - i -= j42 - copy(dAtA[i:], dAtA43[:j42]) - i = encodeVarintEvents(dAtA, i, uint64(j42)) + i -= j43 + copy(dAtA[i:], dAtA44[:j43]) + i = encodeVarintEvents(dAtA, i, uint64(j43)) i-- dAtA[i] = 0xa } @@ -7227,6 +7323,43 @@ func (m *JobRunCancelled) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *JobRunTerminated) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *JobRunTerminated) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *JobRunTerminated) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.RunId) > 0 { + i -= len(m.RunId) + copy(dAtA[i:], m.RunId) + i = encodeVarintEvents(dAtA, i, uint64(len(m.RunId))) + i-- + dAtA[i] = 0x12 + } + if len(m.JobId) > 0 { + i -= len(m.JobId) + copy(dAtA[i:], m.JobId) + i = encodeVarintEvents(dAtA, i, uint64(len(m.JobId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func encodeVarintEvents(dAtA []byte, offset int, v uint64) int { offset -= sovEvents(v) base := offset @@ -7563,6 +7696,18 @@ func (m *EventSequence_Event_JobValidated) Size() (n int) { } return n } +func (m *EventSequence_Event_JobRunTerminated) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.JobRunTerminated != nil { + l = m.JobRunTerminated.Size() + n += 2 + l + sovEvents(uint64(l)) + } + return n +} func (m *ResourceUtilisation) Size() (n int) { if m == nil { return 0 @@ -8768,6 +8913,23 @@ func (m *JobRunCancelled) Size() (n int) { return n } +func (m *JobRunTerminated) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.JobId) + if l > 0 { + n += 1 + l + sovEvents(uint64(l)) + } + l = len(m.RunId) + if l > 0 { + n += 1 + l + sovEvents(uint64(l)) + } + return n +} + func sovEvents(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } @@ -9856,6 +10018,41 @@ func (m *EventSequence_Event) Unmarshal(dAtA []byte) error { } m.Event = &EventSequence_Event_JobValidated{v} iNdEx = postIndex + case 26: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field JobRunTerminated", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthEvents + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthEvents + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &JobRunTerminated{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Event = &EventSequence_Event_JobRunTerminated{v} + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipEvents(dAtA[iNdEx:]) @@ -17456,6 +17653,120 @@ func (m *JobRunCancelled) Unmarshal(dAtA []byte) error { } return nil } +func (m *JobRunTerminated) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: JobRunTerminated: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: JobRunTerminated: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field JobId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthEvents + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthEvents + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.JobId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RunId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthEvents + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthEvents + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RunId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipEvents(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthEvents + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipEvents(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/pkg/armadaevents/events.proto b/pkg/armadaevents/events.proto index db0096c6489..05ae95c6f55 100644 --- a/pkg/armadaevents/events.proto +++ b/pkg/armadaevents/events.proto @@ -91,6 +91,7 @@ message EventSequence { JobPreemptionRequested jobPreemptionRequested = 23; JobRunCancelled jobRunCancelled = 24; JobValidated jobValidated = 25; + JobRunTerminated jobRunTerminated = 26; } } // The system is namespaced by queue, and all events are associated with a job set. @@ -585,3 +586,13 @@ message JobRunCancelled { string job_id = 3; string run_id= 4; } + +// JobRunTerminated is emitted by the executor when it observes that the pod +// for a run has reached a terminal phase in Kubernetes. Only the executor +// produces this event. The timestamp on the enclosing sequence is the moment +// the executor saw the pod terminal, which is after the kubelet has stopped +// the containers. +message JobRunTerminated{ + string job_id = 1; + string run_id = 2; +}