From c9062b235415d15ac645002382d1bbd8a3bccf9e Mon Sep 17 00:00:00 2001 From: jni-bot Date: Wed, 11 Feb 2026 16:54:31 -0800 Subject: [PATCH] Fix slow /api/v1/jobs endpoint by optimizing RunDao queries (#2987) The findByLatestJob query was scanning millions of rows in run_facets and dataset_facets by joining runs_view with jobs_view in the WHERE clause. This caused 7+ minute query times per job, making the jobs list page unusable at scale. Two changes: - Add findCurrentRunByJob using jobs.current_run_uuid (from #2929) for the jobs list page, reducing it to a single indexed UUID lookup - Optimize findByLatestJob to filter on runs.job_uuid (indexed) with symlink resolution, instead of the expensive runs_view/jobs_view join Closes #2987 Signed-off-by: jni-bot --- api/src/main/java/marquez/db/JobDao.java | 4 ++-- api/src/main/java/marquez/db/RunDao.java | 22 ++++++++++++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/api/src/main/java/marquez/db/JobDao.java b/api/src/main/java/marquez/db/JobDao.java index 7b4c086e4d..5accf7afac 100644 --- a/api/src/main/java/marquez/db/JobDao.java +++ b/api/src/main/java/marquez/db/JobDao.java @@ -282,8 +282,8 @@ default List findAllWithRun( .peek( j -> { List runs = - runDao.findByLatestJob( - j.getNamespace().getValue(), j.getName().getValue(), 10, 0); + runDao.findCurrentRunByJob( + j.getNamespace().getValue(), j.getName().getValue(), 1, 0); this.setJobData(runs, j); }) .toList(); diff --git a/api/src/main/java/marquez/db/RunDao.java b/api/src/main/java/marquez/db/RunDao.java index ff67da7bde..ce8440032c 100644 --- a/api/src/main/java/marquez/db/RunDao.java +++ b/api/src/main/java/marquez/db/RunDao.java @@ -511,13 +511,31 @@ default RunRow upsertRunMeta( BASE_FIND_RUN_SQL + """ WHERE r.uuid IN ( - SELECT r.uuid FROM runs_view r - INNER JOIN jobs_view j ON j.namespace_name=r.namespace_name AND j.name=r.job_name + SELECT j.current_run_uuid FROM jobs_view j WHERE j.namespace_name=:namespace AND (j.name=:jobName OR j.name=ANY(j.aliases)) ) ORDER BY transitioned_at DESC, started_at DESC LIMIT :limit OFFSET :offset """) + List findCurrentRunByJob(String namespace, String jobName, int limit, int offset); + + @SqlQuery( + BASE_FIND_RUN_SQL + + """ + WHERE r.job_uuid IN ( + SELECT j.uuid FROM jobs j + WHERE j.uuid IN ( + SELECT jv.uuid FROM jobs_view jv + WHERE jv.namespace_name=:namespace AND (jv.name=:jobName OR jv.name=ANY(jv.aliases)) + ) + OR j.symlink_target_uuid IN ( + SELECT jv.uuid FROM jobs_view jv + WHERE jv.namespace_name=:namespace AND (jv.name=:jobName OR jv.name=ANY(jv.aliases)) + ) + ) + ORDER BY transitioned_at DESC, started_at DESC + LIMIT :limit OFFSET :offset + """) List findByLatestJob(String namespace, String jobName, int limit, int offset); @Builder