fix fetch execution nodes to crawl query

Author: angaz

pkg/database/db.go

@@ -76,18 +76,18 @@ func NewDB(
 		nextCrawlNotEth: nextCrawlNotEth,
 		githubToken:     githubToken,
 
-		executionNodesToCrawlCache: make(chan *NodeToCrawl, 2048),
+		executionNodesToCrawlCache: make(chan *NodeToCrawl, 16392),
 		executionNodesToCrawlLock:  new(sync.Mutex),
-		executionRecentlyCrawled:   fifomemory.New[enode.ID](256),
+		executionRecentlyCrawled:   fifomemory.New[enode.ID](16392),
 
 		consensusNodesToCrawlCache: make(chan *NodeToCrawl, 2048),
 		consensusNodesToCrawlLock:  new(sync.Mutex),
 		consensusRecentlyCrawled:   fifomemory.New[enode.ID](256),
 		// consensusActiveCrawlers:    map[enode.ID]struct{}{},
 
-		discNodesToCrawlCache: make(chan *NodeToCrawl, 2048),
+		discNodesToCrawlCache: make(chan *NodeToCrawl, 16392),
 		discNodesToCrawlLock:  new(sync.Mutex),
-		discRecentlyCrawled:   fifomemory.New[enode.ID](256),
+		discRecentlyCrawled:   fifomemory.New[enode.ID](16392),
 
 		discUpdateCache: fifomemory.New[enode.ID](1024),
 

pkg/database/disc.go

@@ -218,7 +218,7 @@ func (db *DB) fetchExecutionNodesToCrawl(ctx context.Context) error {
 	rows, err := db.pg.Query(
 		ctx,
 		`
-			WITH last_crawled AS (
+			WITH last_crawled AS MATERIALIZED (
 				SELECT node_id
 				FROM crawler.next_disc_crawl
 				WHERE last_found > now() - INTERVAL '48 hours'
@@ -227,12 +227,12 @@ func (db *DB) fetchExecutionNodesToCrawl(ctx context.Context) error {
 				next_node_crawl.next_crawl,
 				node_record
 			FROM crawler.next_node_crawl
-			INNER JOIN crawler.next_disc_crawl USING (node_id)
+			INNER JOIN last_crawled USING (node_id)
 			INNER JOIN disc.nodes USING (node_id)
 			WHERE
 				next_node_crawl.node_type IN ('Unknown', 'Execution')
 			ORDER BY next_node_crawl.next_crawl
-			LIMIT 1024
+			LIMIT 4096
 		`,
 	)
 	if err != nil {