github: Retry RESOURCE_LIMITS_EXCEEDED GraphQL errors with backoff

Author: matias-christensen-skydio

revup/github_real.py

@@ -60,6 +60,7 @@ async def graphql(self, query: str, **kwargs: Any) -> Any:
         max_retries = 3
         base_delay = 1.0
         transient_statuses = {500, 502, 503, 504}
+        retryable_graphql_errors = {"RESOURCE_LIMITS_EXCEEDED"}
 
         headers = {}
         if self.oauth_token:
@@ -104,6 +105,20 @@ async def graphql(self, query: str, **kwargs: Any) -> Any:
                     logging.debug("Response JSON:\n{}".format(pretty_json))
 
                 if "errors" in r:
+                    error_types = {err.get("type", "Unknown") for err in r["errors"]}
+                    if error_types & retryable_graphql_errors:
+                        if attempt < max_retries - 1:
+                            delay = base_delay * (2**attempt)
+                            logging.warning(
+                                "GitHub GraphQL error ({}), retrying in {}s (attempt {}/{})".format(
+                                    ", ".join(error_types & retryable_graphql_errors),
+                                    delay,
+                                    attempt + 1,
+                                    max_retries,
+                                )
+                            )
+                            await asyncio.sleep(delay)
+                            continue
                     raise RevupGithubException(r["errors"])
 
                 if resp.status != 200: