Enhance error handling and logging for JSON and page fetching

lolstaticdata/champions/__main__.py

@@ -2,6 +2,7 @@
 import json
 import time
 import argparse
+import requests
 from bs4 import BeautifulSoup
 
 from ..common import utils
@@ -10,7 +11,11 @@
 
 
 def get_ability_filenames(url):
-    soup = utils.download_soup(url, use_cache=False)
+    try:
+        soup = utils.download_soup(url, use_cache=False)
+    except requests.exceptions.RequestException as e:
+        utils.print_warning(f"Could not load ability icon listing ({url}): {e}")
+        return []
     soup = BeautifulSoup(soup, "lxml")
 
     filenames = []
@@ -55,10 +60,20 @@ def print_runtime():
         process_skins=skins or process_all,
     )
 
-    latest_version = utils.get_latest_patch_version()
-    ddragon_champions = utils.download_json(
-        f"http://ddragon.leagueoflegends.com/cdn/{latest_version}/data/en_US/championFull.json"
-    )["data"]
+    try:
+        latest_version = utils.get_latest_patch_version()
+    except requests.exceptions.RequestException as e:
+        utils.print_error(f"Unable to determine latest patch version from Data Dragon: {e}")
+        return
+
+    print(f"Fetching Champion data for patch version: {latest_version}")
+    try:
+        ddragon_champions = utils.download_json(
+            f"http://ddragon.leagueoflegends.com/cdn/{latest_version}/data/en_US/championFull.json"
+        )["data"]
+    except requests.exceptions.RequestException as e:
+        utils.print_error(f"Unable to load champion metadata for patch {latest_version}: {e}")
+        return
 
     factions = {}
     if lore or process_all:
@@ -71,7 +86,7 @@ def print_runtime():
             for universe_entry in universe_stats:
                 factions[universe_entry["slug"]] = universe_entry["associated-faction-slug"]
         except Exception:
-            print("ERROR: Unable to load/parse universe file, location may have changed")
+            utils.print_error("Unable to load/parse universe file, location may have changed")
 
     ability_key_to_identifier = {
         "P": "passive",
@@ -88,7 +103,7 @@ def print_runtime():
         champion_key = champion.key
 
         if champion_key not in ddragon_champions:
-            print(f"WARNING: Skipping champion '{champion_key}' (not present in Data Dragon metadata).")
+            utils.print_warning(f"Skipping champion '{champion_key}' (not present in Data Dragon metadata).")
             continue
 
         ddragon_champion = ddragon_champions[champion_key]
@@ -140,7 +155,7 @@ def print_runtime():
     if champion and not champions:
         end_time = time.time()
         print_runtime()
-        print(f"WARNING: Champion '{champion}' was not found. Try the exact in-game name, e.g. \"Jarvan IV\".")
+        utils.print_warning(f"Champion '{champion}' was not found. Try the exact in-game name, e.g. \"Jarvan IV\".")
         return
 
     jsonfn = os.path.join(directory, "champions.json")

lolstaticdata/champions/pull_champions_wiki.py

@@ -33,6 +33,8 @@
     to_enum_like,
     download_json,
     strip_lua_comments,
+    print_error,
+    print_warning,
 )
 from .modelchampion import (
     Champion,
@@ -492,7 +494,10 @@ def _pull_champion_ability(self, champion_name, ability_name) -> HTMLAbilityWrap
         try: 
             return HTMLAbilityWrapper(soup)
         except ValueError:
-            print(f"WARNING: Ability data could not be found for {ability_name}. The Wiki page may be empty: https://wiki.leagueoflegends.com/en-us/Template:Data_{champion_name}/{ability_name}")
+            print_warning(
+                f"Ability data could not be found for {ability_name}. "
+                f"The Wiki page may be empty: https://wiki.leagueoflegends.com/en-us/Template:Data_{champion_name}/{ability_name}"
+            )
 
     def _get_ability_effects(self, name, skill_dict):
         effects = []
@@ -721,17 +726,17 @@ def _render_modifiers(self, mods: str, nvalues: int) -> List[Modifier]:
         try:
             parsed_modifiers = ParsingAndRegex.split_modifiers(mods)
         except Exception as error:
-            print("ERROR: FAILURE TO SPLIT MODIFIER")
-            print("ERROR:", error)
+            print_error("FAILURE TO SPLIT MODIFIER")
+            print_error(str(error))
             return modifiers
 
         for lvling in parsed_modifiers:
             try:
                 modifier = self._render_modifier(lvling, nvalues)
                 modifiers.append(modifier)
             except Exception as error:
-                print(f"ERROR: FAILURE TO PARSE MODIFIER:  {lvling}")
-                print("ERROR:", error)
+                print_error(f"FAILURE TO PARSE MODIFIER:  {lvling}")
+                print_error(str(error))
                 modifier = Modifier(
                     values=[0 for _ in range(nvalues)],
                     units=[lvling for _ in range(nvalues)],
@@ -1015,7 +1020,7 @@ def regex_slash_separated(string: str, nvalues: int) -> Tuple[List[str], List[Un
                 if nvalues == 3 and len(values) == 5:
                     values = [values[0], values[2], values[4]]
                 if nvalues is not None and len(values) != nvalues:
-                    print(f"WARNING: Unexpected number of modifier values: {values} (expected {nvalues})")
+                    print_warning(f"Unexpected number of modifier values: {values} (expected {nvalues})")
                 return not_parsed, values
         raise ValueError(f"Could not parse slash-separated string: {string}")
 

lolstaticdata/common/utils.py

@@ -14,6 +14,26 @@
 Json = Union[dict, list, str, int, float, bool, None]
 
 
+ANSI_RED = "\033[31m"
+ANSI_ORANGE = "\033[38;5;214m"
+ANSI_RESET = "\033[0m"
+
+
+def _print_colored(prefix: str, message: str, color: str, code: Union[int, str, None] = None):
+    if code is None:
+        print(f"{color}{prefix}: {message}{ANSI_RESET}")
+    else:
+        print(f"{color}{prefix} [{code}]: {message}{ANSI_RESET}")
+
+
+def print_error(message: str, code: Union[int, str, None] = None):
+    _print_colored("ERROR", message, ANSI_RED, code=code)
+
+
+def print_warning(message: str, code: Union[int, str, None] = None):
+    _print_colored("WARNING", message, ANSI_ORANGE, code=code)
+
+
 def to_enum_like(string: str) -> str:
     return string.upper().replace(" ", "_")
 
@@ -131,6 +151,9 @@ def download_json(url: str, use_cache: bool = True) -> Json:
     else:
         headers = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36"}
         page = requests.get(url, headers=headers)
+        if page.status_code >= 400:
+            print_error(f"HTTP {page.status_code} while fetching JSON: {url}", code=page.status_code)
+            raise requests.exceptions.HTTPError(f"HTTP {page.status_code}: {url}", response=page)
         j = page.json()
         if use_cache:
             with open(fn, "w") as f:
@@ -153,6 +176,9 @@ def download_soup(url: str, use_cache: bool = True, dir: str = f"__cache__"):
             html = f.read()
     else:
         page = requests.get(url)
+        if page.status_code >= 400:
+            print_error(f"HTTP {page.status_code} while fetching page: {url}", code=page.status_code)
+            raise requests.exceptions.HTTPError(f"HTTP {page.status_code}: {url}", response=page)
         # html = page.content.decode(page.encoding)
         html = page.text
         if use_cache:

lolstaticdata/items/__main__.py

@@ -1,13 +1,21 @@
 import os
 import shutil
 import json
+import requests
 
 from .pull_items_wiki import WikiItem, get_item_urls
 from .pull_items_dragon import DragonItem
 from collections import OrderedDict
+from ..common.utils import print_error
 
 def main():
     directory = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../.."))
+    try:
+        latest_version = DragonItem.get_latest_version()
+    except requests.exceptions.RequestException as e:
+        print_error(f"Unable to determine latest patch version from Data Dragon: {e}")
+        return False
+    print(f"Fetching Item data for patch version: {latest_version}")
     if not os.path.exists(os.path.join(directory, "items")):
         os.mkdir(os.path.join(directory, "items"))
 
@@ -16,7 +24,11 @@ def main():
 
     if not os.path.exists(os.path.join(directory, "__wiki__")):
         os.mkdir(os.path.join(directory, "__wiki__"))
-    cdragon = DragonItem.get_cdragon()
+    try:
+        cdragon = DragonItem.get_cdragon()
+    except requests.exceptions.RequestException as e:
+        print_error(f"Unable to load CommunityDragon items list: {e}")
+        return False
     wikiItems = get_item_urls(False)
 
     jsons = {}
@@ -61,8 +73,9 @@ def main():
     with open(jsonfn, "w", encoding="utf8") as f:
         json.dump(jsons, f, indent=2, ensure_ascii=False)
     del jsons
+    return True
 
 
 if __name__ == "__main__":
-    main()
-    print("Hello! What a surprise, it worked!")
+    if main():
+        print("Hello! What a surprise, it worked!")

lolstaticdata/items/pull_items_dragon.py

@@ -1,7 +1,8 @@
 import json
 import os
+import requests
 
-from ..common.utils import download_json
+from ..common.utils import download_json, print_warning
 from .pull_items_wiki import WikiItem
 from .modelitem import Item, Shop
 
@@ -13,9 +14,22 @@ def get_latest_version():
 
 
 class DragonItem:
-    latest_version = get_latest_version()
-    version = latest_version.split(".")
-    version = str(version[0]) + "." + str(version[1])
+    _latest_version = None
+    _version = None
+
+    @classmethod
+    def get_latest_version(cls):
+        if cls._latest_version is None:
+            cls._latest_version = get_latest_version()
+        return cls._latest_version
+
+    @classmethod
+    def get_version(cls):
+        if cls._version is None:
+            latest_version = cls.get_latest_version()
+            version_parts = latest_version.split(".")
+            cls._version = str(version_parts[0]) + "." + str(version_parts[1])
+        return cls._version
 
     @staticmethod
     def get_cdragon():  # cdragon to list
@@ -26,13 +40,25 @@ def get_cdragon():  # cdragon to list
         cdragon = [i for i in j if str(i["id"])]
         return cdragon
 
-    @staticmethod
-    def get_item_plaintext(item):
-        url = f"https://raw.communitydragon.org/{DragonItem.version}/game/en_us/data/menu/en_us/lol.stringtable.json"
-        j = download_json(url, use_cache=True)
+    _stringtable_cache = None
+    _stringtable_loaded = False
+
+    @classmethod
+    def get_item_plaintext(cls, item):
+        if not cls._stringtable_loaded:
+            url = f"https://raw.communitydragon.org/latest/game/en_us/data/menu/en_us/lol.stringtable.json"
+            try:
+                cls._stringtable_cache = download_json(url, use_cache=True)
+            except requests.exceptions.RequestException as e:
+                print_warning(f"Could not fetch CommunityDragon stringtable ({url}): {e}")
+                cls._stringtable_cache = None
+            cls._stringtable_loaded = True
+
+        if cls._stringtable_cache is None:
+            return None
         try:
-            return j['entries']["game_item_plaintext_" + str(item)]
-        except:
+            return cls._stringtable_cache['entries']["game_item_plaintext_" + str(item)]
+        except (KeyError, TypeError):
             return None
 
     @classmethod
@@ -88,7 +114,7 @@ def _get_skin_path(cls, path):
                 path = path.lower()
                 path = (
                     "https://raw.communitydragon.org/{}/plugins/rcp-be-lol-game-data/global/default/assets".format(
-                        DragonItem.version
+                        cls.get_version()
                     )
                     + path
                 )
@@ -101,14 +127,14 @@ def get_json_ddragon(
         cls,
     ):  # Main Function, gets items from ddragon, compares them with cdragon and then gets the items from the wiki
         # I didn't want make a request to cdragon for every item
-        url = "http://ddragon.leagueoflegends.com/cdn/{}/data/en_US/item.json".format(get_latest_version())
+        url = "http://ddragon.leagueoflegends.com/cdn/{}/data/en_US/item.json".format(cls.get_latest_version())
         p = download_json(url, use_cache=True)
         return p["data"]
 
     @classmethod
     def get_ddragon(cls, ddragon: int, p: dict):
         # print(ddragon)
-        baseurl = "http://ddragon.leagueoflegends.com/cdn/{}/img/item/".format(get_latest_version())  # icon base url
+        baseurl = "http://ddragon.leagueoflegends.com/cdn/{}/img/item/".format(cls.get_latest_version())  # icon base url
         icon = baseurl + p[ddragon]["image"]["full"]
         plaintext = p[ddragon]["plaintext"]  # simple description
         purchasable = p[ddragon]["gold"]["purchasable"]  # is this purchasable or is it upgraded (seraph's embrace)

lolstaticdata/items/pull_items_wiki.py

@@ -17,6 +17,7 @@
 from ..common.utils import (
     download_soup,
     strip_lua_comments,
+    print_warning,
 )
 from ..common.modelcommon import (
     ArmorPenetration,
@@ -630,7 +631,7 @@ def _parse_item_data(cls, item_data: dict, item_name:str,wiki_data:dict) -> Item
                     item_data[x] = wiki_data[parent_item][x]
                 except KeyError as e:
                     clear_keys.append(x)
-                    print(f"WARNING: Couldn't find inherited value of '{x}' for {item_name} - inherited from {parent_item}")
+                    print_warning(f"Couldn't find inherited value of '{x}' for {item_name} - inherited from {parent_item}")
                     break
             if x in "effects":
                 for l in item_data[x]: