diff --git a/lolstaticdata/champions/__main__.py b/lolstaticdata/champions/__main__.py index 054770cb..05a3a60f 100644 --- a/lolstaticdata/champions/__main__.py +++ b/lolstaticdata/champions/__main__.py @@ -2,6 +2,7 @@ import json import time import argparse +import requests from bs4 import BeautifulSoup from ..common import utils @@ -10,7 +11,11 @@ def get_ability_filenames(url): - soup = utils.download_soup(url, use_cache=False) + try: + soup = utils.download_soup(url, use_cache=False) + except requests.exceptions.RequestException as e: + utils.print_warning(f"Could not load ability icon listing ({url}): {e}") + return [] soup = BeautifulSoup(soup, "lxml") filenames = [] @@ -55,10 +60,20 @@ def print_runtime(): process_skins=skins or process_all, ) - latest_version = utils.get_latest_patch_version() - ddragon_champions = utils.download_json( - f"http://ddragon.leagueoflegends.com/cdn/{latest_version}/data/en_US/championFull.json" - )["data"] + try: + latest_version = utils.get_latest_patch_version() + except requests.exceptions.RequestException as e: + utils.print_error(f"Unable to determine latest patch version from Data Dragon: {e}") + return + + print(f"Fetching Champion data for patch version: {latest_version}") + try: + ddragon_champions = utils.download_json( + f"http://ddragon.leagueoflegends.com/cdn/{latest_version}/data/en_US/championFull.json" + )["data"] + except requests.exceptions.RequestException as e: + utils.print_error(f"Unable to load champion metadata for patch {latest_version}: {e}") + return factions = {} if lore or process_all: @@ -71,7 +86,7 @@ def print_runtime(): for universe_entry in universe_stats: factions[universe_entry["slug"]] = universe_entry["associated-faction-slug"] except Exception: - print("ERROR: Unable to load/parse universe file, location may have changed") + utils.print_error("Unable to load/parse universe file, location may have changed") ability_key_to_identifier = { "P": "passive", @@ -88,7 +103,7 @@ def print_runtime(): champion_key = champion.key if champion_key not in ddragon_champions: - print(f"WARNING: Skipping champion '{champion_key}' (not present in Data Dragon metadata).") + utils.print_warning(f"Skipping champion '{champion_key}' (not present in Data Dragon metadata).") continue ddragon_champion = ddragon_champions[champion_key] @@ -140,7 +155,7 @@ def print_runtime(): if champion and not champions: end_time = time.time() print_runtime() - print(f"WARNING: Champion '{champion}' was not found. Try the exact in-game name, e.g. \"Jarvan IV\".") + utils.print_warning(f"Champion '{champion}' was not found. Try the exact in-game name, e.g. \"Jarvan IV\".") return jsonfn = os.path.join(directory, "champions.json") diff --git a/lolstaticdata/champions/pull_champions_wiki.py b/lolstaticdata/champions/pull_champions_wiki.py index f02904fc..8ad9bf9d 100644 --- a/lolstaticdata/champions/pull_champions_wiki.py +++ b/lolstaticdata/champions/pull_champions_wiki.py @@ -33,6 +33,8 @@ to_enum_like, download_json, strip_lua_comments, + print_error, + print_warning, ) from .modelchampion import ( Champion, @@ -492,7 +494,10 @@ def _pull_champion_ability(self, champion_name, ability_name) -> HTMLAbilityWrap try: return HTMLAbilityWrapper(soup) except ValueError: - print(f"WARNING: Ability data could not be found for {ability_name}. The Wiki page may be empty: https://wiki.leagueoflegends.com/en-us/Template:Data_{champion_name}/{ability_name}") + print_warning( + f"Ability data could not be found for {ability_name}. " + f"The Wiki page may be empty: https://wiki.leagueoflegends.com/en-us/Template:Data_{champion_name}/{ability_name}" + ) def _get_ability_effects(self, name, skill_dict): effects = [] @@ -721,8 +726,8 @@ def _render_modifiers(self, mods: str, nvalues: int) -> List[Modifier]: try: parsed_modifiers = ParsingAndRegex.split_modifiers(mods) except Exception as error: - print("ERROR: FAILURE TO SPLIT MODIFIER") - print("ERROR:", error) + print_error("FAILURE TO SPLIT MODIFIER") + print_error(str(error)) return modifiers for lvling in parsed_modifiers: @@ -730,8 +735,8 @@ def _render_modifiers(self, mods: str, nvalues: int) -> List[Modifier]: modifier = self._render_modifier(lvling, nvalues) modifiers.append(modifier) except Exception as error: - print(f"ERROR: FAILURE TO PARSE MODIFIER: {lvling}") - print("ERROR:", error) + print_error(f"FAILURE TO PARSE MODIFIER: {lvling}") + print_error(str(error)) modifier = Modifier( values=[0 for _ in range(nvalues)], units=[lvling for _ in range(nvalues)], @@ -1015,7 +1020,7 @@ def regex_slash_separated(string: str, nvalues: int) -> Tuple[List[str], List[Un if nvalues == 3 and len(values) == 5: values = [values[0], values[2], values[4]] if nvalues is not None and len(values) != nvalues: - print(f"WARNING: Unexpected number of modifier values: {values} (expected {nvalues})") + print_warning(f"Unexpected number of modifier values: {values} (expected {nvalues})") return not_parsed, values raise ValueError(f"Could not parse slash-separated string: {string}") diff --git a/lolstaticdata/common/utils.py b/lolstaticdata/common/utils.py index ed7c8a90..b403e809 100644 --- a/lolstaticdata/common/utils.py +++ b/lolstaticdata/common/utils.py @@ -14,6 +14,26 @@ Json = Union[dict, list, str, int, float, bool, None] +ANSI_RED = "\033[31m" +ANSI_ORANGE = "\033[38;5;214m" +ANSI_RESET = "\033[0m" + + +def _print_colored(prefix: str, message: str, color: str, code: Union[int, str, None] = None): + if code is None: + print(f"{color}{prefix}: {message}{ANSI_RESET}") + else: + print(f"{color}{prefix} [{code}]: {message}{ANSI_RESET}") + + +def print_error(message: str, code: Union[int, str, None] = None): + _print_colored("ERROR", message, ANSI_RED, code=code) + + +def print_warning(message: str, code: Union[int, str, None] = None): + _print_colored("WARNING", message, ANSI_ORANGE, code=code) + + def to_enum_like(string: str) -> str: return string.upper().replace(" ", "_") @@ -131,6 +151,9 @@ def download_json(url: str, use_cache: bool = True) -> Json: else: headers = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36"} page = requests.get(url, headers=headers) + if page.status_code >= 400: + print_error(f"HTTP {page.status_code} while fetching JSON: {url}", code=page.status_code) + raise requests.exceptions.HTTPError(f"HTTP {page.status_code}: {url}", response=page) j = page.json() if use_cache: with open(fn, "w") as f: @@ -153,6 +176,9 @@ def download_soup(url: str, use_cache: bool = True, dir: str = f"__cache__"): html = f.read() else: page = requests.get(url) + if page.status_code >= 400: + print_error(f"HTTP {page.status_code} while fetching page: {url}", code=page.status_code) + raise requests.exceptions.HTTPError(f"HTTP {page.status_code}: {url}", response=page) # html = page.content.decode(page.encoding) html = page.text if use_cache: diff --git a/lolstaticdata/items/__main__.py b/lolstaticdata/items/__main__.py index 60d24f01..4b8eb347 100644 --- a/lolstaticdata/items/__main__.py +++ b/lolstaticdata/items/__main__.py @@ -1,13 +1,21 @@ import os import shutil import json +import requests from .pull_items_wiki import WikiItem, get_item_urls from .pull_items_dragon import DragonItem from collections import OrderedDict +from ..common.utils import print_error def main(): directory = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../..")) + try: + latest_version = DragonItem.get_latest_version() + except requests.exceptions.RequestException as e: + print_error(f"Unable to determine latest patch version from Data Dragon: {e}") + return False + print(f"Fetching Item data for patch version: {latest_version}") if not os.path.exists(os.path.join(directory, "items")): os.mkdir(os.path.join(directory, "items")) @@ -16,7 +24,11 @@ def main(): if not os.path.exists(os.path.join(directory, "__wiki__")): os.mkdir(os.path.join(directory, "__wiki__")) - cdragon = DragonItem.get_cdragon() + try: + cdragon = DragonItem.get_cdragon() + except requests.exceptions.RequestException as e: + print_error(f"Unable to load CommunityDragon items list: {e}") + return False wikiItems = get_item_urls(False) jsons = {} @@ -61,8 +73,9 @@ def main(): with open(jsonfn, "w", encoding="utf8") as f: json.dump(jsons, f, indent=2, ensure_ascii=False) del jsons + return True if __name__ == "__main__": - main() - print("Hello! What a surprise, it worked!") + if main(): + print("Hello! What a surprise, it worked!") diff --git a/lolstaticdata/items/pull_items_dragon.py b/lolstaticdata/items/pull_items_dragon.py index 93b57301..d94b46c7 100644 --- a/lolstaticdata/items/pull_items_dragon.py +++ b/lolstaticdata/items/pull_items_dragon.py @@ -1,7 +1,8 @@ import json import os +import requests -from ..common.utils import download_json +from ..common.utils import download_json, print_warning from .pull_items_wiki import WikiItem from .modelitem import Item, Shop @@ -13,9 +14,22 @@ def get_latest_version(): class DragonItem: - latest_version = get_latest_version() - version = latest_version.split(".") - version = str(version[0]) + "." + str(version[1]) + _latest_version = None + _version = None + + @classmethod + def get_latest_version(cls): + if cls._latest_version is None: + cls._latest_version = get_latest_version() + return cls._latest_version + + @classmethod + def get_version(cls): + if cls._version is None: + latest_version = cls.get_latest_version() + version_parts = latest_version.split(".") + cls._version = str(version_parts[0]) + "." + str(version_parts[1]) + return cls._version @staticmethod def get_cdragon(): # cdragon to list @@ -26,13 +40,25 @@ def get_cdragon(): # cdragon to list cdragon = [i for i in j if str(i["id"])] return cdragon - @staticmethod - def get_item_plaintext(item): - url = f"https://raw.communitydragon.org/{DragonItem.version}/game/en_us/data/menu/en_us/lol.stringtable.json" - j = download_json(url, use_cache=True) + _stringtable_cache = None + _stringtable_loaded = False + + @classmethod + def get_item_plaintext(cls, item): + if not cls._stringtable_loaded: + url = f"https://raw.communitydragon.org/latest/game/en_us/data/menu/en_us/lol.stringtable.json" + try: + cls._stringtable_cache = download_json(url, use_cache=True) + except requests.exceptions.RequestException as e: + print_warning(f"Could not fetch CommunityDragon stringtable ({url}): {e}") + cls._stringtable_cache = None + cls._stringtable_loaded = True + + if cls._stringtable_cache is None: + return None try: - return j['entries']["game_item_plaintext_" + str(item)] - except: + return cls._stringtable_cache['entries']["game_item_plaintext_" + str(item)] + except (KeyError, TypeError): return None @classmethod @@ -88,7 +114,7 @@ def _get_skin_path(cls, path): path = path.lower() path = ( "https://raw.communitydragon.org/{}/plugins/rcp-be-lol-game-data/global/default/assets".format( - DragonItem.version + cls.get_version() ) + path ) @@ -101,14 +127,14 @@ def get_json_ddragon( cls, ): # Main Function, gets items from ddragon, compares them with cdragon and then gets the items from the wiki # I didn't want make a request to cdragon for every item - url = "http://ddragon.leagueoflegends.com/cdn/{}/data/en_US/item.json".format(get_latest_version()) + url = "http://ddragon.leagueoflegends.com/cdn/{}/data/en_US/item.json".format(cls.get_latest_version()) p = download_json(url, use_cache=True) return p["data"] @classmethod def get_ddragon(cls, ddragon: int, p: dict): # print(ddragon) - baseurl = "http://ddragon.leagueoflegends.com/cdn/{}/img/item/".format(get_latest_version()) # icon base url + baseurl = "http://ddragon.leagueoflegends.com/cdn/{}/img/item/".format(cls.get_latest_version()) # icon base url icon = baseurl + p[ddragon]["image"]["full"] plaintext = p[ddragon]["plaintext"] # simple description purchasable = p[ddragon]["gold"]["purchasable"] # is this purchasable or is it upgraded (seraph's embrace) diff --git a/lolstaticdata/items/pull_items_wiki.py b/lolstaticdata/items/pull_items_wiki.py index 4113d94e..1444a7c8 100644 --- a/lolstaticdata/items/pull_items_wiki.py +++ b/lolstaticdata/items/pull_items_wiki.py @@ -17,6 +17,7 @@ from ..common.utils import ( download_soup, strip_lua_comments, + print_warning, ) from ..common.modelcommon import ( ArmorPenetration, @@ -630,7 +631,7 @@ def _parse_item_data(cls, item_data: dict, item_name:str,wiki_data:dict) -> Item item_data[x] = wiki_data[parent_item][x] except KeyError as e: clear_keys.append(x) - print(f"WARNING: Couldn't find inherited value of '{x}' for {item_name} - inherited from {parent_item}") + print_warning(f"Couldn't find inherited value of '{x}' for {item_name} - inherited from {parent_item}") break if x in "effects": for l in item_data[x]: