From cc301907c451624a95187e59888c6d9a82c1cf8c Mon Sep 17 00:00:00 2001 From: "Foad S. Farimani" Date: Fri, 3 Oct 2025 21:21:32 +0200 Subject: [PATCH 1/2] Fix imports, add User-Agent headers, update selectors for modern Amazon HTML Fixes #10 Fixes #14 Fixes #16 Fixes #17 Fixes #19 Fixes #27 --- amzsear/core/AmzSear.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/amzsear/core/AmzSear.py b/amzsear/core/AmzSear.py index 88113ee..66319af 100644 --- a/amzsear/core/AmzSear.py +++ b/amzsear/core/AmzSear.py @@ -7,9 +7,9 @@ from amzsear.core.consts import URL_ADDONS, SEARCH_URL, DEFAULT_REGION from amzsear.core.AmzProduct import AmzProduct except ImportError: - from .amzsear.core import build_url - from .amzsear.core.consts import URL_ADDONS, SEARCH_URL, DEFAULT_REGION - from .amzsear.core.AmzProduct import AmzProduct + from . import build_url + from .consts import URL_ADDONS, SEARCH_URL, DEFAULT_REGION + from .AmzProduct import AmzProduct """ The AmzSear object is similar to a Python dict, with each item having a @@ -74,14 +74,24 @@ def get_iter(it): if url != None: url = get_iter(url) self._urls = url - html = [request.urlopen(request.Request(build_url(u), **URL_ADDONS)).read() for u in url] + html = [] + for u in url: + req = request.Request( + build_url(u), + headers={ + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0", + "Accept-Language": "en-US,en;q=0.9", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + } + ) + html.append(request.urlopen(req).read()) if html != None: html = get_iter(html) html_element = [html_module.fromstring(h) for h in html] if html_element != None: html_element = get_iter(html_element) for html_el in html_element: - products = html_el.cssselect('li[id*="result_"]') + products = html_el.cssselect('div[data-asin][data-component-type="s-search-result"]') products = [x for x in products if x.cssselect('h2')] products = [AmzProduct(elem) for elem in products] if products != None: From b1732c38f250d41fb504dc4277635a81eaeb9d5e Mon Sep 17 00:00:00 2001 From: "Foad S. Farimani" Date: Fri, 3 Oct 2025 21:34:37 +0200 Subject: [PATCH 2/2] Update dependencies in setup.py and requirements.txt (lxml, cssselect, lxml_html_clean, requests) --- requirements.txt | 5 ++++- setup.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 10707f8..9652c44 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,4 @@ -lxml==3.8.0 +lxml>=4.9.1 +cssselect>=1.1.0 +lxml_html_clean>=0.1.0 +requests>=2.20.0 \ No newline at end of file diff --git a/setup.py b/setup.py index 571f156..ad708af 100644 --- a/setup.py +++ b/setup.py @@ -16,6 +16,9 @@ ], }, install_requires=[ - 'lxml==3.8', + "lxml>=4.9.1", + "cssselect>=1.1.0", + "lxml_html_clean>=0.1.0", + "requests>=2.20.0", ], )