zyte-common-items ≥ 0.25.0

zytedata · Nov 11, 2024 · 4b771fb · 4b771fb
1 parent dd18ec4
commit 4b771fb
Show file tree

Hide file tree

Showing 6 changed files with 33 additions and 35 deletions.
diff --git a/docs/customization/pages.rst b/docs/customization/pages.rst
@@ -165,4 +165,4 @@ For example:
     class ExampleComSearchRequestTemplatePage(BaseSearchRequestTemplatePage):
         @field
         def url(self):
-            return "https://example.com/search?q={{ keyword|quote_plus }}"
+            return "https://example.com/search?q={{ query|quote_plus }}"
diff --git a/setup.py b/setup.py
@@ -23,7 +23,7 @@
         "scrapy-spider-metadata>=0.2.0",
         "scrapy-zyte-api[provider]>=0.23.0",
         "web-poet>=0.17.1",
-        "zyte-common-items @ git+https://github.com/Gallaecio/zyte-common-items.git@search-request-template-ancestry",
+        "zyte-common-items>=0.25.0",
     ],
     classifiers=[
         "Development Status :: 3 - Alpha",

diff --git a/tests/test_search.py b/tests/test_search.py
@@ -39,7 +39,7 @@
             """,
             {"search_request_builders": ["extruct"]},
             {
-                "url": "https://query.example.com/search?q={{ keyword|quote_plus }}",
+                "url": "https://query.example.com/search?q={{ query|quote_plus }}",
             },
         ),
         # Microdata example from Google
@@ -57,7 +57,7 @@
             """,
             {"search_request_builders": ["extruct"]},
             {
-                "url": "https://query.example.com/search?q={{ keyword|quote_plus }}",
+                "url": "https://query.example.com/search?q={{ query|quote_plus }}",
             },
         ),
         # Non-compliant JSON-LD that uses a JSON array for potentialAction
@@ -89,7 +89,7 @@
             """,
             {"search_request_builders": ["extruct"]},
             {
-                "url": "https://query.example.com/search?q={{ keyword|quote_plus }}",
+                "url": "https://query.example.com/search?q={{ query|quote_plus }}",
             },
         ),
         # Non-default placeholder, JSON-LD
@@ -119,7 +119,7 @@
             """,
             {"search_request_builders": ["extruct"]},
             {
-                "url": "https://query.example.com/search?q={{ keyword|quote_plus }}&dont_replace={search_term_string}",
+                "url": "https://query.example.com/search?q={{ query|quote_plus }}&dont_replace={search_term_string}",
             },
         ),
         # Non-default placeholder, Microdata
@@ -136,7 +136,7 @@
             """,
             {"search_request_builders": ["extruct"]},
             {
-                "url": "https://query.example.com/search?q={{ keyword|quote_plus }}&dont_replace={search_term_string}",
+                "url": "https://query.example.com/search?q={{ query|quote_plus }}&dont_replace={search_term_string}",
             },
         ),
         # JSON-LD, WebSite isPartOf WebPage
@@ -170,7 +170,7 @@
             """,
             {"search_request_builders": ["extruct"]},
             {
-                "url": "https://query.example.com/search?q={{ keyword|quote_plus }}",
+                "url": "https://query.example.com/search?q={{ query|quote_plus }}",
             },
         ),
         # Relative URL, JSON-LD
@@ -200,7 +200,7 @@
             """,
             {"search_request_builders": ["extruct"]},
             {
-                "url": "https://example.com/search?q={{ keyword|quote_plus }}",
+                "url": "https://example.com/search?q={{ query|quote_plus }}",
             },
         ),
         # Relative URL, Microdata
@@ -217,7 +217,7 @@
             """,
             {"search_request_builders": ["extruct"]},
             {
-                "url": "https://example.com/search?q={{ keyword|quote_plus }}",
+                "url": "https://example.com/search?q={{ query|quote_plus }}",
             },
         ),
         # Wrong escaping in JSON-LD
@@ -245,7 +245,7 @@
             """,
             {"search_request_builders": ["extruct"]},
             {
-                "url": "https://example.com/search?a=b&q={{ keyword|quote_plus }}",
+                "url": "https://example.com/search?a=b&q={{ query|quote_plus }}",
             },
         ),
         # Query in path, JSON-LD
@@ -275,7 +275,7 @@
             """,
             {"search_request_builders": ["extruct"]},
             {
-                "url": "https://example.com/s/{{ keyword|urlencode }}",
+                "url": "https://example.com/s/{{ query|urlencode }}",
             },
         ),
         # Relative URL, Microdata
@@ -292,7 +292,7 @@
             """,
             {"search_request_builders": ["extruct"]},
             {
-                "url": "https://example.com/s/{{ keyword|urlencode }}",
+                "url": "https://example.com/s/{{ query|urlencode }}",
             },
         ),
         # No potentialAction, JSON-LD
@@ -415,7 +415,7 @@
             {"search_request_builders": ["extruct"]},
             {"error": "Try enabling browser rendering"},
         ),
-        # No keyword name, JSON-LD
+        # No query variable name, JSON-LD
         (
             b"""
             <html>
@@ -442,10 +442,10 @@
             """,
             {"search_request_builders": ["extruct"]},
             {
-                "url": "https://query.example.com/search?q={{ keyword|quote_plus }}",
+                "url": "https://query.example.com/search?q={{ query|quote_plus }}",
             },
         ),
-        # No keyword name, Microdata
+        # No query variable name, Microdata
         (
             b"""
             <div itemscope itemtype="https://schema.org/WebSite">
@@ -459,7 +459,7 @@
             """,
             {"search_request_builders": ["extruct"]},
             {
-                "url": "https://query.example.com/search?q={{ keyword|quote_plus }}",
+                "url": "https://query.example.com/search?q={{ query|quote_plus }}",
             },
         ),
         # Formasaurus and form heuristics #-----------------------------------#
@@ -476,7 +476,7 @@
                     </form>
                     """,
                     {
-                        "url": "https://example.com?q={{ keyword|quote_plus }}",
+                        "url": "https://example.com?q={{ query|quote_plus }}",
                     },
                 ),
                 # No form
@@ -517,7 +517,7 @@
                 f"""<a href="https://example.com/search?{prefix}{q}=example{suffix}""".encode(),
                 {"search_request_builders": ["link_heuristics"]},
                 {
-                    "url": f"https://example.com/search?{prefix}{q}={{{{ keyword|quote_plus }}}}{suffix}"
+                    "url": f"https://example.com/search?{prefix}{q}={{{{ query|quote_plus }}}}{suffix}"
                 },
             )
             for q in (
@@ -577,14 +577,14 @@
                 # priority, if both the Formasaurus builder and the form
                 # heuristics builder output the same URL, that one is used
                 # instead.
-                ({}, {"url": "https://example.com/form?q={{ keyword|quote_plus }}"}),
+                ({}, {"url": "https://example.com/form?q={{ query|quote_plus }}"}),
                 (
                     {"search_request_builder_strategy": "popular"},
-                    {"url": "https://example.com/form?q={{ keyword|quote_plus }}"},
+                    {"url": "https://example.com/form?q={{ query|quote_plus }}"},
                 ),
                 (
                     {"search_request_builder_strategy": "first"},
-                    {"url": "https://example.com/metadata?q={{ keyword|quote_plus }}"},
+                    {"url": "https://example.com/metadata?q={{ query|quote_plus }}"},
                 ),
                 # Strategies only take into account the specified builders, and
                 # in the supplied order.
@@ -593,7 +593,7 @@
                         "search_request_builder_strategy": "first",
                         "search_request_builders": ["formasaurus", "extruct"],
                     },
-                    {"url": "https://example.com/form?q={{ keyword|quote_plus }}"},
+                    {"url": "https://example.com/form?q={{ query|quote_plus }}"},
                 ),
                 (
                     {
@@ -604,7 +604,7 @@
                             "link_heuristics",
                         ],
                     },
-                    {"url": "https://example.com/metadata?q={{ keyword|quote_plus }}"},
+                    {"url": "https://example.com/metadata?q={{ query|quote_plus }}"},
                 ),
                 # Unsupported strategies trigger a ValueError
                 (

diff --git a/tox.ini b/tox.ini
@@ -31,7 +31,7 @@ deps =
     scrapy-spider-metadata==0.2.0
     scrapy-zyte-api[provider]==0.23.0
     web-poet==0.17.1
-;     zyte-common-items
+    zyte-common-items==0.25.0
 
 [testenv:mypy]
 deps =

diff --git a/zyte_spider_templates/pages/search_request_template.py b/zyte_spider_templates/pages/search_request_template.py
@@ -100,11 +100,11 @@ def _item_from_form_heuristics(self):
         except NotImplementedError:
             raise ValueError("form2request does not support the target search form")
         return SearchRequestTemplate(
-            url=request_data.url.replace(_PLACEHOLDER, "{{ keyword|quote_plus }}"),
+            url=request_data.url.replace(_PLACEHOLDER, "{{ query|quote_plus }}"),
             method=request_data.method,
             headers=request_data.headers,
             body=request_data.body.decode().replace(
-                _PLACEHOLDER, "{{ keyword|quote_plus }}"
+                _PLACEHOLDER, "{{ query|quote_plus }}"
             ),
         )
 
@@ -166,11 +166,9 @@ def _item_from_extruct(self):
                 "Could not find HTML metadata to compose a search request template."
             )
         parts = url_template.split("?", maxsplit=1)
-        parts[0] = parts[0].replace(f"{{{query_field}}}", "{{ keyword|urlencode }}")
+        parts[0] = parts[0].replace(f"{{{query_field}}}", "{{ query|urlencode }}")
         if len(parts) > 1:
-            parts[1] = parts[1].replace(
-                f"{{{query_field}}}", "{{ keyword|quote_plus }}"
-            )
+            parts[1] = parts[1].replace(f"{{{query_field}}}", "{{ query|quote_plus }}")
         url = "?".join(parts)
         url = str(self.response.urljoin(url))
         url = html.unescape(url)
@@ -218,7 +216,7 @@ def _item_from_link_heuristics(self):
             url = add_or_replace_parameters(
                 search_link.url, {k: _PLACEHOLDER for k in search_params}
             )
-            url = url.replace(_PLACEHOLDER, "{{ keyword|quote_plus }}")
+            url = url.replace(_PLACEHOLDER, "{{ query|quote_plus }}")
             return SearchRequestTemplate(
                 url=url,
                 method="GET",
@@ -250,11 +248,11 @@ def _item_from_formasaurus(self):
         except NotImplementedError:
             raise ValueError("form2request does not support the target search form")
         return SearchRequestTemplate(
-            url=request_data.url.replace(_PLACEHOLDER, "{{ keyword|quote_plus }}"),
+            url=request_data.url.replace(_PLACEHOLDER, "{{ query|quote_plus }}"),
             method=request_data.method,
             headers=request_data.headers,
             body=request_data.body.decode().replace(
-                _PLACEHOLDER, "{{ keyword|quote_plus }}"
+                _PLACEHOLDER, "{{ query|quote_plus }}"
             ),
         )
 

diff --git a/zyte_spider_templates/spiders/ecommerce.py b/zyte_spider_templates/spiders/ecommerce.py
@@ -279,7 +279,7 @@ def parse_search_request_template(
         if probability is None or probability <= 0:
             return
         for query in self.args.search_queries:
-            yield search_request_template.request(keyword=query).to_scrapy(
+            yield search_request_template.request(query=query).to_scrapy(
                 callback=self.parse_navigation,
                 meta={
                     "crawling_logs": {"page_type": "productNavigation"},