Create utils.py

utils.py

@@ -0,0 +1,74 @@
+import re
+from decimal import Decimal
+from html import unescape
+from math import atan2, cos, radians, sin, sqrt
+from typing import Any, Dict, List, Union
+from urllib.parse import unquote
+import orjson
+
+from .exceptions import WebscoutE
+
+REGEX_STRIP_TAGS = re.compile("<.*?>")
+
+
+def json_dumps(obj: Any) -> str:
+    try:
+        return orjson.dumps(obj).decode("utf-8")
+    except Exception as ex:
+        raise WebscoutE(f"{type(ex).__name__}: {ex}") from ex
+
+
+def json_loads(obj: Union[str, bytes]) -> Any:
+    try:
+        return orjson.loads(obj)
+    except Exception as ex:
+        raise WebscoutE(f"{type(ex).__name__}: {ex}") from ex
+
+
+def _extract_vqd(html_bytes: bytes, keywords: str) -> str:
+    """Extract vqd from html bytes."""
+    for c1, c1_len, c2 in (
+        (b'vqd="', 5, b'"'),
+        (b"vqd=", 4, b"&"),
+        (b"vqd='", 5, b"'"),
+    ):
+        try:
+            start = html_bytes.index(c1) + c1_len
+            end = html_bytes.index(c2, start)
+            return html_bytes[start:end].decode()
+        except ValueError:
+            pass
+    raise WebscoutE(f"_extract_vqd() {keywords=} Could not extract vqd.")
+
+
+def _text_extract_json(html_bytes: bytes, keywords: str) -> List[Dict[str, str]]:
+    """text(backend="api") -> extract json from html."""
+    try:
+        start = html_bytes.index(b"DDG.pageLayout.load('d',") + 24
+        end = html_bytes.index(b");DDG.duckbar.load(", start)
+        data = html_bytes[start:end]
+        result: List[Dict[str, str]] = json_loads(data)
+        return result
+    except Exception as ex:
+        raise WebscoutE(f"_text_extract_json() {keywords=} {type(ex).__name__}: {ex}") from ex
+    raise WebscoutE(f"_text_extract_json() {keywords=} return None")
+
+
+def _normalize(raw_html: str) -> str:
+    """Strip HTML tags from the raw_html string."""
+    return unescape(REGEX_STRIP_TAGS.sub("", raw_html)) if raw_html else ""
+
+
+def _normalize_url(url: str) -> str:
+    """Unquote URL and replace spaces with '+'."""
+    return unquote(url.replace(" ", "+")) if url else ""
+
+
+def _calculate_distance(lat1: Decimal, lon1: Decimal, lat2: Decimal, lon2: Decimal) -> float:
+    """Calculate distance between two points in km. Haversine formula."""
+    R = 6371.0087714  # Earth's radius in km
+    rlat1, rlon1, rlat2, rlon2 = map(radians, [float(lat1), float(lon1), float(lat2), float(lon2)])
+    dlon, dlat = rlon2 - rlon1, rlat2 - rlat1
+    a = sin(dlat / 2) ** 2 + cos(rlat1) * cos(rlat2) * sin(dlon / 2) ** 2
+    c = 2 * atan2(sqrt(a), sqrt(1 - a))
+    return R * c