From e11425d5f81adab3ab99a5920daf46b6ed042ceb Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sat, 14 Mar 2026 15:46:21 -0700 Subject: [PATCH] Fix relative redirect handling in web page fetcher --- modules/web_search.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/web_search.py b/modules/web_search.py index a4424ee3..6d005496 100644 --- a/modules/web_search.py +++ b/modules/web_search.py @@ -7,7 +7,7 @@ import socket import urllib.request from concurrent.futures import as_completed from datetime import datetime -from urllib.parse import quote_plus, urlparse +from urllib.parse import quote_plus, urljoin, urlparse import requests @@ -55,7 +55,7 @@ def download_web_page(url, timeout=10, include_links=False): for _ in range(max_redirects): response = requests.get(url, headers=headers, timeout=timeout, allow_redirects=False) if response.is_redirect and 'Location' in response.headers: - url = response.headers['Location'] + url = urljoin(url, response.headers['Location']) _validate_url(url) else: break