From f9f6261e747e81c009259275629e35ad0eb4e4da Mon Sep 17 00:00:00 2001 From: Arpit Jain Date: Sun, 13 Sep 2026 13:50:49 -0400 Subject: [PATCH] Make _http_exists send the ranged GET its docstring describes The docstring states the requirement exactly: a ranged GET, so that servers which ignore HEAD still work. The body calls make_head_request, which is requests.head. Two consequences. The Range header is meaningless on a HEAD, so the 206 the function tests for is never observed on that path. And requests.head defaults to allow_redirects=False, so a 302 also reads as missing. That combination takes out the HuggingFace lane entirely: huggingface.py accepts only .bin, every real .bin weight is Git-LFS backed, and HF serves those from a CDN via a 302. Signed-off-by: Arpit Jain --- src/fetchcode/utils.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/fetchcode/utils.py b/src/fetchcode/utils.py index 13f1421..094de93 100644 --- a/src/fetchcode/utils.py +++ b/src/fetchcode/utils.py @@ -250,7 +250,14 @@ def _http_exists(url: str) -> bool: Lightweight existence check using a ranged GET so CDNs/servers that ignore HEAD still work. """ try: - resp = make_head_request(url, headers={"Range": "bytes=0-0"}) - return resp is not None and resp.status_code in (200, 206) + # A ranged GET rather than a HEAD, as the docstring says: HEAD ignores the + # Range header so 206 is never observed on that path, and requests.head + # does not follow redirects, so a CDN 302 reads as "missing". Git-LFS + # backed files are served exactly that way. + resp = requests.get(url, headers={"Range": "bytes=0-0"}, allow_redirects=True, stream=True) + try: + return resp.status_code in (200, 206) + finally: + resp.close() except Exception: return False