Fix: Do not send an empty sec-ch-ua header for empty or unparseable user agents

crawl4ai/async_configs.py

@@ -1166,7 +1166,12 @@ def __init__(
             pass
 
         self.browser_hint = UAGen.generate_client_hints(self.user_agent)
-        self.headers.setdefault("sec-ch-ua", self.browser_hint)
+        # An empty user agent (or one with no recognizable browser) yields an
+        # empty client-hints value; sending an empty sec-ch-ua header misleads
+        # servers, so only auto-add the header when there is a hint to send.
+        # A caller-provided sec-ch-ua is still respected via setdefault.
+        if self.browser_hint:
+            self.headers.setdefault("sec-ch-ua", self.browser_hint)
 
         # Set appropriate browser management flags based on browser_mode
         if self.browser_mode == "builtin":

tests/test_browser_config_empty_sec_ch_ua.py

@@ -0,0 +1,33 @@
+"""Regression tests for crawl4ai issue #2493 (part: empty sec-ch-ua).
+
+BrowserConfig(user_agent="") keeps the browser's native User-Agent, but
+__init__ used to auto-add an empty "sec-ch-ua" header to every request.
+An empty client-hints header must not be sent; a caller-provided value
+must still be respected.
+"""
+
+from crawl4ai.async_configs import BrowserConfig
+
+
+class TestEmptySecChUa:
+    def test_empty_user_agent_adds_no_sec_ch_ua(self):
+        config = BrowserConfig(user_agent="")
+        assert config.browser_hint == ""
+        assert "sec-ch-ua" not in config.headers
+
+    def test_unparseable_user_agent_adds_no_sec_ch_ua(self):
+        config = BrowserConfig(user_agent="MyBot/1.0")
+        assert config.browser_hint == ""
+        assert "sec-ch-ua" not in config.headers
+
+    def test_default_user_agent_still_adds_sec_ch_ua(self):
+        config = BrowserConfig()
+        assert config.browser_hint != ""
+        assert config.headers["sec-ch-ua"] == config.browser_hint
+
+    def test_caller_provided_sec_ch_ua_is_respected(self):
+        config = BrowserConfig(
+            user_agent="",
+            headers={"sec-ch-ua": '"Chromium";v="156"'},
+        )
+        assert config.headers["sec-ch-ua"] == '"Chromium";v="156"'