Merge pull request #2802 from dokson/feature/optional-curl-cffi

Make curl_cffi optional with fallback to requests (closes #2692)

README.md

@@ -53,6 +53,8 @@ Install `yfinance` from PYPI using `pip`:
 $ pip install yfinance
 ```
 
+To install without `curl_cffi` for requests fallback, see [Advanced ▸ Installation](https://ranaroussi.github.io/yfinance/advanced/install.html).
+
 ### [yfinance relies on the community to investigate bugs and contribute code. Here's how you can help.](https://github.com/ranaroussi/yfinance/blob/main/CONTRIBUTING.md)
 
 ---

doc/source/advanced/index.rst

@@ -5,6 +5,7 @@ Advanced
 .. toctree::
    :maxdepth: 2
 
+   install
    logging
    config
    caching

doc/source/advanced/install.rst

@@ -0,0 +1,9 @@
+Installation
+============
+
+To install without ``curl_cffi`` for requests fallback:
+
+.. code-block:: bash
+
+    curl -fsSL https://raw.githubusercontent.com/ranaroussi/yfinance/main/requirements.txt | grep -vi '^curl_cffi' | pip install -r /dev/stdin
+    pip install --no-deps yfinance

tests/test_http.py

@@ -0,0 +1,75 @@
+"""Tests for the HTTP backend abstraction.
+
+Verifies the fallback to plain ``requests`` when ``curl_cffi`` is
+unavailable, and the ``YF_DISABLE_CURL_CFFI`` env-var opt-out.
+"""
+import importlib
+import os
+import sys
+import unittest
+
+
+def _reload_http(curl_cffi_available: bool, disable_env: bool):
+    """Reload ``yfinance._http`` simulating curl_cffi presence / opt-out."""
+    saved_modules = {k: sys.modules[k] for k in list(sys.modules) if k == "yfinance._http"}
+    saved_env = os.environ.get("YF_DISABLE_CURL_CFFI")
+    if disable_env:
+        os.environ["YF_DISABLE_CURL_CFFI"] = "1"
+    elif "YF_DISABLE_CURL_CFFI" in os.environ:
+        del os.environ["YF_DISABLE_CURL_CFFI"]
+
+    saved_curl = sys.modules.get("curl_cffi")
+    if not curl_cffi_available:
+        sys.modules["curl_cffi"] = None  # type: ignore[assignment]
+
+    try:
+        sys.modules.pop("yfinance._http", None)
+        return importlib.import_module("yfinance._http")
+    finally:
+        # Restore originals so other tests aren't affected.
+        if saved_curl is not None:
+            sys.modules["curl_cffi"] = saved_curl
+        elif not curl_cffi_available:
+            sys.modules.pop("curl_cffi", None)
+        if saved_env is None:
+            os.environ.pop("YF_DISABLE_CURL_CFFI", None)
+        else:
+            os.environ["YF_DISABLE_CURL_CFFI"] = saved_env
+        sys.modules.pop("yfinance._http", None)
+        for k, v in saved_modules.items():
+            sys.modules[k] = v
+
+
+class TestHttpBackend(unittest.TestCase):
+
+    def test_fallback_when_curl_cffi_missing(self):
+        import requests as _stdlib_requests
+        mod = _reload_http(curl_cffi_available=False, disable_env=False)
+        self.assertFalse(mod.HAS_CURL_CFFI)
+        session = mod.new_session()
+        self.assertIsInstance(session, _stdlib_requests.Session)
+        self.assertIn("Chrome", session.headers["User-Agent"])
+
+    def test_disable_env_var_forces_fallback(self):
+        import requests as _stdlib_requests
+        mod = _reload_http(curl_cffi_available=True, disable_env=True)
+        self.assertFalse(mod.HAS_CURL_CFFI)
+        self.assertIsInstance(mod.new_session(), _stdlib_requests.Session)
+
+    def test_cookie_jar_works_for_requests_session(self):
+        mod = _reload_http(curl_cffi_available=False, disable_env=False)
+        session = mod.new_session()
+        jar = mod.cookie_jar(session)
+        # requests.Session.cookies subclasses CookieJar directly.
+        from http.cookiejar import CookieJar
+        self.assertIsInstance(jar, CookieJar)
+
+    def test_is_supported_session_accepts_requests_session(self):
+        import requests as _stdlib_requests
+        mod = _reload_http(curl_cffi_available=False, disable_env=False)
+        self.assertTrue(mod.is_supported_session(_stdlib_requests.Session()))
+        self.assertFalse(mod.is_supported_session(object()))
+
+
+if __name__ == "__main__":
+    unittest.main()

yfinance/_http.py

@@ -0,0 +1,101 @@
+"""HTTP backend abstraction.
+
+Prefers ``curl_cffi`` for browser TLS impersonation. Falls back to plain
+``requests`` with a realistic ``User-Agent`` when ``curl_cffi`` cannot be
+imported (e.g. binary not buildable on the host platform — see issue #2692)
+or when ``YF_DISABLE_CURL_CFFI`` is set in the environment (downstream
+packagers may ship without ``curl_cffi`` even if it is installed).
+
+The fallback is best-effort: plain ``requests`` cannot replicate the
+JA3/JA4 fingerprint and HTTP/2 settings that ``curl_cffi`` provides, so
+Yahoo Finance may rate-limit or block this client. ``curl_cffi`` remains
+the preferred backend and the default install dependency.
+"""
+import functools
+import os
+
+from . import utils
+
+_DISABLE = os.environ.get("YF_DISABLE_CURL_CFFI", "").lower() in ("1", "true", "yes")
+
+if not _DISABLE:
+    try:
+        from curl_cffi import requests as _curl_backend
+        _backend = _curl_backend
+        HAS_CURL_CFFI = True
+    except ImportError:
+        import requests as _requests_backend
+        _backend = _requests_backend
+        HAS_CURL_CFFI = False
+else:
+    import requests as _requests_backend
+    _backend = _requests_backend
+    HAS_CURL_CFFI = False
+
+requests = _backend
+HTTPError = _backend.exceptions.HTTPError
+
+_FALLBACK_USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+)
+
+_fallback_warned = False
+
+
+def _warn_once_on_fallback():
+    global _fallback_warned
+    if HAS_CURL_CFFI or _fallback_warned:
+        return
+    _fallback_warned = True
+    utils.get_yf_logger().warning(
+        "curl_cffi not available; falling back to requests without browser TLS "
+        "impersonation. Yahoo Finance may rate-limit or block this client. "
+        "Install curl_cffi (>=0.15) for the supported configuration."
+    )
+
+
+def new_session():
+    """Create a default Session for the active backend."""
+    if HAS_CURL_CFFI:
+        return _backend.Session(impersonate="chrome")
+    _warn_once_on_fallback()
+    s = _backend.Session()
+    s.headers.update({
+        "User-Agent": _FALLBACK_USER_AGENT,
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.5",
+    })
+    return s
+
+
+def cookie_jar(session):
+    """Return the underlying ``http.cookiejar.CookieJar`` for either backend.
+
+    ``curl_cffi`` exposes the jar as ``session.cookies.jar``; ``requests``
+    uses ``session.cookies`` directly (it subclasses ``CookieJar``).
+    """
+    cookies = session.cookies
+    return getattr(cookies, "jar", cookies)
+
+
+@functools.lru_cache(maxsize=1)
+def _supported_session_classes() -> tuple:
+    classes = []
+    try:
+        from curl_cffi.requests.session import Session as _CurlSession
+        classes.append(_CurlSession)
+    except ImportError:
+        pass
+    try:
+        from requests.sessions import Session as _ReqSession
+        classes.append(_ReqSession)
+    except ImportError:
+        pass
+    return tuple(classes)
+
+
+def is_supported_session(obj) -> bool:
+    """True if ``obj`` is a Session from either supported backend."""
+    classes = _supported_session_classes()
+    return bool(classes) and isinstance(obj, classes)

yfinance/base.py

@@ -27,7 +27,7 @@ from urllib.parse import quote as urlencode
 
 import numpy as np
 import pandas as pd
-from curl_cffi import requests
+from ._http import requests, new_session
 
 
 from . import utils, cache
@@ -81,7 +81,7 @@ class TickerBase:
                 ticker = base_symbol
 
         self.ticker = ticker.upper()
-        self.session = session or requests.Session(impersonate="chrome")
+        self.session = session or new_session()
         self._tz = None
 
         self._isin = None

yfinance/data.py

@@ -4,7 +4,7 @@ import socket
 import time as _time
 import json as _json
 
-from curl_cffi import requests
+from ._http import requests, new_session, is_supported_session, cookie_jar
 from urllib.parse import urlsplit, urljoin
 from bs4 import BeautifulSoup
 import datetime
@@ -90,7 +90,7 @@ class YfData(metaclass=SingletonMeta):
         self._cookie_lock = threading.Lock()
 
         self._session = None
-        self._set_session(session or requests.Session(impersonate="chrome"))
+        self._set_session(session or new_session())
 
     def set_login_cookies(self, cookie_t, cookie_y):
         with self._cookie_lock:
@@ -114,11 +114,12 @@ class YfData(metaclass=SingletonMeta):
             # Can't simply use a non-caching session to fetch cookie & crumb,
             # because then the caching-session won't have cookie.
             self._session_is_caching = True
-            # But since switch to curl_cffi, can't use requests_cache with it.
-            raise YFDataException("request_cache sessions don't work with curl_cffi, which is necessary now for Yahoo API. Solution: stop setting session, let YF handle.")
+            # requests_cache wraps the stdlib Session; it doesn't work with
+            # curl_cffi and tends to miss anyway because the Yahoo crumb rotates.
+            raise YFDataException("Caching sessions (e.g. requests_cache) are not supported. Solution: stop setting session, let yfinance handle.")
 
-        if not isinstance(session, requests.session.Session):
-            raise YFDataException(f"Yahoo API requires curl_cffi session not {type(session)}. Solution: stop setting session, let YF handle.")
+        if not is_supported_session(session):
+            raise YFDataException(f"Unsupported session type {type(session)}; expected curl_cffi or requests Session. Solution: stop setting session, let yfinance handle.")
 
         with self._cookie_lock:
             self._session = session
@@ -152,7 +153,7 @@ class YfData(metaclass=SingletonMeta):
     def _save_cookie_curlCffi(self):
         if self._session is None:
             return False
-        cookies = self._session.cookies.jar._cookies
+        cookies = cookie_jar(self._session)._cookies
         if len(cookies) == 0:
             return False
         yh_domains = [k for k in cookies.keys() if 'yahoo' in k]
@@ -188,7 +189,7 @@ class YfData(metaclass=SingletonMeta):
         if expired:
             utils.get_yf_logger().debug('cached cookie expired')
             return False
-        self._session.cookies.jar._cookies.update(cookies)
+        cookie_jar(self._session)._cookies.update(cookies)
         self._cookie = cookie
         return True
 

yfinance/multi.py

@@ -29,7 +29,7 @@ from typing import Union
 
 import multitasking as _multitasking
 import pandas as _pd
-from curl_cffi import requests
+from ._http import new_session
 
 from . import Ticker, utils
 from .data import YfData
@@ -122,7 +122,7 @@ def _download_impl(ctx, tickers, start=None, end=None, actions=False, threads=Tr
                    prepost=False, rounding=False, timeout=10, session=None,
                    multi_level_index=True):
     logger = utils.get_yf_logger()
-    session = session or requests.Session(impersonate="chrome")
+    session = session or new_session()
 
     YfData(session=session)
 

yfinance/scrapers/analysis.py

@@ -1,4 +1,4 @@
-import curl_cffi
+from yfinance._http import HTTPError
 import pandas as pd
 
 from yfinance import utils
@@ -187,7 +187,7 @@ class Analysis:
         params_dict = {"modules": modules, "corsDomain": "finance.yahoo.com", "formatted": "false", "symbol": self._symbol}
         try:
             result = self._data.get_raw_json(_QUOTE_SUMMARY_URL_ + f"/{self._symbol}", params=params_dict)
-        except curl_cffi.requests.exceptions.HTTPError as e:
+        except HTTPError as e:
             if not YfConfig.debug.hide_exceptions:
                 raise
             utils.get_yf_logger().error(str(e) + e.response.text)

yfinance/scrapers/history.py

@@ -1,4 +1,4 @@
-from curl_cffi import requests
+from yfinance._http import new_session
 from math import isclose
 import bisect
 import datetime as _datetime
@@ -19,7 +19,7 @@ class PriceHistory:
         self._data = data
         self.ticker = ticker.upper()
         self.tz = tz
-        self.session = session or requests.Session(impersonate="chrome")
+        self.session = session or new_session()
 
         self._history_cache = {}
         self._history_metadata = None

yfinance/scrapers/holders.py

@@ -1,4 +1,4 @@
-import curl_cffi
+from yfinance._http import HTTPError
 import pandas as pd
 
 from yfinance import utils
@@ -71,7 +71,7 @@ class Holders:
     def _fetch_and_parse(self):
         try:
             result = self._fetch()
-        except curl_cffi.requests.exceptions.HTTPError as e:
+        except HTTPError as e:
             if not YfConfig.debug.hide_exceptions:
                 raise
             utils.get_yf_logger().error(str(e) + e.response.text)

yfinance/scrapers/quote.py

@@ -1,4 +1,4 @@
-import curl_cffi
+from yfinance._http import HTTPError
 import datetime
 import json
 import numpy as _np
@@ -594,7 +594,7 @@ class Quote:
         params_dict = {"modules": modules, "corsDomain": "finance.yahoo.com", "formatted": "false", "symbol": self._symbol, "lang": YfConfig.locale.lang, "region": YfConfig.locale.region}
         try:
             result = self._data.get_raw_json(_QUOTE_SUMMARY_URL_ + f"/{self._symbol}", params=params_dict)
-        except curl_cffi.requests.exceptions.HTTPError as e:
+        except HTTPError as e:
             if not YfConfig.debug.hide_exceptions:
                 raise
             utils.get_yf_logger().error(str(e) + e.response.text)
@@ -605,7 +605,7 @@ class Quote:
         params_dict = {"symbols": self._symbol, "formatted": "false", "lang": YfConfig.locale.lang, "region": YfConfig.locale.region}
         try:
             result = self._data.get_raw_json(f"{_QUERY1_URL_}/v7/finance/quote?", params=params_dict)
-        except curl_cffi.requests.exceptions.HTTPError as e:
+        except HTTPError as e:
             if not YfConfig.debug.hide_exceptions:
                 raise
             utils.get_yf_logger().error(str(e) + e.response.text)

yfinance/screener/screener.py

@@ -1,4 +1,4 @@
-import curl_cffi
+from yfinance._http import HTTPError
 from typing import Union
 import warnings
 from json import dumps
@@ -176,7 +176,7 @@ def screen(query: Union[str, EquityQuery, FundQuery, ETFQuery],
         resp = _data.get(url=_PREDEFINED_URL_, params=params_dict)
         try:
             resp.raise_for_status()
-        except curl_cffi.requests.exceptions.HTTPError:
+        except HTTPError:
             if query not in PREDEFINED_SCREENER_QUERIES:
                 print(f"yfinance.screen: '{query}' is probably not a predefined query.")
             raise