diff --git a/README.md b/README.md index ef22d1e..00cf0d8 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,8 @@ Install `yfinance` from PYPI using `pip`: $ pip install yfinance ``` +To install without `curl_cffi` for requests fallback, see [Advanced ▸ Installation](https://ranaroussi.github.io/yfinance/advanced/install.html). + ### [yfinance relies on the community to investigate bugs and contribute code. Here's how you can help.](https://github.com/ranaroussi/yfinance/blob/main/CONTRIBUTING.md) --- diff --git a/doc/source/advanced/index.rst b/doc/source/advanced/index.rst index 9b3dd6b..81c6d28 100644 --- a/doc/source/advanced/index.rst +++ b/doc/source/advanced/index.rst @@ -5,6 +5,7 @@ Advanced .. toctree:: :maxdepth: 2 + install logging config caching diff --git a/doc/source/advanced/install.rst b/doc/source/advanced/install.rst new file mode 100644 index 0000000..320ddc9 --- /dev/null +++ b/doc/source/advanced/install.rst @@ -0,0 +1,9 @@ +Installation +============ + +To install without ``curl_cffi`` for requests fallback: + +.. code-block:: bash + + curl -fsSL https://raw.githubusercontent.com/ranaroussi/yfinance/main/requirements.txt | grep -vi '^curl_cffi' | pip install -r /dev/stdin + pip install --no-deps yfinance diff --git a/tests/test_http.py b/tests/test_http.py new file mode 100644 index 0000000..43f4ba5 --- /dev/null +++ b/tests/test_http.py @@ -0,0 +1,75 @@ +"""Tests for the HTTP backend abstraction. + +Verifies the fallback to plain ``requests`` when ``curl_cffi`` is +unavailable, and the ``YF_DISABLE_CURL_CFFI`` env-var opt-out. +""" +import importlib +import os +import sys +import unittest + + +def _reload_http(curl_cffi_available: bool, disable_env: bool): + """Reload ``yfinance._http`` simulating curl_cffi presence / opt-out.""" + saved_modules = {k: sys.modules[k] for k in list(sys.modules) if k == "yfinance._http"} + saved_env = os.environ.get("YF_DISABLE_CURL_CFFI") + if disable_env: + os.environ["YF_DISABLE_CURL_CFFI"] = "1" + elif "YF_DISABLE_CURL_CFFI" in os.environ: + del os.environ["YF_DISABLE_CURL_CFFI"] + + saved_curl = sys.modules.get("curl_cffi") + if not curl_cffi_available: + sys.modules["curl_cffi"] = None # type: ignore[assignment] + + try: + sys.modules.pop("yfinance._http", None) + return importlib.import_module("yfinance._http") + finally: + # Restore originals so other tests aren't affected. + if saved_curl is not None: + sys.modules["curl_cffi"] = saved_curl + elif not curl_cffi_available: + sys.modules.pop("curl_cffi", None) + if saved_env is None: + os.environ.pop("YF_DISABLE_CURL_CFFI", None) + else: + os.environ["YF_DISABLE_CURL_CFFI"] = saved_env + sys.modules.pop("yfinance._http", None) + for k, v in saved_modules.items(): + sys.modules[k] = v + + +class TestHttpBackend(unittest.TestCase): + + def test_fallback_when_curl_cffi_missing(self): + import requests as _stdlib_requests + mod = _reload_http(curl_cffi_available=False, disable_env=False) + self.assertFalse(mod.HAS_CURL_CFFI) + session = mod.new_session() + self.assertIsInstance(session, _stdlib_requests.Session) + self.assertIn("Chrome", session.headers["User-Agent"]) + + def test_disable_env_var_forces_fallback(self): + import requests as _stdlib_requests + mod = _reload_http(curl_cffi_available=True, disable_env=True) + self.assertFalse(mod.HAS_CURL_CFFI) + self.assertIsInstance(mod.new_session(), _stdlib_requests.Session) + + def test_cookie_jar_works_for_requests_session(self): + mod = _reload_http(curl_cffi_available=False, disable_env=False) + session = mod.new_session() + jar = mod.cookie_jar(session) + # requests.Session.cookies subclasses CookieJar directly. + from http.cookiejar import CookieJar + self.assertIsInstance(jar, CookieJar) + + def test_is_supported_session_accepts_requests_session(self): + import requests as _stdlib_requests + mod = _reload_http(curl_cffi_available=False, disable_env=False) + self.assertTrue(mod.is_supported_session(_stdlib_requests.Session())) + self.assertFalse(mod.is_supported_session(object())) + + +if __name__ == "__main__": + unittest.main() diff --git a/yfinance/_http.py b/yfinance/_http.py new file mode 100644 index 0000000..e064db8 --- /dev/null +++ b/yfinance/_http.py @@ -0,0 +1,101 @@ +"""HTTP backend abstraction. + +Prefers ``curl_cffi`` for browser TLS impersonation. Falls back to plain +``requests`` with a realistic ``User-Agent`` when ``curl_cffi`` cannot be +imported (e.g. binary not buildable on the host platform — see issue #2692) +or when ``YF_DISABLE_CURL_CFFI`` is set in the environment (downstream +packagers may ship without ``curl_cffi`` even if it is installed). + +The fallback is best-effort: plain ``requests`` cannot replicate the +JA3/JA4 fingerprint and HTTP/2 settings that ``curl_cffi`` provides, so +Yahoo Finance may rate-limit or block this client. ``curl_cffi`` remains +the preferred backend and the default install dependency. +""" +import functools +import os + +from . import utils + +_DISABLE = os.environ.get("YF_DISABLE_CURL_CFFI", "").lower() in ("1", "true", "yes") + +if not _DISABLE: + try: + from curl_cffi import requests as _curl_backend + _backend = _curl_backend + HAS_CURL_CFFI = True + except ImportError: + import requests as _requests_backend + _backend = _requests_backend + HAS_CURL_CFFI = False +else: + import requests as _requests_backend + _backend = _requests_backend + HAS_CURL_CFFI = False + +requests = _backend +HTTPError = _backend.exceptions.HTTPError + +_FALLBACK_USER_AGENT = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +) + +_fallback_warned = False + + +def _warn_once_on_fallback(): + global _fallback_warned + if HAS_CURL_CFFI or _fallback_warned: + return + _fallback_warned = True + utils.get_yf_logger().warning( + "curl_cffi not available; falling back to requests without browser TLS " + "impersonation. Yahoo Finance may rate-limit or block this client. " + "Install curl_cffi (>=0.15) for the supported configuration." + ) + + +def new_session(): + """Create a default Session for the active backend.""" + if HAS_CURL_CFFI: + return _backend.Session(impersonate="chrome") + _warn_once_on_fallback() + s = _backend.Session() + s.headers.update({ + "User-Agent": _FALLBACK_USER_AGENT, + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.5", + }) + return s + + +def cookie_jar(session): + """Return the underlying ``http.cookiejar.CookieJar`` for either backend. + + ``curl_cffi`` exposes the jar as ``session.cookies.jar``; ``requests`` + uses ``session.cookies`` directly (it subclasses ``CookieJar``). + """ + cookies = session.cookies + return getattr(cookies, "jar", cookies) + + +@functools.lru_cache(maxsize=1) +def _supported_session_classes() -> tuple: + classes = [] + try: + from curl_cffi.requests.session import Session as _CurlSession + classes.append(_CurlSession) + except ImportError: + pass + try: + from requests.sessions import Session as _ReqSession + classes.append(_ReqSession) + except ImportError: + pass + return tuple(classes) + + +def is_supported_session(obj) -> bool: + """True if ``obj`` is a Session from either supported backend.""" + classes = _supported_session_classes() + return bool(classes) and isinstance(obj, classes) diff --git a/yfinance/base.py b/yfinance/base.py index adb0065..068c1e0 100644 --- a/yfinance/base.py +++ b/yfinance/base.py @@ -27,7 +27,7 @@ from urllib.parse import quote as urlencode import numpy as np import pandas as pd -from curl_cffi import requests +from ._http import requests, new_session from . import utils, cache @@ -81,7 +81,7 @@ class TickerBase: ticker = base_symbol self.ticker = ticker.upper() - self.session = session or requests.Session(impersonate="chrome") + self.session = session or new_session() self._tz = None self._isin = None diff --git a/yfinance/data.py b/yfinance/data.py index 5f2a9ba..a47cbc7 100644 --- a/yfinance/data.py +++ b/yfinance/data.py @@ -4,7 +4,7 @@ import socket import time as _time import json as _json -from curl_cffi import requests +from ._http import requests, new_session, is_supported_session, cookie_jar from urllib.parse import urlsplit, urljoin from bs4 import BeautifulSoup import datetime @@ -91,7 +91,7 @@ class YfData(metaclass=SingletonMeta): self._cookie_lock = threading.Lock() self._session = None - self._set_session(session or requests.Session(impersonate="chrome")) + self._set_session(session or new_session()) def set_login_cookies(self, cookie_t, cookie_y): with self._cookie_lock: @@ -115,11 +115,12 @@ class YfData(metaclass=SingletonMeta): # Can't simply use a non-caching session to fetch cookie & crumb, # because then the caching-session won't have cookie. self._session_is_caching = True - # But since switch to curl_cffi, can't use requests_cache with it. - raise YFDataException("request_cache sessions don't work with curl_cffi, which is necessary now for Yahoo API. Solution: stop setting session, let YF handle.") + # requests_cache wraps the stdlib Session; it doesn't work with + # curl_cffi and tends to miss anyway because the Yahoo crumb rotates. + raise YFDataException("Caching sessions (e.g. requests_cache) are not supported. Solution: stop setting session, let yfinance handle.") - if not isinstance(session, requests.session.Session): - raise YFDataException(f"Yahoo API requires curl_cffi session not {type(session)}. Solution: stop setting session, let YF handle.") + if not is_supported_session(session): + raise YFDataException(f"Unsupported session type {type(session)}; expected curl_cffi or requests Session. Solution: stop setting session, let yfinance handle.") with self._cookie_lock: self._session = session @@ -153,7 +154,7 @@ class YfData(metaclass=SingletonMeta): def _save_cookie_curlCffi(self): if self._session is None: return False - cookies = self._session.cookies.jar._cookies + cookies = cookie_jar(self._session)._cookies if len(cookies) == 0: return False yh_domains = [k for k in cookies.keys() if 'yahoo' in k] @@ -189,7 +190,7 @@ class YfData(metaclass=SingletonMeta): if expired: utils.get_yf_logger().debug('cached cookie expired') return False - self._session.cookies.jar._cookies.update(cookies) + cookie_jar(self._session)._cookies.update(cookies) self._cookie = cookie return True diff --git a/yfinance/multi.py b/yfinance/multi.py index 7b9e8df..8115be0 100644 --- a/yfinance/multi.py +++ b/yfinance/multi.py @@ -29,7 +29,7 @@ from typing import Union import multitasking as _multitasking import pandas as _pd -from curl_cffi import requests +from ._http import new_session from . import Ticker, utils from .data import YfData @@ -122,7 +122,7 @@ def _download_impl(ctx, tickers, start=None, end=None, actions=False, threads=Tr prepost=False, rounding=False, timeout=10, session=None, multi_level_index=True): logger = utils.get_yf_logger() - session = session or requests.Session(impersonate="chrome") + session = session or new_session() YfData(session=session) diff --git a/yfinance/scrapers/analysis.py b/yfinance/scrapers/analysis.py index 444d86f..f32823a 100644 --- a/yfinance/scrapers/analysis.py +++ b/yfinance/scrapers/analysis.py @@ -1,4 +1,4 @@ -import curl_cffi +from yfinance._http import HTTPError import pandas as pd from yfinance import utils @@ -187,7 +187,7 @@ class Analysis: params_dict = {"modules": modules, "corsDomain": "finance.yahoo.com", "formatted": "false", "symbol": self._symbol} try: result = self._data.get_raw_json(_QUOTE_SUMMARY_URL_ + f"/{self._symbol}", params=params_dict) - except curl_cffi.requests.exceptions.HTTPError as e: + except HTTPError as e: if not YfConfig.debug.hide_exceptions: raise utils.get_yf_logger().error(str(e) + e.response.text) diff --git a/yfinance/scrapers/history.py b/yfinance/scrapers/history.py index 0c72fb3..294d6dc 100644 --- a/yfinance/scrapers/history.py +++ b/yfinance/scrapers/history.py @@ -1,4 +1,4 @@ -from curl_cffi import requests +from yfinance._http import new_session from math import isclose import bisect import datetime as _datetime @@ -19,7 +19,7 @@ class PriceHistory: self._data = data self.ticker = ticker.upper() self.tz = tz - self.session = session or requests.Session(impersonate="chrome") + self.session = session or new_session() self._history_cache = {} self._history_metadata = None diff --git a/yfinance/scrapers/holders.py b/yfinance/scrapers/holders.py index 73a8215..d2b030a 100644 --- a/yfinance/scrapers/holders.py +++ b/yfinance/scrapers/holders.py @@ -1,4 +1,4 @@ -import curl_cffi +from yfinance._http import HTTPError import pandas as pd from yfinance import utils @@ -71,7 +71,7 @@ class Holders: def _fetch_and_parse(self): try: result = self._fetch() - except curl_cffi.requests.exceptions.HTTPError as e: + except HTTPError as e: if not YfConfig.debug.hide_exceptions: raise utils.get_yf_logger().error(str(e) + e.response.text) diff --git a/yfinance/scrapers/quote.py b/yfinance/scrapers/quote.py index 686bcdb..72b04ab 100644 --- a/yfinance/scrapers/quote.py +++ b/yfinance/scrapers/quote.py @@ -1,4 +1,4 @@ -import curl_cffi +from yfinance._http import HTTPError import datetime import json import numpy as _np @@ -594,7 +594,7 @@ class Quote: params_dict = {"modules": modules, "corsDomain": "finance.yahoo.com", "formatted": "false", "symbol": self._symbol, "lang": YfConfig.locale.lang, "region": YfConfig.locale.region} try: result = self._data.get_raw_json(_QUOTE_SUMMARY_URL_ + f"/{self._symbol}", params=params_dict) - except curl_cffi.requests.exceptions.HTTPError as e: + except HTTPError as e: if not YfConfig.debug.hide_exceptions: raise utils.get_yf_logger().error(str(e) + e.response.text) @@ -605,7 +605,7 @@ class Quote: params_dict = {"symbols": self._symbol, "formatted": "false", "lang": YfConfig.locale.lang, "region": YfConfig.locale.region} try: result = self._data.get_raw_json(f"{_QUERY1_URL_}/v7/finance/quote?", params=params_dict) - except curl_cffi.requests.exceptions.HTTPError as e: + except HTTPError as e: if not YfConfig.debug.hide_exceptions: raise utils.get_yf_logger().error(str(e) + e.response.text) diff --git a/yfinance/screener/screener.py b/yfinance/screener/screener.py index 8716401..e778350 100644 --- a/yfinance/screener/screener.py +++ b/yfinance/screener/screener.py @@ -1,4 +1,4 @@ -import curl_cffi +from yfinance._http import HTTPError from typing import Union import warnings from json import dumps @@ -176,7 +176,7 @@ def screen(query: Union[str, EquityQuery, FundQuery, ETFQuery], resp = _data.get(url=_PREDEFINED_URL_, params=params_dict) try: resp.raise_for_status() - except curl_cffi.requests.exceptions.HTTPError: + except HTTPError: if query not in PREDEFINED_SCREENER_QUERIES: print(f"yfinance.screen: '{query}' is probably not a predefined query.") raise