Fall back to old bhavcopy format (with ISIN) when udiff unavailable

CHANGELOG.md

@@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.35.5] - 2026-08-25
+
+### Changed
+- `bhavcopy_raw` now falls back to the old bhavcopy format (which includes the
+  ISIN field) instead of the full bhavcopy (`sec_bhavdata_full`) when UDiff is
+  unavailable
+- Added `bhavcopy_old_raw` which fetches the legacy `cm{dd}{MMM}{yyyy}bhav.csv.zip`
+  through the NSE reports API
+  (`/api/reports?archives=[{"name": "CM - Bhavcopy(csv)", ...}]&date={DD-MMM-YYYY}&type=equities&mode=single`)
+- Marked network-dependent tests in `tests/test_bhav.py` as `@pytest.mark.live`
+  and added mocked offline tests for the fallback logic (CI-safe)
+
 ## [0.35.4] - 2026-08-21
 
 ### Fixed

jugaad_data/__init__.py

@@ -1 +1 @@
-__version__ = "0.35.4"
+__version__ = "0.35.5"

jugaad_data/nse/archives.py

@@ -170,6 +170,7 @@ def unzipper(*args, **kwargs):
 
 class NSEArchives:
     base_url = "https://nsearchives.nseindia.com/"
+    reports_url = "https://www.nseindia.com/api/reports"
     """Conventions
            d - 1, 12 (without leading zero)
           dd - 01, 21 (day of the month with leading zero)
@@ -218,8 +219,9 @@ def bhavcopy_raw(self, dt):
         Uses hybrid approach:
         - For dates >= Jul 8, 2024: Attempts to fetch UDiff format from the
           historical UDiFF archive, then the daily-reports API
-        - For older dates or if all UDiff sources fail: Falls back to
-          BHAVDATA-FULL format
+        - If UDiff is not available (older dates or both UDiff sources fail):
+          Falls back to the old bhavcopy format (cm bhavcopy zip) fetched via
+          the NSE reports API. This format includes the ISIN field.
 
         Note: UDiff format has different column structure than old format.
         Data is returned as-is without modification.
@@ -261,10 +263,54 @@ def bhavcopy_raw(self, dt):
             except (ValueError, requests.RequestException, zipfile.BadZipFile):
                 pass
 
-        # Fallback: Use BHAVDATA-FULL (available for all historical dates)
-        return self.full_bhavcopy_raw(dt)
+        # Fallback: Old bhavcopy format (includes ISIN field) via reports API
+        return self.bhavcopy_old_raw(dt)
 
 
+    @unzip
+    def bhavcopy_old_raw(self, dt):
+        """Downloads raw bhavcopy text in the old format (includes ISIN field)
+
+        Fetches the legacy CM bhavcopy zip (cm{dd}{MMM}{yyyy}bhav.csv.zip)
+        through the NSE reports API:
+        https://www.nseindia.com/api/reports?archives=[{"name": "CM - Bhavcopy(csv)",
+        "type": "daily-reports", "category": "capital-market", "section": "equities"}]
+
+        The file is available for dates before the UDiff switch (Jul 2024);
+        NSE returns 404 for newer dates.
+
+        Args:
+            dt (date or datetime): Trading date
+
+        Returns:
+            str: Old-format CSV content with stock market data
+
+        Raises:
+            requests.RequestException: If the file is unavailable for given date
+        """
+        if isinstance(dt, datetime):
+            dt = dt.date()
+        archives = [{"name": "CM - Bhavcopy(csv)",
+                     "type": "daily-reports",
+                     "category": "capital-market",
+                     "section": "equities"}]
+        params = {
+            "archives": json.dumps(archives),
+            "date": dt.strftime("%d-%b-%Y"),
+            "type": "equities",
+            "mode": "single",
+        }
+        headers = {"referer": "https://www.nseindia.com/all-reports"}
+        r = self.s.get(self.reports_url, params=params,
+                       timeout=self.timeout, headers=headers)
+        if r.status_code == 403 and not self.s.cookies:
+            # www.nseindia.com may require cookies from the landing page first
+            self.s.get("https://www.nseindia.com", timeout=self.timeout)
+            r = self.s.get(self.reports_url, params=params,
+                           timeout=self.timeout, headers=headers)
+        r.raise_for_status()
+        return r.content
+
     @unzip
     def bhavcopy_udiff_raw(self, dt):
         """Downloads raw UDiFF bhavcopy text for a specific date
@@ -514,6 +560,7 @@ def bhavcopy_index_save(self, dt, dest, skip_if_present=True):
 bhavcopy_fo_raw = a.bhavcopy_fo_raw
 bhavcopy_fo_save = a.bhavcopy_fo_save
 bhavcopy_udiff_raw = a.bhavcopy_udiff_raw
+bhavcopy_old_raw = a.bhavcopy_old_raw
 ia = NSEIndicesArchives()
 bhavcopy_index_raw = ia.bhavcopy_index_raw
 bhavcopy_index_save = ia.bhavcopy_index_save

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "jugaad-data"
-version = "0.35.4"
+version = "0.35.5"
 requires-python = ">= 3.9"
 authors = [{name = "jugaad-coder", email = "abc@xyz.com"}]
 description = "Free Zerodha API python library"

requirements.dev.txt

@@ -1,4 +1,4 @@
-pytest==7.1.2
+pytest>=8.0
 pytest-testmon
 pytest-watch
 pyfakefs

tests/test_bhav.py

@@ -1,20 +1,67 @@
-from datetime import date
-from jugaad_data.nse import bhavcopy_raw, full_bhavcopy_raw, bhavcopy_fo_raw, bhavcopy_index_raw, expiry_dates 
+import io
+import json
+import zipfile
+from datetime import date, datetime
+from unittest.mock import MagicMock, patch
+
+from jugaad_data.nse import (bhavcopy_raw, full_bhavcopy_raw, bhavcopy_fo_raw,
+                             bhavcopy_index_raw, expiry_dates,
+                             bhavcopy_old_raw, NSEArchives)
 import pytest
 import requests
 
+# Old cm bhavcopy format (fetched via NSE reports API), includes ISIN field
+OLD_BHAVCOPY_HEADER = ("SYMBOL,SERIES,OPEN,HIGH,LOW,CLOSE,LAST,PREVCLOSE,"
+                       "TOTTRDQTY,TOTTRDVAL,TIMESTAMP,TOTALTRADES,ISIN")
+
+def _old_bhavcopy_text():
+    return (OLD_BHAVCOPY_HEADER + "\n"
+            "SBIN,EQ,355.0,360.0,352.1,358.5,357.0,354.3,100000,35800000.0,"
+            "01-JAN-2020,5000,INE062A01020\n")
+
+def _zip_bytes(text):
+    fp = io.BytesIO()
+    with zipfile.ZipFile(fp, "w") as zf:
+        zf.writestr("cm01JAN2020bhav.csv", text)
+    return fp.getvalue()
+
+@pytest.mark.live
 def test_bhavcopy():
     """Test bhavcopy for a historical date (before UDiff format)
-    
-    For dates before Jul 8, 2024, uses BHAVDATA-FULL format.
-    For recent dates, uses UDiff format with different column structure.
+
+    When UDiff is not available, bhavcopy_raw must fall back to the old
+    bhavcopy format (which has the ISIN field) instead of BHAVDATA-FULL.
     """
     r = bhavcopy_raw(date(2020, 1, 1))
-    # BHAVDATA-FULL format header (used for historical dates)
-    header = "SYMBOL, SERIES, DATE1, PREV_CLOSE, OPEN_PRICE, HIGH_PRICE, LOW_PRICE, LAST_PRICE, CLOSE_PRICE, AVG_PRICE, TTL_TRD_QNTY, TURNOVER_LACS, NO_OF_TRADES, DELIV_QTY, DELIV_PER"
+    header = r.splitlines()[0]
     assert "RELIANCE" in r or "SBIN" in r  # At least some stock data present
-    assert header in r
+    # Old format header with ISIN field
+    assert header.startswith("SYMBOL,SERIES")
+    assert "ISIN" in header
+    # Must NOT be the full bhavcopy (BHAVDATA-FULL) format
+    assert "DELIV_QTY" not in header
+
+@pytest.mark.live
+def test_bhavcopy_old_raw():
+    """Test direct download of the old-format bhavcopy via reports API"""
+    import re
+    isin_re = re.compile(r",IN[A-Z0-9]{10},\s*$")  # 12-char ISIN at row end
+    r = bhavcopy_old_raw(date(2020, 1, 1))
+    header = r.splitlines()[0]
+    assert header.startswith(OLD_BHAVCOPY_HEADER[:15])
+    assert "ISIN" in header
+    assert "RELIANCE" in r or "SBIN" in r
+    # Rows must carry an actual ISIN value e.g. INE062A01020 for SBIN
+    assert any(isin_re.search(line) for line in r.splitlines()[1:])
 
+@pytest.mark.live
+def test_bhavcopy_old_raw_not_available_for_udiff_dates():
+    """Old-format file returns 404 for dates after the UDiff switch"""
+    nse = NSEArchives()
+    with pytest.raises(requests.RequestException):
+        nse.bhavcopy_old_raw(date(2024, 7, 10))
+
+@pytest.mark.live
 def test_bhavcopy_recent():
     """Test bhavcopy for recent date using UDiff format
 
@@ -29,6 +76,7 @@ def test_bhavcopy_recent():
     assert "ISIN" in header
 
 
+@pytest.mark.live
 def test_bhavcopy_historical_udiff():
     """Test bhavcopy for a historical UDiFF-era date
 
@@ -52,6 +100,7 @@ def test_bhavcopy_historical_udiff():
 #         r = full_bhavcopy_raw(date(2019,1,1))
 #     assert '2019' in e.value.args[0]    
 
+@pytest.mark.live
 def test_bhavcopy_fo():
     r = bhavcopy_fo_raw(date(2020,1,1))
     header = "INSTRUMENT,SYMBOL,EXPIRY_DT,STRIKE_PR,OPTION_TYP,OPEN,HIGH,LOW,CLOSE,SETTLE_PR,CONTRACTS,VAL_INLAKH,OPEN_INT,CHG_IN_O"
@@ -64,6 +113,7 @@ def test_bhavcopy_fo():
 #     assert "NIFTY" in r
 #     assert header in r
 
+@pytest.mark.live
 def test_expiry_dates():
     dt = date(2020, 9, 28)
     dts = expiry_dates(dt)
@@ -81,6 +131,7 @@ def test_expiry_dates():
     assert date(2020, 11, 26) in dts
 
 
+@pytest.mark.live
 def test_list_available_reports():
     """Test listing available reports from daily-reports API"""
     from jugaad_data.nse import NSEArchives
@@ -95,6 +146,7 @@ def test_list_available_reports():
         pytest.skip("API not available")
 
 
+@pytest.mark.live
 def test_download_report(tmp_path):
     """Test downloading a report via download_report method"""
     from jugaad_data.nse import NSEArchives
@@ -114,7 +166,73 @@ def test_download_report(tmp_path):
     except requests.RequestException:
         pytest.skip("API not available")
 
+# ---------------------------------------------------------------------------
+# Offline (mocked) tests - run in CI without network access
+# ---------------------------------------------------------------------------
+
+def test_bhavcopy_raw_falls_back_to_old_format_when_udiff_unavailable():
+    """When both UDiff sources fail, fall back to old format (with ISIN),
+    not to the full bhavcopy (BHAVDATA-FULL) format."""
+    n = NSEArchives()
+    with patch.object(n, "bhavcopy_udiff_raw",
+                      side_effect=zipfile.BadZipFile("not available")), \
+         patch.object(n.daily_reports, "download_file",
+                      side_effect=ValueError("not found")), \
+         patch.object(n, "bhavcopy_old_raw",
+                      return_value=_old_bhavcopy_text()) as m_old, \
+         patch.object(n, "full_bhavcopy_raw") as m_full:
+        out = n.bhavcopy_raw(date(2024, 7, 10))
+    assert out == _old_bhavcopy_text()
+    m_old.assert_called_once_with(date(2024, 7, 10))
+    m_full.assert_not_called()
+
+def test_bhavcopy_raw_skips_udiff_for_pre_udiff_dates():
+    """Pre-UDiff dates must go straight to the old-format fallback"""
+    n = NSEArchives()
+    m_udiff = MagicMock()
+    with patch.object(n, "bhavcopy_udiff_raw", m_udiff), \
+         patch.object(n, "bhavcopy_old_raw",
+                      return_value=_old_bhavcopy_text()) as m_old:
+        out = n.bhavcopy_raw(date(2020, 1, 1))
+    assert m_udiff.call_count == 0
+    m_old.assert_called_once_with(date(2020, 1, 1))
+    assert "ISIN" in out.splitlines()[0]
+
+def test_bhavcopy_old_raw_request_and_unzip():
+    """bhavcopy_old_raw must hit the NSE reports API with correct params
+    and return the unzipped old-format CSV text"""
+    n = NSEArchives()
+    resp = MagicMock()
+    resp.status_code = 200
+    resp.content = _zip_bytes(_old_bhavcopy_text())
+    with patch.object(n.s, "get", return_value=resp) as m_get:
+        text = n.bhavcopy_old_raw(datetime(2020, 1, 1))
+    url = m_get.call_args[0][0]
+    params = m_get.call_args[1]["params"]
+    assert url == "https://www.nseindia.com/api/reports"
+    archives = json.loads(params["archives"])
+    assert archives[0] == {"name": "CM - Bhavcopy(csv)",
+                           "type": "daily-reports",
+                           "category": "capital-market",
+                           "section": "equities"}
+    assert params["date"] == "01-Jan-2020"
+    assert params["type"] == "equities"
+    assert params["mode"] == "single"
+    assert text.splitlines()[0].startswith(OLD_BHAVCOPY_HEADER[:15])
+    assert "INE062A01020" in text
+
+def test_bhavcopy_old_raw_raises_when_file_missing():
+    """NSE returns 404 when the old-format file does not exist for a date"""
+    n = NSEArchives()
+    resp = MagicMock()
+    resp.status_code = 404
+    resp.raise_for_status.side_effect = requests.exceptions.HTTPError("404")
+    with patch.object(n.s, "get", return_value=resp):
+        with pytest.raises(requests.exceptions.HTTPError):
+            n.bhavcopy_old_raw(date(2024, 7, 10))
+
 """
+@pytest.mark.live
 def test_bhavcopy_on_holiday():
     r = bhavcopy_raw(date(2020,1,5))
     header = "SYMBOL,SERIES,OPEN,HIGH,LOW,CLOSE,LAST,PREVCLOSE,TOTTRDQTY,TOTTRDVAL,TIMESTAMP,TOTALTRADES,ISIN"