fix: Nightscout API silently truncated multi-day fetches to ~3-4 days

_entries_since/_treatments_since requested a single page via `count`, but Nightscout's /api/v1/entries and /api/v1/treatments ignore a large `count` and cap the response around ~1150/~700 documents regardless of how far back the requested period goes. Every report page relying on fetch_entries_range/fetch_treatments_range (7/14/30/90-day selectors) was silently only getting the most recent ~3-4 days, no matter the selected period. Fixed by paginating with `find[date][$lte]` cursors in a new _paginated_get(), walking back until the requested `since` boundary is reached or the server returns a short/empty page (= genuinely no more data, confirmed by manually paginating against the real instance: entries go back 50+ days, treatments only ~4 days on this install). Also added reporting.completeness_warning(), shared by the interactive pages (via ui_common.render_completeness_warning) and the PDF export, that distinguishes "a structural gap at the start of the period" (data wasn't fetched/available) from "scattered gaps below the clinical AGP coverage threshold" (Danne et al. 2017) — the previous single generic "X% sensor coverage" message read as a CGM/sensor problem even when the real cause was this fetch bug.
2026-07-31 14:21:35 +02:00
parent fe6a471f17
commit 29d20ca441
13 changed files with 322 additions and 30 deletions
@@ -134,3 +134,124 @@ def test_fetch_treatments_range_filters_to_date_boundaries(monkeypatch):
    result = ns.fetch_treatments_range(date(2026, 1, 1), date(2026, 1, 31))
    assert len(result) == 1
    assert result.iloc[0]["insulin"] == pytest.approx(4.0)
+
+
+def _entry(dt, sgv=100):
+    return {"date": int(dt.timestamp() * 1000), "sgv": sgv, "dateString": dt.isoformat()}
+
+
+def test_paginated_get_stops_when_a_short_page_is_returned(monkeypatch):
+    """Nightscout kan minder dan _PAGE_SIZE teruggeven omdat er simpelweg niet meer data is.
+
+    Dit was precies de bug: een enkel groot count-verzoek werd stilzwijgend
+    afgeknipt. Met paginering moet een pagina die kleiner is dan de gevraagde
+    paginagrootte de paginering laten stoppen — dat betekent "geen oudere
+    data beschikbaar", niet "er ging iets mis".
+    """
+    monkeypatch.setenv("NIGHTSCOUT_URL", "https://example.com")
+    now = datetime.now(timezone.utc)
+    payload = [_entry(now - timedelta(minutes=5 * i)) for i in range(5)]  # << _PAGE_SIZE
+
+    calls = []
+    monkeypatch.setattr(
+        ns.requests,
+        "get",
+        lambda *a, **k: (calls.append(k), FakeResponse(payload))[1],
+    )
+
+    result = ns._paginated_get("/api/v1/entries/sgv.json", now - timedelta(days=1))
+    assert len(result) == 5
+    assert len(calls) == 1  # één pagina was genoeg, geen verdere paginering nodig
+
+
+def test_paginated_get_walks_back_multiple_pages_until_since_is_reached(monkeypatch):
+    """De kern-regressietest voor de bug: data die verder terugligt dan één
+    pagina (~1000 documenten) moet toch volledig worden opgehaald door
+    meerdere pagina's met find[date][$lte] te doorlopen."""
+    monkeypatch.setenv("NIGHTSCOUT_URL", "https://example.com")
+    now = datetime.now(timezone.utc)
+
+    # 3 volle pagina's (elk _PAGE_SIZE groot, 5 minuten uit elkaar, aflopend in tijd)
+    # + een laatste kleinere pagina die de paginering laat stoppen.
+    page_size = ns._PAGE_SIZE
+    pages = []
+    cursor = now
+    for _ in range(3):
+        page = [_entry(cursor - timedelta(minutes=5 * i)) for i in range(page_size)]
+        pages.append(page)
+        cursor = cursor - timedelta(minutes=5 * (page_size - 1)) - timedelta(minutes=5)
+    pages.append([_entry(cursor - timedelta(minutes=5 * i)) for i in range(10)])
+
+    call_count = {"n": 0}
+
+    def fake_get(*_a, **_k):
+        response = FakeResponse(pages[call_count["n"]])
+        call_count["n"] += 1
+        return response
+
+    monkeypatch.setattr(ns.requests, "get", fake_get)
+
+    since = now - timedelta(days=365)  # ruim voorbij alle test-pagina's
+    result = ns._paginated_get("/api/v1/entries/sgv.json", since)
+
+    assert call_count["n"] == 4  # 3 volle pagina's + 1 korte slotpagina
+    assert len(result) == 3 * page_size + 10
+
+
+def test_paginated_get_stops_at_since_boundary_without_fetching_further(monkeypatch):
+    """Zodra een pagina data ouder dan `since` bevat, hoeft er geen volgende
+    pagina meer opgehaald te worden, ook al was de pagina zelf vol."""
+    monkeypatch.setenv("NIGHTSCOUT_URL", "https://example.com")
+    now = datetime.now(timezone.utc)
+    page_size = ns._PAGE_SIZE
+    # deze ene pagina spant ruim meer dan een dag, dus bevat al data voorbij `since`
+    page = [_entry(now - timedelta(minutes=5 * i)) for i in range(page_size)]
+
+    calls = {"n": 0}
+
+    def fake_get(*_a, **_k):
+        calls["n"] += 1
+        return FakeResponse(page)
+
+    monkeypatch.setattr(ns.requests, "get", fake_get)
+
+    since = now - timedelta(hours=6)  # ruim binnen de eerste pagina
+    result = ns._paginated_get("/api/v1/entries/sgv.json", since)
+
+    assert calls["n"] == 1
+    assert len(result) == page_size  # filtering op since gebeurt pas in _entries_since, niet hier
+
+
+def test_fetch_entries_range_retrieves_data_spanning_multiple_pages(monkeypatch):
+    """End-to-end regressietest: een 90-dagenrapport moet echt 90 dagen data
+    krijgen, niet stilzwijgend worden afgeknipt tot de meest recente pagina."""
+    monkeypatch.setenv("NIGHTSCOUT_URL", "https://example.com")
+    monkeypatch.delenv("NIGHTSCOUT_TOKEN", raising=False)
+    monkeypatch.delenv("NIGHTSCOUT_API_SECRET", raising=False)
+
+    end_date = date(2026, 3, 31)
+    start_date = date(2026, 1, 1)  # 90 dagen
+
+    # simuleer Nightscout: elke 5 minuten een meting over de volle periode,
+    # opgeknipt in door de server afgedwongen pagina's van _PAGE_SIZE
+    until = datetime.combine(end_date, datetime.max.time(), tzinfo=timezone.utc)
+    since = datetime.combine(start_date, datetime.min.time(), tzinfo=timezone.utc)
+    all_entries = []
+    t = until
+    while t >= since:
+        all_entries.append(_entry(t))
+        t -= timedelta(minutes=5)
+
+    page_size = ns._PAGE_SIZE
+
+    def fake_get(*_args, **kwargs):
+        cursor_ms = kwargs["params"].get("find[date][$lte]")
+        pool = all_entries if cursor_ms is None else [e for e in all_entries if e["date"] <= cursor_ms]
+        return FakeResponse(pool[:page_size])
+
+    monkeypatch.setattr(ns.requests, "get", fake_get)
+
+    result = ns.fetch_entries_range(start_date, end_date)
+    assert len(result) == len(all_entries)
+    assert result["timestamp"].min().date() == start_date
+    assert result["timestamp"].max().date() == end_date
@@ -1,9 +1,12 @@
+from datetime import date
+
 import pandas as pd
 import pytest

 from reporting import (
    agp_percentile_bands,
    bolus_stats,
+    completeness_warning,
    day_of_week_pattern,
    detect_events,
    gmi_percent,
@@ -93,6 +96,42 @@ def test_sensor_active_percent_handles_gaps():
    assert result == pytest.approx(50.0, abs=1.0)


+def test_completeness_warning_none_for_empty_glucose():
+    empty = pd.DataFrame(columns=["timestamp", "mmol"])
+    assert completeness_warning(empty, "2026-01-01", "2026-01-14") == "Geen glucosedata gevonden in deze periode."
+
+
+def test_completeness_warning_none_when_period_is_fully_covered():
+    # 14 volle dagen aan 5-minuten-metingen, precies de gekozen periode
+    timestamps = pd.date_range("2026-01-01 00:00", "2026-01-14 23:55", freq="5min")
+    glucose = pd.DataFrame({"timestamp": timestamps, "mmol": [6.0] * len(timestamps)})
+    assert completeness_warning(glucose, date(2026, 1, 1), date(2026, 1, 14)) is None
+
+
+def test_completeness_warning_flags_missing_days_at_start_of_period():
+    # gevraagd: 1 t/m 14 januari, maar data begint pas op 11 januari (de
+    # exacte bug: alleen de laatste ~4 dagen van een langere periode zijn
+    # opgehaald)
+    timestamps = pd.date_range("2026-01-11 00:00", "2026-01-14 23:55", freq="5min")
+    glucose = pd.DataFrame({"timestamp": timestamps, "mmol": [6.0] * len(timestamps)})
+    message = completeness_warning(glucose, date(2026, 1, 1), date(2026, 1, 14))
+    assert message is not None
+    assert "11-01-2026" in message
+    assert "10 dag" in message  # 1 t/m 10 januari ontbreken volledig = 10 dagen
+    assert "sensorhiaten" in message
+
+
+def test_completeness_warning_flags_low_coverage_without_start_gap():
+    # data start op tijd, maar met veel verspreide hiaten -> lage dekking
+    # zonder dat er een aaneengesloten gat aan het begin is
+    timestamps = pd.date_range("2026-01-01 00:00", periods=20, freq="5min")
+    glucose = pd.DataFrame({"timestamp": timestamps, "mmol": [6.0] * 20})
+    message = completeness_warning(glucose, date(2026, 1, 1), date(2026, 1, 14))
+    assert message is not None
+    assert "sensordekking" in message
+    assert "sensorhiaten" not in message  # dit is de generieke, niet de "ontbreekt volledig"-boodschap
+
+
 def test_detect_events_ignores_dip_shorter_than_minimum_duration():
    # dip van 10 minuten (< 15 min drempel) mag niet meetellen
    glucose = _series(