fix: treatments endpoint needs a created_at cursor, not date

Investigated the "only ~4 days of treatments" report via direct
MongoDB access (issue #13): the collection has 123k+ documents, no
capping, no TTL index — the full year of history is in the database.

The actual cause: /api/v1/treatments.json filters/sorts on the string
field `created_at`, not the numeric `date` field /api/v1/entries uses.
Paginating with find[date][$lte] (or no filter at all) silently caps
treatments to an internal ~4-day window even though every document has
a valid `date` field; find[created_at][$lte] correctly walks back
through the full history.

_paginated_get() now takes an explicit cursor_field per endpoint and
always sends the cursor from the first request (never omitted), so
that hidden default window can't trigger. Verified against the real
instance: a 92-day treatments fetch now returns 39,403 rows (was ~700).
This commit is contained in:
2026-08-01 21:43:08 +02:00
parent 55883480c0
commit 699848b479
2 changed files with 153 additions and 24 deletions
+95 -3
View File
@@ -159,7 +159,9 @@ def test_paginated_get_stops_when_a_short_page_is_returned(monkeypatch):
lambda *a, **k: (calls.append(k), FakeResponse(payload))[1],
)
result = ns._paginated_get("/api/v1/entries/sgv.json", now - timedelta(days=1))
result = ns._paginated_get(
"/api/v1/entries/sgv.json", now - timedelta(days=1), None, ns.ENTRIES_CURSOR_FIELD
)
assert len(result) == 5
assert len(calls) == 1 # één pagina was genoeg, geen verdere paginering nodig
@@ -192,7 +194,9 @@ def test_paginated_get_walks_back_multiple_pages_until_since_is_reached(monkeypa
monkeypatch.setattr(ns.requests, "get", fake_get)
since = now - timedelta(days=365) # ruim voorbij alle test-pagina's
result = ns._paginated_get("/api/v1/entries/sgv.json", since)
result = ns._paginated_get(
"/api/v1/entries/sgv.json", since, None, ns.ENTRIES_CURSOR_FIELD
)
assert call_count["n"] == 4 # 3 volle pagina's + 1 korte slotpagina
assert len(result) == 3 * page_size + 10
@@ -216,7 +220,9 @@ def test_paginated_get_stops_at_since_boundary_without_fetching_further(monkeypa
monkeypatch.setattr(ns.requests, "get", fake_get)
since = now - timedelta(hours=6) # ruim binnen de eerste pagina
result = ns._paginated_get("/api/v1/entries/sgv.json", since)
result = ns._paginated_get(
"/api/v1/entries/sgv.json", since, None, ns.ENTRIES_CURSOR_FIELD
)
assert calls["n"] == 1
assert len(result) == page_size # filtering op since gebeurt pas in _entries_since, niet hier
@@ -255,3 +261,89 @@ def test_fetch_entries_range_retrieves_data_spanning_multiple_pages(monkeypatch)
assert len(result) == len(all_entries)
assert result["timestamp"].min().date() == start_date
assert result["timestamp"].max().date() == end_date
def _treatment(dt, event_type="Temp Basal"):
return {
"created_at": dt.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z",
"date": int(dt.timestamp() * 1000),
"eventType": event_type,
}
def test_fetch_treatments_range_retrieves_full_history_even_with_a_hidden_server_side_window(
monkeypatch,
):
"""Regressietest voor de echte productiebug (2026-07-31): treatments.json
filtert/sorteert intern op `created_at` (string), niet op `date`
(numeriek) zoals entries. Filteren op `date` — of helemaal geen filter
meegeven — leverde stilzwijgend hooguit de laatste ~4 dagen op, ook al
bevatten de documenten een geldig `date`-veld en stond de rest van de
geschiedenis (maanden) gewoon in de database (geverifieerd rechtstreeks
in MongoDB: 123k+ documenten, geen TTL-index, geen capped collection).
Deze test simuleert precies dat servergedrag: een find[date]-filter (of
het ontbreken van elk filter) geeft alleen de laatste 4 dagen terug;
alleen find[created_at] geeft de volledige, langere geschiedenis.
"""
monkeypatch.setenv("NIGHTSCOUT_URL", "https://example.com")
monkeypatch.delenv("NIGHTSCOUT_TOKEN", raising=False)
monkeypatch.delenv("NIGHTSCOUT_API_SECRET", raising=False)
end_date = date(2026, 7, 31)
start_date = date(2026, 5, 1) # 92 dagen, ruim voorbij het 4-dagen-venster
until = datetime.combine(end_date, datetime.max.time(), tzinfo=timezone.utc)
since = datetime.combine(start_date, datetime.min.time(), tzinfo=timezone.utc)
all_treatments = []
t = until
while t >= since:
all_treatments.append(_treatment(t))
t -= timedelta(minutes=30)
hidden_window_start = until - timedelta(days=4)
page_size = ns._PAGE_SIZE
def fake_get(*_args, **kwargs):
params = kwargs["params"]
cursor = params.get("find[created_at][$lte]")
if cursor is None:
# geen (of het verkeerde) filter -> verborgen 4-dagenvenster
pool = [e for e in all_treatments if e["created_at"] >= hidden_window_start.strftime(
"%Y-%m-%dT%H:%M:%S.%f"
)[:-3] + "Z"]
else:
pool = [e for e in all_treatments if e["created_at"] <= cursor]
pool = sorted(pool, key=lambda e: e["created_at"], reverse=True)
return FakeResponse(pool[:page_size])
monkeypatch.setattr(ns.requests, "get", fake_get)
result = ns.fetch_treatments_range(start_date, end_date)
assert len(result) == len(all_treatments)
assert result["timestamp"].min().date() == start_date
assert result["timestamp"].max().date() == end_date
def test_paginated_get_uses_created_at_field_for_treatments_cursor(monkeypatch):
"""Losstaande, gerichte assertie op het daadwerkelijke queryparameter-veld
— moet `find[created_at][$lte]` zijn, nooit `find[date][$lte]`, voor het
treatments-endpoint."""
monkeypatch.setenv("NIGHTSCOUT_URL", "https://example.com")
now = datetime.now(timezone.utc)
payload = [_treatment(now - timedelta(minutes=30 * i)) for i in range(5)]
seen_params = []
def fake_get(*_a, **kwargs):
seen_params.append(kwargs["params"])
return FakeResponse(payload)
monkeypatch.setattr(ns.requests, "get", fake_get)
ns._paginated_get(
"/api/v1/treatments.json", now - timedelta(days=1), None, ns.TREATMENTS_CURSOR_FIELD
)
assert "find[created_at][$lte]" in seen_params[0]
assert "find[date][$lte]" not in seen_params[0]