fix: Nightscout API silently truncated multi-day fetches to ~3-4 days
_entries_since/_treatments_since requested a single page via `count`, but Nightscout's /api/v1/entries and /api/v1/treatments ignore a large `count` and cap the response around ~1150/~700 documents regardless of how far back the requested period goes. Every report page relying on fetch_entries_range/fetch_treatments_range (7/14/30/90-day selectors) was silently only getting the most recent ~3-4 days, no matter the selected period. Fixed by paginating with `find[date][$lte]` cursors in a new _paginated_get(), walking back until the requested `since` boundary is reached or the server returns a short/empty page (= genuinely no more data, confirmed by manually paginating against the real instance: entries go back 50+ days, treatments only ~4 days on this install). Also added reporting.completeness_warning(), shared by the interactive pages (via ui_common.render_completeness_warning) and the PDF export, that distinguishes "a structural gap at the start of the period" (data wasn't fetched/available) from "scattered gaps below the clinical AGP coverage threshold" (Danne et al. 2017) — the previous single generic "X% sensor coverage" message read as a CGM/sensor problem even when the real cause was this fetch bug.
This commit is contained in:
+55
-19
@@ -31,6 +31,19 @@ SENSOR_CHANGE_EVENT_TYPES = {"Sensor Change", "Sensor Start"}
|
|||||||
|
|
||||||
_REQUEST_TIMEOUT = 10
|
_REQUEST_TIMEOUT = 10
|
||||||
|
|
||||||
|
# Nightscout's /api/v1/entries en /api/v1/treatments negeren een groot
|
||||||
|
# `count`-verzoek: het antwoord wordt hoe dan ook stilzwijgend afgeknipt rond
|
||||||
|
# een paar honderd tot ~1150 documenten, ongeacht hoe ver de opgevraagde
|
||||||
|
# periode teruggaat (waargenomen: count=100000 op entries gaf slechts de
|
||||||
|
# meest recente ~1150 metingen terug — nog geen 4 dagen — terwijl er wel
|
||||||
|
# degelijk weken oudere data in Nightscout beschikbaar bleek te zijn). Eén
|
||||||
|
# verzoek is dus niet genoeg voor meerdaagse rapportages; _paginated_get()
|
||||||
|
# loopt terug in de tijd met `find[date][$lte]`-cursors tot de gevraagde
|
||||||
|
# `since`-grens bereikt is of de server minder dan een volle pagina teruggeeft
|
||||||
|
# (= geen oudere data meer beschikbaar, wat verschilt van "afgeknipt").
|
||||||
|
_PAGE_SIZE = 1000
|
||||||
|
_MAX_PAGES = 150 # 150 000 documenten; ruim boven wat een 90-dagenrapport nodig heeft
|
||||||
|
|
||||||
|
|
||||||
class NightscoutConfigError(RuntimeError):
|
class NightscoutConfigError(RuntimeError):
|
||||||
"""NIGHTSCOUT_URL ontbreekt of is ongeldig."""
|
"""NIGHTSCOUT_URL ontbreekt of is ongeldig."""
|
||||||
@@ -75,15 +88,44 @@ def _get(path: str, params: dict) -> list:
|
|||||||
return response.json()
|
return response.json()
|
||||||
|
|
||||||
|
|
||||||
def _entries_since(since: datetime, count: int, until: datetime | None = None) -> pd.DataFrame:
|
def _paginated_get(path: str, since: datetime, until: datetime | None = None) -> list:
|
||||||
"""Haal entries op sinds `since` (en optioneel tot `until`), lokaal gefilterd.
|
"""Haal alle documenten op tussen `since` en `until` (of tot nu), pagina voor pagina.
|
||||||
|
|
||||||
Nightscout geeft resultaten terug in aflopende volgorde; vraag ruim genoeg
|
Zie de toelichting bij _PAGE_SIZE hierboven voor waarom dit niet één
|
||||||
entries op (CGM meet doorgaans elke 5 min) en filter lokaal op tijd. Dit is
|
verzoek met een groot `count` kan zijn (BUG: dat leverde bij een
|
||||||
robuuster dan server-side find[]-query's, die niet op elke Nightscout-versie
|
meerdaags rapport stilzwijgend alleen de laatste ~3-4 dagen op, ongeacht
|
||||||
identiek werken.
|
de gekozen periode). Elke pagina wordt opgevraagd met `find[date][$lte]`
|
||||||
|
net onder de oudste timestamp van de vorige pagina, zodat elk document
|
||||||
|
precies één keer terugkomt. Stopt zodra de oudste timestamp in een pagina
|
||||||
|
op of voorbij `since` ligt, of zodra de server minder dan een volle
|
||||||
|
pagina teruggeeft (dan is er geen oudere data meer, punt).
|
||||||
"""
|
"""
|
||||||
data = _get("/api/v1/entries/sgv.json", {"count": count})
|
since_ms = int(since.timestamp() * 1000)
|
||||||
|
cursor_ms = int(until.timestamp() * 1000) if until is not None else None
|
||||||
|
|
||||||
|
results: list = []
|
||||||
|
for _ in range(_MAX_PAGES):
|
||||||
|
params = {"count": _PAGE_SIZE}
|
||||||
|
if cursor_ms is not None:
|
||||||
|
params["find[date][$lte]"] = cursor_ms
|
||||||
|
page = _get(path, params)
|
||||||
|
if not page:
|
||||||
|
break
|
||||||
|
results.extend(page)
|
||||||
|
|
||||||
|
dates = [item["date"] for item in page if "date" in item]
|
||||||
|
if not dates:
|
||||||
|
break
|
||||||
|
oldest_ms = min(dates)
|
||||||
|
if oldest_ms <= since_ms or len(page) < _PAGE_SIZE:
|
||||||
|
break
|
||||||
|
cursor_ms = oldest_ms - 1
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def _entries_since(since: datetime, until: datetime | None = None) -> pd.DataFrame:
|
||||||
|
"""Haal entries op sinds `since` (en optioneel tot `until`), lokaal gefilterd."""
|
||||||
|
data = _paginated_get("/api/v1/entries/sgv.json", since, until)
|
||||||
|
|
||||||
if not data:
|
if not data:
|
||||||
return pd.DataFrame(columns=["timestamp", "mmol"])
|
return pd.DataFrame(columns=["timestamp", "mmol"])
|
||||||
@@ -97,9 +139,9 @@ def _entries_since(since: datetime, count: int, until: datetime | None = None) -
|
|||||||
return df[["timestamp", "mmol"]].sort_values("timestamp").reset_index(drop=True)
|
return df[["timestamp", "mmol"]].sort_values("timestamp").reset_index(drop=True)
|
||||||
|
|
||||||
|
|
||||||
def _treatments_since(since: datetime, count: int, until: datetime | None = None) -> pd.DataFrame:
|
def _treatments_since(since: datetime, until: datetime | None = None) -> pd.DataFrame:
|
||||||
"""Haal treatments op sinds `since` (en optioneel tot `until`), lokaal gefilterd."""
|
"""Haal treatments op sinds `since` (en optioneel tot `until`), lokaal gefilterd."""
|
||||||
data = _get("/api/v1/treatments.json", {"count": count})
|
data = _paginated_get("/api/v1/treatments.json", since, until)
|
||||||
|
|
||||||
if not data:
|
if not data:
|
||||||
return pd.DataFrame(columns=["timestamp", "eventType", "insulin", "carbs"])
|
return pd.DataFrame(columns=["timestamp", "eventType", "insulin", "carbs"])
|
||||||
@@ -129,8 +171,7 @@ def fetch_entries(hours: int = 24) -> pd.DataFrame:
|
|||||||
gesorteerd op tijd.
|
gesorteerd op tijd.
|
||||||
"""
|
"""
|
||||||
since = datetime.now(timezone.utc) - timedelta(hours=hours)
|
since = datetime.now(timezone.utc) - timedelta(hours=hours)
|
||||||
count = max(300, hours * 15)
|
return _entries_since(since)
|
||||||
return _entries_since(since, count)
|
|
||||||
|
|
||||||
|
|
||||||
def fetch_treatments(hours: int = 24) -> pd.DataFrame:
|
def fetch_treatments(hours: int = 24) -> pd.DataFrame:
|
||||||
@@ -141,8 +182,7 @@ def fetch_treatments(hours: int = 24) -> pd.DataFrame:
|
|||||||
'insulin' en 'carbs' (de laatste twee NaN waar niet van toepassing).
|
'insulin' en 'carbs' (de laatste twee NaN waar niet van toepassing).
|
||||||
"""
|
"""
|
||||||
since = datetime.now(timezone.utc) - timedelta(hours=hours)
|
since = datetime.now(timezone.utc) - timedelta(hours=hours)
|
||||||
count = max(200, hours * 10)
|
return _treatments_since(since)
|
||||||
return _treatments_since(since, count)
|
|
||||||
|
|
||||||
|
|
||||||
def fetch_entries_range(start_date: date, end_date: date) -> pd.DataFrame:
|
def fetch_entries_range(start_date: date, end_date: date) -> pd.DataFrame:
|
||||||
@@ -154,9 +194,7 @@ def fetch_entries_range(start_date: date, end_date: date) -> pd.DataFrame:
|
|||||||
"""
|
"""
|
||||||
since = datetime.combine(start_date, time.min, tzinfo=timezone.utc)
|
since = datetime.combine(start_date, time.min, tzinfo=timezone.utc)
|
||||||
until = datetime.combine(end_date, time.max, tzinfo=timezone.utc)
|
until = datetime.combine(end_date, time.max, tzinfo=timezone.utc)
|
||||||
days = (end_date - start_date).days + 1
|
return _entries_since(since, until)
|
||||||
count = min(days * 300, 100_000) # ruime marge boven 5-min-interval
|
|
||||||
return _entries_since(since, count, until)
|
|
||||||
|
|
||||||
|
|
||||||
def fetch_treatments_range(start_date: date, end_date: date) -> pd.DataFrame:
|
def fetch_treatments_range(start_date: date, end_date: date) -> pd.DataFrame:
|
||||||
@@ -166,6 +204,4 @@ def fetch_treatments_range(start_date: date, end_date: date) -> pd.DataFrame:
|
|||||||
"""
|
"""
|
||||||
since = datetime.combine(start_date, time.min, tzinfo=timezone.utc)
|
since = datetime.combine(start_date, time.min, tzinfo=timezone.utc)
|
||||||
until = datetime.combine(end_date, time.max, tzinfo=timezone.utc)
|
until = datetime.combine(end_date, time.max, tzinfo=timezone.utc)
|
||||||
days = (end_date - start_date).days + 1
|
return _treatments_since(since, until)
|
||||||
count = min(days * 50, 50_000)
|
|
||||||
return _treatments_since(since, count, until)
|
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ from reporting import (
|
|||||||
sensor_active_percent,
|
sensor_active_percent,
|
||||||
tir_breakdown_5band,
|
tir_breakdown_5band,
|
||||||
)
|
)
|
||||||
from ui_common import load_range_data, select_period_days
|
from ui_common import load_range_data, render_completeness_warning, select_period_days
|
||||||
|
|
||||||
st.set_page_config(page_title="Glucose dashboard — Samenvatting", page_icon="📊", layout="wide")
|
st.set_page_config(page_title="Glucose dashboard — Samenvatting", page_icon="📊", layout="wide")
|
||||||
require_auth()
|
require_auth()
|
||||||
@@ -32,12 +32,8 @@ if glucose.empty:
|
|||||||
st.info("Geen glucosemetingen gevonden in deze periode.")
|
st.info("Geen glucosemetingen gevonden in deze periode.")
|
||||||
st.stop()
|
st.stop()
|
||||||
|
|
||||||
|
render_completeness_warning(glucose, start, end)
|
||||||
active_pct = sensor_active_percent(glucose, start, end)
|
active_pct = sensor_active_percent(glucose, start, end)
|
||||||
if active_pct < 70:
|
|
||||||
st.warning(
|
|
||||||
f"Slechts {active_pct:.0f}% sensordekking in deze periode (streefwaarde ≥70%). "
|
|
||||||
"De onderstaande cijfers zijn mogelijk minder betrouwbaar door de vele hiaten."
|
|
||||||
)
|
|
||||||
|
|
||||||
mean_mmol = glucose["mmol"].mean()
|
mean_mmol = glucose["mmol"].mean()
|
||||||
gmi = gmi_percent(mean_mmol)
|
gmi = gmi_percent(mean_mmol)
|
||||||
|
|||||||
@@ -6,7 +6,13 @@ from auth import require_auth
|
|||||||
from data_processing import TARGET_HIGH, TARGET_LOW
|
from data_processing import TARGET_HIGH, TARGET_LOW
|
||||||
from nightscout_client import NightscoutConfigError
|
from nightscout_client import NightscoutConfigError
|
||||||
from reporting import agp_percentile_bands
|
from reporting import agp_percentile_bands
|
||||||
from ui_common import get_colors, hex_to_rgba, load_range_data, select_period_days
|
from ui_common import (
|
||||||
|
get_colors,
|
||||||
|
hex_to_rgba,
|
||||||
|
load_range_data,
|
||||||
|
render_completeness_warning,
|
||||||
|
select_period_days,
|
||||||
|
)
|
||||||
|
|
||||||
st.set_page_config(page_title="Glucose dashboard — AGP profiel", page_icon="📊", layout="wide")
|
st.set_page_config(page_title="Glucose dashboard — AGP profiel", page_icon="📊", layout="wide")
|
||||||
require_auth()
|
require_auth()
|
||||||
@@ -32,6 +38,8 @@ if glucose.empty:
|
|||||||
st.info("Geen glucosemetingen gevonden in deze periode.")
|
st.info("Geen glucosemetingen gevonden in deze periode.")
|
||||||
st.stop()
|
st.stop()
|
||||||
|
|
||||||
|
render_completeness_warning(glucose, start, end)
|
||||||
|
|
||||||
bands = agp_percentile_bands(glucose, bin_minutes=30)
|
bands = agp_percentile_bands(glucose, bin_minutes=30)
|
||||||
colors = get_colors()
|
colors = get_colors()
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,12 @@ from plotly.subplots import make_subplots
|
|||||||
from auth import require_auth
|
from auth import require_auth
|
||||||
from data_processing import TARGET_HIGH, TARGET_LOW
|
from data_processing import TARGET_HIGH, TARGET_LOW
|
||||||
from nightscout_client import NightscoutConfigError
|
from nightscout_client import NightscoutConfigError
|
||||||
from ui_common import get_colors, load_range_data, select_period_days
|
from ui_common import (
|
||||||
|
get_colors,
|
||||||
|
load_range_data,
|
||||||
|
render_completeness_warning,
|
||||||
|
select_period_days,
|
||||||
|
)
|
||||||
|
|
||||||
st.set_page_config(page_title="Glucose dashboard — Dagoverzicht", page_icon="📊", layout="wide")
|
st.set_page_config(page_title="Glucose dashboard — Dagoverzicht", page_icon="📊", layout="wide")
|
||||||
require_auth()
|
require_auth()
|
||||||
@@ -34,6 +39,8 @@ if glucose.empty:
|
|||||||
st.info("Geen glucosemetingen gevonden in deze periode.")
|
st.info("Geen glucosemetingen gevonden in deze periode.")
|
||||||
st.stop()
|
st.stop()
|
||||||
|
|
||||||
|
render_completeness_warning(glucose, start, end)
|
||||||
|
|
||||||
colors = get_colors()
|
colors = get_colors()
|
||||||
glucose = glucose.copy()
|
glucose = glucose.copy()
|
||||||
glucose["date"] = glucose["timestamp"].dt.date
|
glucose["date"] = glucose["timestamp"].dt.date
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ from reporting import LOW, VERY_LOW
|
|||||||
from ui_common import (
|
from ui_common import (
|
||||||
get_colors,
|
get_colors,
|
||||||
load_range_data,
|
load_range_data,
|
||||||
|
render_completeness_warning,
|
||||||
render_event_level,
|
render_event_level,
|
||||||
select_period_days,
|
select_period_days,
|
||||||
)
|
)
|
||||||
@@ -36,6 +37,8 @@ if glucose.empty:
|
|||||||
st.info("Geen glucosemetingen gevonden in deze periode.")
|
st.info("Geen glucosemetingen gevonden in deze periode.")
|
||||||
st.stop()
|
st.stop()
|
||||||
|
|
||||||
|
render_completeness_warning(glucose, start, end)
|
||||||
|
|
||||||
colors = get_colors()
|
colors = get_colors()
|
||||||
|
|
||||||
render_event_level(glucose, LOW, "below", f"Level 1 (<{LOW} mmol/L)", days, colors)
|
render_event_level(glucose, LOW, "below", f"Level 1 (<{LOW} mmol/L)", days, colors)
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ from reporting import HIGH, VERY_HIGH
|
|||||||
from ui_common import (
|
from ui_common import (
|
||||||
get_colors,
|
get_colors,
|
||||||
load_range_data,
|
load_range_data,
|
||||||
|
render_completeness_warning,
|
||||||
render_event_level,
|
render_event_level,
|
||||||
select_period_days,
|
select_period_days,
|
||||||
)
|
)
|
||||||
@@ -35,6 +36,8 @@ if glucose.empty:
|
|||||||
st.info("Geen glucosemetingen gevonden in deze periode.")
|
st.info("Geen glucosemetingen gevonden in deze periode.")
|
||||||
st.stop()
|
st.stop()
|
||||||
|
|
||||||
|
render_completeness_warning(glucose, start, end)
|
||||||
|
|
||||||
colors = get_colors()
|
colors = get_colors()
|
||||||
|
|
||||||
render_event_level(glucose, HIGH, "above", f"Level 1 (>{HIGH} mmol/L)", days, colors)
|
render_event_level(glucose, HIGH, "above", f"Level 1 (>{HIGH} mmol/L)", days, colors)
|
||||||
|
|||||||
+8
-1
@@ -12,7 +12,12 @@ from reporting import (
|
|||||||
overnight_daytime_split,
|
overnight_daytime_split,
|
||||||
site_interval_stats,
|
site_interval_stats,
|
||||||
)
|
)
|
||||||
from ui_common import get_colors, load_range_data, select_period_days
|
from ui_common import (
|
||||||
|
get_colors,
|
||||||
|
load_range_data,
|
||||||
|
render_completeness_warning,
|
||||||
|
select_period_days,
|
||||||
|
)
|
||||||
|
|
||||||
st.set_page_config(page_title="Glucose dashboard — Trends", page_icon="📊", layout="wide")
|
st.set_page_config(page_title="Glucose dashboard — Trends", page_icon="📊", layout="wide")
|
||||||
require_auth()
|
require_auth()
|
||||||
@@ -35,6 +40,8 @@ if glucose.empty:
|
|||||||
st.info("Geen glucosemetingen gevonden in deze periode.")
|
st.info("Geen glucosemetingen gevonden in deze periode.")
|
||||||
st.stop()
|
st.stop()
|
||||||
|
|
||||||
|
render_completeness_warning(glucose, start, end)
|
||||||
|
|
||||||
colors = get_colors()
|
colors = get_colors()
|
||||||
events = classify_treatments(treatments)
|
events = classify_treatments(treatments)
|
||||||
|
|
||||||
|
|||||||
+3
-1
@@ -5,7 +5,7 @@ from auth import require_auth
|
|||||||
from nightscout_client import NightscoutConfigError
|
from nightscout_client import NightscoutConfigError
|
||||||
from pdf_export import AVAILABLE_SECTIONS, SECTION_EVENTS, build_report_pdf
|
from pdf_export import AVAILABLE_SECTIONS, SECTION_EVENTS, build_report_pdf
|
||||||
from reporting import HIGH, LOW, VERY_HIGH, VERY_LOW, detect_events, summarize_events
|
from reporting import HIGH, LOW, VERY_HIGH, VERY_LOW, detect_events, summarize_events
|
||||||
from ui_common import load_range_data, select_period_days
|
from ui_common import load_range_data, render_completeness_warning, select_period_days
|
||||||
|
|
||||||
st.set_page_config(page_title="Glucose dashboard — Export", page_icon="📊", layout="wide")
|
st.set_page_config(page_title="Glucose dashboard — Export", page_icon="📊", layout="wide")
|
||||||
require_auth()
|
require_auth()
|
||||||
@@ -29,6 +29,8 @@ if glucose.empty:
|
|||||||
st.info("Geen glucosemetingen gevonden in deze periode.")
|
st.info("Geen glucosemetingen gevonden in deze periode.")
|
||||||
st.stop()
|
st.stop()
|
||||||
|
|
||||||
|
render_completeness_warning(glucose, start, end)
|
||||||
|
|
||||||
events_summary = {}
|
events_summary = {}
|
||||||
if SECTION_EVENTS in sections:
|
if SECTION_EVENTS in sections:
|
||||||
levels = {
|
levels = {
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ from fpdf.enums import XPos, YPos
|
|||||||
|
|
||||||
from reporting import (
|
from reporting import (
|
||||||
agp_percentile_bands,
|
agp_percentile_bands,
|
||||||
|
completeness_warning,
|
||||||
gmi_percent,
|
gmi_percent,
|
||||||
percent_cv,
|
percent_cv,
|
||||||
sensor_active_percent,
|
sensor_active_percent,
|
||||||
@@ -64,6 +65,13 @@ def _add_summary_section(pdf: FPDF, glucose: pd.DataFrame, start, end) -> None:
|
|||||||
|
|
||||||
pdf.set_font("Helvetica", "B", 13)
|
pdf.set_font("Helvetica", "B", 13)
|
||||||
pdf.cell(0, 8, "Samenvatting", **_LINE_KWARGS)
|
pdf.cell(0, 8, "Samenvatting", **_LINE_KWARGS)
|
||||||
|
|
||||||
|
warning = completeness_warning(glucose, start, end)
|
||||||
|
if warning:
|
||||||
|
pdf.set_font("Helvetica", "BI", 10)
|
||||||
|
pdf.multi_cell(0, 5, warning)
|
||||||
|
pdf.ln(2)
|
||||||
|
|
||||||
pdf.set_font("Helvetica", "", 11)
|
pdf.set_font("Helvetica", "", 11)
|
||||||
pdf.cell(0, 6, f"Sensordekking: {active_pct:.0f}%", **_LINE_KWARGS)
|
pdf.cell(0, 6, f"Sensordekking: {active_pct:.0f}%", **_LINE_KWARGS)
|
||||||
pdf.cell(0, 6, f"Gemiddelde glucose: {mean_mmol:.1f} mmol/L", **_LINE_KWARGS)
|
pdf.cell(0, 6, f"Gemiddelde glucose: {mean_mmol:.1f} mmol/L", **_LINE_KWARGS)
|
||||||
|
|||||||
@@ -95,6 +95,56 @@ def sensor_active_percent(glucose: pd.DataFrame, start, end, expected_interval_m
|
|||||||
return round(min(len(glucose) / expected_readings, 1.0) * 100, 1)
|
return round(min(len(glucose) / expected_readings, 1.0) * 100, 1)
|
||||||
|
|
||||||
|
|
||||||
|
def completeness_warning(glucose: pd.DataFrame, start, end) -> str | None:
|
||||||
|
"""Bouw een waarschuwingstekst als er te weinig data is voor de gekozen periode.
|
||||||
|
|
||||||
|
Pure functie zodat zowel de interactieve pagina's (ui_common.py, als
|
||||||
|
st.warning) als het PDF-rapport (pdf_export.py, als tekstregel) exact
|
||||||
|
dezelfde beoordeling en bewoording gebruiken — een rapport dat naar het
|
||||||
|
diabetesteam gaat moet dezelfde disclaimer bevatten als wat de gebruiker
|
||||||
|
zelf op het scherm ziet.
|
||||||
|
|
||||||
|
Onderscheidt twee oorzaken:
|
||||||
|
- een structureel gat aan het BEGIN van de periode (de vroegste
|
||||||
|
binnengekomen meting ligt later dan de gekozen startdatum) — data is
|
||||||
|
simpelweg niet opgehaald/beschikbaar voor dat stuk, geen sensorhiaat.
|
||||||
|
- een lagere dekking dan de klinische AGP-ondergrens (Danne et al. 2017)
|
||||||
|
zonder zo'n gat aan het begin — verspreide hiaten.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
None als er geen waarschuwing nodig is; anders de te tonen tekst.
|
||||||
|
"""
|
||||||
|
if glucose.empty:
|
||||||
|
return "Geen glucosedata gevonden in deze periode."
|
||||||
|
|
||||||
|
start_date = pd.Timestamp(start).date()
|
||||||
|
end_date = pd.Timestamp(end).date()
|
||||||
|
active_pct = sensor_active_percent(glucose, start, end)
|
||||||
|
earliest = glucose["timestamp"].min().date()
|
||||||
|
missing_days_at_start = max((earliest - start_date).days, 0)
|
||||||
|
|
||||||
|
if missing_days_at_start > 0:
|
||||||
|
return (
|
||||||
|
f"Nightscout heeft pas data vanaf {earliest:%d-%m-%Y} teruggegeven — "
|
||||||
|
f"de eerste {missing_days_at_start} dag(en) van de gekozen periode "
|
||||||
|
f"({start_date:%d-%m-%Y} t/m {end_date:%d-%m-%Y}) ontbreken volledig, niet alleen "
|
||||||
|
f"losse sensorhiaten. Sensordekking over de wél opgehaalde periode: "
|
||||||
|
f"{active_pct:.0f}%. Onderstaande cijfers zijn dus berekend over een "
|
||||||
|
"kortere periode dan geselecteerd."
|
||||||
|
)
|
||||||
|
if active_pct < 70:
|
||||||
|
# ">=" i.p.v. "≥": deze tekst komt ook in het PDF-rapport terecht
|
||||||
|
# (pdf_export.py), en fpdf2's ingebouwde Helvetica-font ondersteunt
|
||||||
|
# alleen Latin-1 — zie gitea issue #11 voor het opruimen hiervan
|
||||||
|
# (Unicode-lettertype embedden) zodra dat de moeite waard is.
|
||||||
|
return (
|
||||||
|
f"Slechts {active_pct:.0f}% sensordekking in deze periode (streefwaarde "
|
||||||
|
">=70% voor een klinisch betrouwbaar rapport, Danne et al. 2017). De "
|
||||||
|
"onderstaande cijfers zijn mogelijk minder betrouwbaar door de vele hiaten."
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _state_blocks(df: pd.DataFrame, out: pd.Series) -> list[dict]:
|
def _state_blocks(df: pd.DataFrame, out: pd.Series) -> list[dict]:
|
||||||
"""Groepeer aaneengesloten metingen met dezelfde in/out-status.
|
"""Groepeer aaneengesloten metingen met dezelfde in/out-status.
|
||||||
|
|
||||||
|
|||||||
@@ -134,3 +134,124 @@ def test_fetch_treatments_range_filters_to_date_boundaries(monkeypatch):
|
|||||||
result = ns.fetch_treatments_range(date(2026, 1, 1), date(2026, 1, 31))
|
result = ns.fetch_treatments_range(date(2026, 1, 1), date(2026, 1, 31))
|
||||||
assert len(result) == 1
|
assert len(result) == 1
|
||||||
assert result.iloc[0]["insulin"] == pytest.approx(4.0)
|
assert result.iloc[0]["insulin"] == pytest.approx(4.0)
|
||||||
|
|
||||||
|
|
||||||
|
def _entry(dt, sgv=100):
|
||||||
|
return {"date": int(dt.timestamp() * 1000), "sgv": sgv, "dateString": dt.isoformat()}
|
||||||
|
|
||||||
|
|
||||||
|
def test_paginated_get_stops_when_a_short_page_is_returned(monkeypatch):
|
||||||
|
"""Nightscout kan minder dan _PAGE_SIZE teruggeven omdat er simpelweg niet meer data is.
|
||||||
|
|
||||||
|
Dit was precies de bug: een enkel groot count-verzoek werd stilzwijgend
|
||||||
|
afgeknipt. Met paginering moet een pagina die kleiner is dan de gevraagde
|
||||||
|
paginagrootte de paginering laten stoppen — dat betekent "geen oudere
|
||||||
|
data beschikbaar", niet "er ging iets mis".
|
||||||
|
"""
|
||||||
|
monkeypatch.setenv("NIGHTSCOUT_URL", "https://example.com")
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
payload = [_entry(now - timedelta(minutes=5 * i)) for i in range(5)] # << _PAGE_SIZE
|
||||||
|
|
||||||
|
calls = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
ns.requests,
|
||||||
|
"get",
|
||||||
|
lambda *a, **k: (calls.append(k), FakeResponse(payload))[1],
|
||||||
|
)
|
||||||
|
|
||||||
|
result = ns._paginated_get("/api/v1/entries/sgv.json", now - timedelta(days=1))
|
||||||
|
assert len(result) == 5
|
||||||
|
assert len(calls) == 1 # één pagina was genoeg, geen verdere paginering nodig
|
||||||
|
|
||||||
|
|
||||||
|
def test_paginated_get_walks_back_multiple_pages_until_since_is_reached(monkeypatch):
|
||||||
|
"""De kern-regressietest voor de bug: data die verder terugligt dan één
|
||||||
|
pagina (~1000 documenten) moet toch volledig worden opgehaald door
|
||||||
|
meerdere pagina's met find[date][$lte] te doorlopen."""
|
||||||
|
monkeypatch.setenv("NIGHTSCOUT_URL", "https://example.com")
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
# 3 volle pagina's (elk _PAGE_SIZE groot, 5 minuten uit elkaar, aflopend in tijd)
|
||||||
|
# + een laatste kleinere pagina die de paginering laat stoppen.
|
||||||
|
page_size = ns._PAGE_SIZE
|
||||||
|
pages = []
|
||||||
|
cursor = now
|
||||||
|
for _ in range(3):
|
||||||
|
page = [_entry(cursor - timedelta(minutes=5 * i)) for i in range(page_size)]
|
||||||
|
pages.append(page)
|
||||||
|
cursor = cursor - timedelta(minutes=5 * (page_size - 1)) - timedelta(minutes=5)
|
||||||
|
pages.append([_entry(cursor - timedelta(minutes=5 * i)) for i in range(10)])
|
||||||
|
|
||||||
|
call_count = {"n": 0}
|
||||||
|
|
||||||
|
def fake_get(*_a, **_k):
|
||||||
|
response = FakeResponse(pages[call_count["n"]])
|
||||||
|
call_count["n"] += 1
|
||||||
|
return response
|
||||||
|
|
||||||
|
monkeypatch.setattr(ns.requests, "get", fake_get)
|
||||||
|
|
||||||
|
since = now - timedelta(days=365) # ruim voorbij alle test-pagina's
|
||||||
|
result = ns._paginated_get("/api/v1/entries/sgv.json", since)
|
||||||
|
|
||||||
|
assert call_count["n"] == 4 # 3 volle pagina's + 1 korte slotpagina
|
||||||
|
assert len(result) == 3 * page_size + 10
|
||||||
|
|
||||||
|
|
||||||
|
def test_paginated_get_stops_at_since_boundary_without_fetching_further(monkeypatch):
|
||||||
|
"""Zodra een pagina data ouder dan `since` bevat, hoeft er geen volgende
|
||||||
|
pagina meer opgehaald te worden, ook al was de pagina zelf vol."""
|
||||||
|
monkeypatch.setenv("NIGHTSCOUT_URL", "https://example.com")
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
page_size = ns._PAGE_SIZE
|
||||||
|
# deze ene pagina spant ruim meer dan een dag, dus bevat al data voorbij `since`
|
||||||
|
page = [_entry(now - timedelta(minutes=5 * i)) for i in range(page_size)]
|
||||||
|
|
||||||
|
calls = {"n": 0}
|
||||||
|
|
||||||
|
def fake_get(*_a, **_k):
|
||||||
|
calls["n"] += 1
|
||||||
|
return FakeResponse(page)
|
||||||
|
|
||||||
|
monkeypatch.setattr(ns.requests, "get", fake_get)
|
||||||
|
|
||||||
|
since = now - timedelta(hours=6) # ruim binnen de eerste pagina
|
||||||
|
result = ns._paginated_get("/api/v1/entries/sgv.json", since)
|
||||||
|
|
||||||
|
assert calls["n"] == 1
|
||||||
|
assert len(result) == page_size # filtering op since gebeurt pas in _entries_since, niet hier
|
||||||
|
|
||||||
|
|
||||||
|
def test_fetch_entries_range_retrieves_data_spanning_multiple_pages(monkeypatch):
|
||||||
|
"""End-to-end regressietest: een 90-dagenrapport moet echt 90 dagen data
|
||||||
|
krijgen, niet stilzwijgend worden afgeknipt tot de meest recente pagina."""
|
||||||
|
monkeypatch.setenv("NIGHTSCOUT_URL", "https://example.com")
|
||||||
|
monkeypatch.delenv("NIGHTSCOUT_TOKEN", raising=False)
|
||||||
|
monkeypatch.delenv("NIGHTSCOUT_API_SECRET", raising=False)
|
||||||
|
|
||||||
|
end_date = date(2026, 3, 31)
|
||||||
|
start_date = date(2026, 1, 1) # 90 dagen
|
||||||
|
|
||||||
|
# simuleer Nightscout: elke 5 minuten een meting over de volle periode,
|
||||||
|
# opgeknipt in door de server afgedwongen pagina's van _PAGE_SIZE
|
||||||
|
until = datetime.combine(end_date, datetime.max.time(), tzinfo=timezone.utc)
|
||||||
|
since = datetime.combine(start_date, datetime.min.time(), tzinfo=timezone.utc)
|
||||||
|
all_entries = []
|
||||||
|
t = until
|
||||||
|
while t >= since:
|
||||||
|
all_entries.append(_entry(t))
|
||||||
|
t -= timedelta(minutes=5)
|
||||||
|
|
||||||
|
page_size = ns._PAGE_SIZE
|
||||||
|
|
||||||
|
def fake_get(*_args, **kwargs):
|
||||||
|
cursor_ms = kwargs["params"].get("find[date][$lte]")
|
||||||
|
pool = all_entries if cursor_ms is None else [e for e in all_entries if e["date"] <= cursor_ms]
|
||||||
|
return FakeResponse(pool[:page_size])
|
||||||
|
|
||||||
|
monkeypatch.setattr(ns.requests, "get", fake_get)
|
||||||
|
|
||||||
|
result = ns.fetch_entries_range(start_date, end_date)
|
||||||
|
assert len(result) == len(all_entries)
|
||||||
|
assert result["timestamp"].min().date() == start_date
|
||||||
|
assert result["timestamp"].max().date() == end_date
|
||||||
|
|||||||
@@ -1,9 +1,12 @@
|
|||||||
|
from datetime import date
|
||||||
|
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from reporting import (
|
from reporting import (
|
||||||
agp_percentile_bands,
|
agp_percentile_bands,
|
||||||
bolus_stats,
|
bolus_stats,
|
||||||
|
completeness_warning,
|
||||||
day_of_week_pattern,
|
day_of_week_pattern,
|
||||||
detect_events,
|
detect_events,
|
||||||
gmi_percent,
|
gmi_percent,
|
||||||
@@ -93,6 +96,42 @@ def test_sensor_active_percent_handles_gaps():
|
|||||||
assert result == pytest.approx(50.0, abs=1.0)
|
assert result == pytest.approx(50.0, abs=1.0)
|
||||||
|
|
||||||
|
|
||||||
|
def test_completeness_warning_none_for_empty_glucose():
|
||||||
|
empty = pd.DataFrame(columns=["timestamp", "mmol"])
|
||||||
|
assert completeness_warning(empty, "2026-01-01", "2026-01-14") == "Geen glucosedata gevonden in deze periode."
|
||||||
|
|
||||||
|
|
||||||
|
def test_completeness_warning_none_when_period_is_fully_covered():
|
||||||
|
# 14 volle dagen aan 5-minuten-metingen, precies de gekozen periode
|
||||||
|
timestamps = pd.date_range("2026-01-01 00:00", "2026-01-14 23:55", freq="5min")
|
||||||
|
glucose = pd.DataFrame({"timestamp": timestamps, "mmol": [6.0] * len(timestamps)})
|
||||||
|
assert completeness_warning(glucose, date(2026, 1, 1), date(2026, 1, 14)) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_completeness_warning_flags_missing_days_at_start_of_period():
|
||||||
|
# gevraagd: 1 t/m 14 januari, maar data begint pas op 11 januari (de
|
||||||
|
# exacte bug: alleen de laatste ~4 dagen van een langere periode zijn
|
||||||
|
# opgehaald)
|
||||||
|
timestamps = pd.date_range("2026-01-11 00:00", "2026-01-14 23:55", freq="5min")
|
||||||
|
glucose = pd.DataFrame({"timestamp": timestamps, "mmol": [6.0] * len(timestamps)})
|
||||||
|
message = completeness_warning(glucose, date(2026, 1, 1), date(2026, 1, 14))
|
||||||
|
assert message is not None
|
||||||
|
assert "11-01-2026" in message
|
||||||
|
assert "10 dag" in message # 1 t/m 10 januari ontbreken volledig = 10 dagen
|
||||||
|
assert "sensorhiaten" in message
|
||||||
|
|
||||||
|
|
||||||
|
def test_completeness_warning_flags_low_coverage_without_start_gap():
|
||||||
|
# data start op tijd, maar met veel verspreide hiaten -> lage dekking
|
||||||
|
# zonder dat er een aaneengesloten gat aan het begin is
|
||||||
|
timestamps = pd.date_range("2026-01-01 00:00", periods=20, freq="5min")
|
||||||
|
glucose = pd.DataFrame({"timestamp": timestamps, "mmol": [6.0] * 20})
|
||||||
|
message = completeness_warning(glucose, date(2026, 1, 1), date(2026, 1, 14))
|
||||||
|
assert message is not None
|
||||||
|
assert "sensordekking" in message
|
||||||
|
assert "sensorhiaten" not in message # dit is de generieke, niet de "ontbreekt volledig"-boodschap
|
||||||
|
|
||||||
|
|
||||||
def test_detect_events_ignores_dip_shorter_than_minimum_duration():
|
def test_detect_events_ignores_dip_shorter_than_minimum_duration():
|
||||||
# dip van 10 minuten (< 15 min drempel) mag niet meetellen
|
# dip van 10 minuten (< 15 min drempel) mag niet meetellen
|
||||||
glucose = _series(
|
glucose = _series(
|
||||||
|
|||||||
+13
-1
@@ -15,7 +15,7 @@ import plotly.graph_objects as go
|
|||||||
import streamlit as st
|
import streamlit as st
|
||||||
|
|
||||||
from nightscout_client import fetch_entries_range, fetch_treatments_range
|
from nightscout_client import fetch_entries_range, fetch_treatments_range
|
||||||
from reporting import detect_events, summarize_events
|
from reporting import completeness_warning, detect_events, summarize_events
|
||||||
|
|
||||||
MMOL_HOVERTEMPLATE = "%{x|%d-%m %H:%M} · %{y:.1f} mmol/L<extra></extra>"
|
MMOL_HOVERTEMPLATE = "%{x|%d-%m %H:%M} · %{y:.1f} mmol/L<extra></extra>"
|
||||||
|
|
||||||
@@ -98,6 +98,18 @@ def load_range_data(start: date, end: date):
|
|||||||
return glucose, treatments
|
return glucose, treatments
|
||||||
|
|
||||||
|
|
||||||
|
def render_completeness_warning(glucose: pd.DataFrame, start: date, end: date) -> None:
|
||||||
|
"""Toon reporting.completeness_warning()'s oordeel als st.warning, indien nodig.
|
||||||
|
|
||||||
|
Gebruikt door elke rapportagepagina, direct na de load_range_data-call.
|
||||||
|
De beoordelingslogica zelf staat in reporting.py (pure functie) zodat het
|
||||||
|
PDF-rapport (pdf_export.py) exact dezelfde disclaimer kan hergebruiken.
|
||||||
|
"""
|
||||||
|
message = completeness_warning(glucose, start, end)
|
||||||
|
if message:
|
||||||
|
st.warning(message)
|
||||||
|
|
||||||
|
|
||||||
def render_event_level(
|
def render_event_level(
|
||||||
glucose: pd.DataFrame,
|
glucose: pd.DataFrame,
|
||||||
threshold: float,
|
threshold: float,
|
||||||
|
|||||||
Reference in New Issue
Block a user