From 95d365d9faae2d220119571a5ce7fb5d93d77951 Mon Sep 17 00:00:00 2001 From: Lars Date: Wed, 28 Oct 2026 11:14:37 +0100 Subject: [PATCH] Schedule media retention at 03:00 Europe/Berlin. The folded python -c command crashed the cron container before any run. Content-report rate limiting now applies only after the good-faith declaration. Co-authored-by: Cursor --- backend/routers/content_reports.py | 2 +- backend/scripts/media_retention_job.py | 4 +- backend/scripts/media_retention_scheduler.py | 121 ++++++++++++++ .../tests/test_media_retention_scheduler.py | 157 ++++++++++++++++++ backend/tests/test_p13_content_reports.py | 18 +- backend/version.py | 20 ++- docker-compose.yml | 16 +- 7 files changed, 317 insertions(+), 21 deletions(-) create mode 100644 backend/scripts/media_retention_scheduler.py create mode 100644 backend/tests/test_media_retention_scheduler.py diff --git a/backend/routers/content_reports.py b/backend/routers/content_reports.py index 30c9362..3c6c2e7 100644 --- a/backend/routers/content_reports.py +++ b/backend/routers/content_reports.py @@ -456,12 +456,12 @@ def submit_content_report( - Journaleintrag im Medien-Audit-Log (wenn target_type=media_asset) - E-Mail-Bestaetigung an Melder + Benachrichtigung an Plattform-Admins (best-effort) """ - limiter._check_request_limit(request, _content_report_rate_marker, False) if not body.good_faith_confirmed: raise HTTPException( status_code=400, detail="Die Gutglaubenserklärung (good_faith_confirmed) muss bestätigt werden.", ) + limiter._check_request_limit(request, _content_report_rate_marker, False) priority = "high" if body.report_reason in HIGH_PRIORITY_REASONS else "normal" diff --git a/backend/scripts/media_retention_job.py b/backend/scripts/media_retention_job.py index 2ef70cc..9de197c 100644 --- a/backend/scripts/media_retention_job.py +++ b/backend/scripts/media_retention_job.py @@ -2,12 +2,12 @@ """ Automatische Medien-Retention (Papierkorb Stufe 1→2, Purge wenn fällig). -Lauf z. B. täglich per Cron: +Einzellauf (der tägliche Planer ist media_retention_scheduler.py): cd /path/to/backend && python scripts/media_retention_job.py Umgebung wie Backend (DB_*), optional: MEDIA_TRASH_SOFT_TO_HIDDEN_DAYS (Default 30) - MEDIA_TRASH_HIDDEN_TO_PURGE_DAYS (Default 90) + MEDIA_TRASH_HIDDEN_TO_PURGE_DAYS (Default 30) """ from __future__ import annotations diff --git a/backend/scripts/media_retention_scheduler.py b/backend/scripts/media_retention_scheduler.py new file mode 100644 index 0000000..7ced2b3 --- /dev/null +++ b/backend/scripts/media_retention_scheduler.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Täglicher Start von media_retention_job.py um 03:00 Europe/Berlin. + +Der Prozess löscht beim Start nichts. Ein Lauf sofort nach dem Start nur mit +RETENTION_RUN_ON_START=1. Ein fehlgeschlagener Lauf bleibt im Log +(RETENTION_FAIL); der Prozess bleibt aktiv, damit Docker ihn nicht sofort +erneut startet. +""" +from __future__ import annotations + +import logging +import os +import subprocess +import sys +import time +from datetime import datetime, timedelta +from zoneinfo import ZoneInfo + +LOG = logging.getLogger("retention.scheduler") + +DEFAULT_TZ = "Europe/Berlin" +_RUN_ON_START = "RETENTION_RUN_ON_START" +_TZ_ENV = "RETENTION_TZ" + + +def retention_zone() -> ZoneInfo: + name = os.environ.get(_TZ_ENV, DEFAULT_TZ).strip() or DEFAULT_TZ + return ZoneInfo(name) + + +def next_run_at(now: datetime, *, hour: int = 3) -> datetime: + """Nächste Uhrzeit `hour`:00 in der Zeitzone von `now`. + + `now` muss zeitzonenbewusst sein. Die Zone von `now` ist die Planungszone. + """ + if now.tzinfo is None: + raise ValueError("now muss eine Zeitzone tragen") + zone = now.tzinfo + local = now.astimezone(zone) + day = local.date() + target = datetime(day.year, day.month, day.day, hour, 0, 0, tzinfo=zone) + if target.timestamp() <= local.timestamp(): + day = day + timedelta(days=1) + target = datetime(day.year, day.month, day.day, hour, 0, 0, tzinfo=zone) + return target + + +def seconds_until_next_0300(now: datetime, *, hour: int = 3) -> float: + """Sekunden bis zum nächsten Lauf, als absolute Dauer. + + `target - now` zieht bei gleicher tzinfo die Wanduhr ab und liegt an + Umstellungstagen um eine Stunde daneben. `timestamp()` rechnet in UTC. + """ + target = next_run_at(now, hour=hour) + local = now.astimezone(target.tzinfo) + return target.timestamp() - local.timestamp() + + +def _job_path() -> str: + return os.path.join(os.path.dirname(os.path.abspath(__file__)), "media_retention_job.py") + + +def _run_on_start() -> bool: + return os.environ.get(_RUN_ON_START, "").strip() == "1" + + +def run_job() -> int: + """Startet den Retention-Job. Führt keine Löschung in diesem Prozess aus.""" + LOG.info("Retention-Lauf startet") + completed = subprocess.run([sys.executable, "-u", _job_path()], check=False) + if completed.returncode != 0: + LOG.error("RETENTION_FAIL exit=%s", completed.returncode) + else: + LOG.info("RETENTION_OK") + return int(completed.returncode) + + +def serve( + *, + iterations: int | None = None, + sleep=time.sleep, + clock=None, + run=None, +) -> None: + zone = retention_zone() + if clock is None: + def clock() -> datetime: + return datetime.now(zone) + + execute = run if run is not None else run_job + if _run_on_start(): + LOG.info("Sofortlauf wegen RETENTION_RUN_ON_START=1") + execute() + + completed = 0 + while iterations is None or completed < iterations: + target = next_run_at(clock()) + LOG.info("Nächster Retention-Lauf %s", target.isoformat(timespec="seconds")) + while True: + remaining = target.timestamp() - clock().timestamp() + if remaining <= 1: + break + LOG.info("Warte %.0f Sekunden bis %s", remaining, target.isoformat(timespec="seconds")) + sleep(remaining) + execute() + completed += 1 + + +def main() -> int: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", + stream=sys.stdout, + force=True, + ) + serve() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/tests/test_media_retention_scheduler.py b/backend/tests/test_media_retention_scheduler.py new file mode 100644 index 0000000..3f41a19 --- /dev/null +++ b/backend/tests/test_media_retention_scheduler.py @@ -0,0 +1,157 @@ +"""Scheduler für den Papierkorb-Job: 03:00 Europe/Berlin, ohne echten Löschlauf.""" +from __future__ import annotations + +import logging +from datetime import datetime, timedelta +from pathlib import Path +from zoneinfo import ZoneInfo + +import pytest + +from scripts.media_retention_scheduler import ( + next_run_at, + run_job, + seconds_until_next_0300, + serve, +) + +BERLIN = ZoneInfo("Europe/Berlin") +REPO = Path(__file__).resolve().parents[2] +COMPOSE = REPO / "docker-compose.yml" + + +def test_winter_afternoon_waits_until_next_morning() -> None: + now = datetime(2026, 1, 15, 10, 0, tzinfo=BERLIN) + assert next_run_at(now).isoformat() == "2026-01-16T03:00:00+01:00" + assert seconds_until_next_0300(now) == 17 * 3600 + + +def test_summer_before_three_is_same_morning() -> None: + now = datetime(2026, 7, 15, 2, 0, tzinfo=BERLIN) + assert next_run_at(now).isoformat() == "2026-07-15T03:00:00+02:00" + assert seconds_until_next_0300(now) == 3600 + + +def test_exactly_0300_waits_until_next_day() -> None: + now = datetime(2026, 1, 15, 3, 0, tzinfo=BERLIN) + assert next_run_at(now).isoformat() == "2026-01-16T03:00:00+01:00" + assert seconds_until_next_0300(now) == 24 * 3600 + + +def test_spring_forward_sleeps_absolute_22_hours() -> None: + now = datetime(2026, 3, 28, 4, 0, tzinfo=BERLIN) + target = next_run_at(now) + assert target.isoformat() == "2026-03-29T03:00:00+02:00" + assert seconds_until_next_0300(now) == 22 * 3600 + assert (target - now).total_seconds() == 23 * 3600 + + +def test_fall_back_sleeps_absolute_24_hours() -> None: + now = datetime(2026, 10, 24, 4, 0, tzinfo=BERLIN) + target = next_run_at(now) + assert target.isoformat() == "2026-10-25T03:00:00+01:00" + assert seconds_until_next_0300(now) == 24 * 3600 + assert (target - now).total_seconds() == 23 * 3600 + + +def test_naive_datetime_is_rejected() -> None: + with pytest.raises(ValueError): + next_run_at(datetime(2026, 1, 15, 10, 0)) + + +def test_serve_sleeps_before_first_run(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("RETENTION_RUN_ON_START", raising=False) + state = {"now": datetime(2026, 1, 15, 10, 0, tzinfo=BERLIN)} + events: list[tuple] = [] + + def clock() -> datetime: + return state["now"] + + def sleep(seconds: float) -> None: + events.append(("sleep", seconds)) + state["now"] = state["now"] + timedelta(seconds=seconds) + + def run() -> None: + events.append(("run",)) + + serve(iterations=1, sleep=sleep, clock=clock, run=run) + assert events == [("sleep", 17 * 3600), ("run",)] + + +def test_serve_can_run_once_on_start(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("RETENTION_RUN_ON_START", "1") + state = {"now": datetime(2026, 7, 15, 2, 0, tzinfo=BERLIN)} + events: list[str] = [] + + def sleep(seconds: float) -> None: + events.append("sleep") + state["now"] = state["now"] + timedelta(seconds=seconds) + + serve( + iterations=1, + sleep=sleep, + clock=lambda: state["now"], + run=lambda: events.append("run"), + ) + assert events == ["run", "sleep", "run"] + + +def test_serve_continues_after_failed_run(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("RETENTION_RUN_ON_START", raising=False) + state = {"now": datetime(2026, 1, 15, 10, 0, tzinfo=BERLIN)} + runs: list[int] = [] + + def sleep(seconds: float) -> None: + state["now"] = state["now"] + timedelta(seconds=seconds) + + def run() -> int: + runs.append(1) + return 1 + + serve(iterations=2, sleep=sleep, clock=lambda: state["now"], run=run) + assert runs == [1, 1] + + +def test_run_job_logs_failure_without_starting_real_job( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + class Done: + returncode = 1 + + monkeypatch.setattr( + "scripts.media_retention_scheduler.subprocess.run", + lambda *args, **kwargs: Done(), + ) + with caplog.at_level(logging.INFO, logger="retention.scheduler"): + assert run_job() == 1 + assert "RETENTION_FAIL exit=1" in caplog.text + + +def test_run_job_logs_success( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + class Done: + returncode = 0 + + seen: list[list[str]] = [] + + def fake_run(cmd, check=False): + seen.append(list(cmd)) + return Done() + + monkeypatch.setattr("scripts.media_retention_scheduler.subprocess.run", fake_run) + with caplog.at_level(logging.INFO, logger="retention.scheduler"): + assert run_job() == 0 + assert seen and seen[0][-1].endswith("media_retention_job.py") + assert "RETENTION_OK" in caplog.text + assert "RETENTION_FAIL" not in caplog.text + + +def test_compose_starts_scheduler_script() -> None: + text = COMPOSE.read_text(encoding="utf-8") + start = text.index(" retention-cron:") + block = text[start : text.index("\nvolumes:", start)] + assert 'command: ["python", "-u", "scripts/media_retention_scheduler.py"]' in block + assert "python -c" not in block + assert 'PYTHONUNBUFFERED: "1"' in block + assert "RETENTION_RUN_ON_START" not in block diff --git a/backend/tests/test_p13_content_reports.py b/backend/tests/test_p13_content_reports.py index eee82a2..2cf6525 100644 --- a/backend/tests/test_p13_content_reports.py +++ b/backend/tests/test_p13_content_reports.py @@ -73,8 +73,24 @@ def test_submit_requires_good_faith(): good_faith_confirmed=False, ) from routers.content_reports import submit_content_report + from starlette.requests import Request + + request = Request( + { + "type": "http", + "http_version": "1.1", + "method": "POST", + "scheme": "http", + "path": "/api/content-reports", + "raw_path": b"/api/content-reports", + "query_string": b"", + "headers": [], + "client": ("127.0.0.1", 50000), + "server": ("testserver", 80), + } + ) with pytest.raises(HTTPException) as exc: - submit_content_report(body) + submit_content_report(request, body) assert exc.value.status_code == 400 assert "Gutglauben" in exc.value.detail diff --git a/backend/version.py b/backend/version.py index 5a671ec..561ac59 100644 --- a/backend/version.py +++ b/backend/version.py @@ -1,7 +1,7 @@ # Shinkan Jinkendo Version Information -APP_VERSION = "0.8.243" -BUILD_DATE = "2026-10-13" +APP_VERSION = "0.8.245" +BUILD_DATE = "2026-10-28" DB_SCHEMA_VERSION = "20261012096" MODULE_VERSIONS = { @@ -49,10 +49,24 @@ MODULE_VERSIONS = { "membership": "1.0.0", "catalogs": "1.5.0", # Updated: Trainer Contexts API (Migration 012) "maturity_models": "1.4.0", # matrix_stack_bundle: vollständiger Katalog+Modelle+Bindings Export/Import - "content_reports": "1.5.2", # POST /content-reports: 10 Meldungen pro Stunde und IP + "content_reports": "1.5.3", # Gutglauben vor dem Stundenlimit; Direktaufruf im Test mit Request } CHANGELOG = [ + { + "version": "0.8.245", + "date": "2026-10-28", + "changes": [ + "Papierkorb-Retention läuft täglich um 03:00 Uhr Europe/Berlin. Ein fehlgeschlagener Lauf bleibt im Container-Log sichtbar und löscht beim Containerstart nichts.", + ], + }, + { + "version": "0.8.244", + "date": "2026-10-13", + "changes": [ + "Inhaltsmeldungen ohne Gutglaubenserklärung zählen nicht auf das Stundenlimit.", + ], + }, { "version": "0.8.243", "date": "2026-10-13", diff --git a/docker-compose.yml b/docker-compose.yml index 56fabb0..b2f05ba 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -89,20 +89,7 @@ services: context: ./backend dockerfile: Dockerfile container_name: shinkan-retention-cron - command: > - python -c " - import time, subprocess, sys, os, datetime - def next_3am(): - now = datetime.datetime.now() - target = now.replace(hour=3, minute=0, second=0, microsecond=0) - if target <= now: - target += datetime.timedelta(days=1) - return (target - now).total_seconds() - subprocess.run([sys.executable, 'scripts/media_retention_job.py'], check=False) - while True: - time.sleep(next_3am()) - subprocess.run([sys.executable, 'scripts/media_retention_job.py'], check=False) - " + command: ["python", "-u", "scripts/media_retention_scheduler.py"] working_dir: /app environment: DB_HOST: postgres @@ -110,6 +97,7 @@ services: DB_NAME: shinkan DB_USER: shinkan_user DB_PASSWORD: ${DB_PASSWORD} + PYTHONUNBUFFERED: "1" MEDIA_ROOT: "${MEDIA_ROOT:-/app/media}" # Loeschkonzept (Audit P-03b): 30 Tage Soft-Trash, dann 30 Tage Hidden, dann Purge (gesamt 60 Tage). MEDIA_TRASH_SOFT_TO_HIDDEN_DAYS: "${MEDIA_TRASH_SOFT_TO_HIDDEN_DAYS:-30}"