This commit is contained in:
Lars committed 2026-08-29 11:04:34 +02:00
1 parent 4f37991ad1
commit be11f82f84
39 files changed
+2490 -239

No files matched your search

+6 -123
View File
@@ -1,4 +1,4 @@
"""Local journal shape after the model. Does not invent content."""
"""Local journal shape after the model. Does not invent or rewrite content."""
from __future__ import annotations
import re
@@ -10,26 +10,6 @@ NEW_PHASE = re.compile(
r"Als\s+(?:wir|ich)\b)",
re.I,
)
_LETTER = r"A-Za-zÄÖÜäöüß"
_WORD = re.compile(rf"[{_LETTER}]+")
DATIVE_PREPS = {
"mit", "bei", "von", "zu", "nach", "aus", "außer", "gegenüber", "gemäß",
}
NOM_CUES = {
"dass", "daß", "ob", "weil", "wenn", "als", "während", "obwohl", "damit",
"nachdem", "bevor", "sodass", "sofern",
}
THING_GOVERNORS = {
"esse", "essen", "isst", "aß", "aßest", "gegessen",
"koche", "kochen", "kochte", "gekocht",
"trinke", "trinken", "trank", "getrunken",
"kaufe", "kaufen", "kaufte", "gekauft",
"bestelle", "bestellen", "bestellte",
"hole", "holen", "holte",
}
FEM_HINTS = {"sie", "ihr", "ihre", "ihrem", "ihren", "ihrer", "ihres"}
MASC_HINTS = {"er", "ihn", "ihm", "seine", "seinem", "seinen", "seiner", "seines"}
KEEP_NAMES_PER_PARAGRAPH = 2
def _strip_roles(text: str) -> str:
@@ -55,6 +35,7 @@ def is_verbatim_join(body: str, user_bodies: list[str]) -> bool:
def paragraphize(body: str) -> str:
"""Optional local preview helper. Must not rewrite an accepted model draft."""
text = (body or "").strip()
if "\n\n" in text:
return text
@@ -75,99 +56,6 @@ def paragraphize(body: str) -> str:
return "\n\n".join(chunks)
def _words_before(text: str, index: int, n: int = 4) -> list[str]:
return [word.lower() for word in _WORD.findall(text[:index])[-n:]]
def _words_after(text: str, index: int, n: int = 3) -> list[str]:
return [word.lower() for word in _WORD.findall(text[index:])[:n]]
def _is_thing_mention(text: str, start: int, end: int) -> bool:
prev = _words_before(text, start)
nxt = _words_after(text, end)
return any(word in THING_GOVERNORS for word in prev + nxt)
def _sentence_start(text: str, index: int) -> bool:
prefix = text[:index].rstrip()
return not prefix or prefix[-1] in ".!?\n"
def infer_person_gender(label: str, source_texts: list[str]) -> str | None:
blob = " ".join(source_texts or [])
if not label or not blob:
return None
pattern = re.compile(rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])", re.I)
first = pattern.search(blob)
if not first:
return None
after = [word.lower() for word in _WORD.findall(blob[first.start():])]
fem = sum(1 for word in after if word in FEM_HINTS)
masc = sum(1 for word in after if word in MASC_HINTS)
if fem > masc:
return "f"
if masc > fem:
return "m"
return None
def _pronoun(gender: str, prev: str, sentence_start: bool) -> str:
dative = prev in DATIVE_PREPS
if gender == "f":
form = "ihr" if dative else "sie"
elif sentence_start or prev in NOM_CUES:
form = "er"
elif dative:
form = "ihm"
else:
form = "ihn"
if sentence_start:
return form[:1].upper() + form[1:]
return form
def naturalize_person_mentions(body: str, labels: list[str], source_texts: list[str]) -> str:
"""After demask: keep the name once or twice per passage, then pronouns. No new facts."""
text = body or ""
people = []
for label in sorted({item.strip() for item in (labels or []) if (item or "").strip()}, key=len, reverse=True):
gender = infer_person_gender(label, source_texts)
if gender:
people.append((label, gender))
if not people:
return text
paragraphs = re.split(r"(\n\n+)", text)
out: list[str] = []
for block in paragraphs:
if not block or block.startswith("\n"):
out.append(block)
continue
piece = block
for label, gender in people:
pattern = re.compile(rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])", re.I)
matches = [
match
for match in pattern.finditer(piece)
if not _is_thing_mention(piece, match.start(), match.end())
]
if len(matches) <= KEEP_NAMES_PER_PARAGRAPH:
continue
replacements: list[tuple[int, int, str]] = []
kept = 0
for match in matches:
kept += 1
if kept <= KEEP_NAMES_PER_PARAGRAPH:
continue
prev = (_words_before(piece, match.start(), 1) or [""])[0]
form = _pronoun(gender, prev, _sentence_start(piece, match.start()))
replacements.append((match.start(), match.end(), form))
for start, end, form in reversed(replacements):
piece = piece[:start] + form + piece[end:]
out.append(piece)
return "".join(out)
def shape_journal(
title: str,
body: str,
@@ -176,15 +64,10 @@ def shape_journal(
*,
source: str = "model",
) -> tuple[str, str]:
"""Local post-shape. Never replaces accepted model wording with source turns."""
turns = _turns(user_bodies)
text = (body or "").strip()
"""Safe technical trim only. Never replaces names, pronouns, or accepted wording."""
del user_bodies, person_labels
heading = (title or "").strip()
text = _strip_roles(text)
if source != "fallback":
text = _strip_roles(body or "").strip()
if source == "fallback" and "\n\n" not in text:
text = paragraphize(text)
elif "\n\n" not in text:
text = paragraphize(text)
if person_labels:
text = naturalize_person_mentions(text, person_labels, turns)
return heading, text