MVP 0.9
This commit is contained in:
1 parent
a8caeb0dd4
commit
fd7102e3b3
95 files changed
+16520
-193
No files matched your search
@@ -0,0 +1,120 @@
|
||||
"""Bind German 3rd-person pronouns to the last named person. Local; no new Detect tokens."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
PLACEHOLDER = re.compile(r"\[\[(PERSON:[^\]]+)\]\]")
|
||||
PIECE = re.compile(r"\[\[PERSON:[^\]]+\]\]|[A-Za-zÄÖÜäöüß]+")
|
||||
|
||||
FEM_CUES = {
|
||||
"frau", "partnerin", "tochter", "mutter", "schwester", "freundin",
|
||||
"oma", "tante",
|
||||
}
|
||||
MASC_CUES = {
|
||||
"herr", "mann", "partner", "sohn", "vater", "bruder", "freund",
|
||||
"opa", "onkel",
|
||||
}
|
||||
FEM_PRON = {"sie", "ihr", "ihre", "ihrem", "ihren", "ihrer", "ihres"}
|
||||
MASC_PRON = {"er", "ihn", "ihm", "seine", "seinem", "seinen", "seiner", "seines"}
|
||||
IHR_ADDRESS = {
|
||||
"seid", "habt", "wart", "könnt", "müsst", "sollt", "wollt", "dürft",
|
||||
"hättet", "wäret", "werdet",
|
||||
}
|
||||
PLURAL_VERB = {
|
||||
"sind", "waren", "haben", "hatten", "werden", "wurden",
|
||||
"können", "müssen", "wollen", "sollen",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class BindState:
|
||||
last: str | None = None
|
||||
last_f: str | None = None
|
||||
last_m: str | None = None
|
||||
|
||||
|
||||
def _gender_from_prev(pieces: list[re.Match], index: int) -> str | None:
|
||||
seen: list[str] = []
|
||||
pos = index - 1
|
||||
while pos >= 0 and len(seen) < 3:
|
||||
raw = pieces[pos].group(0)
|
||||
if PLACEHOLDER.fullmatch(raw):
|
||||
break
|
||||
seen.append(raw.lower())
|
||||
pos -= 1
|
||||
window = set(seen)
|
||||
if window & FEM_CUES:
|
||||
return "f"
|
||||
if window & MASC_CUES:
|
||||
return "m"
|
||||
return None
|
||||
|
||||
|
||||
def _referent(state: BindState, kind: str) -> str | None:
|
||||
if kind == "f" and state.last_f:
|
||||
return state.last_f
|
||||
if kind == "m" and state.last_m:
|
||||
return state.last_m
|
||||
return state.last
|
||||
|
||||
|
||||
def _is_address_ihr(prev: str, nxt: str) -> bool:
|
||||
if nxt in IHR_ADDRESS or prev in IHR_ADDRESS:
|
||||
return True
|
||||
if prev.endswith("t") and len(prev) >= 4:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def bind_pronouns(text: str, state: BindState | None = None) -> tuple[str, BindState]:
|
||||
state = state or BindState()
|
||||
pieces = list(PIECE.finditer(text or ""))
|
||||
replacements: list[tuple[int, int, str]] = []
|
||||
for i, match in enumerate(pieces):
|
||||
raw = match.group(0)
|
||||
person = PLACEHOLDER.fullmatch(raw)
|
||||
if person:
|
||||
token = person.group(1)
|
||||
state.last = token
|
||||
gender = _gender_from_prev(pieces, i)
|
||||
if gender == "f":
|
||||
state.last_f = token
|
||||
elif gender == "m":
|
||||
state.last_m = token
|
||||
continue
|
||||
low = raw.lower()
|
||||
prev = pieces[i - 1].group(0).lower() if i else ""
|
||||
nxt = pieces[i + 1].group(0).lower() if i + 1 < len(pieces) else ""
|
||||
if low == "ihr" and _is_address_ihr(prev, nxt):
|
||||
continue
|
||||
if low == "sie" and nxt in PLURAL_VERB:
|
||||
continue
|
||||
kind = "f" if low in FEM_PRON else "m" if low in MASC_PRON else ""
|
||||
if not kind:
|
||||
continue
|
||||
ref = _referent(state, kind)
|
||||
if not ref:
|
||||
continue
|
||||
replacements.append((match.start(), match.end(), f"[[{ref}]]"))
|
||||
if not replacements:
|
||||
return text or "", state
|
||||
out = text or ""
|
||||
for start, end, repl in reversed(replacements):
|
||||
out = out[:start] + repl + out[end:]
|
||||
return out, state
|
||||
|
||||
|
||||
def bind_user_lines(text: str) -> str:
|
||||
"""Only user: lines; state carries across those lines. Instructions stay untouched."""
|
||||
state = BindState()
|
||||
chunks: list[str] = []
|
||||
for line in (text or "").splitlines(keepends=True):
|
||||
ended = line.endswith("\n")
|
||||
body = line[:-1] if ended else line
|
||||
if body.startswith("user:"):
|
||||
bound, state = bind_pronouns(body[5:], state)
|
||||
chunks.append("user:" + bound + ("\n" if ended else ""))
|
||||
else:
|
||||
chunks.append(line)
|
||||
return "".join(chunks)
|
||||
Reference in new issue
Block a user