joshlaymon/Illustrations
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update web/core/scripture_normalizer.py

Browse Source
This commit is contained in:
Joshua Laymon 2025-08-14 01:49:30 +00:00
parent c2943a59f8
commit 260d70f4cd
1 changed files with 35 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
web/core/scripture_normalizer.py
View File

@ -25,7 +25,7 @@ BOOK_CANON = {
"nehemiah": "Neh.", "neh": "Neh.",
"esther": "Esth.", "esth": "Esth.",
"job": "Job",
"psalm": "Ps.", "psalms": "Ps.", "ps": "Ps.",
"psalm": "Ps.", "psalms": "Ps.", "ps": "Ps.", "psa": "Ps.",
"proverbs": "Prov.", "prov": "Prov.",
"ecclesiastes": "Eccl.", "eccles": "Eccl.", "eccl": "Eccl.",
"song of solomon": "Song", "song of songs": "Song", "song": "Song",
@ -47,10 +47,10 @@ BOOK_CANON = {
"zechariah": "Zech.", "zech": "Zech.",
"malachi": "Mal.", "mal": "Mal.",
# NT
"matthew": "Matt.", "matt": "Matt.",
"mark": "Mark",
"luke": "Luke",
"john": "John",
"matthew": "Matt.", "matt": "Matt.", "mt": "Matt.",
"mark": "Mark", "mk": "Mark",
"luke": "Luke", "lk": "Luke",
"john": "John", "jn": "John",
"acts": "Acts",
"romans": "Rom.", "rom": "Rom.",
"1 corinthians": "1 Cor.", "i corinthians": "1 Cor.", "1 cor": "1 Cor.", "1 cor.": "1 Cor.",
@ -59,11 +59,11 @@ BOOK_CANON = {
"ephesians": "Eph.", "eph": "Eph.", "eph.": "Eph.",
"philippians": "Phil.", "phil": "Phil.", "philippians 216": "Phil.", # common import glitch
"colossians": "Col.", "col": "Col.",
"1 thessalonians": "1 Thess.", "i thessalonians": "1 Thess.", "1 thess": "1 Thess.",
"2 thessalonians": "2 Thess.", "ii thessalonians": "2 Thess.", "2 thess": "2 Thess.",
"1 timothy": "1 Tim.", "i timothy": "1 Tim.", "1 tim": "1 Tim.",
"2 timothy": "2 Tim.", "ii timothy": "2 Tim.", "2 tim": "2 Tim.",
"titus": "Titus",
"1 thessalonians": "1 Thess.", "i thessalonians": "1 Thess.", "1 thess": "1 Thess.", "1 thes": "1 Thess.",
"2 thessalonians": "2 Thess.", "ii thessalonians": "2 Thess.", "2 thess": "2 Thess.", "2 thes": "2 Thess.",
"1 timothy": "1 Tim.", "i timothy": "1 Tim.", "1 tim": "1 Tim.", "1 ti": "1 Tim.",
"2 timothy": "2 Tim.", "ii timothy": "2 Tim.", "2 tim": "2 Tim.", "2 ti": "2 Tim.",
"titus": "Titus", "ti": "Tim.", # 'Ti' is usually Timothy in your data
"philemon": "Philem.", "philem": "Philem.",
"hebrews": "Heb.", "heb": "Heb.",
"james": "Jas.", "jas": "Jas.",
@ -74,6 +74,9 @@ BOOK_CANON = {
"3 john": "3 John", "iii john": "3 John",
"jude": "Jude",
"revelation": "Rev.", "rev": "Rev.",
# very short generic stems
"cor": "Cor.", "co": "Cor.",
"thess": "Thess.", "thes": "Thess.",
}
# add “numbered+book” fallbacks like “1 cor”, “2 ki”, “1 chron”, etc.
@ -81,17 +84,22 @@ def _variants() -> Dict[str, str]:
base = dict(BOOK_CANON)
numbered = [
("samuel", "Sam."), ("kings", "Ki."), ("chronicles", "Chron."),
("corinthians", "Cor."), ("thessalonians", "Thess."),
("timothy", "Tim."), ("peter", "Pet."), ("john", "John"),
("corinthians", "Cor."), ("cor", "Cor."), ("co", "Cor."),
("thessalonians", "Thess."), ("thess", "Thess."), ("thes", "Thess."),
("timothy", "Tim."), ("ti", "Tim."),
("peter", "Pet."), ("john", "John"),
]
for n in ("1", "i"):
for name, abbr in numbered:
base[f"{n} {name}"] = f"1 {abbr}"
base[f"{n}{name}"] = f"1 {abbr}"
for n in ("2", "ii"):
for name, abbr in numbered:
base[f"{n} {name}"] = f"2 {abbr}"
base[f"{n}{name}"] = f"2 {abbr}"
for n in ("3", "iii"):
base[f"{n} john"] = "3 John"
base[f"{n}john"] = "3 John"
# very common shorthands
base["ps"] = "Ps."
base["prov"] = "Prov."
@ -102,6 +110,11 @@ def _variants() -> Dict[str, str]:
BOOK_MAP = _variants()
# Extra explicit short forms that don't naturally fall out of the variant builder.
BOOK_MAP.update({
"1 co": "1 Cor.", "2 co": "2 Cor.", "1co": "1 Cor.", "2co": "2 Cor.",
})
# strip cruft words like "Read", "chapter"
CRUFT_RE = re.compile(r"\b(read|see|chap(?:ter)?|ch)\b\.?", re.I)
@ -169,35 +182,30 @@ def _parse_segment(seg: str, last_book: str | None) -> Tuple[str | None, str | N
if m:
raw = ((m.group("num") or "").strip() + " " + (m.group("book") or "").strip()).strip()
raw = raw.replace(" ", " ")
# if the "book" word is obviously a book
canon = _canon_book(raw)
if canon:
book = canon
rest = s[m.end():].strip(",;: .")
else:
# maybe m just matched a word that's not a book; keep whole as rest
rest = s
if not book:
book = last_book # inherit prior
book = last_book # inherit prior (e.g., "14:20" after "1 Cor. 13:11")
# now rest should hold "4:6,7-9" or "21" etc — normalize spaces
rest = rest.replace(" ", "")
# fix cases like "2: 24" -> "2:24"
rest = re.sub(r":\s+", ":", rest)
rest = re.sub(r"\s+", "", rest)
rest = re.sub(r":\s+", ":", rest) # "2: 24" -> "2:24"
# allow chapter-only
if not rest:
cv = None
else:
# validate basic shape
if C_V_RE.search(rest):
cv = rest
# normalize commas around verses e.g. "6, 7" -> "6,7"
cv = cv.replace(" ", "")
# normalize commas to include a following space: "6,7" -> "6, 7"
cv = re.sub(r"\s*,\s*", ", ", cv)
else:
# weird text (like just a word) — treat as chapter-only number if any
m2 = re.search(r"\d+(?::[\d,\-]+)?", rest)
cv = m2.group(0).replace(" ", "") if m2 else None
cv = re.sub(r"\s*,\s*", ", ", m2.group(0)) if m2 else None
return (book, cv)
@ -221,27 +229,22 @@ def normalize_scripture_field(text: str) -> Tuple[str, List[str]]:
if not book and not cv:
continue
if book and not cv:
# only a book (e.g., "Acts")
out.append(book)
last_book = book
continue
if not book and cv:
# verses but no book — cannot link properly; warn and skip
warnings.append(f"Missing book for '{piece.strip()}'")
continue
# normalize verse separators "1, 2" -> "1,2" already done; ensure ranges use hyphen
cv = cv.replace("–", "-").replace("—", "-")
cv = (cv or "").replace("–", "-").replace("—", "-")
cv = re.sub(r"\s+", " ", cv).strip()
# build
out.append(f"{book} {cv}")
out.append(f"{book} {cv}" if cv else f"{book}")
last_book = book
# de-dup whitespace, join with semicolons
norm = "; ".join(o.strip() for o in out if o.strip())
norm = norm.strip(" ;,")
# final tiny cleanup
norm = re.sub(r"\s+", " ", norm)
return (norm, warnings)