# core/scripture_normalizer.py from __future__ import annotations import re from typing import Dict, List, Tuple # ----------------------------- # Canonical book abbreviations # ----------------------------- BOOK_CANON: Dict[str, str] = { # OT "genesis": "Gen.", "gen": "Gen.", "genesisesis": "Gen.", "exodus": "Ex.", "ex": "Ex.", "leviticus": "Lev.", "lev": "Lev.", "numbers": "Num.", "num": "Num.", "nums": "Num.", "deuteronomy": "Deut.", "deut": "Deut.", "deu": "Deut.", "deutronomy": "Deut.", "deut.": "Deut.", "joshua": "Josh.", "josh": "Josh.", "judges": "Judg.", "judg": "Judg.", "ruth": "Ruth", "1 samuel": "1 Sam.", "i samuel": "1 Sam.", "1 sam": "1 Sam.", "2 samuel": "2 Sam.", "ii samuel": "2 Sam.", "2 sam": "2 Sam.", "1 kings": "1 Ki.", "i kings": "1 Ki.", "1 ki": "1 Ki.", "1kgs": "1 Ki.", "1 kgs": "1 Ki.", "2 kings": "2 Ki.", "ii kings": "2 Ki.", "2 ki": "2 Ki.", "2kgs": "2 Ki.", "2 kgs": "2 Ki.", "1 chronicles": "1 Chron.", "i chronicles": "1 Chron.", "1 chron": "1 Chron.", "2 chronicles": "2 Chron.", "ii chronicles": "2 Chron.", "2 chron": "2 Chron.", # short “Ch/Chr” forms (plus numbered) "ch": "Chron.", "chr": "Chron.", "1 ch": "1 Chron.", "1 chr": "1 Chron.", "2 ch": "2 Chron.", "2 chr": "2 Chron.", "ezra": "Ezra", "nehemiah": "Neh.", "neh": "Neh.", "esther": "Esth.", "esth": "Esth.", "job": "Job", "psalm": "Ps.", "psalms": "Ps.", "ps": "Ps.", "proverbs": "Prov.", "prov": "Prov.", "ecclesiastes": "Eccl.", "eccles": "Eccl.", "eccl": "Eccl.", "song of solomon": "Song", "song of songs": "Song", "song": "Song", "isaiah": "Isa.", "isa": "Isa.", "jeremiah": "Jer.", "jer": "Jer.", "jer.": "Jer.", "lamentations": "Lam.", "lam": "Lam.", "ezekiel": "Ezek.", "ezek": "Ezek.", "daniel": "Dan.", "dan": "Dan.", "hosea": "Hos.", "hos": "Hos.", "joel": "Joel", "amos": "Amos", "obadiah": "Obad.", "obad": "Obad.", "jonah": "Jon.", "jon": "Jon.", "micah": "Mic.", "mic": "Mic.", "nahum": "Nah.", "nah": "Nah.", "habakkuk": "Hab.", "hab": "Hab.", "zephaniah": "Zeph.", "zeph": "Zeph.", "haggai": "Hag.", "hag": "Hag.", "zechariah": "Zech.", "zech": "Zech.", "malachi": "Mal.", "mal": "Mal.", # NT "matthew": "Matt.", "matt": "Matt.", "mt": "Matt.", "mark": "Mark", "mk": "Mark.", "luke": "Luke", "lk": "Luke", "john": "John", "jn": "John", "acts": "Acts", "romans": "Rom.", "rom": "Rom.", "1 corinthians": "1 Cor.", "i corinthians": "1 Cor.", "1 cor": "1 Cor.", "1 cor.": "1 Cor.", "2 corinthians": "2 Cor.", "ii corinthians": "2 Cor.", "2 cor": "2 Cor.", "2 cor.": "2 Cor.", "galatians": "Gal.", "gal": "Gal.", "ephesians": "Eph.", "eph": "Eph.", "eph.": "Eph.", "philippians": "Phil.", "phil": "Phil.", "philippians 216": "Phil.", "colossians": "Col.", "col": "Col.", "1 thessalonians": "1 Thess.", "i thessalonians": "1 Thess.", "1 thess": "1 Thess.", "2 thessalonians": "2 Thess.", "ii thessalonians": "2 Thess.", "2 thess": "2 Thess.", "1 timothy": "1 Tim.", "i timothy": "1 Tim.", "1 tim": "1 Tim.", "2 timothy": "2 Tim.", "ii timothy": "2 Tim.", "2 tim": "2 Tim.", "titus": "Titus", "philemon": "Philem.", "philem": "Philem.", "hebrews": "Heb.", "heb": "Heb.", "james": "Jas.", "jas": "Jas.", "1 peter": "1 Pet.", "i peter": "1 Pet.", "1 pet": "1 Pet.", "2 peter": "2 Pet.", "ii peter": "2 Pet.", "2 pet": "2 Pet.", "1 john": "1 John", "i john": "1 John", "2 john": "2 John", "ii john": "2 John", "3 john": "3 John", "iii john": "3 John", "jude": "Jude", "revelation": "Rev.", "rev": "Rev.", } # For ultra-compact "1pe" / "2co" / "1ti" / "2th" etc. NUMBERED_SHORT = { "sa": "Sam.", "sam": "Sam.", "ki": "Ki.", "kgs": "Ki.", "kg": "Ki.", "ch": "Chron.", "chr": "Chron.", "co": "Cor.", "cor": "Cor.", "th": "Thess.", "ths": "Thess.", "ti": "Tim.", "tim": "Tim.", "pe": "Pet.", "pet": "Pet.", "jn": "John", "jo": "John", "john": "John", } # strip cruft words like "Read", "chapter" CRUFT_RE = re.compile(r"\b(read|see|chap(?:ter)?|ch)\b\.?", re.I) # book-like prefix (optional leading number) BOOK_RE = re.compile( r""" ^\s* (?:(?P[1-3]|i{1,3})\s*)? # 1/2/3 or i/ii/iii (?P[A-Za-z\.]+(?:\s+[A-Za-z\.]+){0,2}) \s* """, re.X, ) # chapter/verse piece C_V_RE = re.compile( r""" (?: (?P\d+) (?::(?P[\d,\-\u2013\u2014\s]+))? ) """, re.X, ) def _clean_text(s: str) -> str: s = s.replace("\xa0", " ").replace("\u2009", " ").replace("\u202f", " ") s = CRUFT_RE.sub("", s) s = s.replace("..", ".").replace("—", "-").replace("–", "-") s = re.sub(r"\s+", " ", s) return s.strip(" ;,.\t\r\n ") def _canon_book(book_raw: str, num: str | None = None) -> str | None: """ Normalize a book token (with or without a numeric prefix). Handles tight forms like '1pe', '2co', '1ch', etc. """ if not book_raw and not num: return None # Build a normalized key with number + book words raw = ((num or "") + " " + (book_raw or "")).strip() key = raw.lower().replace(".", " ") key = re.sub(r"\s+", " ", key) # If it's like '1pe' (no space), insert a space key = re.sub(r"^([1-3]|i{1,3})([a-z])", r"\1 \2", key) # Try exact phrase first (e.g., "1 peter", "2 cor") if key in BOOK_CANON: return BOOK_CANON[key] # Try without number (e.g., "peter", "cor") parts = key.split(" ", 1) if len(parts) == 2 and parts[1] in BOOK_CANON and parts[0] in {"1","2","3","i","ii","iii"}: canon_no_num = BOOK_CANON[parts[1]] # replace leading number in canon if present (for John we keep "1 John") if canon_no_num.startswith(("1 ", "2 ", "3 ")): # if canon is already numbered (like "1 John"), prefer that return canon_no_num lead = parts[0] lead = {"i":"1","ii":"2","iii":"3"}.get(lead, lead) return f"{lead} {canon_no_num}" # Handle compact numbered shorts like '1 pe', '2 co', '1 ch', '2 th' m = re.match(r"^(?P[1-3]|i{1,3})\s*(?P[a-z]{1,4})$", key) if m: n = m.group("n") s = m.group("s") n = {"i":"1","ii":"2","iii":"3"}.get(n, n) if s in NUMBERED_SHORT: return f"{n} {NUMBERED_SHORT[s]}" # Lastly try pure book without number if key in BOOK_CANON: return BOOK_CANON[key] return None def _parse_segment(seg: str, last_book: str | None) -> Tuple[str | None, str | None, bool]: """ Parse one semicolon-delimited segment. Returns (book_canon, cv_string, preserve_original_if_unknown) """ original = seg s = _clean_text(seg) if not s: return (None, None, False) # Detect a leading number stuck to letters like "1co", "2pe" m_tight = re.match(r"^\s*(?P[1-3]|i{1,3})\s*(?P[a-z]{1,4})\b\.?", s, flags=re.I) if m_tight: num = m_tight.group("num") letters = m_tight.group("letters") canon = _canon_book(letters, num=num) if canon: rest = s[m_tight.end():].strip(",;: .") book = canon else: # not recognized—keep whole piece verbatim return (None, original.strip(), True) else: # General book matcher m = BOOK_RE.match(s) book = None rest = s if m: num = m.group("num") raw_book = (m.group("book") or "").strip() canon = _canon_book(raw_book, num=num or None) if canon: book = canon rest = s[m.end():].strip(",;: .") else: # Not a recognized book: if we already have a last_book, treat # this as CV only; otherwise preserve the original piece. if last_book: book = last_book else: return (None, original.strip(), True) else: # No obvious book – inherit if we can, else preserve original if last_book: book = last_book else: return (None, original.strip(), True) # Normalize chapter/verse part rest = rest.replace(" ", "") rest = re.sub(r":\s+", ":", rest) if not rest: cv = None else: if C_V_RE.search(rest): cv = rest.replace(" ", "").replace("–", "-").replace("—", "-") else: m2 = re.search(r"\d+(?::[\d,\-]+)?", rest) cv = m2.group(0).replace(" ", "") if m2 else None return (book, cv, False) def normalize_scripture_field(text: str) -> Tuple[str, List[str]]: """ Normalize a scripture_raw string. Returns (normalized_text, warnings). Unknown segments are preserved as-is and reported in warnings. """ warnings: List[str] = [] if not text: return ("", warnings) pieces = [p for p in re.split(r"\s*;\s*", text) if p and p.strip()] out: List[str] = [] last_book: str | None = None for piece in pieces: book, cv, preserve = _parse_segment(piece, last_book) if preserve: out.append(piece.strip()) warnings.append(f"Unrecognized segment kept as-is: '{piece.strip()}'") continue if not book and not cv: continue if book and not cv: out.append(book) last_book = book continue if not book and cv: # Shouldn't really happen now; keep as-is out.append(piece.strip()) warnings.append(f"Missing book for '{piece.strip()}'") continue out.append(f"{book} {cv}") last_book = book norm = "; ".join(o.strip() for o in out if o.strip()) norm = re.sub(r"\s+", " ", norm).strip(" ;,") return (norm, warnings)