590 lines
18 KiB
Python
590 lines
18 KiB
Python
from django.shortcuts import render, redirect, get_object_or_404
|
||
from django.contrib.auth import authenticate, login
|
||
from django.contrib.auth.decorators import login_required, user_passes_test
|
||
from django.http import HttpResponse
|
||
from django.contrib import messages
|
||
from django.db.models import Q
|
||
from django.views.decorators.http import require_http_methods
|
||
from datetime import date, timedelta
|
||
import csv
|
||
import re
|
||
import random
|
||
|
||
from .models import Entry
|
||
from .forms import ImportForm, EntryForm
|
||
from .utils import terms, has_wildcards, wildcard_to_regex, import_csv_bytes
|
||
from .scripture_normalizer import normalize_scripture_field # <-- NEW
|
||
|
||
# Order + labels used in the Search UI
|
||
FIELD_ORDER = [
|
||
("subject", "Subject"),
|
||
("illustration", "Illustration"),
|
||
("application", "Application"),
|
||
("scripture_raw", "Scripture"),
|
||
("source", "Source"),
|
||
("talk_title", "Talk Title"),
|
||
("talk_number", "Talk Number"),
|
||
("entry_code", "Code"),
|
||
]
|
||
|
||
|
||
def is_admin(user):
|
||
return user.is_superuser or user.is_staff
|
||
|
||
|
||
def login_view(request):
|
||
if request.user.is_authenticated:
|
||
return redirect("search")
|
||
ctx = {}
|
||
if request.method == "POST":
|
||
u = request.POST.get("username")
|
||
p = request.POST.get("password")
|
||
user = authenticate(request, username=u, password=p)
|
||
if user:
|
||
login(request, user)
|
||
return redirect("search")
|
||
ctx["error"] = "Invalid credentials"
|
||
return render(request, "login.html", ctx)
|
||
|
||
|
||
def entry_context(entry, result_ids):
|
||
"""
|
||
Build the navigation + chips context for the entry pages.
|
||
"""
|
||
count = len(result_ids or [])
|
||
if entry and result_ids and entry.id in result_ids:
|
||
position = result_ids.index(entry.id) + 1
|
||
else:
|
||
position = 1
|
||
|
||
subject_list = [t.strip() for t in (entry.subject or "").split(",") if t.strip()]
|
||
scripture_list = [
|
||
t.strip() for t in (entry.scripture_raw or "").split(";") if t.strip()
|
||
]
|
||
return {
|
||
"entry": entry,
|
||
"locked": True,
|
||
"position": position,
|
||
"count": count,
|
||
"subject_list": subject_list,
|
||
"scripture_list": scripture_list,
|
||
}
|
||
|
||
|
||
@login_required
|
||
def search_page(request):
|
||
"""
|
||
Search-first landing. Defaults to Subject, Illustration, Application.
|
||
Supports:
|
||
- quoted phrases
|
||
- * and ? wildcards (regex); if regex returns zero, falls back to icontains
|
||
- AND across tokens, OR across the selected fields
|
||
"""
|
||
default_fields = {
|
||
"subject": True,
|
||
"illustration": True,
|
||
"application": True,
|
||
"scripture_raw": False,
|
||
"source": False,
|
||
"talk_title": False,
|
||
"talk_number": False,
|
||
"entry_code": False,
|
||
}
|
||
|
||
form_submitted = ("q" in request.GET) or any(k in request.GET for k in default_fields)
|
||
if form_submitted:
|
||
selected = {k: (k in request.GET) for k in default_fields}
|
||
else:
|
||
selected = default_fields.copy()
|
||
|
||
field_options = [
|
||
{"name": k, "label": label, "checked": bool(selected.get(k))}
|
||
for k, label in FIELD_ORDER
|
||
]
|
||
|
||
q = (request.GET.get("q") or "").strip()
|
||
if q:
|
||
tokens = terms(q)
|
||
fields = [f for f, sel in selected.items() if sel] or ["subject"]
|
||
|
||
qs = Entry.objects.all()
|
||
used_regex = False
|
||
for tok in tokens:
|
||
clause = Q()
|
||
if has_wildcards(tok):
|
||
used_regex = True
|
||
pattern = wildcard_to_regex(tok)
|
||
for f in fields:
|
||
clause |= Q(**{f + "__iregex": pattern})
|
||
else:
|
||
for f in fields:
|
||
clause |= Q(**{f + "__icontains": tok})
|
||
qs = qs.filter(clause)
|
||
|
||
ids = list(qs.order_by("-date_added", "-id").values_list("id", flat=True))
|
||
|
||
if used_regex and not ids:
|
||
qs = Entry.objects.all()
|
||
for tok in tokens:
|
||
clause = Q()
|
||
tok_stripped = tok.replace("*", "").replace("?", "")
|
||
for f in fields:
|
||
clause |= Q(**{f + "__icontains": tok_stripped})
|
||
qs = qs.filter(clause)
|
||
ids = list(qs.order_by("-date_added", "-id").values_list("id", flat=True))
|
||
|
||
try:
|
||
print(f"[search] q={q!r} tokens={tokens} fields={fields} count={len(ids)}")
|
||
except Exception:
|
||
pass
|
||
|
||
request.session["result_ids"] = ids
|
||
count = len(ids)
|
||
|
||
if count:
|
||
entry = Entry.objects.get(pk=ids[0])
|
||
ctx = entry_context(entry, ids)
|
||
ctx.update({"from_search": True})
|
||
return render(request, "entry_view.html", ctx)
|
||
|
||
total = Entry.objects.count()
|
||
return render(
|
||
request,
|
||
"search.html",
|
||
{
|
||
"q": q,
|
||
"selected": selected,
|
||
"field_options": field_options,
|
||
"total": total,
|
||
"ran_search": True,
|
||
"result_count": 0,
|
||
},
|
||
)
|
||
|
||
total = Entry.objects.count()
|
||
return render(
|
||
request,
|
||
"search.html",
|
||
{
|
||
"q": q,
|
||
"selected": selected,
|
||
"field_options": field_options,
|
||
"total": total,
|
||
"ran_search": False,
|
||
},
|
||
)
|
||
|
||
|
||
@login_required
|
||
def nav_next(request):
|
||
ids = request.session.get("result_ids", [])
|
||
if not ids:
|
||
return redirect("search")
|
||
idx = int(request.GET.get("i", "0"))
|
||
idx = min(idx + 1, len(ids) - 1)
|
||
entry = get_object_or_404(Entry, pk=ids[idx])
|
||
return render(request, "entry_view.html", entry_context(entry, ids))
|
||
|
||
|
||
@login_required
|
||
def nav_prev(request):
|
||
ids = request.session.get("result_ids", [])
|
||
if not ids:
|
||
return redirect("search")
|
||
idx = int(request.GET.get("i", "0"))
|
||
idx = max(idx - 1, 0)
|
||
entry = get_object_or_404(Entry, pk=ids[idx])
|
||
return render(request, "entry_view.html", entry_context(entry, ids))
|
||
|
||
|
||
@login_required
|
||
def entry_view(request, entry_id):
|
||
ids = request.session.get("result_ids", [])
|
||
entry = get_object_or_404(Entry, pk=entry_id)
|
||
return render(request, "entry_view.html", entry_context(entry, ids))
|
||
|
||
|
||
@login_required
|
||
def entry_add(request):
|
||
"""
|
||
Create a brand new Entry using the same EntryForm you use for editing.
|
||
Since EntryForm is a regular Form (not a ModelForm), we copy fields manually.
|
||
"""
|
||
if request.method == "POST":
|
||
form = EntryForm(request.POST)
|
||
if form.is_valid():
|
||
entry = Entry()
|
||
for k, v in form.cleaned_data.items():
|
||
setattr(entry, k, v)
|
||
entry.save()
|
||
messages.success(request, "New entry added.")
|
||
return redirect("entry_view", entry_id=entry.id)
|
||
else:
|
||
form = EntryForm()
|
||
|
||
return render(request, "entry_add.html", {"form": form})
|
||
|
||
|
||
@login_required
|
||
def entry_edit(request, entry_id):
|
||
ids = request.session.get("result_ids", [])
|
||
entry = get_object_or_404(Entry, pk=entry_id)
|
||
|
||
if request.method == "POST":
|
||
form = EntryForm(request.POST)
|
||
if form.is_valid():
|
||
for k, v in form.cleaned_data.items():
|
||
setattr(entry, k, v)
|
||
entry.save()
|
||
messages.success(request, "Entry saved.")
|
||
return redirect("entry_view", entry_id=entry.id)
|
||
else:
|
||
form = EntryForm(
|
||
initial={
|
||
"subject": entry.subject,
|
||
"illustration": entry.illustration,
|
||
"application": entry.application,
|
||
"scripture_raw": entry.scripture_raw,
|
||
"source": entry.source,
|
||
"talk_number": entry.talk_number,
|
||
"talk_title": entry.talk_title,
|
||
"entry_code": entry.entry_code,
|
||
"date_added": entry.date_added,
|
||
"date_edited": entry.date_edited,
|
||
}
|
||
)
|
||
|
||
ctx = {"entry": entry, "form": form}
|
||
ctx.update(entry_context(entry, ids))
|
||
return render(request, "entry_edit.html", ctx)
|
||
|
||
|
||
@login_required
|
||
def entry_delete(request, entry_id):
|
||
entry = get_object_or_404(Entry, pk=entry_id)
|
||
if request.method == "POST":
|
||
entry.delete()
|
||
messages.success(request, "Entry deleted.")
|
||
return redirect("search")
|
||
return render(request, "entry_delete_confirm.html", {"entry": entry})
|
||
|
||
|
||
@login_required
|
||
@user_passes_test(is_admin)
|
||
def import_wizard(request):
|
||
if request.method == "POST":
|
||
form = ImportForm(request.POST, request.FILES)
|
||
if form.is_valid():
|
||
try:
|
||
report = import_csv_bytes(
|
||
form.cleaned_data["file"].read(),
|
||
dry_run=form.cleaned_data["dry_run"],
|
||
)
|
||
return render(
|
||
request,
|
||
"import_result.html",
|
||
{"report": report, "dry_run": form.cleaned_data["dry_run"]},
|
||
)
|
||
except Exception as e:
|
||
messages.error(request, f"Import failed: {e}")
|
||
else:
|
||
form = ImportForm()
|
||
return render(request, "import_wizard.html", {"form": form})
|
||
|
||
|
||
@login_required
|
||
@user_passes_test(is_admin)
|
||
def export_csv(request):
|
||
ts = date.today().strftime("%Y-%m-%d")
|
||
response = HttpResponse(content_type="text/csv")
|
||
response["Content-Disposition"] = (
|
||
f'attachment; filename="illustrations_backup_{ts}.csv"'
|
||
)
|
||
w = csv.writer(response)
|
||
w.writerow(
|
||
[
|
||
"Subject",
|
||
"Illustration",
|
||
"Application",
|
||
"Scripture",
|
||
"Source",
|
||
"Talk Number",
|
||
"Talk Title",
|
||
"Code",
|
||
"Date",
|
||
"Date Edited",
|
||
]
|
||
)
|
||
for e in Entry.objects.all().order_by("id"):
|
||
w.writerow(
|
||
[
|
||
e.subject,
|
||
e.illustration,
|
||
e.application,
|
||
e.scripture_raw,
|
||
e.source,
|
||
e.talk_number if e.talk_number is not None else "",
|
||
e.talk_title,
|
||
e.entry_code,
|
||
e.date_added.isoformat() if e.date_added else "",
|
||
e.date_edited.isoformat() if e.date_edited else "",
|
||
]
|
||
)
|
||
return response
|
||
|
||
|
||
@login_required
|
||
def stats_page(request):
|
||
total = Entry.objects.count()
|
||
today = date.today()
|
||
last30 = Entry.objects.filter(date_added__gte=today - timedelta(days=30)).count()
|
||
last365 = Entry.objects.filter(date_added__gte=today - timedelta(days=365)).count()
|
||
|
||
from collections import Counter
|
||
|
||
months = []
|
||
y = today.year
|
||
m = today.month
|
||
for i in range(12):
|
||
mm = m - i
|
||
yy = y
|
||
while mm <= 0:
|
||
mm += 12
|
||
yy -= 1
|
||
from datetime import date as _d
|
||
start = _d(yy, mm, 1)
|
||
end = _d(yy + 1, 1, 1) if mm == 12 else _d(yy, mm + 1, 1)
|
||
label = f"{yy}-{mm:02d}"
|
||
months.append((label, start, end))
|
||
months = list(reversed(months))
|
||
|
||
series = [
|
||
(label, Entry.objects.filter(date_added__gte=start, date_added__lt=end).count())
|
||
for label, start, end in months
|
||
]
|
||
peak = max((v for _, v in series), default=1)
|
||
heights = [
|
||
(label, value, 8 + int((value / peak) * 100) if peak else 8)
|
||
for label, value in series
|
||
]
|
||
|
||
counts = Counter()
|
||
for subj in Entry.objects.exclude(subject="").values_list("subject", flat=True):
|
||
for tag in [t.strip() for t in subj.split(",") if t.strip()]:
|
||
counts[tag.lower()] += 1
|
||
top_subjects = [{"name": n.title(), "count": c} for n, c in counts.most_common(10)]
|
||
|
||
return render(
|
||
request,
|
||
"stats.html",
|
||
{
|
||
"total": total,
|
||
"last30": last30,
|
||
"last365": last365,
|
||
"series": series,
|
||
"heights": heights,
|
||
"top_subjects": top_subjects,
|
||
},
|
||
)
|
||
|
||
|
||
# ========= NEW: Scripture Normalizer endpoint =========
|
||
|
||
@login_required
|
||
@user_passes_test(is_admin)
|
||
@require_http_methods(["GET", "POST"])
|
||
def normalize_scripture(request):
|
||
"""
|
||
GET -> dry-run preview (summary + first 100 examples)
|
||
POST -> apply changes to all entries' scripture_raw (batched)
|
||
Optional ?limit= for preview subset.
|
||
"""
|
||
apply = request.method == "POST"
|
||
limit = int(request.GET.get("limit", "0") or "0")
|
||
|
||
qs = Entry.objects.all().order_by("id")
|
||
if limit:
|
||
qs = qs[:limit]
|
||
|
||
changed = 0
|
||
warnings_total = 0
|
||
preview = []
|
||
|
||
if apply:
|
||
# write in batches to keep transactions short
|
||
from django.db import transaction
|
||
batch, pending = 500, []
|
||
for e in qs.iterator():
|
||
original = (e.scripture_raw or "").strip()
|
||
normalized, warns = normalize_scripture_field(original)
|
||
warnings_total += len(warns)
|
||
if normalized != original:
|
||
changed += 1
|
||
preview.append((e.id, original, normalized))
|
||
e.scripture_raw = normalized
|
||
pending.append(e)
|
||
if len(pending) >= batch:
|
||
with transaction.atomic():
|
||
for obj in pending:
|
||
obj.save(update_fields=["scripture_raw"])
|
||
pending.clear()
|
||
if pending:
|
||
with transaction.atomic():
|
||
for obj in pending:
|
||
obj.save(update_fields=["scripture_raw"])
|
||
else:
|
||
# dry-run only
|
||
for e in qs.iterator():
|
||
original = (e.scripture_raw or "").strip()
|
||
normalized, warns = normalize_scripture_field(original)
|
||
warnings_total += len(warns)
|
||
if normalized != original:
|
||
changed += 1
|
||
preview.append((e.id, original, normalized))
|
||
|
||
preview = preview[:100] # keep the table reasonable
|
||
|
||
messages.info(
|
||
request,
|
||
f"{'Applied' if apply else 'Dry‑run'}: {changed} entries "
|
||
f"{'changed' if apply else 'would change'}; {warnings_total} warnings."
|
||
)
|
||
return render(
|
||
request,
|
||
"normalize_result.html",
|
||
{
|
||
"applied": apply,
|
||
"changed": changed,
|
||
"warnings_total": warnings_total,
|
||
"preview": preview,
|
||
"limit": limit,
|
||
},
|
||
)
|
||
@login_required
|
||
def stats_page(request):
|
||
from collections import Counter
|
||
total = Entry.objects.count()
|
||
today = date.today()
|
||
last30 = Entry.objects.filter(date_added__gte=today - timedelta(days=30)).count()
|
||
last365 = Entry.objects.filter(date_added__gte=today - timedelta(days=365)).count()
|
||
|
||
# ---- Adds per month (existing logic) ----
|
||
months = []
|
||
y = today.year
|
||
m = today.month
|
||
for i in range(12):
|
||
mm = m - i
|
||
yy = y
|
||
while mm <= 0:
|
||
mm += 12
|
||
yy -= 1
|
||
from datetime import date as _d
|
||
start = _d(yy, mm, 1)
|
||
end = _d(yy + 1, 1, 1) if mm == 12 else _d(yy, mm + 1, 1)
|
||
label = f"{yy}-{mm:02d}"
|
||
months.append((label, start, end))
|
||
months = list(reversed(months))
|
||
|
||
series = [
|
||
(label, Entry.objects.filter(date_added__gte=start, date_added__lt=end).count())
|
||
for label, start, end in months
|
||
]
|
||
peak = max((v for _, v in series), default=1)
|
||
heights = [(label, value, 8 + int((value / peak) * 100) if peak else 8)
|
||
for label, value in series]
|
||
|
||
# ---- Top subjects (existing logic) ----
|
||
counts = Counter()
|
||
for subj in Entry.objects.exclude(subject="").values_list("subject", flat=True):
|
||
for tag in [t.strip() for t in subj.split(",") if t.strip()]:
|
||
counts[tag.lower()] += 1
|
||
top_subjects = [{"name": n.title(), "count": c} for n, c in counts.most_common(10)]
|
||
|
||
# ---- Scripture analytics (NEW) ----
|
||
# Expect canonical like: "Matt. 5:14; Ps. 1:1,2; 1 Cor. 13:4-7"
|
||
# Split on semicolons; capture book and chap/verses if present.
|
||
BOOK_RE = re.compile(
|
||
r"^\s*(?P<book>(?:[1-3]\s+)?[A-Za-z\.]+(?:\s+[A-Za-z\.]+){0,2})"
|
||
r"(?:\s+(?P<cv>\d+(?::[\d,\-\u2013\u2014]+)?))?\s*$"
|
||
)
|
||
|
||
books_counter = Counter()
|
||
refs_counter = Counter()
|
||
ref_per_entry_counts = []
|
||
entries_with_scripture = 0
|
||
|
||
scriptures = Entry.objects.exclude(scripture_raw="") \
|
||
.values_list("scripture_raw", flat=True)
|
||
|
||
for raw in scriptures:
|
||
pieces = [p for p in re.split(r"\s*;\s*", raw or "") if p.strip()]
|
||
if not pieces:
|
||
continue
|
||
entries_with_scripture += 1
|
||
refs_this_entry = 0
|
||
|
||
for p in pieces:
|
||
m = BOOK_RE.match(p)
|
||
if not m:
|
||
continue
|
||
book = (m.group("book") or "").strip()
|
||
cv = (m.group("cv") or "").strip()
|
||
if not book:
|
||
continue
|
||
books_counter[book] += 1
|
||
if cv:
|
||
refs_counter[f"{book} {cv}"] += 1
|
||
refs_this_entry += 1
|
||
|
||
ref_per_entry_counts.append(refs_this_entry)
|
||
|
||
top_books = books_counter.most_common(10)
|
||
top_refs = refs_counter.most_common(10)
|
||
avg_refs_per_entry = round(
|
||
(sum(ref_per_entry_counts) / len(ref_per_entry_counts))
|
||
if ref_per_entry_counts else 0.0, 2
|
||
)
|
||
book_distribution = books_counter.most_common(30) # handy for future charts
|
||
|
||
return render(
|
||
request,
|
||
"stats.html",
|
||
{
|
||
"total": total,
|
||
"last30": last30,
|
||
"last365": last365,
|
||
"series": series,
|
||
"heights": heights,
|
||
"top_subjects": top_subjects,
|
||
|
||
# NEW context for the template
|
||
"entries_with_scripture": entries_with_scripture,
|
||
"avg_refs_per_entry": avg_refs_per_entry,
|
||
"top_books": top_books,
|
||
"top_refs": top_refs,
|
||
"book_distribution": book_distribution,
|
||
},
|
||
)
|
||
|
||
@login_required
|
||
def search_page(request):
|
||
# ... your existing search logic above ...
|
||
|
||
# --- Illustration of the Day (simple, no persistence required) ---
|
||
illustration_entry = None
|
||
qs_day = Entry.objects.exclude(illustration__isnull=True).exclude(illustration="")
|
||
if qs_day.exists():
|
||
chosen = qs_day.order_by("?").first() # simple random row
|
||
illustration_entry = {
|
||
"illustration": chosen.illustration or "",
|
||
"application": chosen.application or "",
|
||
}
|
||
|
||
return render(
|
||
request,
|
||
"search.html",
|
||
{
|
||
# ... your existing context keys ...
|
||
"illustration_of_the_day": illustration_entry, # << add this
|
||
},
|
||
) |