diff --git a/web/core/views.py b/web/core/views.py index a450a6b..27ebac5 100644 --- a/web/core/views.py +++ b/web/core/views.py @@ -329,103 +329,101 @@ def entry_delete(request, entry_id): @login_required @user_passes_test(is_admin) def import_wizard(request): - # Safety: expected header list + # Safety: expected header list (matches DB/order the importer expects) _EXPECTED_HEADERS = [ "Subject", "Illustration", "Application", "Scripture", "Source", "Talk Title", "Talk Number", "Code", "Date", "Date Edited", ] -if request.method == "POST": - form = ImportForm(request.POST, request.FILES) - if form.is_valid(): - try: - raw = form.cleaned_data["file"].read() - - import io, csv as _csv - - # Decode once (BOM-safe) - text = raw.decode("utf-8-sig", errors="replace") - - # Try to sniff a dialect; fall back to Excel-style CSV + if request.method == "POST": + form = ImportForm(request.POST, request.FILES) + if form.is_valid(): try: - first_line = text.splitlines()[0] if text else "" - dialect = _csv.Sniffer().sniff(first_line) if first_line else _csv.excel - except Exception: - dialect = _csv.excel + raw = form.cleaned_data["file"].read() - rdr = _csv.reader(io.StringIO(text), dialect) - rows = list(rdr) - if not rows: - raise ValueError("The CSV file appears to be empty.") + import io + import csv as _csv - # Expected header (DB field order) - expected = [ - "Subject", "Illustration", "Application", "Scripture", "Source", - "Talk Title", "Talk Number", "Code", "Date", "Date Edited", - ] - expected_norm = [h.lower() for h in expected] + # Decode once (BOM‑safe) + text = raw.decode("utf-8-sig", errors="replace") - # Header cleaner: fixes r:"Talk Title", stray quotes, spaces, case - def _clean_header(s): - s = "" if s is None else str(s) - s = s.strip() - if s.lower().startswith("r:") or s.lower().startswith("r="): - s = s[2:].lstrip() - if (len(s) >= 2) and (s[0] == s[-1]) and s[0] in ('"', "'"): - s = s[1:-1] - return s.strip().lower() + # Try to sniff a dialect; fall back to Excel-style CSV + try: + first_line = text.splitlines()[0] if text else "" + dialect = _csv.Sniffer().sniff(first_line) if first_line else _csv.excel + except Exception: + dialect = _csv.excel - first = rows[0] - norm_first = [_clean_header(c) for c in first] + rdr = _csv.reader(io.StringIO(text), dialect) + rows = list(rdr) + if not rows: + raise ValueError("The CSV file appears to be empty.") - # If first row isn’t our header but length matches, inject one - header_ok = (norm_first == expected_norm) - if not header_ok and len(first) == len(expected): - rows.insert(0, expected) - elif not header_ok and len(first) != len(expected): - # Try common alternate delimiters if column count is off - for delim in (";", "\t"): - rdr2 = _csv.reader(io.StringIO(text), delimiter=delim) - test_rows = list(rdr2) - if test_rows and len(test_rows[0]) == len(expected): - rows = test_rows - first = rows[0] - norm_first = [_clean_header(c) for c in first] - header_ok = (norm_first == expected_norm) - if not header_ok: - rows.insert(0, expected) - break + expected = _EXPECTED_HEADERS + expected_norm = [h.lower() for h in expected] - # Re-encode a sanitized CSV for the existing importer - out = io.StringIO() - w = _csv.writer(out) - for r in rows: - w.writerow(r) - fixed_raw = out.getvalue().encode("utf-8") + # Header cleaner: fixes r:"Talk Title", stray quotes, spaces, case + def _clean_header(s): + s = "" if s is None else str(s) + s = s.strip() + if s.lower().startswith("r:") or s.lower().startswith("r="): + s = s[2:].lstrip() + if (len(s) >= 2) and (s[0] == s[-1]) and s[0] in ('"', "'"): + s = s[1:-1] + return s.strip().lower() - # Keep utils in sync for importer variants that read EXPECTED_HEADERS - from . import utils as core_utils - core_utils.EXPECTED_HEADERS = expected + first = rows[0] + norm_first = [_clean_header(c) for c in first] - # Hand off to the robust importer you already have - report = import_csv_bytes(fixed_raw, dry_run=form.cleaned_data["dry_run"]) or {} - report["header_ok"] = header_ok - if not header_ok: - messages.warning( + # If first row isn’t our header but length matches, inject one + header_ok = (norm_first == expected_norm) + if not header_ok and len(first) == len(expected): + rows.insert(0, expected) + elif not header_ok and len(first) != len(expected): + # Try common alternate delimiters if column count is off + for delim in (";", "\t"): + rdr2 = _csv.reader(io.StringIO(text), delimiter=delim) + test_rows = list(rdr2) + if test_rows and len(test_rows[0]) == len(expected): + rows = test_rows + first = rows[0] + norm_first = [_clean_header(c) for c in first] + header_ok = (norm_first == expected_norm) + if not header_ok: + rows.insert(0, expected) + break + + # Re-encode a sanitized CSV for the existing importer + out = io.StringIO() + w = _csv.writer(out) + for r in rows: + w.writerow(r) + fixed_raw = out.getvalue().encode("utf-8") + + # Keep utils in sync for importer variants that read EXPECTED_HEADERS + from . import utils as core_utils + core_utils.EXPECTED_HEADERS = expected + + # Hand off to the robust importer you already have + report = import_csv_bytes(fixed_raw, dry_run=form.cleaned_data["dry_run"]) or {} + report["header_ok"] = header_ok + if not header_ok: + messages.warning( + request, + "The first row didn’t match the expected header; a clean header was injected automatically." + ) + + return render( request, - "The first row didn’t match the expected header; a clean header was injected automatically." + "import_result.html", + {"report": report, "dry_run": form.cleaned_data["dry_run"]}, ) + except Exception as e: + messages.error(request, f"Import failed: {e}") + else: + form = ImportForm() - return render( - request, - "import_result.html", - {"report": report, "dry_run": form.cleaned_data["dry_run"]}, - ) - except Exception as e: - messages.error(request, f"Import failed: {e}") -else: - form = ImportForm() -return render(request, "import_wizard.html", {"form": form}) + return render(request, "import_wizard.html", {"form": form}) @login_required @user_passes_test(is_admin)