""" Invoice Picker: Rechnungen aus Link-/Portal-Mails ziehen und für den Versand an Accountable vormerken. Läuft unabhängig vom Alt-Scan (/emails/scan-all); Dedupe über Message-ID, Postfach wird nur gelesen (readonly). CLI (Cron/Rückblick): python -m backend.grabber --since 2025-01-01 """ import argparse import email import hashlib import os import re from datetime import date, datetime, timedelta from typing import Optional from sqlalchemy.orm import Session from backend.fetchers.direct import fetch_pdf from backend.models import Outbound from backend.routers.emails import INVOICE_KEYWORDS, classify_mail, connect_imap, decode_hdr, parse_email_date from backend.routers.settings import get_imap_config # ponytail: feste Ordnerliste (Strato, Stand 13.11.2026); Sent/Spam/Drafts bewusst nicht FOLDERS = ("INBOX", "Archive", "Archive.2026", "Trash") def _cutoff() -> Optional[date]: """Stichtag: Mails davor = Rückblick (Freigabe nötig). Ohne GRAB_CUTOFF geht nichts automatisch raus.""" v = os.getenv("GRAB_CUTOFF") return datetime.strptime(v, "%Y-%m-%d").date() if v else None def _status_for(email_date: Optional[datetime]) -> str: c = _cutoff() return "queued" if c and email_date and email_date.date() >= c else "pending_review" def _save(pdf: bytes, base: str, email_date: Optional[datetime], sender: str) -> tuple: sha = hashlib.sha256(pdf).hexdigest() d = email_date or datetime.utcnow() domain = re.sub(r"[^\w.-]", "", (re.search(r"@([\w.-]+)", sender) or [None, "unbekannt"])[1]) folder = os.path.join(base, "picker", f"{d:%Y}", f"{d:%m}") os.makedirs(folder, exist_ok=True) path = os.path.join(folder, f"{d:%Y-%m-%d}_{domain}_{sha[:8]}.pdf") with open(path, "wb") as f: f.write(pdf) return path, sha def process_message(db: Session, msg, folder: str, base: str) -> Optional[str]: """Eine Mail verarbeiten -> neuer Outbound-Status oder None (übersprungen).""" subject = decode_hdr(msg.get("Subject", "")) if not any(kw in subject.lower() for kw in INVOICE_KEYWORDS): return None sender = decode_hdr(msg.get("From", "")) email_date = parse_email_date(msg.get("Date", "")) mid = (msg.get("Message-ID") or "").strip() or hashlib.sha256(f"{folder}{email_date}{subject}".encode()).hexdigest() row = db.query(Outbound).filter(Outbound.message_id == mid).first() if row and row.status != "manual": # "manual" wird bei jedem Lauf neu versucht (neue Adapter greifen rückwirkend) return None mail_class, url = classify_mail(msg) if mail_class not in ("link", "portal"): # Anhänge leitet Markus bereits selbst an Accountable weiter return None row = row or Outbound(message_id=mid) row.email_folder, row.email_date, row.email_from = folder, email_date, sender row.email_subject, row.mail_class, row.link_url = subject, mail_class, url pdf = None if mail_class == "link": try: pdf = fetch_pdf(url) row.error = None if pdf else "Link liefert kein PDF (Login nötig?)" except Exception as e: row.error = f"Download-Fehler: {e}"[:500] else: row.error = "Portal-Abruf noch nicht automatisiert" # Phase 3: Playwright-Adapter if pdf: row.file_path, row.sha256 = _save(pdf, base, email_date, sender) dup = db.query(Outbound.id).filter(Outbound.sha256 == row.sha256).first() row.status = "duplicate" if dup else _status_for(email_date) else: row.status = "manual" db.add(row) db.commit() return row.status def run(db: Session, since: date, state: Optional[dict] = None) -> dict: state = state if state is not None else {} state.setdefault("errors", []) counts = state.setdefault("counts", {}) cfg = get_imap_config(db) m = connect_imap(cfg["host"], cfg["port"], cfg["user"], cfg["password"]) try: for folder in FOLDERS: if m.select(f'"{folder}"', readonly=True)[0] != "OK": continue ids = m.search(None, f"SINCE {since:%d-%b-%Y}")[1][0].split() if not ids: continue # erst nur Betreff holen (klein), Volltext nur bei Rechnungs-Stichwort hdrs = [h for k in range(0, len(ids), 500) for h in m.fetch(b",".join(ids[k:k + 500]), "(BODY.PEEK[HEADER.FIELDS (SUBJECT)])")[1]] ids = [h[0].split()[0] for h in hdrs if isinstance(h, tuple) and any(kw in decode_hdr(email.message_from_bytes(h[1]).get("Subject", "")).lower() for kw in INVOICE_KEYWORDS)] for i in ids: try: typ, data = m.fetch(i, "(BODY.PEEK[])") if typ != "OK" or not data or not isinstance(data[0], tuple): continue status = process_message(db, email.message_from_bytes(data[0][1]), folder, cfg["storage_path"]) if status: counts[status] = counts.get(status, 0) + 1 except Exception as e: db.rollback() state["errors"].append(f"{folder}/{i.decode()}: {e}"[:300]) state["processed"] = state.get("processed", 0) + 1 finally: try: m.logout() except Exception: pass return state if __name__ == "__main__": from backend.database import SessionLocal p = argparse.ArgumentParser() p.add_argument("--since", help="YYYY-MM-DD (Default: vor 3 Tagen)") a = p.parse_args() since = datetime.strptime(a.since, "%Y-%m-%d").date() if a.since else date.today() - timedelta(days=3) db = SessionLocal() try: print(run(db, since)) finally: db.close()