""" Invoice Picker: Rechnungen aus Link-/Portal-Mails ziehen und für den Versand an Accountable vormerken. Läuft unabhängig vom Alt-Scan (/emails/scan-all); Dedupe über Message-ID, Postfach wird nur gelesen (readonly). CLI (Cron/Rückblick): python -m backend.grabber --since 2025-01-01 """ import argparse import email import hashlib import os import re from datetime import date, datetime, timedelta from typing import Optional from sqlalchemy.orm import Session from backend.fetchers import aws from backend.fetchers.direct import fetch_pdf from backend.models import Outbound from backend.routers.emails import INVOICE_KEYWORDS, classify_mail, connect_imap, decode_hdr, parse_email_date from backend.routers.settings import get_imap_config # ponytail: feste Ordnerliste (Strato, Stand 13.11.2026); Sent/Spam/Drafts bewusst nicht FOLDERS = ("INBOX", "Archive", "Archive.2026", "Trash") def _cutoff() -> Optional[date]: """Stichtag: Mails davor = Rückblick (Freigabe nötig). Ohne GRAB_CUTOFF geht nichts automatisch raus.""" v = os.getenv("GRAB_CUTOFF") return datetime.strptime(v, "%Y-%m-%d").date() if v else None def _status_for(email_date: Optional[datetime]) -> str: c = _cutoff() return "queued" if c and email_date and email_date.date() >= c else "pending_review" def _save(pdf: bytes, base: str, email_date: Optional[datetime], sender: str) -> tuple: sha = hashlib.sha256(pdf).hexdigest() d = email_date or datetime.utcnow() domain = re.sub(r"[^\w.-]", "", _domain(sender)) or "unbekannt" folder = os.path.join(base, "picker", f"{d:%Y}", f"{d:%m}") os.makedirs(folder, exist_ok=True) path = os.path.join(folder, f"{d:%Y-%m-%d}_{domain}_{sha[:8]}.pdf") with open(path, "wb") as f: f.write(pdf) return path, sha def process_message(db: Session, msg, folder: str, base: str) -> Optional[str]: """Eine Mail verarbeiten -> neuer Outbound-Status oder None (übersprungen).""" subject = decode_hdr(msg.get("Subject", "")) if not any(kw in subject.lower() for kw in INVOICE_KEYWORDS): return None sender = decode_hdr(msg.get("From", "")) email_date = parse_email_date(msg.get("Date", "")) mid = (msg.get("Message-ID") or "").strip() or hashlib.sha256(f"{folder}{email_date}{subject}".encode()).hexdigest() row = db.query(Outbound).filter(Outbound.message_id == mid).first() if row and row.status != "manual": # "manual" wird bei jedem Lauf neu versucht (neue Adapter greifen rückwirkend) return None d = _domain(sender) if aws.ENABLED and any(d == x or d.endswith("." + x) for x in aws.SENDER_DOMAINS): # AWS kommt über die API (sync_aws) if row: row.status, row.error = "via_api", None db.commit() return None mail_class, url = classify_mail(msg) if mail_class not in ("link", "portal"): # Anhänge leitet Markus bereits selbst an Accountable weiter return None row = row or Outbound(message_id=mid) row.email_folder, row.email_date, row.email_from = folder, email_date, sender row.email_subject, row.mail_class, row.link_url = subject, mail_class, url pdf = None if mail_class == "link": try: pdf = fetch_pdf(url) row.error = None if pdf else "Link liefert kein PDF (Login nötig?)" except Exception as e: row.error = f"Download-Fehler: {e}"[:500] else: row.error = "Portal-Abruf noch nicht automatisiert" # Phase 3: Playwright-Adapter if pdf: _store(db, row, pdf, base) else: row.status = "manual" db.add(row) db.commit() return row.status def _store(db: Session, row: Outbound, pdf: bytes, base: str): row.file_path, row.sha256 = _save(pdf, base, row.email_date, row.email_from or "") dup = db.query(Outbound.id).filter(Outbound.sha256 == row.sha256, Outbound.id != row.id).first() row.status = "duplicate" if dup else _status_for(row.email_date) def sync_aws(db: Session, since: date, base: str, counts: dict): """AWS-Rechnungen über die Invoicing-API holen (message_id = aws-invoice:).""" known = {r[0].split(":", 1)[1] for r in db.query(Outbound.message_id) .filter(Outbound.message_id.like("aws-invoice:%")).all()} for inv in aws.fetch_invoices(since, known): issued = inv["issued"].replace(tzinfo=None) if inv["issued"] else None row = Outbound(message_id=f"aws-invoice:{inv['invoice_id']}", email_folder="api", email_date=issued, email_from="AWS ", mail_class="api", email_subject=f"{inv['entity']} Rechnung {inv['invoice_id']} ({inv['period']})") _store(db, row, inv["pdf"], base) db.add(row) db.commit() counts[row.status] = counts.get(row.status, 0) + 1 def _domain(sender: str) -> str: m = re.search(r"@([\w.-]+)", sender) return m[1].lower() if m else "" def run(db: Session, since: date, state: Optional[dict] = None) -> dict: state = state if state is not None else {} state.setdefault("errors", []) counts = state.setdefault("counts", {}) cfg = get_imap_config(db) m = connect_imap(cfg["host"], cfg["port"], cfg["user"], cfg["password"]) try: for folder in FOLDERS: if m.select(f'"{folder}"', readonly=True)[0] != "OK": continue ids = m.search(None, f"SINCE {since:%d-%b-%Y}")[1][0].split() if not ids: continue # erst nur Betreff holen (klein), Volltext nur bei Rechnungs-Stichwort hdrs = [h for k in range(0, len(ids), 500) for h in m.fetch(b",".join(ids[k:k + 500]), "(BODY.PEEK[HEADER.FIELDS (SUBJECT)])")[1]] ids = [h[0].split()[0] for h in hdrs if isinstance(h, tuple) and any(kw in decode_hdr(email.message_from_bytes(h[1]).get("Subject", "")).lower() for kw in INVOICE_KEYWORDS)] for i in ids: try: typ, data = m.fetch(i, "(BODY.PEEK[])") if typ != "OK" or not data or not isinstance(data[0], tuple): continue status = process_message(db, email.message_from_bytes(data[0][1]), folder, cfg["storage_path"]) if status: counts[status] = counts.get(status, 0) + 1 except Exception as e: db.rollback() state["errors"].append(f"{folder}/{i.decode()}: {e}"[:300]) state["processed"] = state.get("processed", 0) + 1 finally: try: m.logout() except Exception: pass if aws.ENABLED: try: sync_aws(db, since, cfg["storage_path"], counts) except Exception as e: db.rollback() state["errors"].append(f"AWS-API: {e}"[:300]) return state if __name__ == "__main__": from backend.database import SessionLocal p = argparse.ArgumentParser() p.add_argument("--since", help="YYYY-MM-DD (Default: vor 3 Tagen)") a = p.parse_args() since = datetime.strptime(a.since, "%Y-%m-%d").date() if a.since else date.today() - timedelta(days=3) db = SessionLocal() try: print(run(db, since)) finally: db.close()