""" Invoice Picker: Rechnungen aus Link-/Portal-Mails ziehen und für den Versand an Accountable vormerken. Läuft unabhängig vom Alt-Scan (/emails/scan-all); Dedupe über Message-ID, Postfach wird nur gelesen (readonly). CLI (Cron/Rückblick): python -m backend.grabber --since 2025-01-01 Wochenlauf (Cron NucBox So 06:00): python -m backend.grabber --send --report """ import argparse import email import hashlib import os import re from datetime import date, datetime, timedelta from typing import Optional from sqlalchemy.orm import Session from backend.fetchers import aws, whop from backend.fetchers.direct import fetch_pdf from backend.models import Outbound from backend.routers.emails import INVOICE_KEYWORDS, classify_mail, connect_imap, decode_hdr, parse_email_date from backend.routers.settings import get_imap_config # ponytail: feste Ordnerliste (Strato, Stand 13.11.2026); Sent/Spam/Drafts bewusst nicht FOLDERS = ("INBOX", "Archive", "Archive.2026", "Trash") def _cutoff() -> Optional[date]: """Stichtag: Mails davor = Rückblick (Freigabe nötig). Ohne GRAB_CUTOFF geht nichts automatisch raus.""" v = os.getenv("GRAB_CUTOFF") return datetime.strptime(v, "%Y-%m-%d").date() if v else None def _status_for(email_date: Optional[datetime]) -> str: c = _cutoff() return "queued" if c and email_date and email_date.date() >= c else "pending_review" def _save(pdf: bytes, base: str, email_date: Optional[datetime], sender: str) -> tuple: sha = hashlib.sha256(pdf).hexdigest() d = email_date or datetime.utcnow() domain = re.sub(r"[^\w.-]", "", _domain(sender)) or "unbekannt" folder = os.path.join(base, "picker", f"{d:%Y}", f"{d:%m}") os.makedirs(folder, exist_ok=True) path = os.path.join(folder, f"{d:%Y-%m-%d}_{domain}_{sha[:8]}.pdf") with open(path, "wb") as f: f.write(pdf) return path, sha def process_message(db: Session, msg, folder: str, base: str) -> Optional[str]: """Eine Mail verarbeiten -> neuer Outbound-Status oder None (übersprungen).""" subject = decode_hdr(msg.get("Subject", "")) if not any(kw in subject.lower() for kw in INVOICE_KEYWORDS): return None sender = decode_hdr(msg.get("From", "")) email_date = parse_email_date(msg.get("Date", "")) mid = (msg.get("Message-ID") or "").strip() or hashlib.sha256(f"{folder}{email_date}{subject}".encode()).hexdigest() row = db.query(Outbound).filter(Outbound.message_id == mid).first() if row and row.status != "manual": # "manual" wird bei jedem Lauf neu versucht (neue Adapter greifen rückwirkend) return None d = _domain(sender) robot = (aws.ENABLED and _is(d, aws.SENDER_DOMAINS)) or (whop.ENABLED and _is(d, whop.SENDER_DOMAINS)) if robot: # AWS kommt über die API (sync_aws), Whop über den Roboter (sync_whop) if row: row.status, row.error = "via_api", None db.commit() return None mail_class, url = classify_mail(msg) if mail_class not in ("link", "portal"): # Anhänge leitet Markus bereits selbst an Accountable weiter return None row = row or Outbound(message_id=mid) row.email_folder, row.email_date, row.email_from = folder, email_date, sender row.email_subject, row.mail_class, row.link_url = subject, mail_class, url pdf = None if mail_class == "link": try: pdf = fetch_pdf(url) row.error = None if pdf else "Link liefert kein PDF (Login nötig?)" except Exception as e: row.error = f"Download-Fehler: {e}"[:500] else: row.error = "Portal-Abruf noch nicht automatisiert" # Phase 3: Playwright-Adapter if pdf: _store(db, row, pdf, base) else: row.status = "manual" db.add(row) db.commit() return row.status def _store(db: Session, row: Outbound, pdf: bytes, base: str): row.file_path, row.sha256 = _save(pdf, base, row.email_date, row.email_from or "") dup = db.query(Outbound.id).filter(Outbound.sha256 == row.sha256, Outbound.id != row.id).first() row.status = "duplicate" if dup else _status_for(row.email_date) def sync_aws(db: Session, since: date, base: str, counts: dict): """AWS-Rechnungen über die Invoicing-API holen (message_id = aws-invoice:).""" known = {r[0].split(":", 1)[1] for r in db.query(Outbound.message_id) .filter(Outbound.message_id.like("aws-invoice:%")).all()} for inv in aws.fetch_invoices(since, known): issued = inv["issued"].replace(tzinfo=None) if inv["issued"] else None row = Outbound(message_id=f"aws-invoice:{inv['invoice_id']}", email_folder="api", email_date=issued, email_from="AWS ", mail_class="api", email_subject=f"{inv['entity']} Rechnung {inv['invoice_id']} ({inv['period']})") _store(db, row, inv["pdf"], base) db.add(row) db.commit() counts[row.status] = counts.get(row.status, 0) + 1 def sync_whop(db: Session, cfg: dict, since: date, counts: dict): """Whop-Ads-Belege per Roboter (message_id = whop-receipt:).""" known = {r[0].split(":", 1)[1] for r in db.query(Outbound.message_id) .filter(Outbound.message_id.like("whop-receipt:%")).all()} todo = {r["pay_id"]: r for r in whop.receipts(cfg, since) if r["pay_id"] not in known} if not todo: return pdfs = whop.fetch_pdfs(cfg, list(todo), os.path.join(cfg["storage_path"], ".whop-state.json")) for pid, pdf in pdfs.items(): r = todo[pid] row = Outbound(message_id=f"whop-receipt:{pid}", email_folder="whop-portal", email_date=r["date"], email_from=r["sender"], mail_class="portal", link_url=f"https://whop.com/billing/receipt/{pid}/", email_subject=f"Whop Ads Rechnung {pid} (${r['amount']})") db.add(row) db.flush() _store(db, row, pdf, cfg["storage_path"]) db.commit() counts[row.status] = counts.get(row.status, 0) + 1 if len(pdfs) < len(todo): counts["whop_fehlt"] = len(todo) - len(pdfs) def _is(domain: str, domains) -> bool: return any(domain == x or domain.endswith("." + x) for x in domains) def _domain(sender: str) -> str: m = re.search(r"@([\w.-]+)", sender) return m[1].lower() if m else "" GMAIL_UNTIL = date(2026, 6, 1) # seit Mai 2026 läuft info@ nur noch über Strato def _mailboxes(db: Session, since: date): """Strato (Settings/Env) + Gmail nur für Rückblicke vor Juni 2026 (info@ lief bis Ende Mai über Google).""" cfg = get_imap_config(db) yield cfg, FOLDERS if os.getenv("GMAIL_IMAP_PASSWORD") and since < GMAIL_UNTIL: # ponytail: nur "Alle Nachrichten" (\All) statt jedes Label einzeln; Dedupe über Message-ID yield {**cfg, "host": "imap.gmail.com", "port": 993, "user": os.getenv("GMAIL_IMAP_USER", "info@c-stone-dev.com"), "password": os.environ["GMAIL_IMAP_PASSWORD"]}, None def _all_mail(m) -> tuple: for line in m.list()[1]: s = line.decode(errors="ignore") if "\\All" in s: return (s.rsplit(' "/" ', 1)[-1].strip().strip('"'),) return ("INBOX",) def run(db: Session, since: date, state: Optional[dict] = None) -> dict: state = state if state is not None else {} state.setdefault("errors", []) counts = state.setdefault("counts", {}) for cfg, folders in _mailboxes(db, since): try: _scan(db, cfg, folders, since, state, counts) except Exception as e: state["errors"].append(f"{cfg['host']}: {e}"[:300]) if whop.ENABLED: try: sync_whop(db, get_imap_config(db), since, counts) except Exception as e: db.rollback() state["errors"].append(f"Whop-Roboter: {e}"[:300]) if aws.ENABLED: try: sync_aws(db, since, get_imap_config(db)["storage_path"], counts) except Exception as e: db.rollback() state["errors"].append(f"AWS-API: {e}"[:300]) return state def _scan(db: Session, cfg: dict, folders, since: date, state: dict, counts: dict): m = connect_imap(cfg["host"], cfg["port"], cfg["user"], cfg["password"]) try: for folder in folders or _all_mail(m): if m.select(f'"{folder}"', readonly=True)[0] != "OK": continue ids = m.search(None, f"SINCE {since:%d-%b-%Y}")[1][0].split() if not ids: continue # erst nur Betreff holen (klein), Volltext nur bei Rechnungs-Stichwort hdrs = [h for k in range(0, len(ids), 500) for h in m.fetch(b",".join(ids[k:k + 500]), "(BODY.PEEK[HEADER.FIELDS (SUBJECT)])")[1]] ids = [h[0].split()[0] for h in hdrs if isinstance(h, tuple) and any(kw in decode_hdr(email.message_from_bytes(h[1]).get("Subject", "")).lower() for kw in INVOICE_KEYWORDS)] for i in ids: try: typ, data = m.fetch(i, "(BODY.PEEK[])") if typ != "OK" or not data or not isinstance(data[0], tuple): continue status = process_message(db, email.message_from_bytes(data[0][1]), folder, cfg["storage_path"]) if status: counts[status] = counts.get(status, 0) + 1 except Exception as e: db.rollback() state["errors"].append(f"{folder}/{i.decode()}: {e}"[:300]) state["processed"] = state.get("processed", 0) + 1 finally: try: m.logout() except Exception: pass if __name__ == "__main__": from backend.database import SessionLocal p = argparse.ArgumentParser() p.add_argument("--since", help="YYYY-MM-DD (Default: vor 10 Tagen, passt zum Wochenlauf)") p.add_argument("--send", action="store_true", help="danach status=queued an Accountable senden") p.add_argument("--report", action="store_true", help="Kurzbericht per Mail an IMAP_USER") a = p.parse_args() since = datetime.strptime(a.since, "%Y-%m-%d").date() if a.since else date.today() - timedelta(days=10) db = SessionLocal() try: state = run(db, since) if a.send: from backend import mailer state["versand"] = mailer.send_queued(db) if a.report: from backend import mailer mailer.send_report(db, state, since) print(state) finally: db.close()