c-stone-invoice-check/backend/grabber.py
Markus Kruse c6b84db0d2 feat(picker): Whop-Roboter (Login per Mail-Code, PDF via receipt_pdf)
Belegnummern aus Order-Confirmation-Mails, Login-Cookies im Volume,
Whop-Mails nicht mehr als manual. Chromium im Backend-Image.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Wd9aeH8MKgesfbsaS3kpnx
2026-11-15 09:57:35 +01:00

245 lines
10 KiB
Python

"""
Invoice Picker: Rechnungen aus Link-/Portal-Mails ziehen und für den Versand an Accountable vormerken.
Läuft unabhängig vom Alt-Scan (/emails/scan-all); Dedupe über Message-ID, Postfach wird nur gelesen (readonly).
CLI (Cron/Rückblick): python -m backend.grabber --since 2025-01-01
Wochenlauf (Cron NucBox So 06:00): python -m backend.grabber --send --report
"""
import argparse
import email
import hashlib
import os
import re
from datetime import date, datetime, timedelta
from typing import Optional
from sqlalchemy.orm import Session
from backend.fetchers import aws, whop
from backend.fetchers.direct import fetch_pdf
from backend.models import Outbound
from backend.routers.emails import INVOICE_KEYWORDS, classify_mail, connect_imap, decode_hdr, parse_email_date
from backend.routers.settings import get_imap_config
# ponytail: feste Ordnerliste (Strato, Stand 13.11.2026); Sent/Spam/Drafts bewusst nicht
FOLDERS = ("INBOX", "Archive", "Archive.2026", "Trash")
def _cutoff() -> Optional[date]:
"""Stichtag: Mails davor = Rückblick (Freigabe nötig). Ohne GRAB_CUTOFF geht nichts automatisch raus."""
v = os.getenv("GRAB_CUTOFF")
return datetime.strptime(v, "%Y-%m-%d").date() if v else None
def _status_for(email_date: Optional[datetime]) -> str:
c = _cutoff()
return "queued" if c and email_date and email_date.date() >= c else "pending_review"
def _save(pdf: bytes, base: str, email_date: Optional[datetime], sender: str) -> tuple:
sha = hashlib.sha256(pdf).hexdigest()
d = email_date or datetime.utcnow()
domain = re.sub(r"[^\w.-]", "", _domain(sender)) or "unbekannt"
folder = os.path.join(base, "picker", f"{d:%Y}", f"{d:%m}")
os.makedirs(folder, exist_ok=True)
path = os.path.join(folder, f"{d:%Y-%m-%d}_{domain}_{sha[:8]}.pdf")
with open(path, "wb") as f:
f.write(pdf)
return path, sha
def process_message(db: Session, msg, folder: str, base: str) -> Optional[str]:
"""Eine Mail verarbeiten -> neuer Outbound-Status oder None (übersprungen)."""
subject = decode_hdr(msg.get("Subject", ""))
if not any(kw in subject.lower() for kw in INVOICE_KEYWORDS):
return None
sender = decode_hdr(msg.get("From", ""))
email_date = parse_email_date(msg.get("Date", ""))
mid = (msg.get("Message-ID") or "").strip() or hashlib.sha256(f"{folder}{email_date}{subject}".encode()).hexdigest()
row = db.query(Outbound).filter(Outbound.message_id == mid).first()
if row and row.status != "manual": # "manual" wird bei jedem Lauf neu versucht (neue Adapter greifen rückwirkend)
return None
d = _domain(sender)
robot = (aws.ENABLED and _is(d, aws.SENDER_DOMAINS)) or (whop.ENABLED and _is(d, whop.SENDER_DOMAINS))
if robot: # AWS kommt über die API (sync_aws), Whop über den Roboter (sync_whop)
if row:
row.status, row.error = "via_api", None
db.commit()
return None
mail_class, url = classify_mail(msg)
if mail_class not in ("link", "portal"): # Anhänge leitet Markus bereits selbst an Accountable weiter
return None
row = row or Outbound(message_id=mid)
row.email_folder, row.email_date, row.email_from = folder, email_date, sender
row.email_subject, row.mail_class, row.link_url = subject, mail_class, url
pdf = None
if mail_class == "link":
try:
pdf = fetch_pdf(url)
row.error = None if pdf else "Link liefert kein PDF (Login nötig?)"
except Exception as e:
row.error = f"Download-Fehler: {e}"[:500]
else:
row.error = "Portal-Abruf noch nicht automatisiert" # Phase 3: Playwright-Adapter
if pdf:
_store(db, row, pdf, base)
else:
row.status = "manual"
db.add(row)
db.commit()
return row.status
def _store(db: Session, row: Outbound, pdf: bytes, base: str):
row.file_path, row.sha256 = _save(pdf, base, row.email_date, row.email_from or "")
dup = db.query(Outbound.id).filter(Outbound.sha256 == row.sha256, Outbound.id != row.id).first()
row.status = "duplicate" if dup else _status_for(row.email_date)
def sync_aws(db: Session, since: date, base: str, counts: dict):
"""AWS-Rechnungen über die Invoicing-API holen (message_id = aws-invoice:<id>)."""
known = {r[0].split(":", 1)[1] for r in db.query(Outbound.message_id)
.filter(Outbound.message_id.like("aws-invoice:%")).all()}
for inv in aws.fetch_invoices(since, known):
issued = inv["issued"].replace(tzinfo=None) if inv["issued"] else None
row = Outbound(message_id=f"aws-invoice:{inv['invoice_id']}", email_folder="api", email_date=issued,
email_from="AWS <invoicing@aws.com>", mail_class="api",
email_subject=f"{inv['entity']} Rechnung {inv['invoice_id']} ({inv['period']})")
_store(db, row, inv["pdf"], base)
db.add(row)
db.commit()
counts[row.status] = counts.get(row.status, 0) + 1
def sync_whop(db: Session, cfg: dict, since: date, counts: dict):
"""Whop-Ads-Belege per Roboter (message_id = whop-receipt:<pay_id>)."""
known = {r[0].split(":", 1)[1] for r in db.query(Outbound.message_id)
.filter(Outbound.message_id.like("whop-receipt:%")).all()}
todo = {r["pay_id"]: r for r in whop.receipts(cfg, since) if r["pay_id"] not in known}
if not todo:
return
pdfs = whop.fetch_pdfs(cfg, list(todo), os.path.join(cfg["storage_path"], ".whop-state.json"))
for pid, pdf in pdfs.items():
r = todo[pid]
row = Outbound(message_id=f"whop-receipt:{pid}", email_folder="whop-portal", email_date=r["date"],
email_from=r["sender"], mail_class="portal", link_url=f"https://whop.com/billing/receipt/{pid}/",
email_subject=f"Whop Ads Rechnung {pid} (${r['amount']})")
db.add(row)
db.flush()
_store(db, row, pdf, cfg["storage_path"])
db.commit()
counts[row.status] = counts.get(row.status, 0) + 1
if len(pdfs) < len(todo):
counts["whop_fehlt"] = len(todo) - len(pdfs)
def _is(domain: str, domains) -> bool:
return any(domain == x or domain.endswith("." + x) for x in domains)
def _domain(sender: str) -> str:
m = re.search(r"@([\w.-]+)", sender)
return m[1].lower() if m else ""
GMAIL_UNTIL = date(2026, 6, 1) # seit Mai 2026 läuft info@ nur noch über Strato
def _mailboxes(db: Session, since: date):
"""Strato (Settings/Env) + Gmail nur für Rückblicke vor Juni 2026 (info@ lief bis Ende Mai über Google)."""
cfg = get_imap_config(db)
yield cfg, FOLDERS
if os.getenv("GMAIL_IMAP_PASSWORD") and since < GMAIL_UNTIL:
# ponytail: nur "Alle Nachrichten" (\All) statt jedes Label einzeln; Dedupe über Message-ID
yield {**cfg, "host": "imap.gmail.com", "port": 993, "user": os.getenv("GMAIL_IMAP_USER", "info@c-stone-dev.com"),
"password": os.environ["GMAIL_IMAP_PASSWORD"]}, None
def _all_mail(m) -> tuple:
for line in m.list()[1]:
s = line.decode(errors="ignore")
if "\\All" in s:
return (s.rsplit(' "/" ', 1)[-1].strip().strip('"'),)
return ("INBOX",)
def run(db: Session, since: date, state: Optional[dict] = None) -> dict:
state = state if state is not None else {}
state.setdefault("errors", [])
counts = state.setdefault("counts", {})
for cfg, folders in _mailboxes(db, since):
try:
_scan(db, cfg, folders, since, state, counts)
except Exception as e:
state["errors"].append(f"{cfg['host']}: {e}"[:300])
if whop.ENABLED:
try:
sync_whop(db, get_imap_config(db), since, counts)
except Exception as e:
db.rollback()
state["errors"].append(f"Whop-Roboter: {e}"[:300])
if aws.ENABLED:
try:
sync_aws(db, since, get_imap_config(db)["storage_path"], counts)
except Exception as e:
db.rollback()
state["errors"].append(f"AWS-API: {e}"[:300])
return state
def _scan(db: Session, cfg: dict, folders, since: date, state: dict, counts: dict):
m = connect_imap(cfg["host"], cfg["port"], cfg["user"], cfg["password"])
try:
for folder in folders or _all_mail(m):
if m.select(f'"{folder}"', readonly=True)[0] != "OK":
continue
ids = m.search(None, f"SINCE {since:%d-%b-%Y}")[1][0].split()
if not ids:
continue
# erst nur Betreff holen (klein), Volltext nur bei Rechnungs-Stichwort
hdrs = [h for k in range(0, len(ids), 500)
for h in m.fetch(b",".join(ids[k:k + 500]), "(BODY.PEEK[HEADER.FIELDS (SUBJECT)])")[1]]
ids = [h[0].split()[0] for h in hdrs if isinstance(h, tuple)
and any(kw in decode_hdr(email.message_from_bytes(h[1]).get("Subject", "")).lower()
for kw in INVOICE_KEYWORDS)]
for i in ids:
try:
typ, data = m.fetch(i, "(BODY.PEEK[])")
if typ != "OK" or not data or not isinstance(data[0], tuple):
continue
status = process_message(db, email.message_from_bytes(data[0][1]), folder, cfg["storage_path"])
if status:
counts[status] = counts.get(status, 0) + 1
except Exception as e:
db.rollback()
state["errors"].append(f"{folder}/{i.decode()}: {e}"[:300])
state["processed"] = state.get("processed", 0) + 1
finally:
try:
m.logout()
except Exception:
pass
if __name__ == "__main__":
from backend.database import SessionLocal
p = argparse.ArgumentParser()
p.add_argument("--since", help="YYYY-MM-DD (Default: vor 10 Tagen, passt zum Wochenlauf)")
p.add_argument("--send", action="store_true", help="danach status=queued an Accountable senden")
p.add_argument("--report", action="store_true", help="Kurzbericht per Mail an IMAP_USER")
a = p.parse_args()
since = datetime.strptime(a.since, "%Y-%m-%d").date() if a.since else date.today() - timedelta(days=10)
db = SessionLocal()
try:
state = run(db, since)
if a.send:
from backend import mailer
state["versand"] = mailer.send_queued(db)
if a.report:
from backend import mailer
mailer.send_report(db, state, since)
print(state)
finally:
db.close()