c-stone-invoice-check/backend/grabber.py
Markus Kruse 4dff8234f1 feat(picker): Wochenlauf --send/--report, Gmail nur fuer Rueckblick
Gmail-Postfach seit 05/2026 ungenutzt: nur noch bei since < 2026-06-01.
CLI-Default since = vor 10 Tagen; Bericht an eigenes Postfach.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Wd9aeH8MKgesfbsaS3kpnx
2026-11-15 09:53:55 +01:00

212 lines
8.9 KiB
Python

"""
Invoice Picker: Rechnungen aus Link-/Portal-Mails ziehen und für den Versand an Accountable vormerken.
Läuft unabhängig vom Alt-Scan (/emails/scan-all); Dedupe über Message-ID, Postfach wird nur gelesen (readonly).
CLI (Cron/Rückblick): python -m backend.grabber --since 2025-01-01
Wochenlauf (Cron NucBox So 06:00): python -m backend.grabber --send --report
"""
import argparse
import email
import hashlib
import os
import re
from datetime import date, datetime, timedelta
from typing import Optional
from sqlalchemy.orm import Session
from backend.fetchers import aws
from backend.fetchers.direct import fetch_pdf
from backend.models import Outbound
from backend.routers.emails import INVOICE_KEYWORDS, classify_mail, connect_imap, decode_hdr, parse_email_date
from backend.routers.settings import get_imap_config
# ponytail: feste Ordnerliste (Strato, Stand 13.11.2026); Sent/Spam/Drafts bewusst nicht
FOLDERS = ("INBOX", "Archive", "Archive.2026", "Trash")
def _cutoff() -> Optional[date]:
"""Stichtag: Mails davor = Rückblick (Freigabe nötig). Ohne GRAB_CUTOFF geht nichts automatisch raus."""
v = os.getenv("GRAB_CUTOFF")
return datetime.strptime(v, "%Y-%m-%d").date() if v else None
def _status_for(email_date: Optional[datetime]) -> str:
c = _cutoff()
return "queued" if c and email_date and email_date.date() >= c else "pending_review"
def _save(pdf: bytes, base: str, email_date: Optional[datetime], sender: str) -> tuple:
sha = hashlib.sha256(pdf).hexdigest()
d = email_date or datetime.utcnow()
domain = re.sub(r"[^\w.-]", "", _domain(sender)) or "unbekannt"
folder = os.path.join(base, "picker", f"{d:%Y}", f"{d:%m}")
os.makedirs(folder, exist_ok=True)
path = os.path.join(folder, f"{d:%Y-%m-%d}_{domain}_{sha[:8]}.pdf")
with open(path, "wb") as f:
f.write(pdf)
return path, sha
def process_message(db: Session, msg, folder: str, base: str) -> Optional[str]:
"""Eine Mail verarbeiten -> neuer Outbound-Status oder None (übersprungen)."""
subject = decode_hdr(msg.get("Subject", ""))
if not any(kw in subject.lower() for kw in INVOICE_KEYWORDS):
return None
sender = decode_hdr(msg.get("From", ""))
email_date = parse_email_date(msg.get("Date", ""))
mid = (msg.get("Message-ID") or "").strip() or hashlib.sha256(f"{folder}{email_date}{subject}".encode()).hexdigest()
row = db.query(Outbound).filter(Outbound.message_id == mid).first()
if row and row.status != "manual": # "manual" wird bei jedem Lauf neu versucht (neue Adapter greifen rückwirkend)
return None
d = _domain(sender)
if aws.ENABLED and any(d == x or d.endswith("." + x) for x in aws.SENDER_DOMAINS): # AWS kommt über die API (sync_aws)
if row:
row.status, row.error = "via_api", None
db.commit()
return None
mail_class, url = classify_mail(msg)
if mail_class not in ("link", "portal"): # Anhänge leitet Markus bereits selbst an Accountable weiter
return None
row = row or Outbound(message_id=mid)
row.email_folder, row.email_date, row.email_from = folder, email_date, sender
row.email_subject, row.mail_class, row.link_url = subject, mail_class, url
pdf = None
if mail_class == "link":
try:
pdf = fetch_pdf(url)
row.error = None if pdf else "Link liefert kein PDF (Login nötig?)"
except Exception as e:
row.error = f"Download-Fehler: {e}"[:500]
else:
row.error = "Portal-Abruf noch nicht automatisiert" # Phase 3: Playwright-Adapter
if pdf:
_store(db, row, pdf, base)
else:
row.status = "manual"
db.add(row)
db.commit()
return row.status
def _store(db: Session, row: Outbound, pdf: bytes, base: str):
row.file_path, row.sha256 = _save(pdf, base, row.email_date, row.email_from or "")
dup = db.query(Outbound.id).filter(Outbound.sha256 == row.sha256, Outbound.id != row.id).first()
row.status = "duplicate" if dup else _status_for(row.email_date)
def sync_aws(db: Session, since: date, base: str, counts: dict):
"""AWS-Rechnungen über die Invoicing-API holen (message_id = aws-invoice:<id>)."""
known = {r[0].split(":", 1)[1] for r in db.query(Outbound.message_id)
.filter(Outbound.message_id.like("aws-invoice:%")).all()}
for inv in aws.fetch_invoices(since, known):
issued = inv["issued"].replace(tzinfo=None) if inv["issued"] else None
row = Outbound(message_id=f"aws-invoice:{inv['invoice_id']}", email_folder="api", email_date=issued,
email_from="AWS <invoicing@aws.com>", mail_class="api",
email_subject=f"{inv['entity']} Rechnung {inv['invoice_id']} ({inv['period']})")
_store(db, row, inv["pdf"], base)
db.add(row)
db.commit()
counts[row.status] = counts.get(row.status, 0) + 1
def _domain(sender: str) -> str:
m = re.search(r"@([\w.-]+)", sender)
return m[1].lower() if m else ""
GMAIL_UNTIL = date(2026, 6, 1) # seit Mai 2026 läuft info@ nur noch über Strato
def _mailboxes(db: Session, since: date):
"""Strato (Settings/Env) + Gmail nur für Rückblicke vor Juni 2026 (info@ lief bis Ende Mai über Google)."""
cfg = get_imap_config(db)
yield cfg, FOLDERS
if os.getenv("GMAIL_IMAP_PASSWORD") and since < GMAIL_UNTIL:
# ponytail: nur "Alle Nachrichten" (\All) statt jedes Label einzeln; Dedupe über Message-ID
yield {**cfg, "host": "imap.gmail.com", "port": 993, "user": os.getenv("GMAIL_IMAP_USER", "info@c-stone-dev.com"),
"password": os.environ["GMAIL_IMAP_PASSWORD"]}, None
def _all_mail(m) -> tuple:
for line in m.list()[1]:
s = line.decode(errors="ignore")
if "\\All" in s:
return (s.rsplit(' "/" ', 1)[-1].strip().strip('"'),)
return ("INBOX",)
def run(db: Session, since: date, state: Optional[dict] = None) -> dict:
state = state if state is not None else {}
state.setdefault("errors", [])
counts = state.setdefault("counts", {})
for cfg, folders in _mailboxes(db, since):
try:
_scan(db, cfg, folders, since, state, counts)
except Exception as e:
state["errors"].append(f"{cfg['host']}: {e}"[:300])
if aws.ENABLED:
try:
sync_aws(db, since, get_imap_config(db)["storage_path"], counts)
except Exception as e:
db.rollback()
state["errors"].append(f"AWS-API: {e}"[:300])
return state
def _scan(db: Session, cfg: dict, folders, since: date, state: dict, counts: dict):
m = connect_imap(cfg["host"], cfg["port"], cfg["user"], cfg["password"])
try:
for folder in folders or _all_mail(m):
if m.select(f'"{folder}"', readonly=True)[0] != "OK":
continue
ids = m.search(None, f"SINCE {since:%d-%b-%Y}")[1][0].split()
if not ids:
continue
# erst nur Betreff holen (klein), Volltext nur bei Rechnungs-Stichwort
hdrs = [h for k in range(0, len(ids), 500)
for h in m.fetch(b",".join(ids[k:k + 500]), "(BODY.PEEK[HEADER.FIELDS (SUBJECT)])")[1]]
ids = [h[0].split()[0] for h in hdrs if isinstance(h, tuple)
and any(kw in decode_hdr(email.message_from_bytes(h[1]).get("Subject", "")).lower()
for kw in INVOICE_KEYWORDS)]
for i in ids:
try:
typ, data = m.fetch(i, "(BODY.PEEK[])")
if typ != "OK" or not data or not isinstance(data[0], tuple):
continue
status = process_message(db, email.message_from_bytes(data[0][1]), folder, cfg["storage_path"])
if status:
counts[status] = counts.get(status, 0) + 1
except Exception as e:
db.rollback()
state["errors"].append(f"{folder}/{i.decode()}: {e}"[:300])
state["processed"] = state.get("processed", 0) + 1
finally:
try:
m.logout()
except Exception:
pass
if __name__ == "__main__":
from backend.database import SessionLocal
p = argparse.ArgumentParser()
p.add_argument("--since", help="YYYY-MM-DD (Default: vor 10 Tagen, passt zum Wochenlauf)")
p.add_argument("--send", action="store_true", help="danach status=queued an Accountable senden")
p.add_argument("--report", action="store_true", help="Kurzbericht per Mail an IMAP_USER")
a = p.parse_args()
since = datetime.strptime(a.since, "%Y-%m-%d").date() if a.since else date.today() - timedelta(days=10)
db = SessionLocal()
try:
state = run(db, since)
if a.send:
from backend import mailer
state["versand"] = mailer.send_queued(db)
if a.report:
from backend import mailer
mailer.send_report(db, state, since)
print(state)
finally:
db.close()