c-stone-invoice-check/backend/grabber.py
2026-11-13 23:07:22 +01:00

137 lines
5.7 KiB
Python

"""
Invoice Picker: Rechnungen aus Link-/Portal-Mails ziehen und für den Versand an Accountable vormerken.
Läuft unabhängig vom Alt-Scan (/emails/scan-all); Dedupe über Message-ID, Postfach wird nur gelesen (readonly).
CLI (Cron/Rückblick): python -m backend.grabber --since 2025-01-01
"""
import argparse
import email
import hashlib
import os
import re
from datetime import date, datetime, timedelta
from typing import Optional
from sqlalchemy.orm import Session
from backend.fetchers.direct import fetch_pdf
from backend.models import Outbound
from backend.routers.emails import INVOICE_KEYWORDS, classify_mail, connect_imap, decode_hdr, parse_email_date
from backend.routers.settings import get_imap_config
# ponytail: feste Ordnerliste (Strato, Stand 13.11.2026); Sent/Spam/Drafts bewusst nicht
FOLDERS = ("INBOX", "Archive", "Archive.2026", "Trash")
def _cutoff() -> Optional[date]:
"""Stichtag: Mails davor = Rückblick (Freigabe nötig). Ohne GRAB_CUTOFF geht nichts automatisch raus."""
v = os.getenv("GRAB_CUTOFF")
return datetime.strptime(v, "%Y-%m-%d").date() if v else None
def _status_for(email_date: Optional[datetime]) -> str:
c = _cutoff()
return "queued" if c and email_date and email_date.date() >= c else "pending_review"
def _save(pdf: bytes, base: str, email_date: Optional[datetime], sender: str) -> tuple:
sha = hashlib.sha256(pdf).hexdigest()
d = email_date or datetime.utcnow()
domain = re.sub(r"[^\w.-]", "", (re.search(r"@([\w.-]+)", sender) or [None, "unbekannt"])[1])
folder = os.path.join(base, "picker", f"{d:%Y}", f"{d:%m}")
os.makedirs(folder, exist_ok=True)
path = os.path.join(folder, f"{d:%Y-%m-%d}_{domain}_{sha[:8]}.pdf")
with open(path, "wb") as f:
f.write(pdf)
return path, sha
def process_message(db: Session, msg, folder: str, base: str) -> Optional[str]:
"""Eine Mail verarbeiten -> neuer Outbound-Status oder None (übersprungen)."""
subject = decode_hdr(msg.get("Subject", ""))
if not any(kw in subject.lower() for kw in INVOICE_KEYWORDS):
return None
sender = decode_hdr(msg.get("From", ""))
email_date = parse_email_date(msg.get("Date", ""))
mid = (msg.get("Message-ID") or "").strip() or hashlib.sha256(f"{folder}{email_date}{subject}".encode()).hexdigest()
row = db.query(Outbound).filter(Outbound.message_id == mid).first()
if row and row.status != "manual": # "manual" wird bei jedem Lauf neu versucht (neue Adapter greifen rückwirkend)
return None
mail_class, url = classify_mail(msg)
if mail_class not in ("link", "portal"): # Anhänge leitet Markus bereits selbst an Accountable weiter
return None
row = row or Outbound(message_id=mid)
row.email_folder, row.email_date, row.email_from = folder, email_date, sender
row.email_subject, row.mail_class, row.link_url = subject, mail_class, url
pdf = None
if mail_class == "link":
try:
pdf = fetch_pdf(url)
row.error = None if pdf else "Link liefert kein PDF (Login nötig?)"
except Exception as e:
row.error = f"Download-Fehler: {e}"[:500]
else:
row.error = "Portal-Abruf noch nicht automatisiert" # Phase 3: Playwright-Adapter
if pdf:
row.file_path, row.sha256 = _save(pdf, base, email_date, sender)
dup = db.query(Outbound.id).filter(Outbound.sha256 == row.sha256).first()
row.status = "duplicate" if dup else _status_for(email_date)
else:
row.status = "manual"
db.add(row)
db.commit()
return row.status
def run(db: Session, since: date, state: Optional[dict] = None) -> dict:
state = state if state is not None else {}
state.setdefault("errors", [])
counts = state.setdefault("counts", {})
cfg = get_imap_config(db)
m = connect_imap(cfg["host"], cfg["port"], cfg["user"], cfg["password"])
try:
for folder in FOLDERS:
if m.select(f'"{folder}"', readonly=True)[0] != "OK":
continue
ids = m.search(None, f"SINCE {since:%d-%b-%Y}")[1][0].split()
if not ids:
continue
# erst nur Betreff holen (klein), Volltext nur bei Rechnungs-Stichwort
hdrs = [h for k in range(0, len(ids), 500)
for h in m.fetch(b",".join(ids[k:k + 500]), "(BODY.PEEK[HEADER.FIELDS (SUBJECT)])")[1]]
ids = [h[0].split()[0] for h in hdrs if isinstance(h, tuple)
and any(kw in decode_hdr(email.message_from_bytes(h[1]).get("Subject", "")).lower()
for kw in INVOICE_KEYWORDS)]
for i in ids:
try:
typ, data = m.fetch(i, "(BODY.PEEK[])")
if typ != "OK" or not data or not isinstance(data[0], tuple):
continue
status = process_message(db, email.message_from_bytes(data[0][1]), folder, cfg["storage_path"])
if status:
counts[status] = counts.get(status, 0) + 1
except Exception as e:
db.rollback()
state["errors"].append(f"{folder}/{i.decode()}: {e}"[:300])
state["processed"] = state.get("processed", 0) + 1
finally:
try:
m.logout()
except Exception:
pass
return state
if __name__ == "__main__":
from backend.database import SessionLocal
p = argparse.ArgumentParser()
p.add_argument("--since", help="YYYY-MM-DD (Default: vor 3 Tagen)")
a = p.parse_args()
since = datetime.strptime(a.since, "%Y-%m-%d").date() if a.since else date.today() - timedelta(days=3)
db = SessionLocal()
try:
print(run(db, since))
finally:
db.close()