- Klassifikator: Direktlink nur Allowlist/https-PDF, Login-/Konsolen-Links -> portal - grabber.py: INBOX/Archive/Trash readonly, Dedupe Message-ID + SHA-256 - Stichtag GRAB_CUTOFF: Rückblick = pending_review, neu = queued - mailer.py: SMTP an ACCOUNTABLE_INBOX (ohne Adresse kein Versand) - /outbound: Liste/xlsx, PDF-Vorschau, approve, run Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
135 lines
5.5 KiB
Python
135 lines
5.5 KiB
Python
"""
|
|
Invoice Picker: Rechnungen aus Link-/Portal-Mails ziehen und für den Versand an Accountable vormerken.
|
|
Läuft unabhängig vom Alt-Scan (/emails/scan-all); Dedupe über Message-ID, Postfach wird nur gelesen (readonly).
|
|
|
|
CLI (Cron/Rückblick): python -m backend.grabber --since 2025-01-01
|
|
"""
|
|
|
|
import argparse
|
|
import email
|
|
import hashlib
|
|
import os
|
|
import re
|
|
from datetime import date, datetime, timedelta
|
|
from typing import Optional
|
|
|
|
from sqlalchemy.orm import Session
|
|
|
|
from backend.fetchers.direct import fetch_pdf
|
|
from backend.models import Outbound
|
|
from backend.routers.emails import INVOICE_KEYWORDS, classify_mail, connect_imap, decode_hdr, parse_email_date
|
|
from backend.routers.settings import get_imap_config
|
|
|
|
# ponytail: feste Ordnerliste (Strato, Stand 13.11.2026); Sent/Spam/Drafts bewusst nicht
|
|
FOLDERS = ("INBOX", "Archive", "Archive.2026", "Trash")
|
|
|
|
|
|
def _cutoff() -> Optional[date]:
|
|
"""Stichtag: Mails davor = Rückblick (Freigabe nötig). Ohne GRAB_CUTOFF geht nichts automatisch raus."""
|
|
v = os.getenv("GRAB_CUTOFF")
|
|
return datetime.strptime(v, "%Y-%m-%d").date() if v else None
|
|
|
|
|
|
def _status_for(email_date: Optional[datetime]) -> str:
|
|
c = _cutoff()
|
|
return "queued" if c and email_date and email_date.date() >= c else "pending_review"
|
|
|
|
|
|
def _save(pdf: bytes, base: str, email_date: Optional[datetime], sender: str) -> tuple:
|
|
sha = hashlib.sha256(pdf).hexdigest()
|
|
d = email_date or datetime.utcnow()
|
|
domain = re.sub(r"[^\w.-]", "", (re.search(r"@([\w.-]+)", sender) or [None, "unbekannt"])[1])
|
|
folder = os.path.join(base, "picker", f"{d:%Y}", f"{d:%m}")
|
|
os.makedirs(folder, exist_ok=True)
|
|
path = os.path.join(folder, f"{d:%Y-%m-%d}_{domain}_{sha[:8]}.pdf")
|
|
with open(path, "wb") as f:
|
|
f.write(pdf)
|
|
return path, sha
|
|
|
|
|
|
def process_message(db: Session, msg, folder: str, base: str) -> Optional[str]:
|
|
"""Eine Mail verarbeiten -> neuer Outbound-Status oder None (übersprungen)."""
|
|
subject = decode_hdr(msg.get("Subject", ""))
|
|
if not any(kw in subject.lower() for kw in INVOICE_KEYWORDS):
|
|
return None
|
|
sender = decode_hdr(msg.get("From", ""))
|
|
email_date = parse_email_date(msg.get("Date", ""))
|
|
mid = (msg.get("Message-ID") or "").strip() or hashlib.sha256(f"{folder}{email_date}{subject}".encode()).hexdigest()
|
|
if db.query(Outbound.id).filter(Outbound.message_id == mid).first():
|
|
return None
|
|
mail_class, url = classify_mail(msg)
|
|
if mail_class not in ("link", "portal"): # Anhänge leitet Markus bereits selbst an Accountable weiter
|
|
return None
|
|
row = Outbound(message_id=mid, email_folder=folder, email_date=email_date, email_from=sender,
|
|
email_subject=subject, mail_class=mail_class, link_url=url)
|
|
pdf = None
|
|
if mail_class == "link":
|
|
try:
|
|
pdf = fetch_pdf(url)
|
|
row.error = None if pdf else "Link liefert kein PDF (Login nötig?)"
|
|
except Exception as e:
|
|
row.error = f"Download-Fehler: {e}"[:500]
|
|
else:
|
|
row.error = "Portal-Abruf noch nicht automatisiert" # Phase 3: Playwright-Adapter
|
|
if pdf:
|
|
row.file_path, row.sha256 = _save(pdf, base, email_date, sender)
|
|
dup = db.query(Outbound.id).filter(Outbound.sha256 == row.sha256).first()
|
|
row.status = "duplicate" if dup else _status_for(email_date)
|
|
else:
|
|
row.status = "manual"
|
|
db.add(row)
|
|
db.commit()
|
|
return row.status
|
|
|
|
|
|
def run(db: Session, since: date, state: Optional[dict] = None) -> dict:
|
|
state = state if state is not None else {}
|
|
state.setdefault("errors", [])
|
|
counts = state.setdefault("counts", {})
|
|
cfg = get_imap_config(db)
|
|
m = connect_imap(cfg["host"], cfg["port"], cfg["user"], cfg["password"])
|
|
try:
|
|
for folder in FOLDERS:
|
|
if m.select(f'"{folder}"', readonly=True)[0] != "OK":
|
|
continue
|
|
ids = m.search(None, f"SINCE {since:%d-%b-%Y}")[1][0].split()
|
|
if not ids:
|
|
continue
|
|
# erst nur Betreff holen (klein), Volltext nur bei Rechnungs-Stichwort
|
|
hdrs = [h for k in range(0, len(ids), 500)
|
|
for h in m.fetch(b",".join(ids[k:k + 500]), "(BODY.PEEK[HEADER.FIELDS (SUBJECT)])")[1]]
|
|
ids = [h[0].split()[0] for h in hdrs if isinstance(h, tuple)
|
|
and any(kw in decode_hdr(email.message_from_bytes(h[1]).get("Subject", "")).lower()
|
|
for kw in INVOICE_KEYWORDS)]
|
|
for i in ids:
|
|
try:
|
|
typ, data = m.fetch(i, "(BODY.PEEK[])")
|
|
if typ != "OK" or not data or not isinstance(data[0], tuple):
|
|
continue
|
|
status = process_message(db, email.message_from_bytes(data[0][1]), folder, cfg["storage_path"])
|
|
if status:
|
|
counts[status] = counts.get(status, 0) + 1
|
|
except Exception as e:
|
|
db.rollback()
|
|
state["errors"].append(f"{folder}/{i.decode()}: {e}"[:300])
|
|
state["processed"] = state.get("processed", 0) + 1
|
|
finally:
|
|
try:
|
|
m.logout()
|
|
except Exception:
|
|
pass
|
|
return state
|
|
|
|
|
|
if __name__ == "__main__":
|
|
from backend.database import SessionLocal
|
|
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--since", help="YYYY-MM-DD (Default: vor 3 Tagen)")
|
|
a = p.parse_args()
|
|
since = datetime.strptime(a.since, "%Y-%m-%d").date() if a.since else date.today() - timedelta(days=3)
|
|
db = SessionLocal()
|
|
try:
|
|
print(run(db, since))
|
|
finally:
|
|
db.close()
|