feat(picker): Grabber (Direktlink-Download), Outbound-Queue, Accountable-Versand mit Freigabe

- Klassifikator: Direktlink nur Allowlist/https-PDF, Login-/Konsolen-Links -> portal
- grabber.py: INBOX/Archive/Trash readonly, Dedupe Message-ID + SHA-256
- Stichtag GRAB_CUTOFF: Rückblick = pending_review, neu = queued
- mailer.py: SMTP an ACCOUNTABLE_INBOX (ohne Adresse kein Versand)
- /outbound: Liste/xlsx, PDF-Vorschau, approve, run

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Markus Kruse 2026-11-13 21:55:17 +01:00
parent 54563f355e
commit db8dc1dae3
10 changed files with 384 additions and 9 deletions

View file

View file

@ -0,0 +1,18 @@
"""Stufe 2: Rechnungs-PDF per Direktlink laden (Stripe, Paddle, ThriveCart, *.pdf)."""
import urllib.request
from typing import Optional
MAX_BYTES = 20 * 1024 * 1024
def fetch_pdf(url: str, timeout: int = 30) -> Optional[bytes]:
"""-> PDF-Bytes oder None (kein PDF, z.B. HTML-Login-Seite). URL ist bereits per DIRECT_LINK geprüft (nur https)."""
if not url.lower().startswith("https://"):
return None
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 (C-Stone Invoice Picker)"})
with urllib.request.urlopen(req, timeout=timeout) as r:
data = r.read(MAX_BYTES + 1)
if len(data) > MAX_BYTES or not data.startswith(b"%PDF-"):
return None
return data

135
backend/grabber.py Normal file
View file

@ -0,0 +1,135 @@
"""
Invoice Picker: Rechnungen aus Link-/Portal-Mails ziehen und für den Versand an Accountable vormerken.
Läuft unabhängig vom Alt-Scan (/emails/scan-all); Dedupe über Message-ID, Postfach wird nur gelesen (readonly).
CLI (Cron/Rückblick): python -m backend.grabber --since 2025-01-01
"""
import argparse
import email
import hashlib
import os
import re
from datetime import date, datetime, timedelta
from typing import Optional
from sqlalchemy.orm import Session
from backend.fetchers.direct import fetch_pdf
from backend.models import Outbound
from backend.routers.emails import INVOICE_KEYWORDS, classify_mail, connect_imap, decode_hdr, parse_email_date
from backend.routers.settings import get_imap_config
# ponytail: feste Ordnerliste (Strato, Stand 13.11.2026); Sent/Spam/Drafts bewusst nicht
FOLDERS = ("INBOX", "Archive", "Archive.2026", "Trash")
def _cutoff() -> Optional[date]:
"""Stichtag: Mails davor = Rückblick (Freigabe nötig). Ohne GRAB_CUTOFF geht nichts automatisch raus."""
v = os.getenv("GRAB_CUTOFF")
return datetime.strptime(v, "%Y-%m-%d").date() if v else None
def _status_for(email_date: Optional[datetime]) -> str:
c = _cutoff()
return "queued" if c and email_date and email_date.date() >= c else "pending_review"
def _save(pdf: bytes, base: str, email_date: Optional[datetime], sender: str) -> tuple:
sha = hashlib.sha256(pdf).hexdigest()
d = email_date or datetime.utcnow()
domain = re.sub(r"[^\w.-]", "", (re.search(r"@([\w.-]+)", sender) or [None, "unbekannt"])[1])
folder = os.path.join(base, "picker", f"{d:%Y}", f"{d:%m}")
os.makedirs(folder, exist_ok=True)
path = os.path.join(folder, f"{d:%Y-%m-%d}_{domain}_{sha[:8]}.pdf")
with open(path, "wb") as f:
f.write(pdf)
return path, sha
def process_message(db: Session, msg, folder: str, base: str) -> Optional[str]:
"""Eine Mail verarbeiten -> neuer Outbound-Status oder None (übersprungen)."""
subject = decode_hdr(msg.get("Subject", ""))
if not any(kw in subject.lower() for kw in INVOICE_KEYWORDS):
return None
sender = decode_hdr(msg.get("From", ""))
email_date = parse_email_date(msg.get("Date", ""))
mid = (msg.get("Message-ID") or "").strip() or hashlib.sha256(f"{folder}{email_date}{subject}".encode()).hexdigest()
if db.query(Outbound.id).filter(Outbound.message_id == mid).first():
return None
mail_class, url = classify_mail(msg)
if mail_class not in ("link", "portal"): # Anhänge leitet Markus bereits selbst an Accountable weiter
return None
row = Outbound(message_id=mid, email_folder=folder, email_date=email_date, email_from=sender,
email_subject=subject, mail_class=mail_class, link_url=url)
pdf = None
if mail_class == "link":
try:
pdf = fetch_pdf(url)
row.error = None if pdf else "Link liefert kein PDF (Login nötig?)"
except Exception as e:
row.error = f"Download-Fehler: {e}"[:500]
else:
row.error = "Portal-Abruf noch nicht automatisiert" # Phase 3: Playwright-Adapter
if pdf:
row.file_path, row.sha256 = _save(pdf, base, email_date, sender)
dup = db.query(Outbound.id).filter(Outbound.sha256 == row.sha256).first()
row.status = "duplicate" if dup else _status_for(email_date)
else:
row.status = "manual"
db.add(row)
db.commit()
return row.status
def run(db: Session, since: date, state: Optional[dict] = None) -> dict:
state = state if state is not None else {}
state.setdefault("errors", [])
counts = state.setdefault("counts", {})
cfg = get_imap_config(db)
m = connect_imap(cfg["host"], cfg["port"], cfg["user"], cfg["password"])
try:
for folder in FOLDERS:
if m.select(f'"{folder}"', readonly=True)[0] != "OK":
continue
ids = m.search(None, f"SINCE {since:%d-%b-%Y}")[1][0].split()
if not ids:
continue
# erst nur Betreff holen (klein), Volltext nur bei Rechnungs-Stichwort
hdrs = [h for k in range(0, len(ids), 500)
for h in m.fetch(b",".join(ids[k:k + 500]), "(BODY.PEEK[HEADER.FIELDS (SUBJECT)])")[1]]
ids = [h[0].split()[0] for h in hdrs if isinstance(h, tuple)
and any(kw in decode_hdr(email.message_from_bytes(h[1]).get("Subject", "")).lower()
for kw in INVOICE_KEYWORDS)]
for i in ids:
try:
typ, data = m.fetch(i, "(BODY.PEEK[])")
if typ != "OK" or not data or not isinstance(data[0], tuple):
continue
status = process_message(db, email.message_from_bytes(data[0][1]), folder, cfg["storage_path"])
if status:
counts[status] = counts.get(status, 0) + 1
except Exception as e:
db.rollback()
state["errors"].append(f"{folder}/{i.decode()}: {e}"[:300])
state["processed"] = state.get("processed", 0) + 1
finally:
try:
m.logout()
except Exception:
pass
return state
if __name__ == "__main__":
from backend.database import SessionLocal
p = argparse.ArgumentParser()
p.add_argument("--since", help="YYYY-MM-DD (Default: vor 3 Tagen)")
a = p.parse_args()
since = datetime.strptime(a.since, "%Y-%m-%d").date() if a.since else date.today() - timedelta(days=3)
db = SessionLocal()
try:
print(run(db, since))
finally:
db.close()

51
backend/mailer.py Normal file
View file

@ -0,0 +1,51 @@
"""Versand gezogener Rechnungen an Accountable (Beleg-Eingangsadresse) per SMTP."""
import os
import smtplib
from datetime import datetime
from email.message import EmailMessage
from sqlalchemy.orm import Session
from backend.models import Outbound
def _cfg() -> dict:
return {
"host": os.getenv("SMTP_HOST", "smtp.strato.de"),
"port": int(os.getenv("SMTP_PORT", "465")),
"user": os.getenv("SMTP_USER") or os.getenv("IMAP_USER", ""),
"password": os.getenv("SMTP_PASSWORD") or os.getenv("IMAP_PASSWORD", ""),
"to": os.getenv("ACCOUNTABLE_INBOX", ""),
}
def send_queued(db: Session) -> dict:
"""Alle Outbound-Zeilen mit status=queued senden (eine Mail pro Rechnung). Ohne ACCOUNTABLE_INBOX: nichts."""
c = _cfg()
if not c["to"]:
return {"sent": 0, "failed": 0, "skipped": "ACCOUNTABLE_INBOX fehlt"}
rows = db.query(Outbound).filter(Outbound.status == "queued").all()
sent = failed = 0
if not rows:
return {"sent": 0, "failed": 0}
with smtplib.SMTP_SSL(c["host"], c["port"], timeout=60) as s:
s.login(c["user"], c["password"])
for r in rows:
try:
msg = EmailMessage()
msg["From"], msg["To"] = c["user"], c["to"]
msg["Subject"] = f"Rechnung: {r.email_subject or os.path.basename(r.file_path)}"[:200]
msg.set_content(f"Automatisch abgerufen (Invoice Picker).\nOriginal-Mail: {r.email_date:%d.%m.%Y} von {r.email_from}"
if r.email_date else "Automatisch abgerufen (Invoice Picker).")
with open(r.file_path, "rb") as f:
msg.add_attachment(f.read(), maintype="application", subtype="pdf",
filename=os.path.basename(r.file_path))
s.send_message(msg)
r.status, r.sent_at, r.error = "sent", datetime.utcnow(), None
sent += 1
except Exception as e:
r.status, r.error = "failed", f"Versand: {e}"[:500]
failed += 1
db.commit()
return {"sent": sent, "failed": failed}

View file

@ -12,10 +12,10 @@ from sqlalchemy import text
from backend.database import get_db, engine, Base from backend.database import get_db, engine, Base
# Models importieren für Alembic # Models importieren für Alembic
from backend.models import Invoice, InvoiceDetail, Base as ModelsBase from backend.models import Invoice, InvoiceDetail, Outbound, Base as ModelsBase
# Routers # Routers
from backend.routers import emails, invoices, auth from backend.routers import emails, invoices, auth, outbound
from backend.routers import settings as settings_router from backend.routers import settings as settings_router
# App erstellen # App erstellen
@ -76,6 +76,7 @@ app.include_router(auth.router, prefix="/auth", tags=["authentication"])
app.include_router(emails.router, prefix="/emails", tags=["emails"]) app.include_router(emails.router, prefix="/emails", tags=["emails"])
app.include_router(invoices.router, prefix="/invoices", tags=["invoices"]) app.include_router(invoices.router, prefix="/invoices", tags=["invoices"])
app.include_router(settings_router.router, prefix="/settings", tags=["settings"]) app.include_router(settings_router.router, prefix="/settings", tags=["settings"])
app.include_router(outbound.router, prefix="/outbound", tags=["invoice-picker"])
if __name__ == "__main__": if __name__ == "__main__":

View file

@ -105,6 +105,27 @@ class InvoiceDetail(Base):
return f"<InvoiceDetail {self.invoice_id}>" return f"<InvoiceDetail {self.invoice_id}>"
class Outbound(Base):
"""Rechnung aus Link-/Portal-Mail (Invoice Picker) -> Versand an Accountable.
status: manual (Abruf fehlgeschlagen/Portal) | duplicate | pending_review (Rückblick) | queued | sent | failed"""
__tablename__ = "outbound"
id = Column(Integer, primary_key=True, index=True)
message_id = Column(String, unique=True, index=True) # Message-ID-Header: stabil über Ordner/Verschieben
email_folder = Column(String, nullable=True)
email_date = Column(DateTime, nullable=True)
email_from = Column(String, nullable=True)
email_subject = Column(String, nullable=True)
mail_class = Column(String, nullable=True) # "link" | "portal"
link_url = Column(Text, nullable=True)
file_path = Column(String, nullable=True)
sha256 = Column(String, index=True, nullable=True)
status = Column(String, index=True)
error = Column(Text, nullable=True)
sent_at = Column(DateTime, nullable=True)
created_at = Column(DateTime, default=datetime.utcnow)
class Setting(Base): class Setting(Base):
__tablename__ = "settings" __tablename__ = "settings"

View file

@ -5,6 +5,7 @@ IMAP Email-Verarbeitung und Filterung
import imaplib import imaplib
import email import email
import html
import os import os
import re import re
import tempfile import tempfile
@ -58,16 +59,23 @@ INVOICE_STORAGE_PATH = os.getenv("INVOICE_STORAGE_PATH", "/tmp/invoice-storage")
LINK_RE = re.compile(r"https?://[^\s\"'<>)\]]+", re.I) LINK_RE = re.compile(r"https?://[^\s\"'<>)\]]+", re.I)
LINK_HINT = re.compile(r"invoice|rechnung|receipt|beleg|billing|download|\.pdf|stripe\.com|paddle\.com", re.I) # "link" = PDF ohne Login abrufbar. Gleichzeitig Allowlist für den Server-Download (nur https, kein beliebiger Host).
DIRECT_LINK = re.compile(
r"^https://((pay|invoice)\.stripe\.com|invoices\.thrivecart\.com|store\.payproglobal\.com|[\w-]+\.paddle\.com)/"
r"|^https://\S+\.pdf([?#]|$)",
re.I,
)
# Rechnungs-Link hinter Login (Konsole, Kundenportal) -> "portal"
LINK_HINT = re.compile(r"invoice|rechnung|receipt|beleg|billing|download|\.pdf", re.I)
PORTAL_HINT = re.compile( PORTAL_HINT = re.compile(
r"in ihrem (kunden)?(konto|portal)|im (kunden)?portal|your account|sign in|log ?in|anmelden|einloggen|" r"in ihrem (kunden)?(konto|portal)|im (kunden)?portal|your account|sign in|log ?in|anmelden|einloggen|"
r"view (your )?(invoice|receipt|bill)|rechnung (ist )?(verfügbar|bereit)|invoice is available", r"view (your )?(invoice|receipt|bill)|rechnung (ist )?(verfügbar|bereit)|invoice is available|zahlungsbeleg",
re.I, re.I,
) )
def classify_mail(msg) -> tuple: def classify_mail(msg) -> tuple:
"""-> (mail_class, link_url). Reihenfolge: Anhang > Link > Portal-Hinweis > ignore.""" """-> (mail_class, link_url). Reihenfolge: Anhang > Direktlink > Portal (Login-Link/Hinweis) > ignore."""
text = "" text = ""
for part in msg.walk(): for part in msg.walk():
fn = part.get_filename() fn = part.get_filename()
@ -77,10 +85,11 @@ def classify_mail(msg) -> tuple:
payload = part.get_payload(decode=True) or b"" payload = part.get_payload(decode=True) or b""
text += payload.decode(part.get_content_charset() or "utf-8", "replace") + "\n" text += payload.decode(part.get_content_charset() or "utf-8", "replace") + "\n"
# ponytail: Regex statt HTML-Parser; Footer-Links ("invoice" im Pfad) können False-Positives liefern # ponytail: Regex statt HTML-Parser; Footer-Links ("invoice" im Pfad) können False-Positives liefern
url = next((u for u in LINK_RE.findall(text) if LINK_HINT.search(u)), None) urls = [html.unescape(u) for u in LINK_RE.findall(text)]
url = next((u for u in urls if DIRECT_LINK.search(u)), None)
if url: if url:
return "link", url return "link", url
if PORTAL_HINT.search(re.sub(r"<[^>]+>", " ", text)): if any(LINK_HINT.search(u) for u in urls) or PORTAL_HINT.search(re.sub(r"<[^>]+>", " ", text)):
return "portal", None return "portal", None
return "ignore", None return "ignore", None
@ -109,7 +118,7 @@ def search_invoices(mail: imaplib.IMAP4_SSL, folder: str = "INBOX") -> List[str]
msg = email.message_from_bytes(msg_data[0][1]) msg = email.message_from_bytes(msg_data[0][1])
subject = msg.get("Subject", "") subject = msg.get("Subject", "")
from_addr = msg.get("From", "") from_addr = decode_hdr(msg.get("From", ""))
# Subject decodieren # Subject decodieren
subject = decode_hdr(subject) subject = decode_hdr(subject)
@ -355,7 +364,7 @@ def _scan_imap(mail_conn, db: Session, state: dict, since_date: Optional[str] =
db.commit() db.commit()
continue continue
subject = msg.get("Subject", "") subject = msg.get("Subject", "")
from_addr = msg.get("From", "") from_addr = decode_hdr(msg.get("From", ""))
date_str = msg.get("Date", "") date_str = msg.get("Date", "")
email_date = parse_email_date(date_str) email_date = parse_email_date(date_str)
subject = decode_hdr(subject) subject = decode_hdr(subject)

View file

@ -0,0 +1,97 @@
"""Invoice Picker: gezogene Rechnungen ansehen, Rückblick freigeben, an Accountable senden."""
import io
import os
import uuid
from datetime import datetime
from typing import List, Optional
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
from fastapi.responses import FileResponse, StreamingResponse
from pydantic import BaseModel
from sqlalchemy.orm import Session
from backend import grabber, mailer
from backend.database import SessionLocal, get_db
from backend.models import Outbound
router = APIRouter()
_runs: dict = {} # ponytail: in-memory Laufstatus, ein Prozess
def _bg_run(run_id: str, since):
db = SessionLocal()
try:
grabber.run(db, since, _runs[run_id])
_runs[run_id]["send"] = mailer.send_queued(db)
except Exception as e:
_runs[run_id]["errors"].append(str(e))
finally:
_runs[run_id]["status"] = "done"
db.close()
@router.post("/run")
async def start_run(background_tasks: BackgroundTasks, since: str):
"""Postfach ab since (YYYY-MM-DD) prüfen, Rechnungen ziehen, freigegebene/neue senden."""
if any(r["status"] == "running" for r in _runs.values()):
raise HTTPException(409, "Lauf aktiv")
run_id = str(uuid.uuid4())
_runs[run_id] = {"status": "running", "errors": [], "counts": {}, "start": datetime.utcnow().isoformat()}
background_tasks.add_task(_bg_run, run_id, datetime.strptime(since, "%Y-%m-%d").date())
return {"run_id": run_id}
@router.get("/run/{run_id}")
async def run_status(run_id: str):
if run_id not in _runs:
raise HTTPException(404, "Lauf nicht gefunden")
return _runs[run_id]
@router.get("")
async def list_outbound(status: Optional[str] = None, format: str = "json", db: Session = Depends(get_db)):
q = db.query(Outbound)
if status:
q = q.filter(Outbound.status == status)
rows = q.order_by(Outbound.email_date.desc()).all()
data = [{"id": r.id, "status": r.status, "datum": f"{r.email_date:%Y-%m-%d}" if r.email_date else "",
"absender": r.email_from, "betreff": r.email_subject, "klasse": r.mail_class,
"link": r.link_url or "", "datei": os.path.basename(r.file_path or ""), "fehler": r.error or ""}
for r in rows]
if format != "xlsx":
return data
from openpyxl import Workbook
wb = Workbook()
ws = wb.active
ws.append(list(data[0].keys()) if data else ["leer"])
for d in data:
ws.append(list(d.values()))
buf = io.BytesIO()
wb.save(buf)
buf.seek(0)
return StreamingResponse(buf, media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
headers={"Content-Disposition": "attachment; filename=invoice-picker.xlsx"})
@router.get("/{oid}/pdf")
async def get_pdf(oid: int, db: Session = Depends(get_db)):
r = db.get(Outbound, oid)
if not r or not r.file_path or not os.path.exists(r.file_path):
raise HTTPException(404, "kein PDF")
return FileResponse(r.file_path, media_type="application/pdf")
class Approve(BaseModel):
ids: Optional[List[int]] = None # None = alle wartenden
@router.post("/approve")
async def approve(body: Approve, db: Session = Depends(get_db)):
"""Rückblick freigeben (pending_review -> queued) und sofort senden."""
q = db.query(Outbound).filter(Outbound.status == "pending_review")
if body.ids is not None:
q = q.filter(Outbound.id.in_(body.ids))
n = q.update({Outbound.status: "queued"}, synchronize_session=False)
db.commit()
return {"approved": n, **mailer.send_queued(db)}

View file

@ -14,3 +14,12 @@ def test_classify():
portal = "Content-Type: text/plain\n\nIhre Rechnung ist verfügbar. Bitte in Ihrem Konto herunterladen." portal = "Content-Type: text/plain\n\nIhre Rechnung ist verfügbar. Bitte in Ihrem Konto herunterladen."
assert classify_mail(_m(portal)) == ("portal", None) assert classify_mail(_m(portal)) == ("portal", None)
assert classify_mail(_m("Content-Type: text/plain\n\nHallo"))[0] == "ignore" assert classify_mail(_m("Content-Type: text/plain\n\nHallo"))[0] == "ignore"
# Login-/Konsolen-Links sind kein Direktdownload -> portal
aws = "Content-Type: text/plain\n\nYour bill: https://console.aws.amazon.com/billing/home#/bills"
assert classify_mail(_m(aws)) == ("portal", None)
assert classify_mail(_m("Content-Type: text/plain\n\nIhr Zahlungsbeleg liegt bereit."))[0] == "portal"
# http (kein TLS) wird nie direkt geladen
assert classify_mail(_m("Content-Type: text/plain\n\nRechnung http://x.example/r.pdf"))[0] == "portal"
# HTML-Entities im Link werden aufgelöst
h = 'Content-Type: text/html\n\n<a href="https://x.example/r.pdf?a=1&amp;b=2">Rechnung</a>'
assert classify_mail(_m(h)) == ("link", "https://x.example/r.pdf?a=1&b=2")

View file

@ -0,0 +1,34 @@
import email
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from backend import grabber
from backend.models import Base, Outbound
LINK = ("Message-ID: <{mid}>\nSubject: Ihre Rechnung\nFrom: Shop <billing@shop.example>\n"
"Date: {date}\nContent-Type: text/plain\n\nDownload: https://pay.stripe.com/invoice/x/pdf")
PORTAL = ("Message-ID: <p1>\nSubject: Rechnung verfügbar\nFrom: AWS <no@aws.example>\n"
"Date: Mon, 3 Nov 2025 10:00:00 +0000\nContent-Type: text/plain\n\nSee https://console.aws.amazon.com/billing")
def test_process_message(tmp_path, monkeypatch):
db = sessionmaker(bind=create_engine("sqlite://"))()
Base.metadata.create_all(db.get_bind())
monkeypatch.setattr(grabber, "fetch_pdf", lambda url: b"%PDF-1.4 same")
monkeypatch.setenv("GRAB_CUTOFF", "2026-11-13")
run = lambda raw: grabber.process_message(db, email.message_from_string(raw), "INBOX", str(tmp_path))
assert run(LINK.format(mid="a", date="Mon, 3 Nov 2025 10:00:00 +0000")) == "pending_review" # Rückblick
assert run(LINK.format(mid="a", date="Mon, 3 Nov 2025 10:00:00 +0000")) is None # gleiche Message-ID
assert run(LINK.format(mid="b", date="Mon, 3 Nov 2025 10:00:00 +0000")) == "duplicate" # gleiche PDF
monkeypatch.setattr(grabber, "fetch_pdf", lambda url: b"%PDF-1.4 neu")
assert run(LINK.format(mid="c", date="Mon, 16 Nov 2026 10:00:00 +0000")) == "queued" # nach Stichtag
assert run(PORTAL) == "manual"
assert run("Subject: Newsletter\n\nhallo") is None
assert db.query(Outbound).count() == 4
assert (tmp_path / "picker" / "2025" / "11").is_dir()
monkeypatch.delenv("GRAB_CUTOFF") # ohne Stichtag nie automatisch senden
monkeypatch.setattr(grabber, "fetch_pdf", lambda url: b"%PDF-1.4 dritte")
assert run(LINK.format(mid="d", date="Mon, 16 Nov 2026 10:00:00 +0000")) == "pending_review"