diff --git a/backend/fetchers/__init__.py b/backend/fetchers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/fetchers/direct.py b/backend/fetchers/direct.py new file mode 100644 index 0000000..76a398d --- /dev/null +++ b/backend/fetchers/direct.py @@ -0,0 +1,18 @@ +"""Stufe 2: Rechnungs-PDF per Direktlink laden (Stripe, Paddle, ThriveCart, *.pdf).""" + +import urllib.request +from typing import Optional + +MAX_BYTES = 20 * 1024 * 1024 + + +def fetch_pdf(url: str, timeout: int = 30) -> Optional[bytes]: + """-> PDF-Bytes oder None (kein PDF, z.B. HTML-Login-Seite). URL ist bereits per DIRECT_LINK geprüft (nur https).""" + if not url.lower().startswith("https://"): + return None + req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 (C-Stone Invoice Picker)"}) + with urllib.request.urlopen(req, timeout=timeout) as r: + data = r.read(MAX_BYTES + 1) + if len(data) > MAX_BYTES or not data.startswith(b"%PDF-"): + return None + return data diff --git a/backend/grabber.py b/backend/grabber.py new file mode 100644 index 0000000..a10077e --- /dev/null +++ b/backend/grabber.py @@ -0,0 +1,135 @@ +""" +Invoice Picker: Rechnungen aus Link-/Portal-Mails ziehen und für den Versand an Accountable vormerken. +Läuft unabhängig vom Alt-Scan (/emails/scan-all); Dedupe über Message-ID, Postfach wird nur gelesen (readonly). + +CLI (Cron/Rückblick): python -m backend.grabber --since 2025-01-01 +""" + +import argparse +import email +import hashlib +import os +import re +from datetime import date, datetime, timedelta +from typing import Optional + +from sqlalchemy.orm import Session + +from backend.fetchers.direct import fetch_pdf +from backend.models import Outbound +from backend.routers.emails import INVOICE_KEYWORDS, classify_mail, connect_imap, decode_hdr, parse_email_date +from backend.routers.settings import get_imap_config + +# ponytail: feste Ordnerliste (Strato, Stand 13.11.2026); Sent/Spam/Drafts bewusst nicht +FOLDERS = ("INBOX", "Archive", "Archive.2026", "Trash") + + +def _cutoff() -> Optional[date]: + """Stichtag: Mails davor = Rückblick (Freigabe nötig). Ohne GRAB_CUTOFF geht nichts automatisch raus.""" + v = os.getenv("GRAB_CUTOFF") + return datetime.strptime(v, "%Y-%m-%d").date() if v else None + + +def _status_for(email_date: Optional[datetime]) -> str: + c = _cutoff() + return "queued" if c and email_date and email_date.date() >= c else "pending_review" + + +def _save(pdf: bytes, base: str, email_date: Optional[datetime], sender: str) -> tuple: + sha = hashlib.sha256(pdf).hexdigest() + d = email_date or datetime.utcnow() + domain = re.sub(r"[^\w.-]", "", (re.search(r"@([\w.-]+)", sender) or [None, "unbekannt"])[1]) + folder = os.path.join(base, "picker", f"{d:%Y}", f"{d:%m}") + os.makedirs(folder, exist_ok=True) + path = os.path.join(folder, f"{d:%Y-%m-%d}_{domain}_{sha[:8]}.pdf") + with open(path, "wb") as f: + f.write(pdf) + return path, sha + + +def process_message(db: Session, msg, folder: str, base: str) -> Optional[str]: + """Eine Mail verarbeiten -> neuer Outbound-Status oder None (übersprungen).""" + subject = decode_hdr(msg.get("Subject", "")) + if not any(kw in subject.lower() for kw in INVOICE_KEYWORDS): + return None + sender = decode_hdr(msg.get("From", "")) + email_date = parse_email_date(msg.get("Date", "")) + mid = (msg.get("Message-ID") or "").strip() or hashlib.sha256(f"{folder}{email_date}{subject}".encode()).hexdigest() + if db.query(Outbound.id).filter(Outbound.message_id == mid).first(): + return None + mail_class, url = classify_mail(msg) + if mail_class not in ("link", "portal"): # Anhänge leitet Markus bereits selbst an Accountable weiter + return None + row = Outbound(message_id=mid, email_folder=folder, email_date=email_date, email_from=sender, + email_subject=subject, mail_class=mail_class, link_url=url) + pdf = None + if mail_class == "link": + try: + pdf = fetch_pdf(url) + row.error = None if pdf else "Link liefert kein PDF (Login nötig?)" + except Exception as e: + row.error = f"Download-Fehler: {e}"[:500] + else: + row.error = "Portal-Abruf noch nicht automatisiert" # Phase 3: Playwright-Adapter + if pdf: + row.file_path, row.sha256 = _save(pdf, base, email_date, sender) + dup = db.query(Outbound.id).filter(Outbound.sha256 == row.sha256).first() + row.status = "duplicate" if dup else _status_for(email_date) + else: + row.status = "manual" + db.add(row) + db.commit() + return row.status + + +def run(db: Session, since: date, state: Optional[dict] = None) -> dict: + state = state if state is not None else {} + state.setdefault("errors", []) + counts = state.setdefault("counts", {}) + cfg = get_imap_config(db) + m = connect_imap(cfg["host"], cfg["port"], cfg["user"], cfg["password"]) + try: + for folder in FOLDERS: + if m.select(f'"{folder}"', readonly=True)[0] != "OK": + continue + ids = m.search(None, f"SINCE {since:%d-%b-%Y}")[1][0].split() + if not ids: + continue + # erst nur Betreff holen (klein), Volltext nur bei Rechnungs-Stichwort + hdrs = [h for k in range(0, len(ids), 500) + for h in m.fetch(b",".join(ids[k:k + 500]), "(BODY.PEEK[HEADER.FIELDS (SUBJECT)])")[1]] + ids = [h[0].split()[0] for h in hdrs if isinstance(h, tuple) + and any(kw in decode_hdr(email.message_from_bytes(h[1]).get("Subject", "")).lower() + for kw in INVOICE_KEYWORDS)] + for i in ids: + try: + typ, data = m.fetch(i, "(BODY.PEEK[])") + if typ != "OK" or not data or not isinstance(data[0], tuple): + continue + status = process_message(db, email.message_from_bytes(data[0][1]), folder, cfg["storage_path"]) + if status: + counts[status] = counts.get(status, 0) + 1 + except Exception as e: + db.rollback() + state["errors"].append(f"{folder}/{i.decode()}: {e}"[:300]) + state["processed"] = state.get("processed", 0) + 1 + finally: + try: + m.logout() + except Exception: + pass + return state + + +if __name__ == "__main__": + from backend.database import SessionLocal + + p = argparse.ArgumentParser() + p.add_argument("--since", help="YYYY-MM-DD (Default: vor 3 Tagen)") + a = p.parse_args() + since = datetime.strptime(a.since, "%Y-%m-%d").date() if a.since else date.today() - timedelta(days=3) + db = SessionLocal() + try: + print(run(db, since)) + finally: + db.close() diff --git a/backend/mailer.py b/backend/mailer.py new file mode 100644 index 0000000..53141b7 --- /dev/null +++ b/backend/mailer.py @@ -0,0 +1,51 @@ +"""Versand gezogener Rechnungen an Accountable (Beleg-Eingangsadresse) per SMTP.""" + +import os +import smtplib +from datetime import datetime +from email.message import EmailMessage + +from sqlalchemy.orm import Session + +from backend.models import Outbound + + +def _cfg() -> dict: + return { + "host": os.getenv("SMTP_HOST", "smtp.strato.de"), + "port": int(os.getenv("SMTP_PORT", "465")), + "user": os.getenv("SMTP_USER") or os.getenv("IMAP_USER", ""), + "password": os.getenv("SMTP_PASSWORD") or os.getenv("IMAP_PASSWORD", ""), + "to": os.getenv("ACCOUNTABLE_INBOX", ""), + } + + +def send_queued(db: Session) -> dict: + """Alle Outbound-Zeilen mit status=queued senden (eine Mail pro Rechnung). Ohne ACCOUNTABLE_INBOX: nichts.""" + c = _cfg() + if not c["to"]: + return {"sent": 0, "failed": 0, "skipped": "ACCOUNTABLE_INBOX fehlt"} + rows = db.query(Outbound).filter(Outbound.status == "queued").all() + sent = failed = 0 + if not rows: + return {"sent": 0, "failed": 0} + with smtplib.SMTP_SSL(c["host"], c["port"], timeout=60) as s: + s.login(c["user"], c["password"]) + for r in rows: + try: + msg = EmailMessage() + msg["From"], msg["To"] = c["user"], c["to"] + msg["Subject"] = f"Rechnung: {r.email_subject or os.path.basename(r.file_path)}"[:200] + msg.set_content(f"Automatisch abgerufen (Invoice Picker).\nOriginal-Mail: {r.email_date:%d.%m.%Y} von {r.email_from}" + if r.email_date else "Automatisch abgerufen (Invoice Picker).") + with open(r.file_path, "rb") as f: + msg.add_attachment(f.read(), maintype="application", subtype="pdf", + filename=os.path.basename(r.file_path)) + s.send_message(msg) + r.status, r.sent_at, r.error = "sent", datetime.utcnow(), None + sent += 1 + except Exception as e: + r.status, r.error = "failed", f"Versand: {e}"[:500] + failed += 1 + db.commit() + return {"sent": sent, "failed": failed} diff --git a/backend/main.py b/backend/main.py index 565bce1..807e905 100644 --- a/backend/main.py +++ b/backend/main.py @@ -12,10 +12,10 @@ from sqlalchemy import text from backend.database import get_db, engine, Base # Models importieren für Alembic -from backend.models import Invoice, InvoiceDetail, Base as ModelsBase +from backend.models import Invoice, InvoiceDetail, Outbound, Base as ModelsBase # Routers -from backend.routers import emails, invoices, auth +from backend.routers import emails, invoices, auth, outbound from backend.routers import settings as settings_router # App erstellen @@ -76,6 +76,7 @@ app.include_router(auth.router, prefix="/auth", tags=["authentication"]) app.include_router(emails.router, prefix="/emails", tags=["emails"]) app.include_router(invoices.router, prefix="/invoices", tags=["invoices"]) app.include_router(settings_router.router, prefix="/settings", tags=["settings"]) +app.include_router(outbound.router, prefix="/outbound", tags=["invoice-picker"]) if __name__ == "__main__": diff --git a/backend/models.py b/backend/models.py index 6dc649c..3b8b118 100644 --- a/backend/models.py +++ b/backend/models.py @@ -105,6 +105,27 @@ class InvoiceDetail(Base): return f"" +class Outbound(Base): + """Rechnung aus Link-/Portal-Mail (Invoice Picker) -> Versand an Accountable. + status: manual (Abruf fehlgeschlagen/Portal) | duplicate | pending_review (Rückblick) | queued | sent | failed""" + __tablename__ = "outbound" + + id = Column(Integer, primary_key=True, index=True) + message_id = Column(String, unique=True, index=True) # Message-ID-Header: stabil über Ordner/Verschieben + email_folder = Column(String, nullable=True) + email_date = Column(DateTime, nullable=True) + email_from = Column(String, nullable=True) + email_subject = Column(String, nullable=True) + mail_class = Column(String, nullable=True) # "link" | "portal" + link_url = Column(Text, nullable=True) + file_path = Column(String, nullable=True) + sha256 = Column(String, index=True, nullable=True) + status = Column(String, index=True) + error = Column(Text, nullable=True) + sent_at = Column(DateTime, nullable=True) + created_at = Column(DateTime, default=datetime.utcnow) + + class Setting(Base): __tablename__ = "settings" diff --git a/backend/routers/emails.py b/backend/routers/emails.py index 930f614..a618711 100644 --- a/backend/routers/emails.py +++ b/backend/routers/emails.py @@ -5,6 +5,7 @@ IMAP Email-Verarbeitung und Filterung import imaplib import email +import html import os import re import tempfile @@ -58,16 +59,23 @@ INVOICE_STORAGE_PATH = os.getenv("INVOICE_STORAGE_PATH", "/tmp/invoice-storage") LINK_RE = re.compile(r"https?://[^\s\"'<>)\]]+", re.I) -LINK_HINT = re.compile(r"invoice|rechnung|receipt|beleg|billing|download|\.pdf|stripe\.com|paddle\.com", re.I) +# "link" = PDF ohne Login abrufbar. Gleichzeitig Allowlist für den Server-Download (nur https, kein beliebiger Host). +DIRECT_LINK = re.compile( + r"^https://((pay|invoice)\.stripe\.com|invoices\.thrivecart\.com|store\.payproglobal\.com|[\w-]+\.paddle\.com)/" + r"|^https://\S+\.pdf([?#]|$)", + re.I, +) +# Rechnungs-Link hinter Login (Konsole, Kundenportal) -> "portal" +LINK_HINT = re.compile(r"invoice|rechnung|receipt|beleg|billing|download|\.pdf", re.I) PORTAL_HINT = re.compile( r"in ihrem (kunden)?(konto|portal)|im (kunden)?portal|your account|sign in|log ?in|anmelden|einloggen|" - r"view (your )?(invoice|receipt|bill)|rechnung (ist )?(verfügbar|bereit)|invoice is available", + r"view (your )?(invoice|receipt|bill)|rechnung (ist )?(verfügbar|bereit)|invoice is available|zahlungsbeleg", re.I, ) def classify_mail(msg) -> tuple: - """-> (mail_class, link_url). Reihenfolge: Anhang > Link > Portal-Hinweis > ignore.""" + """-> (mail_class, link_url). Reihenfolge: Anhang > Direktlink > Portal (Login-Link/Hinweis) > ignore.""" text = "" for part in msg.walk(): fn = part.get_filename() @@ -77,10 +85,11 @@ def classify_mail(msg) -> tuple: payload = part.get_payload(decode=True) or b"" text += payload.decode(part.get_content_charset() or "utf-8", "replace") + "\n" # ponytail: Regex statt HTML-Parser; Footer-Links ("invoice" im Pfad) können False-Positives liefern - url = next((u for u in LINK_RE.findall(text) if LINK_HINT.search(u)), None) + urls = [html.unescape(u) for u in LINK_RE.findall(text)] + url = next((u for u in urls if DIRECT_LINK.search(u)), None) if url: return "link", url - if PORTAL_HINT.search(re.sub(r"<[^>]+>", " ", text)): + if any(LINK_HINT.search(u) for u in urls) or PORTAL_HINT.search(re.sub(r"<[^>]+>", " ", text)): return "portal", None return "ignore", None @@ -109,7 +118,7 @@ def search_invoices(mail: imaplib.IMAP4_SSL, folder: str = "INBOX") -> List[str] msg = email.message_from_bytes(msg_data[0][1]) subject = msg.get("Subject", "") - from_addr = msg.get("From", "") + from_addr = decode_hdr(msg.get("From", "")) # Subject decodieren subject = decode_hdr(subject) @@ -355,7 +364,7 @@ def _scan_imap(mail_conn, db: Session, state: dict, since_date: Optional[str] = db.commit() continue subject = msg.get("Subject", "") - from_addr = msg.get("From", "") + from_addr = decode_hdr(msg.get("From", "")) date_str = msg.get("Date", "") email_date = parse_email_date(date_str) subject = decode_hdr(subject) diff --git a/backend/routers/outbound.py b/backend/routers/outbound.py new file mode 100644 index 0000000..7ab63f3 --- /dev/null +++ b/backend/routers/outbound.py @@ -0,0 +1,97 @@ +"""Invoice Picker: gezogene Rechnungen ansehen, Rückblick freigeben, an Accountable senden.""" + +import io +import os +import uuid +from datetime import datetime +from typing import List, Optional + +from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException +from fastapi.responses import FileResponse, StreamingResponse +from pydantic import BaseModel +from sqlalchemy.orm import Session + +from backend import grabber, mailer +from backend.database import SessionLocal, get_db +from backend.models import Outbound + +router = APIRouter() +_runs: dict = {} # ponytail: in-memory Laufstatus, ein Prozess + + +def _bg_run(run_id: str, since): + db = SessionLocal() + try: + grabber.run(db, since, _runs[run_id]) + _runs[run_id]["send"] = mailer.send_queued(db) + except Exception as e: + _runs[run_id]["errors"].append(str(e)) + finally: + _runs[run_id]["status"] = "done" + db.close() + + +@router.post("/run") +async def start_run(background_tasks: BackgroundTasks, since: str): + """Postfach ab since (YYYY-MM-DD) prüfen, Rechnungen ziehen, freigegebene/neue senden.""" + if any(r["status"] == "running" for r in _runs.values()): + raise HTTPException(409, "Lauf aktiv") + run_id = str(uuid.uuid4()) + _runs[run_id] = {"status": "running", "errors": [], "counts": {}, "start": datetime.utcnow().isoformat()} + background_tasks.add_task(_bg_run, run_id, datetime.strptime(since, "%Y-%m-%d").date()) + return {"run_id": run_id} + + +@router.get("/run/{run_id}") +async def run_status(run_id: str): + if run_id not in _runs: + raise HTTPException(404, "Lauf nicht gefunden") + return _runs[run_id] + + +@router.get("") +async def list_outbound(status: Optional[str] = None, format: str = "json", db: Session = Depends(get_db)): + q = db.query(Outbound) + if status: + q = q.filter(Outbound.status == status) + rows = q.order_by(Outbound.email_date.desc()).all() + data = [{"id": r.id, "status": r.status, "datum": f"{r.email_date:%Y-%m-%d}" if r.email_date else "", + "absender": r.email_from, "betreff": r.email_subject, "klasse": r.mail_class, + "link": r.link_url or "", "datei": os.path.basename(r.file_path or ""), "fehler": r.error or ""} + for r in rows] + if format != "xlsx": + return data + from openpyxl import Workbook + wb = Workbook() + ws = wb.active + ws.append(list(data[0].keys()) if data else ["leer"]) + for d in data: + ws.append(list(d.values())) + buf = io.BytesIO() + wb.save(buf) + buf.seek(0) + return StreamingResponse(buf, media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + headers={"Content-Disposition": "attachment; filename=invoice-picker.xlsx"}) + + +@router.get("/{oid}/pdf") +async def get_pdf(oid: int, db: Session = Depends(get_db)): + r = db.get(Outbound, oid) + if not r or not r.file_path or not os.path.exists(r.file_path): + raise HTTPException(404, "kein PDF") + return FileResponse(r.file_path, media_type="application/pdf") + + +class Approve(BaseModel): + ids: Optional[List[int]] = None # None = alle wartenden + + +@router.post("/approve") +async def approve(body: Approve, db: Session = Depends(get_db)): + """Rückblick freigeben (pending_review -> queued) und sofort senden.""" + q = db.query(Outbound).filter(Outbound.status == "pending_review") + if body.ids is not None: + q = q.filter(Outbound.id.in_(body.ids)) + n = q.update({Outbound.status: "queued"}, synchronize_session=False) + db.commit() + return {"approved": n, **mailer.send_queued(db)} diff --git a/backend/tests/test_classify.py b/backend/tests/test_classify.py index abc67fc..f77485f 100644 --- a/backend/tests/test_classify.py +++ b/backend/tests/test_classify.py @@ -14,3 +14,12 @@ def test_classify(): portal = "Content-Type: text/plain\n\nIhre Rechnung ist verfügbar. Bitte in Ihrem Konto herunterladen." assert classify_mail(_m(portal)) == ("portal", None) assert classify_mail(_m("Content-Type: text/plain\n\nHallo"))[0] == "ignore" + # Login-/Konsolen-Links sind kein Direktdownload -> portal + aws = "Content-Type: text/plain\n\nYour bill: https://console.aws.amazon.com/billing/home#/bills" + assert classify_mail(_m(aws)) == ("portal", None) + assert classify_mail(_m("Content-Type: text/plain\n\nIhr Zahlungsbeleg liegt bereit."))[0] == "portal" + # http (kein TLS) wird nie direkt geladen + assert classify_mail(_m("Content-Type: text/plain\n\nRechnung http://x.example/r.pdf"))[0] == "portal" + # HTML-Entities im Link werden aufgelöst + h = 'Content-Type: text/html\n\nRechnung' + assert classify_mail(_m(h)) == ("link", "https://x.example/r.pdf?a=1&b=2") diff --git a/backend/tests/test_outbound.py b/backend/tests/test_outbound.py new file mode 100644 index 0000000..edfe10b --- /dev/null +++ b/backend/tests/test_outbound.py @@ -0,0 +1,34 @@ +import email + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from backend import grabber +from backend.models import Base, Outbound + +LINK = ("Message-ID: <{mid}>\nSubject: Ihre Rechnung\nFrom: Shop \n" + "Date: {date}\nContent-Type: text/plain\n\nDownload: https://pay.stripe.com/invoice/x/pdf") +PORTAL = ("Message-ID: \nSubject: Rechnung verfügbar\nFrom: AWS \n" + "Date: Mon, 3 Nov 2025 10:00:00 +0000\nContent-Type: text/plain\n\nSee https://console.aws.amazon.com/billing") + + +def test_process_message(tmp_path, monkeypatch): + db = sessionmaker(bind=create_engine("sqlite://"))() + Base.metadata.create_all(db.get_bind()) + monkeypatch.setattr(grabber, "fetch_pdf", lambda url: b"%PDF-1.4 same") + monkeypatch.setenv("GRAB_CUTOFF", "2026-11-13") + run = lambda raw: grabber.process_message(db, email.message_from_string(raw), "INBOX", str(tmp_path)) + + assert run(LINK.format(mid="a", date="Mon, 3 Nov 2025 10:00:00 +0000")) == "pending_review" # Rückblick + assert run(LINK.format(mid="a", date="Mon, 3 Nov 2025 10:00:00 +0000")) is None # gleiche Message-ID + assert run(LINK.format(mid="b", date="Mon, 3 Nov 2025 10:00:00 +0000")) == "duplicate" # gleiche PDF + monkeypatch.setattr(grabber, "fetch_pdf", lambda url: b"%PDF-1.4 neu") + assert run(LINK.format(mid="c", date="Mon, 16 Nov 2026 10:00:00 +0000")) == "queued" # nach Stichtag + assert run(PORTAL) == "manual" + assert run("Subject: Newsletter\n\nhallo") is None + assert db.query(Outbound).count() == 4 + assert (tmp_path / "picker" / "2025" / "11").is_dir() + + monkeypatch.delenv("GRAB_CUTOFF") # ohne Stichtag nie automatisch senden + monkeypatch.setattr(grabber, "fetch_pdf", lambda url: b"%PDF-1.4 dritte") + assert run(LINK.format(mid="d", date="Mon, 16 Nov 2026 10:00:00 +0000")) == "pending_review"