From 37e4ed15ccd58ddb78b6e2b09a9dfff782fc37ac Mon Sep 17 00:00:00 2001 From: Markus Kruse Date: Fri, 23 Oct 2026 00:20:24 +0200 Subject: [PATCH] feat(emails): Mail-Klassifikator (Anhang/Link/Portal/ignore) + /emails/classified (JSON/xlsx) Co-Authored-By: Claude Sonnet 5.5 --- backend/main.py | 4 ++ backend/models.py | 2 + backend/routers/emails.py | 73 +++++++++++++++++++++++++++++++++- backend/tests/test_classify.py | 16 ++++++++ 4 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 backend/tests/test_classify.py diff --git a/backend/main.py b/backend/main.py index a4362cf..565bce1 100644 --- a/backend/main.py +++ b/backend/main.py @@ -65,6 +65,10 @@ async def startup_event(): """Datenbank erstellen bei Start""" Base.metadata.create_all(bind=engine) ModelsBase.metadata.create_all(bind=engine) + # create_all ergänzt keine Spalten an bestehenden Tabellen (kein alembic_version in Prod) + with engine.begin() as c: + c.execute(text("ALTER TABLE email_logs ADD COLUMN IF NOT EXISTS mail_class VARCHAR")) + c.execute(text("ALTER TABLE email_logs ADD COLUMN IF NOT EXISTS link_url TEXT")) # Router einbinden diff --git a/backend/models.py b/backend/models.py index 8a63aba..6dc649c 100644 --- a/backend/models.py +++ b/backend/models.py @@ -76,6 +76,8 @@ class EmailLog(Base): processed = Column(Boolean, default=False) processing_result = Column(String, nullable=True) # "invoice_found", "not_an_invoice", "error" invoice_id = Column(Integer, nullable=True) # Verweis auf Invoice-Tabelle + mail_class = Column(String, nullable=True, index=True) # "attachment" | "link" | "portal" | "ignore" + link_url = Column(Text, nullable=True) # erster Rechnungs-Link (nur bei mail_class="link") created_at = Column(DateTime, default=datetime.utcnow) diff --git a/backend/routers/emails.py b/backend/routers/emails.py index 3d3afb1..930f614 100644 --- a/backend/routers/emails.py +++ b/backend/routers/emails.py @@ -57,6 +57,34 @@ INVOICE_KEYWORDS = [ INVOICE_STORAGE_PATH = os.getenv("INVOICE_STORAGE_PATH", "/tmp/invoice-storage") +LINK_RE = re.compile(r"https?://[^\s\"'<>)\]]+", re.I) +LINK_HINT = re.compile(r"invoice|rechnung|receipt|beleg|billing|download|\.pdf|stripe\.com|paddle\.com", re.I) +PORTAL_HINT = re.compile( + r"in ihrem (kunden)?(konto|portal)|im (kunden)?portal|your account|sign in|log ?in|anmelden|einloggen|" + r"view (your )?(invoice|receipt|bill)|rechnung (ist )?(verfügbar|bereit)|invoice is available", + re.I, +) + + +def classify_mail(msg) -> tuple: + """-> (mail_class, link_url). Reihenfolge: Anhang > Link > Portal-Hinweis > ignore.""" + text = "" + for part in msg.walk(): + fn = part.get_filename() + if fn and decode_hdr(fn).lower().endswith(".pdf"): + return "attachment", None + if part.get_content_type() in ("text/plain", "text/html"): + payload = part.get_payload(decode=True) or b"" + text += payload.decode(part.get_content_charset() or "utf-8", "replace") + "\n" + # ponytail: Regex statt HTML-Parser; Footer-Links ("invoice" im Pfad) können False-Positives liefern + url = next((u for u in LINK_RE.findall(text) if LINK_HINT.search(u)), None) + if url: + return "link", url + if PORTAL_HINT.search(re.sub(r"<[^>]+>", " ", text)): + return "portal", None + return "ignore", None + + def connect_imap(host: str, port: int, user: str, password: str) -> imaplib.IMAP4_SSL: """Verbindet mit IMAP-Server""" mail = imaplib.IMAP4_SSL(host, port) @@ -316,12 +344,16 @@ def _scan_imap(mail_conn, db: Session, state: dict, since_date: Optional[str] = for email_id in email_ids: email_id_str = email_id.decode() log_entry = db.query(EmailLog).filter(EmailLog.email_id == email_id_str).first() - if log_entry and log_entry.processed: + if log_entry and log_entry.processed and log_entry.mail_class: continue result2, msg_data = mail_conn.fetch(email_id, '(RFC822)') if result2 != "OK": continue msg = email.message_from_bytes(msg_data[0][1]) + if log_entry: # Altbestand ohne Klasse: nur nachklassifizieren, Anhänge nicht erneut ablegen + log_entry.mail_class, log_entry.link_url = classify_mail(msg) + db.commit() + continue subject = msg.get("Subject", "") from_addr = msg.get("From", "") date_str = msg.get("Date", "") @@ -331,14 +363,17 @@ def _scan_imap(mail_conn, db: Session, state: dict, since_date: Optional[str] = matched_keywords = [kw for kw in INVOICE_KEYWORDS if kw in subject_lower] log_entry = None if matched_keywords: + mail_class, link_url = classify_mail(msg) log_entry = EmailLog( + mail_class=mail_class, + link_url=link_url, email_id=email_id_str, email_subject=subject, email_from=from_addr, email_date=email_date, email_folder="INBOX", matched_keywords=matched_keywords, - has_attachment=True, + has_attachment=mail_class == "attachment", processed=True, ) db.add(log_entry) @@ -432,6 +467,40 @@ async def scan_status(task_id: str): return state +@router.get("/classified") +async def classified(format: str = "json", db: Session = Depends(get_db)): + """Rechnungsmails nach Klasse: attachment | link | portal (ignore ausgeblendet).""" + rows = ( + db.query(EmailLog) + .filter(EmailLog.mail_class.in_(["attachment", "link", "portal"])) + .order_by(EmailLog.mail_class, EmailLog.email_date.desc()) + .all() + ) + data = [ + {"klasse": r.mail_class, "datum": r.email_date.strftime("%Y-%m-%d") if r.email_date else "", + "absender": r.email_from, "betreff": r.email_subject, "link": r.link_url or ""} + for r in rows + ] + if format != "xlsx": + return data + import io + from openpyxl import Workbook + from fastapi.responses import StreamingResponse + wb = Workbook() + wb.remove(wb.active) + for k, title in (("attachment", "a) Anhang"), ("link", "b) Link"), ("portal", "c) Portal")): + ws = wb.create_sheet(title) + ws.append(["Datum", "Absender", "Betreff", "Link"]) + for d in data: + if d["klasse"] == k: + ws.append([d["datum"], d["absender"], d["betreff"], d["link"]]) + buf = io.BytesIO() + wb.save(buf) + buf.seek(0) + return StreamingResponse(buf, media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + headers={"Content-Disposition": "attachment; filename=rechnungsmails.xlsx"}) + + @router.get("/scan-all") async def scan_all_emails(db: Session = Depends(get_db)): """Synchroner Scan (Legacy) — für direkten Aufruf ohne Polling.""" diff --git a/backend/tests/test_classify.py b/backend/tests/test_classify.py new file mode 100644 index 0000000..abc67fc --- /dev/null +++ b/backend/tests/test_classify.py @@ -0,0 +1,16 @@ +import email +from backend.routers.emails import classify_mail + + +def _m(raw): + return email.message_from_string(raw) + + +def test_classify(): + pdf = "Content-Type: multipart/mixed; boundary=b\n\n--b\nContent-Type: text/plain\n\nhi\n--b\nContent-Type: application/pdf\nContent-Disposition: attachment; filename=r.pdf\n\nx\n--b--\n" + assert classify_mail(_m(pdf)) == ("attachment", None) + link = "Content-Type: text/plain\n\nIhre Rechnung: https://pay.stripe.com/invoice/abc/pdf" + assert classify_mail(_m(link))[0] == "link" + portal = "Content-Type: text/plain\n\nIhre Rechnung ist verfügbar. Bitte in Ihrem Konto herunterladen." + assert classify_mail(_m(portal)) == ("portal", None) + assert classify_mail(_m("Content-Type: text/plain\n\nHallo"))[0] == "ignore"