feat(emails): Mail-Klassifikator (Anhang/Link/Portal/ignore) + /emails/classified (JSON/xlsx)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Markus Kruse 2026-10-23 00:20:24 +02:00
parent fcf7eb45d6
commit 37e4ed15cc
4 changed files with 93 additions and 2 deletions

View file

@ -65,6 +65,10 @@ async def startup_event():
"""Datenbank erstellen bei Start"""
Base.metadata.create_all(bind=engine)
ModelsBase.metadata.create_all(bind=engine)
# create_all ergänzt keine Spalten an bestehenden Tabellen (kein alembic_version in Prod)
with engine.begin() as c:
c.execute(text("ALTER TABLE email_logs ADD COLUMN IF NOT EXISTS mail_class VARCHAR"))
c.execute(text("ALTER TABLE email_logs ADD COLUMN IF NOT EXISTS link_url TEXT"))
# Router einbinden

View file

@ -76,6 +76,8 @@ class EmailLog(Base):
processed = Column(Boolean, default=False)
processing_result = Column(String, nullable=True) # "invoice_found", "not_an_invoice", "error"
invoice_id = Column(Integer, nullable=True) # Verweis auf Invoice-Tabelle
mail_class = Column(String, nullable=True, index=True) # "attachment" | "link" | "portal" | "ignore"
link_url = Column(Text, nullable=True) # erster Rechnungs-Link (nur bei mail_class="link")
created_at = Column(DateTime, default=datetime.utcnow)

View file

@ -57,6 +57,34 @@ INVOICE_KEYWORDS = [
INVOICE_STORAGE_PATH = os.getenv("INVOICE_STORAGE_PATH", "/tmp/invoice-storage")
LINK_RE = re.compile(r"https?://[^\s\"'<>)\]]+", re.I)
LINK_HINT = re.compile(r"invoice|rechnung|receipt|beleg|billing|download|\.pdf|stripe\.com|paddle\.com", re.I)
PORTAL_HINT = re.compile(
r"in ihrem (kunden)?(konto|portal)|im (kunden)?portal|your account|sign in|log ?in|anmelden|einloggen|"
r"view (your )?(invoice|receipt|bill)|rechnung (ist )?(verfügbar|bereit)|invoice is available",
re.I,
)
def classify_mail(msg) -> tuple:
"""-> (mail_class, link_url). Reihenfolge: Anhang > Link > Portal-Hinweis > ignore."""
text = ""
for part in msg.walk():
fn = part.get_filename()
if fn and decode_hdr(fn).lower().endswith(".pdf"):
return "attachment", None
if part.get_content_type() in ("text/plain", "text/html"):
payload = part.get_payload(decode=True) or b""
text += payload.decode(part.get_content_charset() or "utf-8", "replace") + "\n"
# ponytail: Regex statt HTML-Parser; Footer-Links ("invoice" im Pfad) können False-Positives liefern
url = next((u for u in LINK_RE.findall(text) if LINK_HINT.search(u)), None)
if url:
return "link", url
if PORTAL_HINT.search(re.sub(r"<[^>]+>", " ", text)):
return "portal", None
return "ignore", None
def connect_imap(host: str, port: int, user: str, password: str) -> imaplib.IMAP4_SSL:
"""Verbindet mit IMAP-Server"""
mail = imaplib.IMAP4_SSL(host, port)
@ -316,12 +344,16 @@ def _scan_imap(mail_conn, db: Session, state: dict, since_date: Optional[str] =
for email_id in email_ids:
email_id_str = email_id.decode()
log_entry = db.query(EmailLog).filter(EmailLog.email_id == email_id_str).first()
if log_entry and log_entry.processed:
if log_entry and log_entry.processed and log_entry.mail_class:
continue
result2, msg_data = mail_conn.fetch(email_id, '(RFC822)')
if result2 != "OK":
continue
msg = email.message_from_bytes(msg_data[0][1])
if log_entry: # Altbestand ohne Klasse: nur nachklassifizieren, Anhänge nicht erneut ablegen
log_entry.mail_class, log_entry.link_url = classify_mail(msg)
db.commit()
continue
subject = msg.get("Subject", "")
from_addr = msg.get("From", "")
date_str = msg.get("Date", "")
@ -331,14 +363,17 @@ def _scan_imap(mail_conn, db: Session, state: dict, since_date: Optional[str] =
matched_keywords = [kw for kw in INVOICE_KEYWORDS if kw in subject_lower]
log_entry = None
if matched_keywords:
mail_class, link_url = classify_mail(msg)
log_entry = EmailLog(
mail_class=mail_class,
link_url=link_url,
email_id=email_id_str,
email_subject=subject,
email_from=from_addr,
email_date=email_date,
email_folder="INBOX",
matched_keywords=matched_keywords,
has_attachment=True,
has_attachment=mail_class == "attachment",
processed=True,
)
db.add(log_entry)
@ -432,6 +467,40 @@ async def scan_status(task_id: str):
return state
@router.get("/classified")
async def classified(format: str = "json", db: Session = Depends(get_db)):
"""Rechnungsmails nach Klasse: attachment | link | portal (ignore ausgeblendet)."""
rows = (
db.query(EmailLog)
.filter(EmailLog.mail_class.in_(["attachment", "link", "portal"]))
.order_by(EmailLog.mail_class, EmailLog.email_date.desc())
.all()
)
data = [
{"klasse": r.mail_class, "datum": r.email_date.strftime("%Y-%m-%d") if r.email_date else "",
"absender": r.email_from, "betreff": r.email_subject, "link": r.link_url or ""}
for r in rows
]
if format != "xlsx":
return data
import io
from openpyxl import Workbook
from fastapi.responses import StreamingResponse
wb = Workbook()
wb.remove(wb.active)
for k, title in (("attachment", "a) Anhang"), ("link", "b) Link"), ("portal", "c) Portal")):
ws = wb.create_sheet(title)
ws.append(["Datum", "Absender", "Betreff", "Link"])
for d in data:
if d["klasse"] == k:
ws.append([d["datum"], d["absender"], d["betreff"], d["link"]])
buf = io.BytesIO()
wb.save(buf)
buf.seek(0)
return StreamingResponse(buf, media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
headers={"Content-Disposition": "attachment; filename=rechnungsmails.xlsx"})
@router.get("/scan-all")
async def scan_all_emails(db: Session = Depends(get_db)):
"""Synchroner Scan (Legacy) — für direkten Aufruf ohne Polling."""

View file

@ -0,0 +1,16 @@
import email
from backend.routers.emails import classify_mail
def _m(raw):
return email.message_from_string(raw)
def test_classify():
pdf = "Content-Type: multipart/mixed; boundary=b\n\n--b\nContent-Type: text/plain\n\nhi\n--b\nContent-Type: application/pdf\nContent-Disposition: attachment; filename=r.pdf\n\nx\n--b--\n"
assert classify_mail(_m(pdf)) == ("attachment", None)
link = "Content-Type: text/plain\n\nIhre Rechnung: https://pay.stripe.com/invoice/abc/pdf"
assert classify_mail(_m(link))[0] == "link"
portal = "Content-Type: text/plain\n\nIhre Rechnung ist verfügbar. Bitte in Ihrem Konto herunterladen."
assert classify_mail(_m(portal)) == ("portal", None)
assert classify_mail(_m("Content-Type: text/plain\n\nHallo"))[0] == "ignore"