feat(emails): Mail-Klassifikator (Anhang/Link/Portal/ignore) + /emails/classified (JSON/xlsx)
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
fcf7eb45d6
commit
37e4ed15cc
4 changed files with 93 additions and 2 deletions
|
|
@ -65,6 +65,10 @@ async def startup_event():
|
||||||
"""Datenbank erstellen bei Start"""
|
"""Datenbank erstellen bei Start"""
|
||||||
Base.metadata.create_all(bind=engine)
|
Base.metadata.create_all(bind=engine)
|
||||||
ModelsBase.metadata.create_all(bind=engine)
|
ModelsBase.metadata.create_all(bind=engine)
|
||||||
|
# create_all ergänzt keine Spalten an bestehenden Tabellen (kein alembic_version in Prod)
|
||||||
|
with engine.begin() as c:
|
||||||
|
c.execute(text("ALTER TABLE email_logs ADD COLUMN IF NOT EXISTS mail_class VARCHAR"))
|
||||||
|
c.execute(text("ALTER TABLE email_logs ADD COLUMN IF NOT EXISTS link_url TEXT"))
|
||||||
|
|
||||||
|
|
||||||
# Router einbinden
|
# Router einbinden
|
||||||
|
|
|
||||||
|
|
@ -76,6 +76,8 @@ class EmailLog(Base):
|
||||||
processed = Column(Boolean, default=False)
|
processed = Column(Boolean, default=False)
|
||||||
processing_result = Column(String, nullable=True) # "invoice_found", "not_an_invoice", "error"
|
processing_result = Column(String, nullable=True) # "invoice_found", "not_an_invoice", "error"
|
||||||
invoice_id = Column(Integer, nullable=True) # Verweis auf Invoice-Tabelle
|
invoice_id = Column(Integer, nullable=True) # Verweis auf Invoice-Tabelle
|
||||||
|
mail_class = Column(String, nullable=True, index=True) # "attachment" | "link" | "portal" | "ignore"
|
||||||
|
link_url = Column(Text, nullable=True) # erster Rechnungs-Link (nur bei mail_class="link")
|
||||||
|
|
||||||
created_at = Column(DateTime, default=datetime.utcnow)
|
created_at = Column(DateTime, default=datetime.utcnow)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -57,6 +57,34 @@ INVOICE_KEYWORDS = [
|
||||||
INVOICE_STORAGE_PATH = os.getenv("INVOICE_STORAGE_PATH", "/tmp/invoice-storage")
|
INVOICE_STORAGE_PATH = os.getenv("INVOICE_STORAGE_PATH", "/tmp/invoice-storage")
|
||||||
|
|
||||||
|
|
||||||
|
LINK_RE = re.compile(r"https?://[^\s\"'<>)\]]+", re.I)
|
||||||
|
LINK_HINT = re.compile(r"invoice|rechnung|receipt|beleg|billing|download|\.pdf|stripe\.com|paddle\.com", re.I)
|
||||||
|
PORTAL_HINT = re.compile(
|
||||||
|
r"in ihrem (kunden)?(konto|portal)|im (kunden)?portal|your account|sign in|log ?in|anmelden|einloggen|"
|
||||||
|
r"view (your )?(invoice|receipt|bill)|rechnung (ist )?(verfügbar|bereit)|invoice is available",
|
||||||
|
re.I,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def classify_mail(msg) -> tuple:
|
||||||
|
"""-> (mail_class, link_url). Reihenfolge: Anhang > Link > Portal-Hinweis > ignore."""
|
||||||
|
text = ""
|
||||||
|
for part in msg.walk():
|
||||||
|
fn = part.get_filename()
|
||||||
|
if fn and decode_hdr(fn).lower().endswith(".pdf"):
|
||||||
|
return "attachment", None
|
||||||
|
if part.get_content_type() in ("text/plain", "text/html"):
|
||||||
|
payload = part.get_payload(decode=True) or b""
|
||||||
|
text += payload.decode(part.get_content_charset() or "utf-8", "replace") + "\n"
|
||||||
|
# ponytail: Regex statt HTML-Parser; Footer-Links ("invoice" im Pfad) können False-Positives liefern
|
||||||
|
url = next((u for u in LINK_RE.findall(text) if LINK_HINT.search(u)), None)
|
||||||
|
if url:
|
||||||
|
return "link", url
|
||||||
|
if PORTAL_HINT.search(re.sub(r"<[^>]+>", " ", text)):
|
||||||
|
return "portal", None
|
||||||
|
return "ignore", None
|
||||||
|
|
||||||
|
|
||||||
def connect_imap(host: str, port: int, user: str, password: str) -> imaplib.IMAP4_SSL:
|
def connect_imap(host: str, port: int, user: str, password: str) -> imaplib.IMAP4_SSL:
|
||||||
"""Verbindet mit IMAP-Server"""
|
"""Verbindet mit IMAP-Server"""
|
||||||
mail = imaplib.IMAP4_SSL(host, port)
|
mail = imaplib.IMAP4_SSL(host, port)
|
||||||
|
|
@ -316,12 +344,16 @@ def _scan_imap(mail_conn, db: Session, state: dict, since_date: Optional[str] =
|
||||||
for email_id in email_ids:
|
for email_id in email_ids:
|
||||||
email_id_str = email_id.decode()
|
email_id_str = email_id.decode()
|
||||||
log_entry = db.query(EmailLog).filter(EmailLog.email_id == email_id_str).first()
|
log_entry = db.query(EmailLog).filter(EmailLog.email_id == email_id_str).first()
|
||||||
if log_entry and log_entry.processed:
|
if log_entry and log_entry.processed and log_entry.mail_class:
|
||||||
continue
|
continue
|
||||||
result2, msg_data = mail_conn.fetch(email_id, '(RFC822)')
|
result2, msg_data = mail_conn.fetch(email_id, '(RFC822)')
|
||||||
if result2 != "OK":
|
if result2 != "OK":
|
||||||
continue
|
continue
|
||||||
msg = email.message_from_bytes(msg_data[0][1])
|
msg = email.message_from_bytes(msg_data[0][1])
|
||||||
|
if log_entry: # Altbestand ohne Klasse: nur nachklassifizieren, Anhänge nicht erneut ablegen
|
||||||
|
log_entry.mail_class, log_entry.link_url = classify_mail(msg)
|
||||||
|
db.commit()
|
||||||
|
continue
|
||||||
subject = msg.get("Subject", "")
|
subject = msg.get("Subject", "")
|
||||||
from_addr = msg.get("From", "")
|
from_addr = msg.get("From", "")
|
||||||
date_str = msg.get("Date", "")
|
date_str = msg.get("Date", "")
|
||||||
|
|
@ -331,14 +363,17 @@ def _scan_imap(mail_conn, db: Session, state: dict, since_date: Optional[str] =
|
||||||
matched_keywords = [kw for kw in INVOICE_KEYWORDS if kw in subject_lower]
|
matched_keywords = [kw for kw in INVOICE_KEYWORDS if kw in subject_lower]
|
||||||
log_entry = None
|
log_entry = None
|
||||||
if matched_keywords:
|
if matched_keywords:
|
||||||
|
mail_class, link_url = classify_mail(msg)
|
||||||
log_entry = EmailLog(
|
log_entry = EmailLog(
|
||||||
|
mail_class=mail_class,
|
||||||
|
link_url=link_url,
|
||||||
email_id=email_id_str,
|
email_id=email_id_str,
|
||||||
email_subject=subject,
|
email_subject=subject,
|
||||||
email_from=from_addr,
|
email_from=from_addr,
|
||||||
email_date=email_date,
|
email_date=email_date,
|
||||||
email_folder="INBOX",
|
email_folder="INBOX",
|
||||||
matched_keywords=matched_keywords,
|
matched_keywords=matched_keywords,
|
||||||
has_attachment=True,
|
has_attachment=mail_class == "attachment",
|
||||||
processed=True,
|
processed=True,
|
||||||
)
|
)
|
||||||
db.add(log_entry)
|
db.add(log_entry)
|
||||||
|
|
@ -432,6 +467,40 @@ async def scan_status(task_id: str):
|
||||||
return state
|
return state
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/classified")
|
||||||
|
async def classified(format: str = "json", db: Session = Depends(get_db)):
|
||||||
|
"""Rechnungsmails nach Klasse: attachment | link | portal (ignore ausgeblendet)."""
|
||||||
|
rows = (
|
||||||
|
db.query(EmailLog)
|
||||||
|
.filter(EmailLog.mail_class.in_(["attachment", "link", "portal"]))
|
||||||
|
.order_by(EmailLog.mail_class, EmailLog.email_date.desc())
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
data = [
|
||||||
|
{"klasse": r.mail_class, "datum": r.email_date.strftime("%Y-%m-%d") if r.email_date else "",
|
||||||
|
"absender": r.email_from, "betreff": r.email_subject, "link": r.link_url or ""}
|
||||||
|
for r in rows
|
||||||
|
]
|
||||||
|
if format != "xlsx":
|
||||||
|
return data
|
||||||
|
import io
|
||||||
|
from openpyxl import Workbook
|
||||||
|
from fastapi.responses import StreamingResponse
|
||||||
|
wb = Workbook()
|
||||||
|
wb.remove(wb.active)
|
||||||
|
for k, title in (("attachment", "a) Anhang"), ("link", "b) Link"), ("portal", "c) Portal")):
|
||||||
|
ws = wb.create_sheet(title)
|
||||||
|
ws.append(["Datum", "Absender", "Betreff", "Link"])
|
||||||
|
for d in data:
|
||||||
|
if d["klasse"] == k:
|
||||||
|
ws.append([d["datum"], d["absender"], d["betreff"], d["link"]])
|
||||||
|
buf = io.BytesIO()
|
||||||
|
wb.save(buf)
|
||||||
|
buf.seek(0)
|
||||||
|
return StreamingResponse(buf, media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||||
|
headers={"Content-Disposition": "attachment; filename=rechnungsmails.xlsx"})
|
||||||
|
|
||||||
|
|
||||||
@router.get("/scan-all")
|
@router.get("/scan-all")
|
||||||
async def scan_all_emails(db: Session = Depends(get_db)):
|
async def scan_all_emails(db: Session = Depends(get_db)):
|
||||||
"""Synchroner Scan (Legacy) — für direkten Aufruf ohne Polling."""
|
"""Synchroner Scan (Legacy) — für direkten Aufruf ohne Polling."""
|
||||||
|
|
|
||||||
16
backend/tests/test_classify.py
Normal file
16
backend/tests/test_classify.py
Normal file
|
|
@ -0,0 +1,16 @@
|
||||||
|
import email
|
||||||
|
from backend.routers.emails import classify_mail
|
||||||
|
|
||||||
|
|
||||||
|
def _m(raw):
|
||||||
|
return email.message_from_string(raw)
|
||||||
|
|
||||||
|
|
||||||
|
def test_classify():
|
||||||
|
pdf = "Content-Type: multipart/mixed; boundary=b\n\n--b\nContent-Type: text/plain\n\nhi\n--b\nContent-Type: application/pdf\nContent-Disposition: attachment; filename=r.pdf\n\nx\n--b--\n"
|
||||||
|
assert classify_mail(_m(pdf)) == ("attachment", None)
|
||||||
|
link = "Content-Type: text/plain\n\nIhre Rechnung: https://pay.stripe.com/invoice/abc/pdf"
|
||||||
|
assert classify_mail(_m(link))[0] == "link"
|
||||||
|
portal = "Content-Type: text/plain\n\nIhre Rechnung ist verfügbar. Bitte in Ihrem Konto herunterladen."
|
||||||
|
assert classify_mail(_m(portal)) == ("portal", None)
|
||||||
|
assert classify_mail(_m("Content-Type: text/plain\n\nHallo"))[0] == "ignore"
|
||||||
Loading…
Add table
Reference in a new issue