feat(emails): Mail-Klassifikator (Anhang/Link/Portal/ignore) + /emails/classified (JSON/xlsx)
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
fcf7eb45d6
commit
37e4ed15cc
4 changed files with 93 additions and 2 deletions
|
|
@ -65,6 +65,10 @@ async def startup_event():
|
|||
"""Datenbank erstellen bei Start"""
|
||||
Base.metadata.create_all(bind=engine)
|
||||
ModelsBase.metadata.create_all(bind=engine)
|
||||
# create_all ergänzt keine Spalten an bestehenden Tabellen (kein alembic_version in Prod)
|
||||
with engine.begin() as c:
|
||||
c.execute(text("ALTER TABLE email_logs ADD COLUMN IF NOT EXISTS mail_class VARCHAR"))
|
||||
c.execute(text("ALTER TABLE email_logs ADD COLUMN IF NOT EXISTS link_url TEXT"))
|
||||
|
||||
|
||||
# Router einbinden
|
||||
|
|
|
|||
|
|
@ -76,6 +76,8 @@ class EmailLog(Base):
|
|||
processed = Column(Boolean, default=False)
|
||||
processing_result = Column(String, nullable=True) # "invoice_found", "not_an_invoice", "error"
|
||||
invoice_id = Column(Integer, nullable=True) # Verweis auf Invoice-Tabelle
|
||||
mail_class = Column(String, nullable=True, index=True) # "attachment" | "link" | "portal" | "ignore"
|
||||
link_url = Column(Text, nullable=True) # erster Rechnungs-Link (nur bei mail_class="link")
|
||||
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
|
|
|
|||
|
|
@ -57,6 +57,34 @@ INVOICE_KEYWORDS = [
|
|||
INVOICE_STORAGE_PATH = os.getenv("INVOICE_STORAGE_PATH", "/tmp/invoice-storage")
|
||||
|
||||
|
||||
LINK_RE = re.compile(r"https?://[^\s\"'<>)\]]+", re.I)
|
||||
LINK_HINT = re.compile(r"invoice|rechnung|receipt|beleg|billing|download|\.pdf|stripe\.com|paddle\.com", re.I)
|
||||
PORTAL_HINT = re.compile(
|
||||
r"in ihrem (kunden)?(konto|portal)|im (kunden)?portal|your account|sign in|log ?in|anmelden|einloggen|"
|
||||
r"view (your )?(invoice|receipt|bill)|rechnung (ist )?(verfügbar|bereit)|invoice is available",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
||||
def classify_mail(msg) -> tuple:
|
||||
"""-> (mail_class, link_url). Reihenfolge: Anhang > Link > Portal-Hinweis > ignore."""
|
||||
text = ""
|
||||
for part in msg.walk():
|
||||
fn = part.get_filename()
|
||||
if fn and decode_hdr(fn).lower().endswith(".pdf"):
|
||||
return "attachment", None
|
||||
if part.get_content_type() in ("text/plain", "text/html"):
|
||||
payload = part.get_payload(decode=True) or b""
|
||||
text += payload.decode(part.get_content_charset() or "utf-8", "replace") + "\n"
|
||||
# ponytail: Regex statt HTML-Parser; Footer-Links ("invoice" im Pfad) können False-Positives liefern
|
||||
url = next((u for u in LINK_RE.findall(text) if LINK_HINT.search(u)), None)
|
||||
if url:
|
||||
return "link", url
|
||||
if PORTAL_HINT.search(re.sub(r"<[^>]+>", " ", text)):
|
||||
return "portal", None
|
||||
return "ignore", None
|
||||
|
||||
|
||||
def connect_imap(host: str, port: int, user: str, password: str) -> imaplib.IMAP4_SSL:
|
||||
"""Verbindet mit IMAP-Server"""
|
||||
mail = imaplib.IMAP4_SSL(host, port)
|
||||
|
|
@ -316,12 +344,16 @@ def _scan_imap(mail_conn, db: Session, state: dict, since_date: Optional[str] =
|
|||
for email_id in email_ids:
|
||||
email_id_str = email_id.decode()
|
||||
log_entry = db.query(EmailLog).filter(EmailLog.email_id == email_id_str).first()
|
||||
if log_entry and log_entry.processed:
|
||||
if log_entry and log_entry.processed and log_entry.mail_class:
|
||||
continue
|
||||
result2, msg_data = mail_conn.fetch(email_id, '(RFC822)')
|
||||
if result2 != "OK":
|
||||
continue
|
||||
msg = email.message_from_bytes(msg_data[0][1])
|
||||
if log_entry: # Altbestand ohne Klasse: nur nachklassifizieren, Anhänge nicht erneut ablegen
|
||||
log_entry.mail_class, log_entry.link_url = classify_mail(msg)
|
||||
db.commit()
|
||||
continue
|
||||
subject = msg.get("Subject", "")
|
||||
from_addr = msg.get("From", "")
|
||||
date_str = msg.get("Date", "")
|
||||
|
|
@ -331,14 +363,17 @@ def _scan_imap(mail_conn, db: Session, state: dict, since_date: Optional[str] =
|
|||
matched_keywords = [kw for kw in INVOICE_KEYWORDS if kw in subject_lower]
|
||||
log_entry = None
|
||||
if matched_keywords:
|
||||
mail_class, link_url = classify_mail(msg)
|
||||
log_entry = EmailLog(
|
||||
mail_class=mail_class,
|
||||
link_url=link_url,
|
||||
email_id=email_id_str,
|
||||
email_subject=subject,
|
||||
email_from=from_addr,
|
||||
email_date=email_date,
|
||||
email_folder="INBOX",
|
||||
matched_keywords=matched_keywords,
|
||||
has_attachment=True,
|
||||
has_attachment=mail_class == "attachment",
|
||||
processed=True,
|
||||
)
|
||||
db.add(log_entry)
|
||||
|
|
@ -432,6 +467,40 @@ async def scan_status(task_id: str):
|
|||
return state
|
||||
|
||||
|
||||
@router.get("/classified")
|
||||
async def classified(format: str = "json", db: Session = Depends(get_db)):
|
||||
"""Rechnungsmails nach Klasse: attachment | link | portal (ignore ausgeblendet)."""
|
||||
rows = (
|
||||
db.query(EmailLog)
|
||||
.filter(EmailLog.mail_class.in_(["attachment", "link", "portal"]))
|
||||
.order_by(EmailLog.mail_class, EmailLog.email_date.desc())
|
||||
.all()
|
||||
)
|
||||
data = [
|
||||
{"klasse": r.mail_class, "datum": r.email_date.strftime("%Y-%m-%d") if r.email_date else "",
|
||||
"absender": r.email_from, "betreff": r.email_subject, "link": r.link_url or ""}
|
||||
for r in rows
|
||||
]
|
||||
if format != "xlsx":
|
||||
return data
|
||||
import io
|
||||
from openpyxl import Workbook
|
||||
from fastapi.responses import StreamingResponse
|
||||
wb = Workbook()
|
||||
wb.remove(wb.active)
|
||||
for k, title in (("attachment", "a) Anhang"), ("link", "b) Link"), ("portal", "c) Portal")):
|
||||
ws = wb.create_sheet(title)
|
||||
ws.append(["Datum", "Absender", "Betreff", "Link"])
|
||||
for d in data:
|
||||
if d["klasse"] == k:
|
||||
ws.append([d["datum"], d["absender"], d["betreff"], d["link"]])
|
||||
buf = io.BytesIO()
|
||||
wb.save(buf)
|
||||
buf.seek(0)
|
||||
return StreamingResponse(buf, media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
headers={"Content-Disposition": "attachment; filename=rechnungsmails.xlsx"})
|
||||
|
||||
|
||||
@router.get("/scan-all")
|
||||
async def scan_all_emails(db: Session = Depends(get_db)):
|
||||
"""Synchroner Scan (Legacy) — für direkten Aufruf ohne Polling."""
|
||||
|
|
|
|||
16
backend/tests/test_classify.py
Normal file
16
backend/tests/test_classify.py
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
import email
|
||||
from backend.routers.emails import classify_mail
|
||||
|
||||
|
||||
def _m(raw):
|
||||
return email.message_from_string(raw)
|
||||
|
||||
|
||||
def test_classify():
|
||||
pdf = "Content-Type: multipart/mixed; boundary=b\n\n--b\nContent-Type: text/plain\n\nhi\n--b\nContent-Type: application/pdf\nContent-Disposition: attachment; filename=r.pdf\n\nx\n--b--\n"
|
||||
assert classify_mail(_m(pdf)) == ("attachment", None)
|
||||
link = "Content-Type: text/plain\n\nIhre Rechnung: https://pay.stripe.com/invoice/abc/pdf"
|
||||
assert classify_mail(_m(link))[0] == "link"
|
||||
portal = "Content-Type: text/plain\n\nIhre Rechnung ist verfügbar. Bitte in Ihrem Konto herunterladen."
|
||||
assert classify_mail(_m(portal)) == ("portal", None)
|
||||
assert classify_mail(_m("Content-Type: text/plain\n\nHallo"))[0] == "ignore"
|
||||
Loading…
Add table
Reference in a new issue