""" C-Stone Invoice Check — Email Router IMAP Email-Verarbeitung und Filterung """ import imaplib import email import html import os import re import tempfile import uuid from datetime import datetime from typing import List, Optional, Dict, Any from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException from sqlalchemy.orm import Session from email.header import decode_header, make_header from backend.database import get_db, SessionLocal from backend.models import EmailLog, Invoice from backend.routers.settings import get_imap_config router = APIRouter() def decode_hdr(value) -> str: """Header robust dekodieren (alle Segmente); kaputte Encodings (unknown-8bit) -> replace statt Abbruch.""" if not value: return "" try: return str(make_header(decode_header(str(value)))) except Exception: return str(value) # ponytail: in-memory scan state, fine for single process _scan_tasks: dict = {} _active_scan_id: Optional[str] = None # IMAP Konfiguration IMAP_HOST = os.getenv("IMAP_HOST", "imap.strato.de") IMAP_PORT = int(os.getenv("IMAP_PORT", 993)) IMAP_USER = os.getenv("IMAP_USER", "") IMAP_PASSWORD = os.getenv("IMAP_PASSWORD", "") # Gmail Konfiguration GMAIL_IMAP_HOST = os.getenv("GMAIL_IMAP_HOST", "imap.gmail.com") GMAIL_IMAP_PORT = int(os.getenv("GMAIL_IMAP_PORT", 993)) GMAIL_IMAP_USER = os.getenv("GMAIL_IMAP_USER", "") GMAIL_IMAP_PASSWORD = os.getenv("GMAIL_IMAP_PASSWORD", "") # Filter-Kriterien (Substring-Match — deckt Plural/Komposita ab) INVOICE_KEYWORDS = [ "rechnung", "rechnungs", "invoice", "receipt", "bill", "factuur", "gutschrift", "quittung", "beleg", "abrechnung", "mahnung", "order confirmation", # Whop ] # Ablagepfad (Evo-X2) - optional INVOICE_STORAGE_PATH = os.getenv("INVOICE_STORAGE_PATH", "/tmp/invoice-storage") LINK_RE = re.compile(r"https?://[^\s\"'<>)\]]+", re.I) # "link" = PDF ohne Login abrufbar. Gleichzeitig Allowlist für den Server-Download (nur https, kein beliebiger Host). DIRECT_LINK = re.compile( r"^https://((pay|invoice)\.stripe\.com|invoices\.thrivecart\.com|store\.payproglobal\.com|[\w-]+\.paddle\.com)/" r"|^https://\S+\.pdf([?#]|$)", re.I, ) # Rechnungs-Link hinter Login (Konsole, Kundenportal) -> "portal" LINK_HINT = re.compile(r"invoice|rechnung|receipt|beleg|billing|download|\.pdf", re.I) # Klick-Tracker bekannter Absender: Ziel erst beim Abruf auflösen (OpenAI -> invoice.stripe.com) # ponytail: Allowlist je Absender; neuer Anbieter mit Tracker = hier eine Zeile ergänzen TRACKERS = re.compile(r"^https?://[\w-]+\.email\.openai\.com/", re.I) INVOICE_ANCHOR = re.compile(r']*href="([^"]+)"[^>]*>(?:(?!).){0,300}?(?:invoice|rechnung|receipt)', re.I | re.S) PORTAL_HINT = re.compile( r"in ihrem (kunden)?(konto|portal)|im (kunden)?portal|your account|sign in|log ?in|anmelden|einloggen|" r"view (your )?(invoice|receipt|bill)|rechnung (ist )?(verfügbar|bereit)|invoice is available|zahlungsbeleg|das ist keine rechnung", # Meta: Beleg ohne PDF, Rechnung nur im Billing Hub re.I, ) def classify_mail(msg) -> tuple: """-> (mail_class, link_url). Reihenfolge: Anhang > Direktlink > Portal (Login-Link/Hinweis) > ignore.""" text = "" for part in msg.walk(): fn = part.get_filename() if fn and decode_hdr(fn).lower().endswith(".pdf"): return "attachment", None if part.get_content_type() in ("text/plain", "text/html"): payload = part.get_payload(decode=True) or b"" text += payload.decode(part.get_content_charset() or "utf-8", "replace") + "\n" # ponytail: Regex statt HTML-Parser; Footer-Links ("invoice" im Pfad) können False-Positives liefern urls = [html.unescape(u) for u in LINK_RE.findall(text)] url = next((u for u in urls if DIRECT_LINK.search(u)), None) or next( (u for u in map(html.unescape, INVOICE_ANCHOR.findall(text)) if TRACKERS.match(u)), None) if url: return "link", url if any(LINK_HINT.search(u) for u in urls) or PORTAL_HINT.search(re.sub(r"<[^>]+>", " ", text)): return "portal", None return "ignore", None def connect_imap(host: str, port: int, user: str, password: str) -> imaplib.IMAP4_SSL: """Verbindet mit IMAP-Server""" mail = imaplib.IMAP4_SSL(host, port) mail.login(user, password) return mail def search_invoices(mail: imaplib.IMAP4_SSL, folder: str = "INBOX") -> List[str]: """Sucht nach Emails mit Rechnungs-Stichworten""" mail.select(folder) result, data = mail.search(None, 'ALL') if result != "OK": return [] email_ids = data[0].split() matching_ids = [] for email_id in email_ids: result, msg_data = mail.fetch(email_id, '(RFC822)') if result != "OK": continue msg = email.message_from_bytes(msg_data[0][1]) subject = msg.get("Subject", "") from_addr = decode_hdr(msg.get("From", "")) # Subject decodieren subject = decode_hdr(subject) # Prüfen auf Keywords subject_lower = subject.lower() if any(kw in subject_lower for kw in INVOICE_KEYWORDS): matching_ids.append(email_id.decode()) return matching_ids def extract_attachments(msg: email.message.Message) -> List[Dict[str, Any]]: """Extrahiert Anhänge aus Email""" attachments = [] for part in msg.walk(): if part.get_content_maintype() == 'multipart': continue if part.get('Content-Disposition') is None: continue filename = part.get_filename() if filename: filename = decode_hdr(filename) content_type = part.get_content_type() payload = part.get_payload(decode=True) attachments.append({ "filename": filename, "content_type": content_type, "data": payload, "size": len(payload) if payload else 0 }) return attachments def parse_email_date(date_str: str) -> Optional[datetime]: """Parses various email date formats""" if not date_str: return None from email.utils import parsedate_to_datetime try: # Versuche email.utils parsedate_to_datetime return parsedate_to_datetime(date_str) except: pass # Fallback: Versuche verschiedene Formate formats = [ "%a, %d %b %Y %H:%M:%S %z", "%a, %d %b %Y %H:%M:%S %z (%Z)", "%d %b %Y %H:%M:%S %z", "%Y-%m-%d %H:%M:%S", ] for fmt in formats: try: return datetime.strptime(date_str.strip(), fmt) except: continue return None @router.get("/search") async def search_emails(db: Session = Depends(get_db)): """Sucht nach Rechnungs-Emails bei Strato""" results = { "processed": 0, "invoices_found": 0, "errors": [] } try: cfg = get_imap_config(db) mail = connect_imap(cfg["host"], cfg["port"], cfg["user"], cfg["password"]) # Alle Ordner durchsuchen for folder in ["INBOX", "Sent", "Gesendet"]: try: email_ids = search_invoices(mail, folder) results[f"{folder}_count"] = len(email_ids) for email_id in email_ids: # Verarbeitung auslagern pass except Exception as e: results["errors"].append(f"Folder {folder}: {str(e)}") mail.logout() results["processed"] = len(email_ids) if 'email_ids' in dir() else 0 except Exception as e: results["errors"].append(f"IMAP Connection failed: {str(e)}") return results @router.get("/process/{email_id}") async def process_email(email_id: str, db: Session = Depends(get_db)): """Verarbeitet einzelne Email""" # TODO: Implementierung pass def extract_and_process_attachments(msg: email.message.Message, email_id: str, db: Session) -> List[Dict[str, Any]]: """Extrahiert und verarbeitet PDF-Anhänge aus Email""" processed = [] for part in msg.walk(): if part.get_content_maintype() == 'multipart': continue if part.get('Content-Disposition') is None: continue filename = part.get_filename() if not filename: continue filename = decode_hdr(filename) content_type = part.get_content_type() # Nur PDFs verarbeiten if content_type != 'application/pdf' and not filename.lower().endswith('.pdf'): continue payload = part.get_payload(decode=True) if not payload: continue # Temporäre Datei erstellen with tempfile.NamedTemporaryFile(suffix='.pdf', delete=False) as tmp: tmp.write(payload) tmp_path = tmp.name try: # PDF verarbeiten from backend.routers.invoices import extract_pdf_data, get_storage_path pdf_data = extract_pdf_data(tmp_path) if pdf_data.get("not_invoice") or "error" in pdf_data: continue # Duplikat-Check: gleiche email_id + filename if db.query(Invoice).filter(Invoice.email_id == email_id, Invoice.file_name == filename).first(): continue if pdf_data.get("amount"): # Rechnung speichern tax_type = pdf_data.get("tax_info", {}).get("tax_type", "unbekannt") tax_rate = pdf_data.get("tax_info", {}).get("tax_rate", 0) # Speicherpfad erstellen storage_year = 2026 # Default, könnte aus Rechnungsdatum extrahiert werden storage_path = get_storage_path(storage_year, tax_type) # PDF in Zielverzeichnis kopieren (optional) base_storage = get_imap_config(db)["storage_path"] if base_storage: os.makedirs(storage_path, exist_ok=True) # Dateiname generieren (Slashes/Sonderzeichen ersetzen) invoice_number = re.sub(r'[/\\:*?"<>|]', '-', pdf_data.get("invoice_number") or "unbekannt") file_name = f"Rechnung_{invoice_number}_{email_id}.pdf" dest_path = os.path.join(storage_path, file_name) import shutil shutil.copy(tmp_path, dest_path) # In DB speichern invoice = Invoice( email_id=email_id, file_path=dest_path, file_name=file_name, storage_year=storage_year, storage_path=storage_path.replace(base_storage, ""), invoice_number=invoice_number, invoice_amount=pdf_data.get("amount"), issuer_name=pdf_data.get("issuer"), invoice_tax_type=tax_type, invoice_tax_rate=tax_rate, processed=True, confidence_score=pdf_data.get("tax_info", {}).get("confidence", 0) ) db.add(invoice) db.commit() processed.append({ "filename": filename, "invoice_number": invoice_number, "amount": pdf_data.get("amount"), "tax_type": tax_type, "saved_to": dest_path }) finally: # Temporäre Datei löschen try: os.unlink(tmp_path) except: pass return processed def _scan_imap(mail_conn, db: Session, state: dict, since_date: Optional[str] = None, until_date: Optional[str] = None): """Verarbeitet eine IMAP-Verbindung und aktualisiert state.""" mail_conn.select("INBOX") # Datumsfilter als IMAP-Kriterium (Format: DD-Mon-YYYY) criteria = [] if since_date: fmt = datetime.strptime(since_date, "%Y-%m-%d").strftime("%d-%b-%Y") criteria.append(f'SINCE {fmt}') if until_date: fmt = datetime.strptime(until_date, "%Y-%m-%d").strftime("%d-%b-%Y") criteria.append(f'BEFORE {fmt}') if criteria: result, data = mail_conn.search(None, *criteria) else: result, data = mail_conn.search(None, 'ALL') if result != "OK": return email_ids = data[0].split() state["total"] = state.get("total", 0) + len(email_ids) for email_id in email_ids: email_id_str = email_id.decode() log_entry = db.query(EmailLog).filter(EmailLog.email_id == email_id_str).first() if log_entry and log_entry.processed and log_entry.mail_class: continue result2, msg_data = mail_conn.fetch(email_id, '(RFC822)') if result2 != "OK": continue msg = email.message_from_bytes(msg_data[0][1]) if log_entry: # Altbestand ohne Klasse: nur nachklassifizieren, Anhänge nicht erneut ablegen log_entry.mail_class, log_entry.link_url = classify_mail(msg) db.commit() continue subject = msg.get("Subject", "") from_addr = decode_hdr(msg.get("From", "")) date_str = msg.get("Date", "") email_date = parse_email_date(date_str) subject = decode_hdr(subject) subject_lower = subject.lower() matched_keywords = [kw for kw in INVOICE_KEYWORDS if kw in subject_lower] log_entry = None if matched_keywords: mail_class, link_url = classify_mail(msg) log_entry = EmailLog( mail_class=mail_class, link_url=link_url, email_id=email_id_str, email_subject=subject, email_from=from_addr, email_date=email_date, email_folder="INBOX", matched_keywords=matched_keywords, has_attachment=mail_class == "attachment", processed=True, ) db.add(log_entry) state["invoices_found"] += 1 try: pdfs = extract_and_process_attachments(msg, email_id_str, db) state["pdfs_processed"] += len(pdfs) if pdfs and log_entry: log_entry.processing_result = f"PDFs processed: {len(pdfs)}" log_entry.processed = True except Exception as pdf_err: state["errors"].append(f"PDF-Fehler Email {email_id_str}: {pdf_err}") db.commit() state["processed"] += 1 mail_conn.logout() def _do_scan(db: Session, state: dict, force: bool = False, since_date: Optional[str] = None, until_date: Optional[str] = None): """Kern-Scan-Logik für Strato + Gmail.""" try: cfg = get_imap_config(db) mail = connect_imap(cfg["host"], cfg["port"], cfg["user"], cfg["password"]) _scan_imap(mail, db, state, since_date, until_date) except Exception as e: import traceback state["errors"].append(f"Strato: {e}\n{traceback.format_exc()}") if GMAIL_IMAP_USER and GMAIL_IMAP_PASSWORD: try: mail = connect_imap(GMAIL_IMAP_HOST, GMAIL_IMAP_PORT, GMAIL_IMAP_USER, GMAIL_IMAP_PASSWORD) _scan_imap(mail, db, state, since_date, until_date) except Exception as e: import traceback state["errors"].append(f"Gmail: {e}\n{traceback.format_exc()}") def _run_scan_background(task_id: str, force: bool, since_date: Optional[str], until_date: Optional[str]): """Hintergrund-Job für Email-Scan (eigene DB-Session).""" global _active_scan_id db = SessionLocal() state = _scan_tasks[task_id] try: _do_scan(db, state, force, since_date, until_date) except Exception as e: state["errors"].append(str(e)) finally: state["status"] = "done" state["end_time"] = datetime.utcnow().isoformat() _active_scan_id = None db.close() @router.post("/scan-all/start") async def start_scan( background_tasks: BackgroundTasks, force: bool = False, since_date: Optional[str] = None, until_date: Optional[str] = None, db: Session = Depends(get_db), ): """Startet Email-Scan im Hintergrund. since_date/until_date: YYYY-MM-DD""" global _active_scan_id # Kein paralleler Scan if _active_scan_id and _scan_tasks.get(_active_scan_id, {}).get("status") == "running": return {"task_id": _active_scan_id, "already_running": True} task_id = str(uuid.uuid4()) _active_scan_id = task_id _scan_tasks[task_id] = { "status": "running", "processed": 0, "total": 0, "invoices_found": 0, "pdfs_processed": 0, "errors": [], "start_time": datetime.utcnow().isoformat(), "end_time": None, } if force: db.query(Invoice).delete() db.query(EmailLog).delete() db.commit() background_tasks.add_task(_run_scan_background, task_id, force, since_date, until_date) return {"task_id": task_id} @router.get("/scan-all/status/{task_id}") async def scan_status(task_id: str): """Gibt Scan-Status zurück.""" state = _scan_tasks.get(task_id) if not state: raise HTTPException(status_code=404, detail="Task nicht gefunden") return state @router.get("/classified") async def classified(format: str = "json", db: Session = Depends(get_db)): """Rechnungsmails nach Klasse: attachment | link | portal (ignore ausgeblendet).""" rows = ( db.query(EmailLog) .filter(EmailLog.mail_class.in_(["attachment", "link", "portal"])) .order_by(EmailLog.mail_class, EmailLog.email_date.desc()) .all() ) data = [ {"klasse": r.mail_class, "datum": r.email_date.strftime("%Y-%m-%d") if r.email_date else "", "absender": r.email_from, "betreff": r.email_subject, "link": r.link_url or ""} for r in rows ] if format != "xlsx": return data import io from openpyxl import Workbook from fastapi.responses import StreamingResponse wb = Workbook() wb.remove(wb.active) for k, title in (("attachment", "a) Anhang"), ("link", "b) Link"), ("portal", "c) Portal")): ws = wb.create_sheet(title) ws.append(["Datum", "Absender", "Betreff", "Link"]) for d in data: if d["klasse"] == k: ws.append([d["datum"], d["absender"], d["betreff"], d["link"]]) buf = io.BytesIO() wb.save(buf) buf.seek(0) return StreamingResponse(buf, media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", headers={"Content-Disposition": "attachment; filename=rechnungsmails.xlsx"}) @router.get("/scan-all") async def scan_all_emails(db: Session = Depends(get_db)): """Synchroner Scan (Legacy) — für direkten Aufruf ohne Polling.""" state = { "status": "running", "processed": 0, "invoices_found": 0, "pdfs_processed": 0, "errors": [], "start_time": datetime.utcnow().isoformat(), } _do_scan(db, state) state["status"] = "done" state["end_time"] = datetime.utcnow().isoformat() return state