c-stone-invoice-check/backend/routers/emails.py
2026-10-22 23:55:45 +02:00

449 lines
15 KiB
Python

"""
C-Stone Invoice Check — Email Router
IMAP Email-Verarbeitung und Filterung
"""
import imaplib
import email
import os
import re
import tempfile
import uuid
from datetime import datetime
from typing import List, Optional, Dict, Any
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
from sqlalchemy.orm import Session
from email.header import decode_header, make_header
from backend.database import get_db, SessionLocal
from backend.models import EmailLog, Invoice
from backend.routers.settings import get_imap_config
router = APIRouter()
def decode_hdr(value) -> str:
"""Header robust dekodieren (alle Segmente); kaputte Encodings (unknown-8bit) -> replace statt Abbruch."""
if not value:
return ""
try:
return str(make_header(decode_header(str(value))))
except Exception:
return str(value)
# ponytail: in-memory scan state, fine for single process
_scan_tasks: dict = {}
_active_scan_id: Optional[str] = None
# IMAP Konfiguration
IMAP_HOST = os.getenv("IMAP_HOST", "imap.strato.de")
IMAP_PORT = int(os.getenv("IMAP_PORT", 993))
IMAP_USER = os.getenv("IMAP_USER", "")
IMAP_PASSWORD = os.getenv("IMAP_PASSWORD", "")
# Gmail Konfiguration
GMAIL_IMAP_HOST = os.getenv("GMAIL_IMAP_HOST", "imap.gmail.com")
GMAIL_IMAP_PORT = int(os.getenv("GMAIL_IMAP_PORT", 993))
GMAIL_IMAP_USER = os.getenv("GMAIL_IMAP_USER", "")
GMAIL_IMAP_PASSWORD = os.getenv("GMAIL_IMAP_PASSWORD", "")
# Filter-Kriterien (Substring-Match — deckt Plural/Komposita ab)
INVOICE_KEYWORDS = [
"rechnung", "rechnungs", "invoice", "receipt", "bill", "factuur",
"gutschrift", "quittung", "beleg", "abrechnung", "mahnung",
]
# Ablagepfad (Evo-X2) - optional
INVOICE_STORAGE_PATH = os.getenv("INVOICE_STORAGE_PATH", "/tmp/invoice-storage")
def connect_imap(host: str, port: int, user: str, password: str) -> imaplib.IMAP4_SSL:
"""Verbindet mit IMAP-Server"""
mail = imaplib.IMAP4_SSL(host, port)
mail.login(user, password)
return mail
def search_invoices(mail: imaplib.IMAP4_SSL, folder: str = "INBOX") -> List[str]:
"""Sucht nach Emails mit Rechnungs-Stichworten"""
mail.select(folder)
result, data = mail.search(None, 'ALL')
if result != "OK":
return []
email_ids = data[0].split()
matching_ids = []
for email_id in email_ids:
result, msg_data = mail.fetch(email_id, '(RFC822)')
if result != "OK":
continue
msg = email.message_from_bytes(msg_data[0][1])
subject = msg.get("Subject", "")
from_addr = msg.get("From", "")
# Subject decodieren
subject = decode_hdr(subject)
# Prüfen auf Keywords
subject_lower = subject.lower()
if any(kw in subject_lower for kw in INVOICE_KEYWORDS):
matching_ids.append(email_id.decode())
return matching_ids
def extract_attachments(msg: email.message.Message) -> List[Dict[str, Any]]:
"""Extrahiert Anhänge aus Email"""
attachments = []
for part in msg.walk():
if part.get_content_maintype() == 'multipart':
continue
if part.get('Content-Disposition') is None:
continue
filename = part.get_filename()
if filename:
filename = decode_hdr(filename)
content_type = part.get_content_type()
payload = part.get_payload(decode=True)
attachments.append({
"filename": filename,
"content_type": content_type,
"data": payload,
"size": len(payload) if payload else 0
})
return attachments
def parse_email_date(date_str: str) -> Optional[datetime]:
"""Parses various email date formats"""
if not date_str:
return None
from email.utils import parsedate_to_datetime
try:
# Versuche email.utils parsedate_to_datetime
return parsedate_to_datetime(date_str)
except:
pass
# Fallback: Versuche verschiedene Formate
formats = [
"%a, %d %b %Y %H:%M:%S %z",
"%a, %d %b %Y %H:%M:%S %z (%Z)",
"%d %b %Y %H:%M:%S %z",
"%Y-%m-%d %H:%M:%S",
]
for fmt in formats:
try:
return datetime.strptime(date_str.strip(), fmt)
except:
continue
return None
@router.get("/search")
async def search_emails(db: Session = Depends(get_db)):
"""Sucht nach Rechnungs-Emails bei Strato"""
results = {
"processed": 0,
"invoices_found": 0,
"errors": []
}
try:
cfg = get_imap_config(db)
mail = connect_imap(cfg["host"], cfg["port"], cfg["user"], cfg["password"])
# Alle Ordner durchsuchen
for folder in ["INBOX", "Sent", "Gesendet"]:
try:
email_ids = search_invoices(mail, folder)
results[f"{folder}_count"] = len(email_ids)
for email_id in email_ids:
# Verarbeitung auslagern
pass
except Exception as e:
results["errors"].append(f"Folder {folder}: {str(e)}")
mail.logout()
results["processed"] = len(email_ids) if 'email_ids' in dir() else 0
except Exception as e:
results["errors"].append(f"IMAP Connection failed: {str(e)}")
return results
@router.get("/process/{email_id}")
async def process_email(email_id: str, db: Session = Depends(get_db)):
"""Verarbeitet einzelne Email"""
# TODO: Implementierung
pass
def extract_and_process_attachments(msg: email.message.Message, email_id: str, db: Session) -> List[Dict[str, Any]]:
"""Extrahiert und verarbeitet PDF-Anhänge aus Email"""
processed = []
for part in msg.walk():
if part.get_content_maintype() == 'multipart':
continue
if part.get('Content-Disposition') is None:
continue
filename = part.get_filename()
if not filename:
continue
filename = decode_hdr(filename)
content_type = part.get_content_type()
# Nur PDFs verarbeiten
if content_type != 'application/pdf' and not filename.lower().endswith('.pdf'):
continue
payload = part.get_payload(decode=True)
if not payload:
continue
# Temporäre Datei erstellen
with tempfile.NamedTemporaryFile(suffix='.pdf', delete=False) as tmp:
tmp.write(payload)
tmp_path = tmp.name
try:
# PDF verarbeiten
from backend.routers.invoices import extract_pdf_data, get_storage_path
pdf_data = extract_pdf_data(tmp_path)
if pdf_data.get("not_invoice") or "error" in pdf_data:
continue
# Duplikat-Check: gleiche email_id + filename
if db.query(Invoice).filter(Invoice.email_id == email_id, Invoice.file_name == filename).first():
continue
if pdf_data.get("amount"):
# Rechnung speichern
tax_type = pdf_data.get("tax_info", {}).get("tax_type", "unbekannt")
tax_rate = pdf_data.get("tax_info", {}).get("tax_rate", 0)
# Speicherpfad erstellen
storage_year = 2026 # Default, könnte aus Rechnungsdatum extrahiert werden
storage_path = get_storage_path(storage_year, tax_type)
# PDF in Zielverzeichnis kopieren (optional)
base_storage = get_imap_config(db)["storage_path"]
if base_storage:
os.makedirs(storage_path, exist_ok=True)
# Dateiname generieren (Slashes/Sonderzeichen ersetzen)
invoice_number = re.sub(r'[/\\:*?"<>|]', '-', pdf_data.get("invoice_number") or "unbekannt")
file_name = f"Rechnung_{invoice_number}_{email_id}.pdf"
dest_path = os.path.join(storage_path, file_name)
import shutil
shutil.copy(tmp_path, dest_path)
# In DB speichern
invoice = Invoice(
email_id=email_id,
file_path=dest_path,
file_name=file_name,
storage_year=storage_year,
storage_path=storage_path.replace(base_storage, ""),
invoice_number=invoice_number,
invoice_amount=pdf_data.get("amount"),
issuer_name=pdf_data.get("issuer"),
invoice_tax_type=tax_type,
invoice_tax_rate=tax_rate,
processed=True,
confidence_score=pdf_data.get("tax_info", {}).get("confidence", 0)
)
db.add(invoice)
db.commit()
processed.append({
"filename": filename,
"invoice_number": invoice_number,
"amount": pdf_data.get("amount"),
"tax_type": tax_type,
"saved_to": dest_path
})
finally:
# Temporäre Datei löschen
try:
os.unlink(tmp_path)
except:
pass
return processed
def _scan_imap(mail_conn, db: Session, state: dict, since_date: Optional[str] = None, until_date: Optional[str] = None):
"""Verarbeitet eine IMAP-Verbindung und aktualisiert state."""
mail_conn.select("INBOX")
# Datumsfilter als IMAP-Kriterium (Format: DD-Mon-YYYY)
criteria = []
if since_date:
fmt = datetime.strptime(since_date, "%Y-%m-%d").strftime("%d-%b-%Y")
criteria.append(f'SINCE {fmt}')
if until_date:
fmt = datetime.strptime(until_date, "%Y-%m-%d").strftime("%d-%b-%Y")
criteria.append(f'BEFORE {fmt}')
if criteria:
result, data = mail_conn.search(None, *criteria)
else:
result, data = mail_conn.search(None, 'ALL')
if result != "OK":
return
email_ids = data[0].split()
state["total"] = state.get("total", 0) + len(email_ids)
for email_id in email_ids:
email_id_str = email_id.decode()
log_entry = db.query(EmailLog).filter(EmailLog.email_id == email_id_str).first()
if log_entry and log_entry.processed:
continue
result2, msg_data = mail_conn.fetch(email_id, '(RFC822)')
if result2 != "OK":
continue
msg = email.message_from_bytes(msg_data[0][1])
subject = msg.get("Subject", "")
from_addr = msg.get("From", "")
date_str = msg.get("Date", "")
email_date = parse_email_date(date_str)
subject = decode_hdr(subject)
subject_lower = subject.lower()
matched_keywords = [kw for kw in INVOICE_KEYWORDS if kw in subject_lower]
log_entry = None
if matched_keywords:
log_entry = EmailLog(
email_id=email_id_str,
email_subject=subject,
email_from=from_addr,
email_date=email_date,
email_folder="INBOX",
matched_keywords=matched_keywords,
has_attachment=True,
processed=True,
)
db.add(log_entry)
state["invoices_found"] += 1
try:
pdfs = extract_and_process_attachments(msg, email_id_str, db)
state["pdfs_processed"] += len(pdfs)
if pdfs and log_entry:
log_entry.processing_result = f"PDFs processed: {len(pdfs)}"
log_entry.processed = True
except Exception as pdf_err:
state["errors"].append(f"PDF-Fehler Email {email_id_str}: {pdf_err}")
db.commit()
state["processed"] += 1
mail_conn.logout()
def _do_scan(db: Session, state: dict, force: bool = False, since_date: Optional[str] = None, until_date: Optional[str] = None):
"""Kern-Scan-Logik für Strato + Gmail."""
try:
cfg = get_imap_config(db)
mail = connect_imap(cfg["host"], cfg["port"], cfg["user"], cfg["password"])
_scan_imap(mail, db, state, since_date, until_date)
except Exception as e:
import traceback
state["errors"].append(f"Strato: {e}\n{traceback.format_exc()}")
if GMAIL_IMAP_USER and GMAIL_IMAP_PASSWORD:
try:
mail = connect_imap(GMAIL_IMAP_HOST, GMAIL_IMAP_PORT, GMAIL_IMAP_USER, GMAIL_IMAP_PASSWORD)
_scan_imap(mail, db, state, since_date, until_date)
except Exception as e:
import traceback
state["errors"].append(f"Gmail: {e}\n{traceback.format_exc()}")
def _run_scan_background(task_id: str, force: bool, since_date: Optional[str], until_date: Optional[str]):
"""Hintergrund-Job für Email-Scan (eigene DB-Session)."""
global _active_scan_id
db = SessionLocal()
state = _scan_tasks[task_id]
try:
_do_scan(db, state, force, since_date, until_date)
except Exception as e:
state["errors"].append(str(e))
finally:
state["status"] = "done"
state["end_time"] = datetime.utcnow().isoformat()
_active_scan_id = None
db.close()
@router.post("/scan-all/start")
async def start_scan(
background_tasks: BackgroundTasks,
force: bool = False,
since_date: Optional[str] = None,
until_date: Optional[str] = None,
db: Session = Depends(get_db),
):
"""Startet Email-Scan im Hintergrund. since_date/until_date: YYYY-MM-DD"""
global _active_scan_id
# Kein paralleler Scan
if _active_scan_id and _scan_tasks.get(_active_scan_id, {}).get("status") == "running":
return {"task_id": _active_scan_id, "already_running": True}
task_id = str(uuid.uuid4())
_active_scan_id = task_id
_scan_tasks[task_id] = {
"status": "running",
"processed": 0,
"total": 0,
"invoices_found": 0,
"pdfs_processed": 0,
"errors": [],
"start_time": datetime.utcnow().isoformat(),
"end_time": None,
}
if force:
db.query(Invoice).delete()
db.query(EmailLog).delete()
db.commit()
background_tasks.add_task(_run_scan_background, task_id, force, since_date, until_date)
return {"task_id": task_id}
@router.get("/scan-all/status/{task_id}")
async def scan_status(task_id: str):
"""Gibt Scan-Status zurück."""
state = _scan_tasks.get(task_id)
if not state:
raise HTTPException(status_code=404, detail="Task nicht gefunden")
return state
@router.get("/scan-all")
async def scan_all_emails(db: Session = Depends(get_db)):
"""Synchroner Scan (Legacy) — für direkten Aufruf ohne Polling."""
state = {
"status": "running",
"processed": 0,
"invoices_found": 0,
"pdfs_processed": 0,
"errors": [],
"start_time": datetime.utcnow().isoformat(),
}
_do_scan(db, state)
state["status"] = "done"
state["end_time"] = datetime.utcnow().isoformat()
return state