449 lines
15 KiB
Python
449 lines
15 KiB
Python
"""
|
|
C-Stone Invoice Check — Email Router
|
|
IMAP Email-Verarbeitung und Filterung
|
|
"""
|
|
|
|
import imaplib
|
|
import email
|
|
import os
|
|
import re
|
|
import tempfile
|
|
import uuid
|
|
from datetime import datetime
|
|
from typing import List, Optional, Dict, Any
|
|
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
|
|
from sqlalchemy.orm import Session
|
|
from email.header import decode_header, make_header
|
|
|
|
from backend.database import get_db, SessionLocal
|
|
from backend.models import EmailLog, Invoice
|
|
from backend.routers.settings import get_imap_config
|
|
|
|
router = APIRouter()
|
|
|
|
|
|
def decode_hdr(value) -> str:
|
|
"""Header robust dekodieren (alle Segmente); kaputte Encodings (unknown-8bit) -> replace statt Abbruch."""
|
|
if not value:
|
|
return ""
|
|
try:
|
|
return str(make_header(decode_header(str(value))))
|
|
except Exception:
|
|
return str(value)
|
|
|
|
# ponytail: in-memory scan state, fine for single process
|
|
_scan_tasks: dict = {}
|
|
_active_scan_id: Optional[str] = None
|
|
|
|
# IMAP Konfiguration
|
|
IMAP_HOST = os.getenv("IMAP_HOST", "imap.strato.de")
|
|
IMAP_PORT = int(os.getenv("IMAP_PORT", 993))
|
|
IMAP_USER = os.getenv("IMAP_USER", "")
|
|
IMAP_PASSWORD = os.getenv("IMAP_PASSWORD", "")
|
|
|
|
# Gmail Konfiguration
|
|
GMAIL_IMAP_HOST = os.getenv("GMAIL_IMAP_HOST", "imap.gmail.com")
|
|
GMAIL_IMAP_PORT = int(os.getenv("GMAIL_IMAP_PORT", 993))
|
|
GMAIL_IMAP_USER = os.getenv("GMAIL_IMAP_USER", "")
|
|
GMAIL_IMAP_PASSWORD = os.getenv("GMAIL_IMAP_PASSWORD", "")
|
|
|
|
# Filter-Kriterien (Substring-Match — deckt Plural/Komposita ab)
|
|
INVOICE_KEYWORDS = [
|
|
"rechnung", "rechnungs", "invoice", "receipt", "bill", "factuur",
|
|
"gutschrift", "quittung", "beleg", "abrechnung", "mahnung",
|
|
]
|
|
|
|
# Ablagepfad (Evo-X2) - optional
|
|
INVOICE_STORAGE_PATH = os.getenv("INVOICE_STORAGE_PATH", "/tmp/invoice-storage")
|
|
|
|
|
|
def connect_imap(host: str, port: int, user: str, password: str) -> imaplib.IMAP4_SSL:
|
|
"""Verbindet mit IMAP-Server"""
|
|
mail = imaplib.IMAP4_SSL(host, port)
|
|
mail.login(user, password)
|
|
return mail
|
|
|
|
|
|
def search_invoices(mail: imaplib.IMAP4_SSL, folder: str = "INBOX") -> List[str]:
|
|
"""Sucht nach Emails mit Rechnungs-Stichworten"""
|
|
mail.select(folder)
|
|
result, data = mail.search(None, 'ALL')
|
|
if result != "OK":
|
|
return []
|
|
|
|
email_ids = data[0].split()
|
|
matching_ids = []
|
|
|
|
for email_id in email_ids:
|
|
result, msg_data = mail.fetch(email_id, '(RFC822)')
|
|
if result != "OK":
|
|
continue
|
|
|
|
msg = email.message_from_bytes(msg_data[0][1])
|
|
subject = msg.get("Subject", "")
|
|
from_addr = msg.get("From", "")
|
|
|
|
# Subject decodieren
|
|
subject = decode_hdr(subject)
|
|
|
|
# Prüfen auf Keywords
|
|
subject_lower = subject.lower()
|
|
if any(kw in subject_lower for kw in INVOICE_KEYWORDS):
|
|
matching_ids.append(email_id.decode())
|
|
|
|
return matching_ids
|
|
|
|
|
|
def extract_attachments(msg: email.message.Message) -> List[Dict[str, Any]]:
|
|
"""Extrahiert Anhänge aus Email"""
|
|
attachments = []
|
|
for part in msg.walk():
|
|
if part.get_content_maintype() == 'multipart':
|
|
continue
|
|
if part.get('Content-Disposition') is None:
|
|
continue
|
|
|
|
filename = part.get_filename()
|
|
if filename:
|
|
filename = decode_hdr(filename)
|
|
|
|
content_type = part.get_content_type()
|
|
payload = part.get_payload(decode=True)
|
|
|
|
attachments.append({
|
|
"filename": filename,
|
|
"content_type": content_type,
|
|
"data": payload,
|
|
"size": len(payload) if payload else 0
|
|
})
|
|
|
|
return attachments
|
|
|
|
|
|
def parse_email_date(date_str: str) -> Optional[datetime]:
|
|
"""Parses various email date formats"""
|
|
if not date_str:
|
|
return None
|
|
|
|
from email.utils import parsedate_to_datetime
|
|
|
|
try:
|
|
# Versuche email.utils parsedate_to_datetime
|
|
return parsedate_to_datetime(date_str)
|
|
except:
|
|
pass
|
|
|
|
# Fallback: Versuche verschiedene Formate
|
|
formats = [
|
|
"%a, %d %b %Y %H:%M:%S %z",
|
|
"%a, %d %b %Y %H:%M:%S %z (%Z)",
|
|
"%d %b %Y %H:%M:%S %z",
|
|
"%Y-%m-%d %H:%M:%S",
|
|
]
|
|
|
|
for fmt in formats:
|
|
try:
|
|
return datetime.strptime(date_str.strip(), fmt)
|
|
except:
|
|
continue
|
|
|
|
return None
|
|
|
|
|
|
@router.get("/search")
|
|
async def search_emails(db: Session = Depends(get_db)):
|
|
"""Sucht nach Rechnungs-Emails bei Strato"""
|
|
results = {
|
|
"processed": 0,
|
|
"invoices_found": 0,
|
|
"errors": []
|
|
}
|
|
|
|
try:
|
|
cfg = get_imap_config(db)
|
|
mail = connect_imap(cfg["host"], cfg["port"], cfg["user"], cfg["password"])
|
|
|
|
# Alle Ordner durchsuchen
|
|
for folder in ["INBOX", "Sent", "Gesendet"]:
|
|
try:
|
|
email_ids = search_invoices(mail, folder)
|
|
results[f"{folder}_count"] = len(email_ids)
|
|
|
|
for email_id in email_ids:
|
|
# Verarbeitung auslagern
|
|
pass
|
|
|
|
except Exception as e:
|
|
results["errors"].append(f"Folder {folder}: {str(e)}")
|
|
|
|
mail.logout()
|
|
results["processed"] = len(email_ids) if 'email_ids' in dir() else 0
|
|
|
|
except Exception as e:
|
|
results["errors"].append(f"IMAP Connection failed: {str(e)}")
|
|
|
|
return results
|
|
|
|
|
|
@router.get("/process/{email_id}")
|
|
async def process_email(email_id: str, db: Session = Depends(get_db)):
|
|
"""Verarbeitet einzelne Email"""
|
|
# TODO: Implementierung
|
|
pass
|
|
|
|
|
|
def extract_and_process_attachments(msg: email.message.Message, email_id: str, db: Session) -> List[Dict[str, Any]]:
|
|
"""Extrahiert und verarbeitet PDF-Anhänge aus Email"""
|
|
processed = []
|
|
|
|
for part in msg.walk():
|
|
if part.get_content_maintype() == 'multipart':
|
|
continue
|
|
if part.get('Content-Disposition') is None:
|
|
continue
|
|
|
|
filename = part.get_filename()
|
|
if not filename:
|
|
continue
|
|
|
|
filename = decode_hdr(filename)
|
|
|
|
content_type = part.get_content_type()
|
|
|
|
# Nur PDFs verarbeiten
|
|
if content_type != 'application/pdf' and not filename.lower().endswith('.pdf'):
|
|
continue
|
|
|
|
payload = part.get_payload(decode=True)
|
|
if not payload:
|
|
continue
|
|
|
|
# Temporäre Datei erstellen
|
|
with tempfile.NamedTemporaryFile(suffix='.pdf', delete=False) as tmp:
|
|
tmp.write(payload)
|
|
tmp_path = tmp.name
|
|
|
|
try:
|
|
# PDF verarbeiten
|
|
from backend.routers.invoices import extract_pdf_data, get_storage_path
|
|
|
|
pdf_data = extract_pdf_data(tmp_path)
|
|
|
|
if pdf_data.get("not_invoice") or "error" in pdf_data:
|
|
continue
|
|
|
|
# Duplikat-Check: gleiche email_id + filename
|
|
if db.query(Invoice).filter(Invoice.email_id == email_id, Invoice.file_name == filename).first():
|
|
continue
|
|
|
|
if pdf_data.get("amount"):
|
|
# Rechnung speichern
|
|
tax_type = pdf_data.get("tax_info", {}).get("tax_type", "unbekannt")
|
|
tax_rate = pdf_data.get("tax_info", {}).get("tax_rate", 0)
|
|
|
|
# Speicherpfad erstellen
|
|
storage_year = 2026 # Default, könnte aus Rechnungsdatum extrahiert werden
|
|
storage_path = get_storage_path(storage_year, tax_type)
|
|
|
|
# PDF in Zielverzeichnis kopieren (optional)
|
|
base_storage = get_imap_config(db)["storage_path"]
|
|
if base_storage:
|
|
os.makedirs(storage_path, exist_ok=True)
|
|
|
|
# Dateiname generieren (Slashes/Sonderzeichen ersetzen)
|
|
invoice_number = re.sub(r'[/\\:*?"<>|]', '-', pdf_data.get("invoice_number") or "unbekannt")
|
|
file_name = f"Rechnung_{invoice_number}_{email_id}.pdf"
|
|
dest_path = os.path.join(storage_path, file_name)
|
|
|
|
import shutil
|
|
shutil.copy(tmp_path, dest_path)
|
|
|
|
# In DB speichern
|
|
invoice = Invoice(
|
|
email_id=email_id,
|
|
file_path=dest_path,
|
|
file_name=file_name,
|
|
storage_year=storage_year,
|
|
storage_path=storage_path.replace(base_storage, ""),
|
|
invoice_number=invoice_number,
|
|
invoice_amount=pdf_data.get("amount"),
|
|
issuer_name=pdf_data.get("issuer"),
|
|
invoice_tax_type=tax_type,
|
|
invoice_tax_rate=tax_rate,
|
|
processed=True,
|
|
confidence_score=pdf_data.get("tax_info", {}).get("confidence", 0)
|
|
)
|
|
db.add(invoice)
|
|
db.commit()
|
|
|
|
processed.append({
|
|
"filename": filename,
|
|
"invoice_number": invoice_number,
|
|
"amount": pdf_data.get("amount"),
|
|
"tax_type": tax_type,
|
|
"saved_to": dest_path
|
|
})
|
|
|
|
finally:
|
|
# Temporäre Datei löschen
|
|
try:
|
|
os.unlink(tmp_path)
|
|
except:
|
|
pass
|
|
|
|
return processed
|
|
|
|
|
|
def _scan_imap(mail_conn, db: Session, state: dict, since_date: Optional[str] = None, until_date: Optional[str] = None):
|
|
"""Verarbeitet eine IMAP-Verbindung und aktualisiert state."""
|
|
mail_conn.select("INBOX")
|
|
# Datumsfilter als IMAP-Kriterium (Format: DD-Mon-YYYY)
|
|
criteria = []
|
|
if since_date:
|
|
fmt = datetime.strptime(since_date, "%Y-%m-%d").strftime("%d-%b-%Y")
|
|
criteria.append(f'SINCE {fmt}')
|
|
if until_date:
|
|
fmt = datetime.strptime(until_date, "%Y-%m-%d").strftime("%d-%b-%Y")
|
|
criteria.append(f'BEFORE {fmt}')
|
|
if criteria:
|
|
result, data = mail_conn.search(None, *criteria)
|
|
else:
|
|
result, data = mail_conn.search(None, 'ALL')
|
|
if result != "OK":
|
|
return
|
|
email_ids = data[0].split()
|
|
state["total"] = state.get("total", 0) + len(email_ids)
|
|
for email_id in email_ids:
|
|
email_id_str = email_id.decode()
|
|
log_entry = db.query(EmailLog).filter(EmailLog.email_id == email_id_str).first()
|
|
if log_entry and log_entry.processed:
|
|
continue
|
|
result2, msg_data = mail_conn.fetch(email_id, '(RFC822)')
|
|
if result2 != "OK":
|
|
continue
|
|
msg = email.message_from_bytes(msg_data[0][1])
|
|
subject = msg.get("Subject", "")
|
|
from_addr = msg.get("From", "")
|
|
date_str = msg.get("Date", "")
|
|
email_date = parse_email_date(date_str)
|
|
subject = decode_hdr(subject)
|
|
subject_lower = subject.lower()
|
|
matched_keywords = [kw for kw in INVOICE_KEYWORDS if kw in subject_lower]
|
|
log_entry = None
|
|
if matched_keywords:
|
|
log_entry = EmailLog(
|
|
email_id=email_id_str,
|
|
email_subject=subject,
|
|
email_from=from_addr,
|
|
email_date=email_date,
|
|
email_folder="INBOX",
|
|
matched_keywords=matched_keywords,
|
|
has_attachment=True,
|
|
processed=True,
|
|
)
|
|
db.add(log_entry)
|
|
state["invoices_found"] += 1
|
|
try:
|
|
pdfs = extract_and_process_attachments(msg, email_id_str, db)
|
|
state["pdfs_processed"] += len(pdfs)
|
|
if pdfs and log_entry:
|
|
log_entry.processing_result = f"PDFs processed: {len(pdfs)}"
|
|
log_entry.processed = True
|
|
except Exception as pdf_err:
|
|
state["errors"].append(f"PDF-Fehler Email {email_id_str}: {pdf_err}")
|
|
db.commit()
|
|
state["processed"] += 1
|
|
mail_conn.logout()
|
|
|
|
|
|
def _do_scan(db: Session, state: dict, force: bool = False, since_date: Optional[str] = None, until_date: Optional[str] = None):
|
|
"""Kern-Scan-Logik für Strato + Gmail."""
|
|
try:
|
|
cfg = get_imap_config(db)
|
|
mail = connect_imap(cfg["host"], cfg["port"], cfg["user"], cfg["password"])
|
|
_scan_imap(mail, db, state, since_date, until_date)
|
|
except Exception as e:
|
|
import traceback
|
|
state["errors"].append(f"Strato: {e}\n{traceback.format_exc()}")
|
|
if GMAIL_IMAP_USER and GMAIL_IMAP_PASSWORD:
|
|
try:
|
|
mail = connect_imap(GMAIL_IMAP_HOST, GMAIL_IMAP_PORT, GMAIL_IMAP_USER, GMAIL_IMAP_PASSWORD)
|
|
_scan_imap(mail, db, state, since_date, until_date)
|
|
except Exception as e:
|
|
import traceback
|
|
state["errors"].append(f"Gmail: {e}\n{traceback.format_exc()}")
|
|
|
|
|
|
def _run_scan_background(task_id: str, force: bool, since_date: Optional[str], until_date: Optional[str]):
|
|
"""Hintergrund-Job für Email-Scan (eigene DB-Session)."""
|
|
global _active_scan_id
|
|
db = SessionLocal()
|
|
state = _scan_tasks[task_id]
|
|
try:
|
|
_do_scan(db, state, force, since_date, until_date)
|
|
except Exception as e:
|
|
state["errors"].append(str(e))
|
|
finally:
|
|
state["status"] = "done"
|
|
state["end_time"] = datetime.utcnow().isoformat()
|
|
_active_scan_id = None
|
|
db.close()
|
|
|
|
|
|
@router.post("/scan-all/start")
|
|
async def start_scan(
|
|
background_tasks: BackgroundTasks,
|
|
force: bool = False,
|
|
since_date: Optional[str] = None,
|
|
until_date: Optional[str] = None,
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Startet Email-Scan im Hintergrund. since_date/until_date: YYYY-MM-DD"""
|
|
global _active_scan_id
|
|
# Kein paralleler Scan
|
|
if _active_scan_id and _scan_tasks.get(_active_scan_id, {}).get("status") == "running":
|
|
return {"task_id": _active_scan_id, "already_running": True}
|
|
task_id = str(uuid.uuid4())
|
|
_active_scan_id = task_id
|
|
_scan_tasks[task_id] = {
|
|
"status": "running",
|
|
"processed": 0,
|
|
"total": 0,
|
|
"invoices_found": 0,
|
|
"pdfs_processed": 0,
|
|
"errors": [],
|
|
"start_time": datetime.utcnow().isoformat(),
|
|
"end_time": None,
|
|
}
|
|
if force:
|
|
db.query(Invoice).delete()
|
|
db.query(EmailLog).delete()
|
|
db.commit()
|
|
background_tasks.add_task(_run_scan_background, task_id, force, since_date, until_date)
|
|
return {"task_id": task_id}
|
|
|
|
|
|
@router.get("/scan-all/status/{task_id}")
|
|
async def scan_status(task_id: str):
|
|
"""Gibt Scan-Status zurück."""
|
|
state = _scan_tasks.get(task_id)
|
|
if not state:
|
|
raise HTTPException(status_code=404, detail="Task nicht gefunden")
|
|
return state
|
|
|
|
|
|
@router.get("/scan-all")
|
|
async def scan_all_emails(db: Session = Depends(get_db)):
|
|
"""Synchroner Scan (Legacy) — für direkten Aufruf ohne Polling."""
|
|
state = {
|
|
"status": "running",
|
|
"processed": 0,
|
|
"invoices_found": 0,
|
|
"pdfs_processed": 0,
|
|
"errors": [],
|
|
"start_time": datetime.utcnow().isoformat(),
|
|
}
|
|
_do_scan(db, state)
|
|
state["status"] = "done"
|
|
state["end_time"] = datetime.utcnow().isoformat()
|
|
return state
|