c-stone-invoice-check/backend/grabber.py

174 lines
7.2 KiB
Python

"""
Invoice Picker: Rechnungen aus Link-/Portal-Mails ziehen und für den Versand an Accountable vormerken.
Läuft unabhängig vom Alt-Scan (/emails/scan-all); Dedupe über Message-ID, Postfach wird nur gelesen (readonly).
CLI (Cron/Rückblick): python -m backend.grabber --since 2025-01-01
"""
import argparse
import email
import hashlib
import os
import re
from datetime import date, datetime, timedelta
from typing import Optional
from sqlalchemy.orm import Session
from backend.fetchers import aws
from backend.fetchers.direct import fetch_pdf
from backend.models import Outbound
from backend.routers.emails import INVOICE_KEYWORDS, classify_mail, connect_imap, decode_hdr, parse_email_date
from backend.routers.settings import get_imap_config
# ponytail: feste Ordnerliste (Strato, Stand 13.11.2026); Sent/Spam/Drafts bewusst nicht
FOLDERS = ("INBOX", "Archive", "Archive.2026", "Trash")
def _cutoff() -> Optional[date]:
"""Stichtag: Mails davor = Rückblick (Freigabe nötig). Ohne GRAB_CUTOFF geht nichts automatisch raus."""
v = os.getenv("GRAB_CUTOFF")
return datetime.strptime(v, "%Y-%m-%d").date() if v else None
def _status_for(email_date: Optional[datetime]) -> str:
c = _cutoff()
return "queued" if c and email_date and email_date.date() >= c else "pending_review"
def _save(pdf: bytes, base: str, email_date: Optional[datetime], sender: str) -> tuple:
sha = hashlib.sha256(pdf).hexdigest()
d = email_date or datetime.utcnow()
domain = re.sub(r"[^\w.-]", "", _domain(sender)) or "unbekannt"
folder = os.path.join(base, "picker", f"{d:%Y}", f"{d:%m}")
os.makedirs(folder, exist_ok=True)
path = os.path.join(folder, f"{d:%Y-%m-%d}_{domain}_{sha[:8]}.pdf")
with open(path, "wb") as f:
f.write(pdf)
return path, sha
def process_message(db: Session, msg, folder: str, base: str) -> Optional[str]:
"""Eine Mail verarbeiten -> neuer Outbound-Status oder None (übersprungen)."""
subject = decode_hdr(msg.get("Subject", ""))
if not any(kw in subject.lower() for kw in INVOICE_KEYWORDS):
return None
sender = decode_hdr(msg.get("From", ""))
email_date = parse_email_date(msg.get("Date", ""))
mid = (msg.get("Message-ID") or "").strip() or hashlib.sha256(f"{folder}{email_date}{subject}".encode()).hexdigest()
row = db.query(Outbound).filter(Outbound.message_id == mid).first()
if row and row.status != "manual": # "manual" wird bei jedem Lauf neu versucht (neue Adapter greifen rückwirkend)
return None
d = _domain(sender)
if aws.ENABLED and any(d == x or d.endswith("." + x) for x in aws.SENDER_DOMAINS): # AWS kommt über die API (sync_aws)
if row:
row.status, row.error = "via_api", None
db.commit()
return None
mail_class, url = classify_mail(msg)
if mail_class not in ("link", "portal"): # Anhänge leitet Markus bereits selbst an Accountable weiter
return None
row = row or Outbound(message_id=mid)
row.email_folder, row.email_date, row.email_from = folder, email_date, sender
row.email_subject, row.mail_class, row.link_url = subject, mail_class, url
pdf = None
if mail_class == "link":
try:
pdf = fetch_pdf(url)
row.error = None if pdf else "Link liefert kein PDF (Login nötig?)"
except Exception as e:
row.error = f"Download-Fehler: {e}"[:500]
else:
row.error = "Portal-Abruf noch nicht automatisiert" # Phase 3: Playwright-Adapter
if pdf:
_store(db, row, pdf, base)
else:
row.status = "manual"
db.add(row)
db.commit()
return row.status
def _store(db: Session, row: Outbound, pdf: bytes, base: str):
row.file_path, row.sha256 = _save(pdf, base, row.email_date, row.email_from or "")
dup = db.query(Outbound.id).filter(Outbound.sha256 == row.sha256, Outbound.id != row.id).first()
row.status = "duplicate" if dup else _status_for(row.email_date)
def sync_aws(db: Session, since: date, base: str, counts: dict):
"""AWS-Rechnungen über die Invoicing-API holen (message_id = aws-invoice:<id>)."""
known = {r[0].split(":", 1)[1] for r in db.query(Outbound.message_id)
.filter(Outbound.message_id.like("aws-invoice:%")).all()}
for inv in aws.fetch_invoices(since, known):
issued = inv["issued"].replace(tzinfo=None) if inv["issued"] else None
row = Outbound(message_id=f"aws-invoice:{inv['invoice_id']}", email_folder="api", email_date=issued,
email_from="AWS <invoicing@aws.com>", mail_class="api",
email_subject=f"{inv['entity']} Rechnung {inv['invoice_id']} ({inv['period']})")
_store(db, row, inv["pdf"], base)
db.add(row)
db.commit()
counts[row.status] = counts.get(row.status, 0) + 1
def _domain(sender: str) -> str:
m = re.search(r"@([\w.-]+)", sender)
return m[1].lower() if m else ""
def run(db: Session, since: date, state: Optional[dict] = None) -> dict:
state = state if state is not None else {}
state.setdefault("errors", [])
counts = state.setdefault("counts", {})
cfg = get_imap_config(db)
m = connect_imap(cfg["host"], cfg["port"], cfg["user"], cfg["password"])
try:
for folder in FOLDERS:
if m.select(f'"{folder}"', readonly=True)[0] != "OK":
continue
ids = m.search(None, f"SINCE {since:%d-%b-%Y}")[1][0].split()
if not ids:
continue
# erst nur Betreff holen (klein), Volltext nur bei Rechnungs-Stichwort
hdrs = [h for k in range(0, len(ids), 500)
for h in m.fetch(b",".join(ids[k:k + 500]), "(BODY.PEEK[HEADER.FIELDS (SUBJECT)])")[1]]
ids = [h[0].split()[0] for h in hdrs if isinstance(h, tuple)
and any(kw in decode_hdr(email.message_from_bytes(h[1]).get("Subject", "")).lower()
for kw in INVOICE_KEYWORDS)]
for i in ids:
try:
typ, data = m.fetch(i, "(BODY.PEEK[])")
if typ != "OK" or not data or not isinstance(data[0], tuple):
continue
status = process_message(db, email.message_from_bytes(data[0][1]), folder, cfg["storage_path"])
if status:
counts[status] = counts.get(status, 0) + 1
except Exception as e:
db.rollback()
state["errors"].append(f"{folder}/{i.decode()}: {e}"[:300])
state["processed"] = state.get("processed", 0) + 1
finally:
try:
m.logout()
except Exception:
pass
if aws.ENABLED:
try:
sync_aws(db, since, cfg["storage_path"], counts)
except Exception as e:
db.rollback()
state["errors"].append(f"AWS-API: {e}"[:300])
return state
if __name__ == "__main__":
from backend.database import SessionLocal
p = argparse.ArgumentParser()
p.add_argument("--since", help="YYYY-MM-DD (Default: vor 3 Tagen)")
a = p.parse_args()
since = datetime.strptime(a.since, "%Y-%m-%d").date() if a.since else date.today() - timedelta(days=3)
db = SessionLocal()
try:
print(run(db, since))
finally:
db.close()