Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Wd9aeH8MKgesfbsaS3kpnx
199 lines
8.2 KiB
Python
199 lines
8.2 KiB
Python
"""
|
|
Invoice Picker: Rechnungen aus Link-/Portal-Mails ziehen und für den Versand an Accountable vormerken.
|
|
Läuft unabhängig vom Alt-Scan (/emails/scan-all); Dedupe über Message-ID, Postfach wird nur gelesen (readonly).
|
|
|
|
CLI (Cron/Rückblick): python -m backend.grabber --since 2025-01-01
|
|
"""
|
|
|
|
import argparse
|
|
import email
|
|
import hashlib
|
|
import os
|
|
import re
|
|
from datetime import date, datetime, timedelta
|
|
from typing import Optional
|
|
|
|
from sqlalchemy.orm import Session
|
|
|
|
from backend.fetchers import aws
|
|
from backend.fetchers.direct import fetch_pdf
|
|
from backend.models import Outbound
|
|
from backend.routers.emails import INVOICE_KEYWORDS, classify_mail, connect_imap, decode_hdr, parse_email_date
|
|
from backend.routers.settings import get_imap_config
|
|
|
|
# ponytail: feste Ordnerliste (Strato, Stand 13.11.2026); Sent/Spam/Drafts bewusst nicht
|
|
FOLDERS = ("INBOX", "Archive", "Archive.2026", "Trash")
|
|
|
|
|
|
def _cutoff() -> Optional[date]:
|
|
"""Stichtag: Mails davor = Rückblick (Freigabe nötig). Ohne GRAB_CUTOFF geht nichts automatisch raus."""
|
|
v = os.getenv("GRAB_CUTOFF")
|
|
return datetime.strptime(v, "%Y-%m-%d").date() if v else None
|
|
|
|
|
|
def _status_for(email_date: Optional[datetime]) -> str:
|
|
c = _cutoff()
|
|
return "queued" if c and email_date and email_date.date() >= c else "pending_review"
|
|
|
|
|
|
def _save(pdf: bytes, base: str, email_date: Optional[datetime], sender: str) -> tuple:
|
|
sha = hashlib.sha256(pdf).hexdigest()
|
|
d = email_date or datetime.utcnow()
|
|
domain = re.sub(r"[^\w.-]", "", _domain(sender)) or "unbekannt"
|
|
folder = os.path.join(base, "picker", f"{d:%Y}", f"{d:%m}")
|
|
os.makedirs(folder, exist_ok=True)
|
|
path = os.path.join(folder, f"{d:%Y-%m-%d}_{domain}_{sha[:8]}.pdf")
|
|
with open(path, "wb") as f:
|
|
f.write(pdf)
|
|
return path, sha
|
|
|
|
|
|
def process_message(db: Session, msg, folder: str, base: str) -> Optional[str]:
|
|
"""Eine Mail verarbeiten -> neuer Outbound-Status oder None (übersprungen)."""
|
|
subject = decode_hdr(msg.get("Subject", ""))
|
|
if not any(kw in subject.lower() for kw in INVOICE_KEYWORDS):
|
|
return None
|
|
sender = decode_hdr(msg.get("From", ""))
|
|
email_date = parse_email_date(msg.get("Date", ""))
|
|
mid = (msg.get("Message-ID") or "").strip() or hashlib.sha256(f"{folder}{email_date}{subject}".encode()).hexdigest()
|
|
row = db.query(Outbound).filter(Outbound.message_id == mid).first()
|
|
if row and row.status != "manual": # "manual" wird bei jedem Lauf neu versucht (neue Adapter greifen rückwirkend)
|
|
return None
|
|
d = _domain(sender)
|
|
if aws.ENABLED and any(d == x or d.endswith("." + x) for x in aws.SENDER_DOMAINS): # AWS kommt über die API (sync_aws)
|
|
if row:
|
|
row.status, row.error = "via_api", None
|
|
db.commit()
|
|
return None
|
|
mail_class, url = classify_mail(msg)
|
|
if mail_class not in ("link", "portal"): # Anhänge leitet Markus bereits selbst an Accountable weiter
|
|
return None
|
|
row = row or Outbound(message_id=mid)
|
|
row.email_folder, row.email_date, row.email_from = folder, email_date, sender
|
|
row.email_subject, row.mail_class, row.link_url = subject, mail_class, url
|
|
pdf = None
|
|
if mail_class == "link":
|
|
try:
|
|
pdf = fetch_pdf(url)
|
|
row.error = None if pdf else "Link liefert kein PDF (Login nötig?)"
|
|
except Exception as e:
|
|
row.error = f"Download-Fehler: {e}"[:500]
|
|
else:
|
|
row.error = "Portal-Abruf noch nicht automatisiert" # Phase 3: Playwright-Adapter
|
|
if pdf:
|
|
_store(db, row, pdf, base)
|
|
else:
|
|
row.status = "manual"
|
|
db.add(row)
|
|
db.commit()
|
|
return row.status
|
|
|
|
|
|
def _store(db: Session, row: Outbound, pdf: bytes, base: str):
|
|
row.file_path, row.sha256 = _save(pdf, base, row.email_date, row.email_from or "")
|
|
dup = db.query(Outbound.id).filter(Outbound.sha256 == row.sha256, Outbound.id != row.id).first()
|
|
row.status = "duplicate" if dup else _status_for(row.email_date)
|
|
|
|
|
|
def sync_aws(db: Session, since: date, base: str, counts: dict):
|
|
"""AWS-Rechnungen über die Invoicing-API holen (message_id = aws-invoice:<id>)."""
|
|
known = {r[0].split(":", 1)[1] for r in db.query(Outbound.message_id)
|
|
.filter(Outbound.message_id.like("aws-invoice:%")).all()}
|
|
for inv in aws.fetch_invoices(since, known):
|
|
issued = inv["issued"].replace(tzinfo=None) if inv["issued"] else None
|
|
row = Outbound(message_id=f"aws-invoice:{inv['invoice_id']}", email_folder="api", email_date=issued,
|
|
email_from="AWS <invoicing@aws.com>", mail_class="api",
|
|
email_subject=f"{inv['entity']} Rechnung {inv['invoice_id']} ({inv['period']})")
|
|
_store(db, row, inv["pdf"], base)
|
|
db.add(row)
|
|
db.commit()
|
|
counts[row.status] = counts.get(row.status, 0) + 1
|
|
|
|
|
|
def _domain(sender: str) -> str:
|
|
m = re.search(r"@([\w.-]+)", sender)
|
|
return m[1].lower() if m else ""
|
|
|
|
|
|
def _mailboxes(db: Session):
|
|
"""Strato (Settings/Env) + optional Gmail (info@ lief bis Ende Mai 2026 über Google)."""
|
|
cfg = get_imap_config(db)
|
|
yield cfg, FOLDERS
|
|
if os.getenv("GMAIL_IMAP_PASSWORD"):
|
|
# ponytail: nur "Alle Nachrichten" (\All) statt jedes Label einzeln; Dedupe über Message-ID
|
|
yield {**cfg, "host": "imap.gmail.com", "port": 993, "user": os.getenv("GMAIL_IMAP_USER", "info@c-stone-dev.com"),
|
|
"password": os.environ["GMAIL_IMAP_PASSWORD"]}, None
|
|
|
|
|
|
def _all_mail(m) -> tuple:
|
|
for line in m.list()[1]:
|
|
s = line.decode(errors="ignore")
|
|
if "\\All" in s:
|
|
return (s.rsplit(' "/" ', 1)[-1].strip().strip('"'),)
|
|
return ("INBOX",)
|
|
|
|
|
|
def run(db: Session, since: date, state: Optional[dict] = None) -> dict:
|
|
state = state if state is not None else {}
|
|
state.setdefault("errors", [])
|
|
counts = state.setdefault("counts", {})
|
|
for cfg, folders in _mailboxes(db):
|
|
try:
|
|
_scan(db, cfg, folders, since, state, counts)
|
|
except Exception as e:
|
|
state["errors"].append(f"{cfg['host']}: {e}"[:300])
|
|
if aws.ENABLED:
|
|
try:
|
|
sync_aws(db, since, get_imap_config(db)["storage_path"], counts)
|
|
except Exception as e:
|
|
db.rollback()
|
|
state["errors"].append(f"AWS-API: {e}"[:300])
|
|
return state
|
|
|
|
|
|
def _scan(db: Session, cfg: dict, folders, since: date, state: dict, counts: dict):
|
|
m = connect_imap(cfg["host"], cfg["port"], cfg["user"], cfg["password"])
|
|
try:
|
|
for folder in folders or _all_mail(m):
|
|
if m.select(f'"{folder}"', readonly=True)[0] != "OK":
|
|
continue
|
|
ids = m.search(None, f"SINCE {since:%d-%b-%Y}")[1][0].split()
|
|
if not ids:
|
|
continue
|
|
# erst nur Betreff holen (klein), Volltext nur bei Rechnungs-Stichwort
|
|
hdrs = [h for k in range(0, len(ids), 500)
|
|
for h in m.fetch(b",".join(ids[k:k + 500]), "(BODY.PEEK[HEADER.FIELDS (SUBJECT)])")[1]]
|
|
ids = [h[0].split()[0] for h in hdrs if isinstance(h, tuple)
|
|
and any(kw in decode_hdr(email.message_from_bytes(h[1]).get("Subject", "")).lower()
|
|
for kw in INVOICE_KEYWORDS)]
|
|
for i in ids:
|
|
try:
|
|
typ, data = m.fetch(i, "(BODY.PEEK[])")
|
|
if typ != "OK" or not data or not isinstance(data[0], tuple):
|
|
continue
|
|
status = process_message(db, email.message_from_bytes(data[0][1]), folder, cfg["storage_path"])
|
|
if status:
|
|
counts[status] = counts.get(status, 0) + 1
|
|
except Exception as e:
|
|
db.rollback()
|
|
state["errors"].append(f"{folder}/{i.decode()}: {e}"[:300])
|
|
state["processed"] = state.get("processed", 0) + 1
|
|
finally:
|
|
try:
|
|
m.logout()
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
if __name__ == "__main__":
|
|
from backend.database import SessionLocal
|
|
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--since", help="YYYY-MM-DD (Default: vor 3 Tagen)")
|
|
a = p.parse_args()
|
|
since = datetime.strptime(a.since, "%Y-%m-%d").date() if a.since else date.today() - timedelta(days=3)
|
|
db = SessionLocal()
|
|
try:
|
|
print(run(db, since))
|
|
finally:
|
|
db.close()
|