Compare commits
No commits in common. "0a6c259c7f0f98da14b53664df3b29ca094137e6" and "54563f355e5705323c100be05f2ed9d3f240c44c" have entirely different histories.
0a6c259c7f
...
54563f355e
12 changed files with 9 additions and 391 deletions
|
|
@ -1,18 +0,0 @@
|
||||||
"""Stufe 2: Rechnungs-PDF per Direktlink laden (Stripe, Paddle, ThriveCart, *.pdf)."""
|
|
||||||
|
|
||||||
import urllib.request
|
|
||||||
from typing import Optional
|
|
||||||
|
|
||||||
MAX_BYTES = 20 * 1024 * 1024
|
|
||||||
|
|
||||||
|
|
||||||
def fetch_pdf(url: str, timeout: int = 30) -> Optional[bytes]:
|
|
||||||
"""-> PDF-Bytes oder None (kein PDF, z.B. HTML-Login-Seite). URL ist bereits per DIRECT_LINK geprüft (nur https)."""
|
|
||||||
if not url.lower().startswith("https://"):
|
|
||||||
return None
|
|
||||||
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 (C-Stone Invoice Picker)"})
|
|
||||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
|
||||||
data = r.read(MAX_BYTES + 1)
|
|
||||||
if len(data) > MAX_BYTES or not data.startswith(b"%PDF-"):
|
|
||||||
return None
|
|
||||||
return data
|
|
||||||
|
|
@ -1,135 +0,0 @@
|
||||||
"""
|
|
||||||
Invoice Picker: Rechnungen aus Link-/Portal-Mails ziehen und für den Versand an Accountable vormerken.
|
|
||||||
Läuft unabhängig vom Alt-Scan (/emails/scan-all); Dedupe über Message-ID, Postfach wird nur gelesen (readonly).
|
|
||||||
|
|
||||||
CLI (Cron/Rückblick): python -m backend.grabber --since 2025-01-01
|
|
||||||
"""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import email
|
|
||||||
import hashlib
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
from datetime import date, datetime, timedelta
|
|
||||||
from typing import Optional
|
|
||||||
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from backend.fetchers.direct import fetch_pdf
|
|
||||||
from backend.models import Outbound
|
|
||||||
from backend.routers.emails import INVOICE_KEYWORDS, classify_mail, connect_imap, decode_hdr, parse_email_date
|
|
||||||
from backend.routers.settings import get_imap_config
|
|
||||||
|
|
||||||
# ponytail: feste Ordnerliste (Strato, Stand 13.11.2026); Sent/Spam/Drafts bewusst nicht
|
|
||||||
FOLDERS = ("INBOX", "Archive", "Archive.2026", "Trash")
|
|
||||||
|
|
||||||
|
|
||||||
def _cutoff() -> Optional[date]:
|
|
||||||
"""Stichtag: Mails davor = Rückblick (Freigabe nötig). Ohne GRAB_CUTOFF geht nichts automatisch raus."""
|
|
||||||
v = os.getenv("GRAB_CUTOFF")
|
|
||||||
return datetime.strptime(v, "%Y-%m-%d").date() if v else None
|
|
||||||
|
|
||||||
|
|
||||||
def _status_for(email_date: Optional[datetime]) -> str:
|
|
||||||
c = _cutoff()
|
|
||||||
return "queued" if c and email_date and email_date.date() >= c else "pending_review"
|
|
||||||
|
|
||||||
|
|
||||||
def _save(pdf: bytes, base: str, email_date: Optional[datetime], sender: str) -> tuple:
|
|
||||||
sha = hashlib.sha256(pdf).hexdigest()
|
|
||||||
d = email_date or datetime.utcnow()
|
|
||||||
domain = re.sub(r"[^\w.-]", "", (re.search(r"@([\w.-]+)", sender) or [None, "unbekannt"])[1])
|
|
||||||
folder = os.path.join(base, "picker", f"{d:%Y}", f"{d:%m}")
|
|
||||||
os.makedirs(folder, exist_ok=True)
|
|
||||||
path = os.path.join(folder, f"{d:%Y-%m-%d}_{domain}_{sha[:8]}.pdf")
|
|
||||||
with open(path, "wb") as f:
|
|
||||||
f.write(pdf)
|
|
||||||
return path, sha
|
|
||||||
|
|
||||||
|
|
||||||
def process_message(db: Session, msg, folder: str, base: str) -> Optional[str]:
|
|
||||||
"""Eine Mail verarbeiten -> neuer Outbound-Status oder None (übersprungen)."""
|
|
||||||
subject = decode_hdr(msg.get("Subject", ""))
|
|
||||||
if not any(kw in subject.lower() for kw in INVOICE_KEYWORDS):
|
|
||||||
return None
|
|
||||||
sender = decode_hdr(msg.get("From", ""))
|
|
||||||
email_date = parse_email_date(msg.get("Date", ""))
|
|
||||||
mid = (msg.get("Message-ID") or "").strip() or hashlib.sha256(f"{folder}{email_date}{subject}".encode()).hexdigest()
|
|
||||||
if db.query(Outbound.id).filter(Outbound.message_id == mid).first():
|
|
||||||
return None
|
|
||||||
mail_class, url = classify_mail(msg)
|
|
||||||
if mail_class not in ("link", "portal"): # Anhänge leitet Markus bereits selbst an Accountable weiter
|
|
||||||
return None
|
|
||||||
row = Outbound(message_id=mid, email_folder=folder, email_date=email_date, email_from=sender,
|
|
||||||
email_subject=subject, mail_class=mail_class, link_url=url)
|
|
||||||
pdf = None
|
|
||||||
if mail_class == "link":
|
|
||||||
try:
|
|
||||||
pdf = fetch_pdf(url)
|
|
||||||
row.error = None if pdf else "Link liefert kein PDF (Login nötig?)"
|
|
||||||
except Exception as e:
|
|
||||||
row.error = f"Download-Fehler: {e}"[:500]
|
|
||||||
else:
|
|
||||||
row.error = "Portal-Abruf noch nicht automatisiert" # Phase 3: Playwright-Adapter
|
|
||||||
if pdf:
|
|
||||||
row.file_path, row.sha256 = _save(pdf, base, email_date, sender)
|
|
||||||
dup = db.query(Outbound.id).filter(Outbound.sha256 == row.sha256).first()
|
|
||||||
row.status = "duplicate" if dup else _status_for(email_date)
|
|
||||||
else:
|
|
||||||
row.status = "manual"
|
|
||||||
db.add(row)
|
|
||||||
db.commit()
|
|
||||||
return row.status
|
|
||||||
|
|
||||||
|
|
||||||
def run(db: Session, since: date, state: Optional[dict] = None) -> dict:
|
|
||||||
state = state if state is not None else {}
|
|
||||||
state.setdefault("errors", [])
|
|
||||||
counts = state.setdefault("counts", {})
|
|
||||||
cfg = get_imap_config(db)
|
|
||||||
m = connect_imap(cfg["host"], cfg["port"], cfg["user"], cfg["password"])
|
|
||||||
try:
|
|
||||||
for folder in FOLDERS:
|
|
||||||
if m.select(f'"{folder}"', readonly=True)[0] != "OK":
|
|
||||||
continue
|
|
||||||
ids = m.search(None, f"SINCE {since:%d-%b-%Y}")[1][0].split()
|
|
||||||
if not ids:
|
|
||||||
continue
|
|
||||||
# erst nur Betreff holen (klein), Volltext nur bei Rechnungs-Stichwort
|
|
||||||
hdrs = [h for k in range(0, len(ids), 500)
|
|
||||||
for h in m.fetch(b",".join(ids[k:k + 500]), "(BODY.PEEK[HEADER.FIELDS (SUBJECT)])")[1]]
|
|
||||||
ids = [h[0].split()[0] for h in hdrs if isinstance(h, tuple)
|
|
||||||
and any(kw in decode_hdr(email.message_from_bytes(h[1]).get("Subject", "")).lower()
|
|
||||||
for kw in INVOICE_KEYWORDS)]
|
|
||||||
for i in ids:
|
|
||||||
try:
|
|
||||||
typ, data = m.fetch(i, "(BODY.PEEK[])")
|
|
||||||
if typ != "OK" or not data or not isinstance(data[0], tuple):
|
|
||||||
continue
|
|
||||||
status = process_message(db, email.message_from_bytes(data[0][1]), folder, cfg["storage_path"])
|
|
||||||
if status:
|
|
||||||
counts[status] = counts.get(status, 0) + 1
|
|
||||||
except Exception as e:
|
|
||||||
db.rollback()
|
|
||||||
state["errors"].append(f"{folder}/{i.decode()}: {e}"[:300])
|
|
||||||
state["processed"] = state.get("processed", 0) + 1
|
|
||||||
finally:
|
|
||||||
try:
|
|
||||||
m.logout()
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
return state
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
from backend.database import SessionLocal
|
|
||||||
|
|
||||||
p = argparse.ArgumentParser()
|
|
||||||
p.add_argument("--since", help="YYYY-MM-DD (Default: vor 3 Tagen)")
|
|
||||||
a = p.parse_args()
|
|
||||||
since = datetime.strptime(a.since, "%Y-%m-%d").date() if a.since else date.today() - timedelta(days=3)
|
|
||||||
db = SessionLocal()
|
|
||||||
try:
|
|
||||||
print(run(db, since))
|
|
||||||
finally:
|
|
||||||
db.close()
|
|
||||||
|
|
@ -1,51 +0,0 @@
|
||||||
"""Versand gezogener Rechnungen an Accountable (Beleg-Eingangsadresse) per SMTP."""
|
|
||||||
|
|
||||||
import os
|
|
||||||
import smtplib
|
|
||||||
from datetime import datetime
|
|
||||||
from email.message import EmailMessage
|
|
||||||
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from backend.models import Outbound
|
|
||||||
|
|
||||||
|
|
||||||
def _cfg() -> dict:
|
|
||||||
return {
|
|
||||||
"host": os.getenv("SMTP_HOST", "smtp.strato.de"),
|
|
||||||
"port": int(os.getenv("SMTP_PORT", "465")),
|
|
||||||
"user": os.getenv("SMTP_USER") or os.getenv("IMAP_USER", ""),
|
|
||||||
"password": os.getenv("SMTP_PASSWORD") or os.getenv("IMAP_PASSWORD", ""),
|
|
||||||
"to": os.getenv("ACCOUNTABLE_INBOX", ""),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def send_queued(db: Session) -> dict:
|
|
||||||
"""Alle Outbound-Zeilen mit status=queued senden (eine Mail pro Rechnung). Ohne ACCOUNTABLE_INBOX: nichts."""
|
|
||||||
c = _cfg()
|
|
||||||
if not c["to"]:
|
|
||||||
return {"sent": 0, "failed": 0, "skipped": "ACCOUNTABLE_INBOX fehlt"}
|
|
||||||
rows = db.query(Outbound).filter(Outbound.status == "queued").all()
|
|
||||||
sent = failed = 0
|
|
||||||
if not rows:
|
|
||||||
return {"sent": 0, "failed": 0}
|
|
||||||
with smtplib.SMTP_SSL(c["host"], c["port"], timeout=60) as s:
|
|
||||||
s.login(c["user"], c["password"])
|
|
||||||
for r in rows:
|
|
||||||
try:
|
|
||||||
msg = EmailMessage()
|
|
||||||
msg["From"], msg["To"] = c["user"], c["to"]
|
|
||||||
msg["Subject"] = f"Rechnung: {r.email_subject or os.path.basename(r.file_path)}"[:200]
|
|
||||||
msg.set_content(f"Automatisch abgerufen (Invoice Picker).\nOriginal-Mail: {r.email_date:%d.%m.%Y} von {r.email_from}"
|
|
||||||
if r.email_date else "Automatisch abgerufen (Invoice Picker).")
|
|
||||||
with open(r.file_path, "rb") as f:
|
|
||||||
msg.add_attachment(f.read(), maintype="application", subtype="pdf",
|
|
||||||
filename=os.path.basename(r.file_path))
|
|
||||||
s.send_message(msg)
|
|
||||||
r.status, r.sent_at, r.error = "sent", datetime.utcnow(), None
|
|
||||||
sent += 1
|
|
||||||
except Exception as e:
|
|
||||||
r.status, r.error = "failed", f"Versand: {e}"[:500]
|
|
||||||
failed += 1
|
|
||||||
db.commit()
|
|
||||||
return {"sent": sent, "failed": failed}
|
|
||||||
|
|
@ -12,10 +12,10 @@ from sqlalchemy import text
|
||||||
from backend.database import get_db, engine, Base
|
from backend.database import get_db, engine, Base
|
||||||
|
|
||||||
# Models importieren für Alembic
|
# Models importieren für Alembic
|
||||||
from backend.models import Invoice, InvoiceDetail, Outbound, Base as ModelsBase
|
from backend.models import Invoice, InvoiceDetail, Base as ModelsBase
|
||||||
|
|
||||||
# Routers
|
# Routers
|
||||||
from backend.routers import emails, invoices, auth, outbound
|
from backend.routers import emails, invoices, auth
|
||||||
from backend.routers import settings as settings_router
|
from backend.routers import settings as settings_router
|
||||||
|
|
||||||
# App erstellen
|
# App erstellen
|
||||||
|
|
@ -76,7 +76,6 @@ app.include_router(auth.router, prefix="/auth", tags=["authentication"])
|
||||||
app.include_router(emails.router, prefix="/emails", tags=["emails"])
|
app.include_router(emails.router, prefix="/emails", tags=["emails"])
|
||||||
app.include_router(invoices.router, prefix="/invoices", tags=["invoices"])
|
app.include_router(invoices.router, prefix="/invoices", tags=["invoices"])
|
||||||
app.include_router(settings_router.router, prefix="/settings", tags=["settings"])
|
app.include_router(settings_router.router, prefix="/settings", tags=["settings"])
|
||||||
app.include_router(outbound.router, prefix="/outbound", tags=["invoice-picker"])
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
|
||||||
|
|
@ -105,27 +105,6 @@ class InvoiceDetail(Base):
|
||||||
return f"<InvoiceDetail {self.invoice_id}>"
|
return f"<InvoiceDetail {self.invoice_id}>"
|
||||||
|
|
||||||
|
|
||||||
class Outbound(Base):
|
|
||||||
"""Rechnung aus Link-/Portal-Mail (Invoice Picker) -> Versand an Accountable.
|
|
||||||
status: manual (Abruf fehlgeschlagen/Portal) | duplicate | pending_review (Rückblick) | queued | sent | failed"""
|
|
||||||
__tablename__ = "outbound"
|
|
||||||
|
|
||||||
id = Column(Integer, primary_key=True, index=True)
|
|
||||||
message_id = Column(String, unique=True, index=True) # Message-ID-Header: stabil über Ordner/Verschieben
|
|
||||||
email_folder = Column(String, nullable=True)
|
|
||||||
email_date = Column(DateTime, nullable=True)
|
|
||||||
email_from = Column(String, nullable=True)
|
|
||||||
email_subject = Column(String, nullable=True)
|
|
||||||
mail_class = Column(String, nullable=True) # "link" | "portal"
|
|
||||||
link_url = Column(Text, nullable=True)
|
|
||||||
file_path = Column(String, nullable=True)
|
|
||||||
sha256 = Column(String, index=True, nullable=True)
|
|
||||||
status = Column(String, index=True)
|
|
||||||
error = Column(Text, nullable=True)
|
|
||||||
sent_at = Column(DateTime, nullable=True)
|
|
||||||
created_at = Column(DateTime, default=datetime.utcnow)
|
|
||||||
|
|
||||||
|
|
||||||
class Setting(Base):
|
class Setting(Base):
|
||||||
__tablename__ = "settings"
|
__tablename__ = "settings"
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,6 @@ IMAP Email-Verarbeitung und Filterung
|
||||||
|
|
||||||
import imaplib
|
import imaplib
|
||||||
import email
|
import email
|
||||||
import html
|
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import tempfile
|
import tempfile
|
||||||
|
|
@ -59,23 +58,16 @@ INVOICE_STORAGE_PATH = os.getenv("INVOICE_STORAGE_PATH", "/tmp/invoice-storage")
|
||||||
|
|
||||||
|
|
||||||
LINK_RE = re.compile(r"https?://[^\s\"'<>)\]]+", re.I)
|
LINK_RE = re.compile(r"https?://[^\s\"'<>)\]]+", re.I)
|
||||||
# "link" = PDF ohne Login abrufbar. Gleichzeitig Allowlist für den Server-Download (nur https, kein beliebiger Host).
|
LINK_HINT = re.compile(r"invoice|rechnung|receipt|beleg|billing|download|\.pdf|stripe\.com|paddle\.com", re.I)
|
||||||
DIRECT_LINK = re.compile(
|
|
||||||
r"^https://((pay|invoice)\.stripe\.com|invoices\.thrivecart\.com|store\.payproglobal\.com|[\w-]+\.paddle\.com)/"
|
|
||||||
r"|^https://\S+\.pdf([?#]|$)",
|
|
||||||
re.I,
|
|
||||||
)
|
|
||||||
# Rechnungs-Link hinter Login (Konsole, Kundenportal) -> "portal"
|
|
||||||
LINK_HINT = re.compile(r"invoice|rechnung|receipt|beleg|billing|download|\.pdf", re.I)
|
|
||||||
PORTAL_HINT = re.compile(
|
PORTAL_HINT = re.compile(
|
||||||
r"in ihrem (kunden)?(konto|portal)|im (kunden)?portal|your account|sign in|log ?in|anmelden|einloggen|"
|
r"in ihrem (kunden)?(konto|portal)|im (kunden)?portal|your account|sign in|log ?in|anmelden|einloggen|"
|
||||||
r"view (your )?(invoice|receipt|bill)|rechnung (ist )?(verfügbar|bereit)|invoice is available|zahlungsbeleg",
|
r"view (your )?(invoice|receipt|bill)|rechnung (ist )?(verfügbar|bereit)|invoice is available",
|
||||||
re.I,
|
re.I,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def classify_mail(msg) -> tuple:
|
def classify_mail(msg) -> tuple:
|
||||||
"""-> (mail_class, link_url). Reihenfolge: Anhang > Direktlink > Portal (Login-Link/Hinweis) > ignore."""
|
"""-> (mail_class, link_url). Reihenfolge: Anhang > Link > Portal-Hinweis > ignore."""
|
||||||
text = ""
|
text = ""
|
||||||
for part in msg.walk():
|
for part in msg.walk():
|
||||||
fn = part.get_filename()
|
fn = part.get_filename()
|
||||||
|
|
@ -85,11 +77,10 @@ def classify_mail(msg) -> tuple:
|
||||||
payload = part.get_payload(decode=True) or b""
|
payload = part.get_payload(decode=True) or b""
|
||||||
text += payload.decode(part.get_content_charset() or "utf-8", "replace") + "\n"
|
text += payload.decode(part.get_content_charset() or "utf-8", "replace") + "\n"
|
||||||
# ponytail: Regex statt HTML-Parser; Footer-Links ("invoice" im Pfad) können False-Positives liefern
|
# ponytail: Regex statt HTML-Parser; Footer-Links ("invoice" im Pfad) können False-Positives liefern
|
||||||
urls = [html.unescape(u) for u in LINK_RE.findall(text)]
|
url = next((u for u in LINK_RE.findall(text) if LINK_HINT.search(u)), None)
|
||||||
url = next((u for u in urls if DIRECT_LINK.search(u)), None)
|
|
||||||
if url:
|
if url:
|
||||||
return "link", url
|
return "link", url
|
||||||
if any(LINK_HINT.search(u) for u in urls) or PORTAL_HINT.search(re.sub(r"<[^>]+>", " ", text)):
|
if PORTAL_HINT.search(re.sub(r"<[^>]+>", " ", text)):
|
||||||
return "portal", None
|
return "portal", None
|
||||||
return "ignore", None
|
return "ignore", None
|
||||||
|
|
||||||
|
|
@ -118,7 +109,7 @@ def search_invoices(mail: imaplib.IMAP4_SSL, folder: str = "INBOX") -> List[str]
|
||||||
|
|
||||||
msg = email.message_from_bytes(msg_data[0][1])
|
msg = email.message_from_bytes(msg_data[0][1])
|
||||||
subject = msg.get("Subject", "")
|
subject = msg.get("Subject", "")
|
||||||
from_addr = decode_hdr(msg.get("From", ""))
|
from_addr = msg.get("From", "")
|
||||||
|
|
||||||
# Subject decodieren
|
# Subject decodieren
|
||||||
subject = decode_hdr(subject)
|
subject = decode_hdr(subject)
|
||||||
|
|
@ -364,7 +355,7 @@ def _scan_imap(mail_conn, db: Session, state: dict, since_date: Optional[str] =
|
||||||
db.commit()
|
db.commit()
|
||||||
continue
|
continue
|
||||||
subject = msg.get("Subject", "")
|
subject = msg.get("Subject", "")
|
||||||
from_addr = decode_hdr(msg.get("From", ""))
|
from_addr = msg.get("From", "")
|
||||||
date_str = msg.get("Date", "")
|
date_str = msg.get("Date", "")
|
||||||
email_date = parse_email_date(date_str)
|
email_date = parse_email_date(date_str)
|
||||||
subject = decode_hdr(subject)
|
subject = decode_hdr(subject)
|
||||||
|
|
|
||||||
|
|
@ -1,97 +0,0 @@
|
||||||
"""Invoice Picker: gezogene Rechnungen ansehen, Rückblick freigeben, an Accountable senden."""
|
|
||||||
|
|
||||||
import io
|
|
||||||
import os
|
|
||||||
import uuid
|
|
||||||
from datetime import datetime
|
|
||||||
from typing import List, Optional
|
|
||||||
|
|
||||||
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
|
|
||||||
from fastapi.responses import FileResponse, StreamingResponse
|
|
||||||
from pydantic import BaseModel
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from backend import grabber, mailer
|
|
||||||
from backend.database import SessionLocal, get_db
|
|
||||||
from backend.models import Outbound
|
|
||||||
|
|
||||||
router = APIRouter()
|
|
||||||
_runs: dict = {} # ponytail: in-memory Laufstatus, ein Prozess
|
|
||||||
|
|
||||||
|
|
||||||
def _bg_run(run_id: str, since):
|
|
||||||
db = SessionLocal()
|
|
||||||
try:
|
|
||||||
grabber.run(db, since, _runs[run_id])
|
|
||||||
_runs[run_id]["send"] = mailer.send_queued(db)
|
|
||||||
except Exception as e:
|
|
||||||
_runs[run_id]["errors"].append(str(e))
|
|
||||||
finally:
|
|
||||||
_runs[run_id]["status"] = "done"
|
|
||||||
db.close()
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/run")
|
|
||||||
async def start_run(background_tasks: BackgroundTasks, since: str):
|
|
||||||
"""Postfach ab since (YYYY-MM-DD) prüfen, Rechnungen ziehen, freigegebene/neue senden."""
|
|
||||||
if any(r["status"] == "running" for r in _runs.values()):
|
|
||||||
raise HTTPException(409, "Lauf aktiv")
|
|
||||||
run_id = str(uuid.uuid4())
|
|
||||||
_runs[run_id] = {"status": "running", "errors": [], "counts": {}, "start": datetime.utcnow().isoformat()}
|
|
||||||
background_tasks.add_task(_bg_run, run_id, datetime.strptime(since, "%Y-%m-%d").date())
|
|
||||||
return {"run_id": run_id}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/run/{run_id}")
|
|
||||||
async def run_status(run_id: str):
|
|
||||||
if run_id not in _runs:
|
|
||||||
raise HTTPException(404, "Lauf nicht gefunden")
|
|
||||||
return _runs[run_id]
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("")
|
|
||||||
async def list_outbound(status: Optional[str] = None, format: str = "json", db: Session = Depends(get_db)):
|
|
||||||
q = db.query(Outbound)
|
|
||||||
if status:
|
|
||||||
q = q.filter(Outbound.status == status)
|
|
||||||
rows = q.order_by(Outbound.email_date.desc()).all()
|
|
||||||
data = [{"id": r.id, "status": r.status, "datum": f"{r.email_date:%Y-%m-%d}" if r.email_date else "",
|
|
||||||
"absender": r.email_from, "betreff": r.email_subject, "klasse": r.mail_class,
|
|
||||||
"link": r.link_url or "", "datei": os.path.basename(r.file_path or ""), "fehler": r.error or ""}
|
|
||||||
for r in rows]
|
|
||||||
if format != "xlsx":
|
|
||||||
return data
|
|
||||||
from openpyxl import Workbook
|
|
||||||
wb = Workbook()
|
|
||||||
ws = wb.active
|
|
||||||
ws.append(list(data[0].keys()) if data else ["leer"])
|
|
||||||
for d in data:
|
|
||||||
ws.append(list(d.values()))
|
|
||||||
buf = io.BytesIO()
|
|
||||||
wb.save(buf)
|
|
||||||
buf.seek(0)
|
|
||||||
return StreamingResponse(buf, media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
||||||
headers={"Content-Disposition": "attachment; filename=invoice-picker.xlsx"})
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/{oid}/pdf")
|
|
||||||
async def get_pdf(oid: int, db: Session = Depends(get_db)):
|
|
||||||
r = db.get(Outbound, oid)
|
|
||||||
if not r or not r.file_path or not os.path.exists(r.file_path):
|
|
||||||
raise HTTPException(404, "kein PDF")
|
|
||||||
return FileResponse(r.file_path, media_type="application/pdf")
|
|
||||||
|
|
||||||
|
|
||||||
class Approve(BaseModel):
|
|
||||||
ids: Optional[List[int]] = None # None = alle wartenden
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/approve")
|
|
||||||
async def approve(body: Approve, db: Session = Depends(get_db)):
|
|
||||||
"""Rückblick freigeben (pending_review -> queued) und sofort senden."""
|
|
||||||
q = db.query(Outbound).filter(Outbound.status == "pending_review")
|
|
||||||
if body.ids is not None:
|
|
||||||
q = q.filter(Outbound.id.in_(body.ids))
|
|
||||||
n = q.update({Outbound.status: "queued"}, synchronize_session=False)
|
|
||||||
db.commit()
|
|
||||||
return {"approved": n, **mailer.send_queued(db)}
|
|
||||||
|
|
@ -14,12 +14,3 @@ def test_classify():
|
||||||
portal = "Content-Type: text/plain\n\nIhre Rechnung ist verfügbar. Bitte in Ihrem Konto herunterladen."
|
portal = "Content-Type: text/plain\n\nIhre Rechnung ist verfügbar. Bitte in Ihrem Konto herunterladen."
|
||||||
assert classify_mail(_m(portal)) == ("portal", None)
|
assert classify_mail(_m(portal)) == ("portal", None)
|
||||||
assert classify_mail(_m("Content-Type: text/plain\n\nHallo"))[0] == "ignore"
|
assert classify_mail(_m("Content-Type: text/plain\n\nHallo"))[0] == "ignore"
|
||||||
# Login-/Konsolen-Links sind kein Direktdownload -> portal
|
|
||||||
aws = "Content-Type: text/plain\n\nYour bill: https://console.aws.amazon.com/billing/home#/bills"
|
|
||||||
assert classify_mail(_m(aws)) == ("portal", None)
|
|
||||||
assert classify_mail(_m("Content-Type: text/plain\n\nIhr Zahlungsbeleg liegt bereit."))[0] == "portal"
|
|
||||||
# http (kein TLS) wird nie direkt geladen
|
|
||||||
assert classify_mail(_m("Content-Type: text/plain\n\nRechnung http://x.example/r.pdf"))[0] == "portal"
|
|
||||||
# HTML-Entities im Link werden aufgelöst
|
|
||||||
h = 'Content-Type: text/html\n\n<a href="https://x.example/r.pdf?a=1&b=2">Rechnung</a>'
|
|
||||||
assert classify_mail(_m(h)) == ("link", "https://x.example/r.pdf?a=1&b=2")
|
|
||||||
|
|
|
||||||
|
|
@ -1,34 +0,0 @@
|
||||||
import email
|
|
||||||
|
|
||||||
from sqlalchemy import create_engine
|
|
||||||
from sqlalchemy.orm import sessionmaker
|
|
||||||
|
|
||||||
from backend import grabber
|
|
||||||
from backend.models import Base, Outbound
|
|
||||||
|
|
||||||
LINK = ("Message-ID: <{mid}>\nSubject: Ihre Rechnung\nFrom: Shop <billing@shop.example>\n"
|
|
||||||
"Date: {date}\nContent-Type: text/plain\n\nDownload: https://pay.stripe.com/invoice/x/pdf")
|
|
||||||
PORTAL = ("Message-ID: <p1>\nSubject: Rechnung verfügbar\nFrom: AWS <no@aws.example>\n"
|
|
||||||
"Date: Mon, 3 Nov 2025 10:00:00 +0000\nContent-Type: text/plain\n\nSee https://console.aws.amazon.com/billing")
|
|
||||||
|
|
||||||
|
|
||||||
def test_process_message(tmp_path, monkeypatch):
|
|
||||||
db = sessionmaker(bind=create_engine("sqlite://"))()
|
|
||||||
Base.metadata.create_all(db.get_bind())
|
|
||||||
monkeypatch.setattr(grabber, "fetch_pdf", lambda url: b"%PDF-1.4 same")
|
|
||||||
monkeypatch.setenv("GRAB_CUTOFF", "2026-11-13")
|
|
||||||
run = lambda raw: grabber.process_message(db, email.message_from_string(raw), "INBOX", str(tmp_path))
|
|
||||||
|
|
||||||
assert run(LINK.format(mid="a", date="Mon, 3 Nov 2025 10:00:00 +0000")) == "pending_review" # Rückblick
|
|
||||||
assert run(LINK.format(mid="a", date="Mon, 3 Nov 2025 10:00:00 +0000")) is None # gleiche Message-ID
|
|
||||||
assert run(LINK.format(mid="b", date="Mon, 3 Nov 2025 10:00:00 +0000")) == "duplicate" # gleiche PDF
|
|
||||||
monkeypatch.setattr(grabber, "fetch_pdf", lambda url: b"%PDF-1.4 neu")
|
|
||||||
assert run(LINK.format(mid="c", date="Mon, 16 Nov 2026 10:00:00 +0000")) == "queued" # nach Stichtag
|
|
||||||
assert run(PORTAL) == "manual"
|
|
||||||
assert run("Subject: Newsletter\n\nhallo") is None
|
|
||||||
assert db.query(Outbound).count() == 4
|
|
||||||
assert (tmp_path / "picker" / "2025" / "11").is_dir()
|
|
||||||
|
|
||||||
monkeypatch.delenv("GRAB_CUTOFF") # ohne Stichtag nie automatisch senden
|
|
||||||
monkeypatch.setattr(grabber, "fetch_pdf", lambda url: b"%PDF-1.4 dritte")
|
|
||||||
assert run(LINK.format(mid="d", date="Mon, 16 Nov 2026 10:00:00 +0000")) == "pending_review"
|
|
||||||
|
|
@ -16,11 +16,6 @@ services:
|
||||||
- IMAP_USER=${IMAP_USER}
|
- IMAP_USER=${IMAP_USER}
|
||||||
- IMAP_PASSWORD=${IMAP_PASSWORD}
|
- IMAP_PASSWORD=${IMAP_PASSWORD}
|
||||||
- INVOICE_STORAGE_PATH=/data/rechnungen
|
- INVOICE_STORAGE_PATH=/data/rechnungen
|
||||||
# Invoice Picker: ohne ACCOUNTABLE_INBOX kein Versand, ohne GRAB_CUTOFF alles nur mit Freigabe
|
|
||||||
- ACCOUNTABLE_INBOX=${ACCOUNTABLE_INBOX:-}
|
|
||||||
- GRAB_CUTOFF=${GRAB_CUTOFF:-}
|
|
||||||
- SMTP_HOST=${SMTP_HOST:-smtp.strato.de}
|
|
||||||
- SMTP_PORT=${SMTP_PORT:-465}
|
|
||||||
depends_on:
|
depends_on:
|
||||||
- db
|
- db
|
||||||
networks:
|
networks:
|
||||||
|
|
|
||||||
|
|
@ -14,8 +14,6 @@ IMAP_HOST=${IMAP_HOST:-imap.strato.de}
|
||||||
IMAP_PORT=${IMAP_PORT:-993}
|
IMAP_PORT=${IMAP_PORT:-993}
|
||||||
IMAP_USER=$IMAP_USER
|
IMAP_USER=$IMAP_USER
|
||||||
IMAP_PASSWORD=$IMAP_PASSWORD
|
IMAP_PASSWORD=$IMAP_PASSWORD
|
||||||
ACCOUNTABLE_INBOX=${ACCOUNTABLE_INBOX:-}
|
|
||||||
GRAB_CUTOFF=${GRAB_CUTOFF:-}
|
|
||||||
ENV
|
ENV
|
||||||
echo ".env geschrieben"
|
echo ".env geschrieben"
|
||||||
[ "${1:-}" = "--env-only" ] && exit 0
|
[ "${1:-}" = "--env-only" ] && exit 0
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue