diff --git a/backend/fetchers/direct.py b/backend/fetchers/direct.py index 76a398d..ca2ae23 100644 --- a/backend/fetchers/direct.py +++ b/backend/fetchers/direct.py @@ -1,17 +1,34 @@ -"""Stufe 2: Rechnungs-PDF per Direktlink laden (Stripe, Paddle, ThriveCart, *.pdf).""" +"""Stufe 2: Rechnungs-PDF per Direktlink laden (Stripe, Paddle, ThriveCart, *.pdf, Tracker-Links z.B. OpenAI).""" +import re import urllib.request from typing import Optional +from urllib.parse import urlsplit + +from backend.routers.emails import DIRECT_LINK, TRACKERS MAX_BYTES = 20 * 1024 * 1024 +UA = {"User-Agent": "Mozilla/5.0 (C-Stone Invoice Picker)"} +STRIPE_HOSTED = re.compile(r"^https://invoice\.stripe\.com/i/(acct_\w+)/(\w+)", re.I) + + +def pdf_url(url: str) -> str: + """Stripe-Rechnungsseite (HTML) -> PDF-Download-URL; alles andere unverändert.""" + m = STRIPE_HOSTED.match(url) + if not m: + return url + q = urlsplit(url).query + return f"https://pay.stripe.com/invoice/{m[1]}/{m[2]}/pdf" + (f"?{q}" if q else "") def fetch_pdf(url: str, timeout: int = 30) -> Optional[bytes]: - """-> PDF-Bytes oder None (kein PDF, z.B. HTML-Login-Seite). URL ist bereits per DIRECT_LINK geprüft (nur https).""" - if not url.lower().startswith("https://"): + """-> PDF-Bytes oder None (kein PDF, z.B. HTML-Login-Seite).""" + if TRACKERS.match(url): # nur Weiterleitung auflösen, Ziel muss wieder auf der Allowlist stehen + with urllib.request.urlopen(urllib.request.Request(url, headers=UA), timeout=timeout) as r: + url = r.geturl() + if not DIRECT_LINK.search(url): return None - req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 (C-Stone Invoice Picker)"}) - with urllib.request.urlopen(req, timeout=timeout) as r: + with urllib.request.urlopen(urllib.request.Request(pdf_url(url), headers=UA), timeout=timeout) as r: data = r.read(MAX_BYTES + 1) if len(data) > MAX_BYTES or not data.startswith(b"%PDF-"): return None diff --git a/backend/grabber.py b/backend/grabber.py index a10077e..71f9841 100644 --- a/backend/grabber.py +++ b/backend/grabber.py @@ -55,13 +55,15 @@ def process_message(db: Session, msg, folder: str, base: str) -> Optional[str]: sender = decode_hdr(msg.get("From", "")) email_date = parse_email_date(msg.get("Date", "")) mid = (msg.get("Message-ID") or "").strip() or hashlib.sha256(f"{folder}{email_date}{subject}".encode()).hexdigest() - if db.query(Outbound.id).filter(Outbound.message_id == mid).first(): + row = db.query(Outbound).filter(Outbound.message_id == mid).first() + if row and row.status != "manual": # "manual" wird bei jedem Lauf neu versucht (neue Adapter greifen rückwirkend) return None mail_class, url = classify_mail(msg) if mail_class not in ("link", "portal"): # Anhänge leitet Markus bereits selbst an Accountable weiter return None - row = Outbound(message_id=mid, email_folder=folder, email_date=email_date, email_from=sender, - email_subject=subject, mail_class=mail_class, link_url=url) + row = row or Outbound(message_id=mid) + row.email_folder, row.email_date, row.email_from = folder, email_date, sender + row.email_subject, row.mail_class, row.link_url = subject, mail_class, url pdf = None if mail_class == "link": try: diff --git a/backend/routers/emails.py b/backend/routers/emails.py index a618711..45dff23 100644 --- a/backend/routers/emails.py +++ b/backend/routers/emails.py @@ -67,6 +67,10 @@ DIRECT_LINK = re.compile( ) # Rechnungs-Link hinter Login (Konsole, Kundenportal) -> "portal" LINK_HINT = re.compile(r"invoice|rechnung|receipt|beleg|billing|download|\.pdf", re.I) +# Klick-Tracker bekannter Absender: Ziel erst beim Abruf auflösen (OpenAI -> invoice.stripe.com) +# ponytail: Allowlist je Absender; neuer Anbieter mit Tracker = hier eine Zeile ergänzen +TRACKERS = re.compile(r"^https?://[\w-]+\.email\.openai\.com/", re.I) +INVOICE_ANCHOR = re.compile(r']*href="([^"]+)"[^>]*>(?:(?!).){0,300}?(?:invoice|rechnung|receipt)', re.I | re.S) PORTAL_HINT = re.compile( r"in ihrem (kunden)?(konto|portal)|im (kunden)?portal|your account|sign in|log ?in|anmelden|einloggen|" r"view (your )?(invoice|receipt|bill)|rechnung (ist )?(verfügbar|bereit)|invoice is available|zahlungsbeleg", @@ -86,7 +90,8 @@ def classify_mail(msg) -> tuple: text += payload.decode(part.get_content_charset() or "utf-8", "replace") + "\n" # ponytail: Regex statt HTML-Parser; Footer-Links ("invoice" im Pfad) können False-Positives liefern urls = [html.unescape(u) for u in LINK_RE.findall(text)] - url = next((u for u in urls if DIRECT_LINK.search(u)), None) + url = next((u for u in urls if DIRECT_LINK.search(u)), None) or next( + (u for u in map(html.unescape, INVOICE_ANCHOR.findall(text)) if TRACKERS.match(u)), None) if url: return "link", url if any(LINK_HINT.search(u) for u in urls) or PORTAL_HINT.search(re.sub(r"<[^>]+>", " ", text)): diff --git a/backend/tests/test_classify.py b/backend/tests/test_classify.py index f77485f..3629fc1 100644 --- a/backend/tests/test_classify.py +++ b/backend/tests/test_classify.py @@ -23,3 +23,15 @@ def test_classify(): # HTML-Entities im Link werden aufgelöst h = 'Content-Type: text/html\n\nRechnung' assert classify_mail(_m(h)) == ("link", "https://x.example/r.pdf?a=1&b=2") + + +def test_tracker_and_stripe(): + from backend.fetchers.direct import pdf_url + h = ('Content-Type: text/html\n\nTerms' + 'View invoice') + assert classify_mail(_m(h)) == ("link", "http://url1.email.openai.com/ls/click?inv=9") + # Tracker fremder Absender wird nicht verfolgt + evil = 'Content-Type: text/html\n\nView invoice' + assert classify_mail(_m(evil))[0] == "portal" + assert pdf_url("https://invoice.stripe.com/i/acct_1A/live_X2?s=em") == "https://pay.stripe.com/invoice/acct_1A/live_X2/pdf?s=em" + assert pdf_url("https://x.example/r.pdf") == "https://x.example/r.pdf"