feat(picker): OpenAI-Adapter (Tracker-Link -> Stripe-PDF), manual-Einträge werden neu versucht
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
ab9f1ab3a4
commit
b7840cdcc8
4 changed files with 45 additions and 9 deletions
|
|
@ -1,17 +1,34 @@
|
|||
"""Stufe 2: Rechnungs-PDF per Direktlink laden (Stripe, Paddle, ThriveCart, *.pdf)."""
|
||||
"""Stufe 2: Rechnungs-PDF per Direktlink laden (Stripe, Paddle, ThriveCart, *.pdf, Tracker-Links z.B. OpenAI)."""
|
||||
|
||||
import re
|
||||
import urllib.request
|
||||
from typing import Optional
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from backend.routers.emails import DIRECT_LINK, TRACKERS
|
||||
|
||||
MAX_BYTES = 20 * 1024 * 1024
|
||||
UA = {"User-Agent": "Mozilla/5.0 (C-Stone Invoice Picker)"}
|
||||
STRIPE_HOSTED = re.compile(r"^https://invoice\.stripe\.com/i/(acct_\w+)/(\w+)", re.I)
|
||||
|
||||
|
||||
def pdf_url(url: str) -> str:
|
||||
"""Stripe-Rechnungsseite (HTML) -> PDF-Download-URL; alles andere unverändert."""
|
||||
m = STRIPE_HOSTED.match(url)
|
||||
if not m:
|
||||
return url
|
||||
q = urlsplit(url).query
|
||||
return f"https://pay.stripe.com/invoice/{m[1]}/{m[2]}/pdf" + (f"?{q}" if q else "")
|
||||
|
||||
|
||||
def fetch_pdf(url: str, timeout: int = 30) -> Optional[bytes]:
|
||||
"""-> PDF-Bytes oder None (kein PDF, z.B. HTML-Login-Seite). URL ist bereits per DIRECT_LINK geprüft (nur https)."""
|
||||
if not url.lower().startswith("https://"):
|
||||
"""-> PDF-Bytes oder None (kein PDF, z.B. HTML-Login-Seite)."""
|
||||
if TRACKERS.match(url): # nur Weiterleitung auflösen, Ziel muss wieder auf der Allowlist stehen
|
||||
with urllib.request.urlopen(urllib.request.Request(url, headers=UA), timeout=timeout) as r:
|
||||
url = r.geturl()
|
||||
if not DIRECT_LINK.search(url):
|
||||
return None
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 (C-Stone Invoice Picker)"})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
with urllib.request.urlopen(urllib.request.Request(pdf_url(url), headers=UA), timeout=timeout) as r:
|
||||
data = r.read(MAX_BYTES + 1)
|
||||
if len(data) > MAX_BYTES or not data.startswith(b"%PDF-"):
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -55,13 +55,15 @@ def process_message(db: Session, msg, folder: str, base: str) -> Optional[str]:
|
|||
sender = decode_hdr(msg.get("From", ""))
|
||||
email_date = parse_email_date(msg.get("Date", ""))
|
||||
mid = (msg.get("Message-ID") or "").strip() or hashlib.sha256(f"{folder}{email_date}{subject}".encode()).hexdigest()
|
||||
if db.query(Outbound.id).filter(Outbound.message_id == mid).first():
|
||||
row = db.query(Outbound).filter(Outbound.message_id == mid).first()
|
||||
if row and row.status != "manual": # "manual" wird bei jedem Lauf neu versucht (neue Adapter greifen rückwirkend)
|
||||
return None
|
||||
mail_class, url = classify_mail(msg)
|
||||
if mail_class not in ("link", "portal"): # Anhänge leitet Markus bereits selbst an Accountable weiter
|
||||
return None
|
||||
row = Outbound(message_id=mid, email_folder=folder, email_date=email_date, email_from=sender,
|
||||
email_subject=subject, mail_class=mail_class, link_url=url)
|
||||
row = row or Outbound(message_id=mid)
|
||||
row.email_folder, row.email_date, row.email_from = folder, email_date, sender
|
||||
row.email_subject, row.mail_class, row.link_url = subject, mail_class, url
|
||||
pdf = None
|
||||
if mail_class == "link":
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -67,6 +67,10 @@ DIRECT_LINK = re.compile(
|
|||
)
|
||||
# Rechnungs-Link hinter Login (Konsole, Kundenportal) -> "portal"
|
||||
LINK_HINT = re.compile(r"invoice|rechnung|receipt|beleg|billing|download|\.pdf", re.I)
|
||||
# Klick-Tracker bekannter Absender: Ziel erst beim Abruf auflösen (OpenAI -> invoice.stripe.com)
|
||||
# ponytail: Allowlist je Absender; neuer Anbieter mit Tracker = hier eine Zeile ergänzen
|
||||
TRACKERS = re.compile(r"^https?://[\w-]+\.email\.openai\.com/", re.I)
|
||||
INVOICE_ANCHOR = re.compile(r'<a\b[^>]*href="([^"]+)"[^>]*>(?:(?!</a>).){0,300}?(?:invoice|rechnung|receipt)', re.I | re.S)
|
||||
PORTAL_HINT = re.compile(
|
||||
r"in ihrem (kunden)?(konto|portal)|im (kunden)?portal|your account|sign in|log ?in|anmelden|einloggen|"
|
||||
r"view (your )?(invoice|receipt|bill)|rechnung (ist )?(verfügbar|bereit)|invoice is available|zahlungsbeleg",
|
||||
|
|
@ -86,7 +90,8 @@ def classify_mail(msg) -> tuple:
|
|||
text += payload.decode(part.get_content_charset() or "utf-8", "replace") + "\n"
|
||||
# ponytail: Regex statt HTML-Parser; Footer-Links ("invoice" im Pfad) können False-Positives liefern
|
||||
urls = [html.unescape(u) for u in LINK_RE.findall(text)]
|
||||
url = next((u for u in urls if DIRECT_LINK.search(u)), None)
|
||||
url = next((u for u in urls if DIRECT_LINK.search(u)), None) or next(
|
||||
(u for u in map(html.unescape, INVOICE_ANCHOR.findall(text)) if TRACKERS.match(u)), None)
|
||||
if url:
|
||||
return "link", url
|
||||
if any(LINK_HINT.search(u) for u in urls) or PORTAL_HINT.search(re.sub(r"<[^>]+>", " ", text)):
|
||||
|
|
|
|||
|
|
@ -23,3 +23,15 @@ def test_classify():
|
|||
# HTML-Entities im Link werden aufgelöst
|
||||
h = 'Content-Type: text/html\n\n<a href="https://x.example/r.pdf?a=1&b=2">Rechnung</a>'
|
||||
assert classify_mail(_m(h)) == ("link", "https://x.example/r.pdf?a=1&b=2")
|
||||
|
||||
|
||||
def test_tracker_and_stripe():
|
||||
from backend.fetchers.direct import pdf_url
|
||||
h = ('Content-Type: text/html\n\n<a href="http://url1.email.openai.com/ls/click?x=1&y=2">Terms</a>'
|
||||
'<a href="http://url1.email.openai.com/ls/click?inv=9">View invoice</a>')
|
||||
assert classify_mail(_m(h)) == ("link", "http://url1.email.openai.com/ls/click?inv=9")
|
||||
# Tracker fremder Absender wird nicht verfolgt
|
||||
evil = 'Content-Type: text/html\n\n<a href="http://url1.email.evil.example/ls/click">View invoice</a>'
|
||||
assert classify_mail(_m(evil))[0] == "portal"
|
||||
assert pdf_url("https://invoice.stripe.com/i/acct_1A/live_X2?s=em") == "https://pay.stripe.com/invoice/acct_1A/live_X2/pdf?s=em"
|
||||
assert pdf_url("https://x.example/r.pdf") == "https://x.example/r.pdf"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue