feat(picker): Whop-Roboter (Login per Mail-Code, PDF via receipt_pdf)
Belegnummern aus Order-Confirmation-Mails, Login-Cookies im Volume, Whop-Mails nicht mehr als manual. Chromium im Backend-Image. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Wd9aeH8MKgesfbsaS3kpnx
This commit is contained in:
parent
4dff8234f1
commit
c6b84db0d2
5 changed files with 204 additions and 2 deletions
|
|
@ -13,6 +13,9 @@ COPY backend/requirements.txt .
|
|||
# Install Python dependencies
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# Chromium für den Whop-Roboter
|
||||
RUN playwright install --with-deps chromium
|
||||
|
||||
# Copy application code
|
||||
COPY backend/ ./backend/
|
||||
COPY .env.example .env
|
||||
|
|
|
|||
142
backend/fetchers/whop.py
Normal file
142
backend/fetchers/whop.py
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
"""Whop-Ads-Belege per Browser-Roboter (Whop hat keine Rechnungs-API).
|
||||
|
||||
1. Belegnummern aus den "Order Confirmation"-Mails im Strato-Postfach ("Receipt XYZ" -> pay_XYZ)
|
||||
2. Login bei Whop per Einmal-Code, den Whop an LOGIN_EMAIL schickt (landet im selben Postfach)
|
||||
3. PDF über whop.com/api/receipt_pdf mit den Login-Cookies
|
||||
|
||||
Login-Cookies liegen in <storage>/.whop-state.json (nur NucBox-Volume, chmod 600), damit nicht jeder Lauf neu einloggt.
|
||||
"""
|
||||
|
||||
import email
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from datetime import date
|
||||
|
||||
from backend.routers.emails import connect_imap, decode_hdr, parse_email_date
|
||||
|
||||
LOGIN_EMAIL = os.getenv("WHOP_LOGIN_EMAIL", "mkruse@c-stone-dev.com")
|
||||
SENDER_DOMAINS = ("whop.com",) # deren Mails deckt der Roboter ab
|
||||
FOLDERS = ("INBOX", "Archive", "Archive.2026", "Trash")
|
||||
PDF_URL = "https://whop.com/api/receipt_pdf/?receiptId={}"
|
||||
|
||||
try:
|
||||
import playwright # noqa: F401
|
||||
ENABLED = os.getenv("WHOP_ROBOT", "1") == "1"
|
||||
except ImportError:
|
||||
ENABLED = False
|
||||
|
||||
|
||||
def _text(msg) -> str:
|
||||
parts = []
|
||||
for p in msg.walk():
|
||||
if p.get_content_type() in ("text/plain", "text/html"):
|
||||
t = p.get_payload(decode=True).decode(p.get_content_charset() or "utf-8", "replace")
|
||||
parts.append(re.sub(r"<[^>]+>", " ", re.sub(r"<style.*?</style>", "", t, flags=re.S)))
|
||||
return re.sub(r"\s+", " ", " ".join(parts))
|
||||
|
||||
|
||||
def parse_receipt(msg):
|
||||
"""Order-Confirmation-Mail -> {pay_id, date, amount, sender} oder None."""
|
||||
t = _text(msg)
|
||||
r = re.search(r"Receipt\s+([A-Za-z0-9]{8,})", t)
|
||||
if not r:
|
||||
return None
|
||||
a = re.search(r"Amount paid\s+\$?\s?([\d.,]+)", t)
|
||||
d = parse_email_date(msg.get("Date", ""))
|
||||
return {"pay_id": "pay_" + r[1], "date": d.replace(tzinfo=None) if d and d.tzinfo else d,
|
||||
"amount": a[1] if a else "?", "sender": decode_hdr(msg.get("From", ""))}
|
||||
|
||||
|
||||
def receipts(cfg: dict, since: date) -> list:
|
||||
m = connect_imap(cfg["host"], cfg["port"], cfg["user"], cfg["password"])
|
||||
out = {}
|
||||
try:
|
||||
for f in FOLDERS:
|
||||
if m.select(f'"{f}"', readonly=True)[0] != "OK":
|
||||
continue
|
||||
for i in m.search(None, f'FROM "whop" SUBJECT "Order Confirmation" SINCE {since:%d-%b-%Y}')[1][0].split():
|
||||
rec = parse_receipt(email.message_from_bytes(m.fetch(i, "(BODY.PEEK[])")[1][0][1]))
|
||||
if rec:
|
||||
out[rec["pay_id"]] = rec
|
||||
finally:
|
||||
m.logout()
|
||||
return list(out.values())
|
||||
|
||||
|
||||
def _code_ids(m) -> set:
|
||||
m.select('"INBOX"', readonly=True)
|
||||
return set(m.search(None, f'FROM "whop" SUBJECT "sign-in code" SINCE {date.today():%d-%b-%Y}')[1][0].split())
|
||||
|
||||
|
||||
def _wait_code(cfg: dict, before: set) -> str:
|
||||
# ponytail: Polling 2 Min; reicht, Whop-Codes kommen in Sekunden
|
||||
for _ in range(24):
|
||||
time.sleep(5)
|
||||
m = connect_imap(cfg["host"], cfg["port"], cfg["user"], cfg["password"])
|
||||
try:
|
||||
new = sorted(_code_ids(m) - before, key=int)
|
||||
if new:
|
||||
subj = decode_hdr(email.message_from_bytes(
|
||||
m.fetch(new[-1], "(BODY.PEEK[HEADER.FIELDS (SUBJECT)])")[1][0][1]).get("Subject", ""))
|
||||
c = re.search(r"\b(\d{6})\b", subj)
|
||||
if c:
|
||||
return c[1]
|
||||
finally:
|
||||
m.logout()
|
||||
raise RuntimeError("Whop: kein Login-Code im Postfach angekommen")
|
||||
|
||||
|
||||
def _login(ctx, cfg: dict):
|
||||
m = connect_imap(cfg["host"], cfg["port"], cfg["user"], cfg["password"])
|
||||
try:
|
||||
before = _code_ids(m)
|
||||
finally:
|
||||
m.logout()
|
||||
pg = ctx.new_page()
|
||||
pg.goto("https://whop.com/login/", wait_until="networkidle", timeout=60000)
|
||||
for name in ("Alle ablehnen", "Reject all"): # Cookie-Banner datensparsam
|
||||
b = pg.get_by_role("button", name=name)
|
||||
if b.count():
|
||||
b.first.click()
|
||||
break
|
||||
pg.fill("input[name=email]", LOGIN_EMAIL)
|
||||
pg.get_by_role("button", name=re.compile("^(Weiter|Continue)$")).click()
|
||||
pg.wait_for_selector("input[name=otp]", timeout=30000)
|
||||
pg.fill("input[name=otp]", _wait_code(cfg, before))
|
||||
pg.wait_for_url(lambda u: "/login" not in u, timeout=60000)
|
||||
pg.close()
|
||||
|
||||
|
||||
def fetch_pdfs(cfg: dict, pay_ids: list, state_path: str) -> dict:
|
||||
"""-> {pay_id: pdf_bytes} für alle abrufbaren Belege. Loggt nur ein, wenn die gespeicherten Cookies nicht reichen."""
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
out = {}
|
||||
with sync_playwright() as p:
|
||||
b = p.chromium.launch()
|
||||
try:
|
||||
ctx = b.new_context(locale="de-DE", storage_state=state_path if os.path.exists(state_path) else None)
|
||||
|
||||
def get(pid):
|
||||
r = ctx.request.get(PDF_URL.format(pid), timeout=60000)
|
||||
body = r.body() if r.ok else b""
|
||||
return body if body.startswith(b"%PDF-") else None
|
||||
|
||||
first = get(pay_ids[0])
|
||||
if first is None:
|
||||
_login(ctx, cfg)
|
||||
first = get(pay_ids[0])
|
||||
if first is None:
|
||||
raise RuntimeError("Whop: Login ok, aber kein PDF erhalten")
|
||||
ctx.storage_state(path=state_path)
|
||||
os.chmod(state_path, 0o600)
|
||||
out[pay_ids[0]] = first
|
||||
for pid in pay_ids[1:]:
|
||||
pdf = get(pid)
|
||||
if pdf:
|
||||
out[pid] = pdf
|
||||
time.sleep(1)
|
||||
finally:
|
||||
b.close()
|
||||
return out
|
||||
|
|
@ -16,7 +16,7 @@ from typing import Optional
|
|||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from backend.fetchers import aws
|
||||
from backend.fetchers import aws, whop
|
||||
from backend.fetchers.direct import fetch_pdf
|
||||
from backend.models import Outbound
|
||||
from backend.routers.emails import INVOICE_KEYWORDS, classify_mail, connect_imap, decode_hdr, parse_email_date
|
||||
|
|
@ -61,7 +61,8 @@ def process_message(db: Session, msg, folder: str, base: str) -> Optional[str]:
|
|||
if row and row.status != "manual": # "manual" wird bei jedem Lauf neu versucht (neue Adapter greifen rückwirkend)
|
||||
return None
|
||||
d = _domain(sender)
|
||||
if aws.ENABLED and any(d == x or d.endswith("." + x) for x in aws.SENDER_DOMAINS): # AWS kommt über die API (sync_aws)
|
||||
robot = (aws.ENABLED and _is(d, aws.SENDER_DOMAINS)) or (whop.ENABLED and _is(d, whop.SENDER_DOMAINS))
|
||||
if robot: # AWS kommt über die API (sync_aws), Whop über den Roboter (sync_whop)
|
||||
if row:
|
||||
row.status, row.error = "via_api", None
|
||||
db.commit()
|
||||
|
|
@ -111,6 +112,32 @@ def sync_aws(db: Session, since: date, base: str, counts: dict):
|
|||
counts[row.status] = counts.get(row.status, 0) + 1
|
||||
|
||||
|
||||
def sync_whop(db: Session, cfg: dict, since: date, counts: dict):
|
||||
"""Whop-Ads-Belege per Roboter (message_id = whop-receipt:<pay_id>)."""
|
||||
known = {r[0].split(":", 1)[1] for r in db.query(Outbound.message_id)
|
||||
.filter(Outbound.message_id.like("whop-receipt:%")).all()}
|
||||
todo = {r["pay_id"]: r for r in whop.receipts(cfg, since) if r["pay_id"] not in known}
|
||||
if not todo:
|
||||
return
|
||||
pdfs = whop.fetch_pdfs(cfg, list(todo), os.path.join(cfg["storage_path"], ".whop-state.json"))
|
||||
for pid, pdf in pdfs.items():
|
||||
r = todo[pid]
|
||||
row = Outbound(message_id=f"whop-receipt:{pid}", email_folder="whop-portal", email_date=r["date"],
|
||||
email_from=r["sender"], mail_class="portal", link_url=f"https://whop.com/billing/receipt/{pid}/",
|
||||
email_subject=f"Whop Ads Rechnung {pid} (${r['amount']})")
|
||||
db.add(row)
|
||||
db.flush()
|
||||
_store(db, row, pdf, cfg["storage_path"])
|
||||
db.commit()
|
||||
counts[row.status] = counts.get(row.status, 0) + 1
|
||||
if len(pdfs) < len(todo):
|
||||
counts["whop_fehlt"] = len(todo) - len(pdfs)
|
||||
|
||||
|
||||
def _is(domain: str, domains) -> bool:
|
||||
return any(domain == x or domain.endswith("." + x) for x in domains)
|
||||
|
||||
|
||||
def _domain(sender: str) -> str:
|
||||
m = re.search(r"@([\w.-]+)", sender)
|
||||
return m[1].lower() if m else ""
|
||||
|
|
@ -146,6 +173,12 @@ def run(db: Session, since: date, state: Optional[dict] = None) -> dict:
|
|||
_scan(db, cfg, folders, since, state, counts)
|
||||
except Exception as e:
|
||||
state["errors"].append(f"{cfg['host']}: {e}"[:300])
|
||||
if whop.ENABLED:
|
||||
try:
|
||||
sync_whop(db, get_imap_config(db), since, counts)
|
||||
except Exception as e:
|
||||
db.rollback()
|
||||
state["errors"].append(f"Whop-Roboter: {e}"[:300])
|
||||
if aws.ENABLED:
|
||||
try:
|
||||
sync_aws(db, since, get_imap_config(db)["storage_path"], counts)
|
||||
|
|
|
|||
|
|
@ -24,3 +24,6 @@ imaplib2==3.6
|
|||
# Allgemein
|
||||
python-dotenv==1.0.1
|
||||
boto3==1.43.36
|
||||
|
||||
# Whop-Roboter (Browser)
|
||||
playwright==1.60.0
|
||||
|
|
|
|||
|
|
@ -76,3 +76,24 @@ def test_gmail_nur_rueckblick_und_bericht(monkeypatch):
|
|||
db.commit()
|
||||
txt = mailer.report_text(db, {"counts": {"manual": 1}, "errors": []}, date(2026, 11, 5))
|
||||
assert "Manuell abholen" in txt and "Rechnung verfügbar" in txt
|
||||
|
||||
|
||||
def test_whop(tmp_path, monkeypatch):
|
||||
from datetime import date
|
||||
from backend.fetchers import whop
|
||||
mail = ("Message-ID: <w1>\nSubject: Whop Ads Order Confirmation\nFrom: Whop Ads via Whop <no-reply@businesses.whop.com>\n"
|
||||
"Date: Thu, 12 Nov 2026 10:00:00 +0000\nContent-Type: text/html\n\n"
|
||||
"<p>Amount paid $15.01</p><p>Receipt NJlZDb5nDSnaFK</p>")
|
||||
rec = whop.parse_receipt(email.message_from_string(mail))
|
||||
assert rec["pay_id"] == "pay_NJlZDb5nDSnaFK" and rec["amount"] == "15.01" and rec["date"].tzinfo is None
|
||||
db = sessionmaker(bind=create_engine("sqlite://"))()
|
||||
Base.metadata.create_all(db.get_bind())
|
||||
monkeypatch.setattr(whop, "receipts", lambda cfg, since: [rec])
|
||||
calls = []
|
||||
monkeypatch.setattr(whop, "fetch_pdfs", lambda cfg, ids, st: calls.append(ids) or {i: b"%PDF-1.4 w" for i in ids})
|
||||
cfg, counts = {"storage_path": str(tmp_path)}, {}
|
||||
grabber.sync_whop(db, cfg, date(2026, 11, 1), counts)
|
||||
grabber.sync_whop(db, cfg, date(2026, 11, 1), counts) # zweiter Lauf: nichts Neues, kein Browser
|
||||
assert counts == {"pending_review": 1} and len(calls) == 1
|
||||
monkeypatch.setattr(whop, "ENABLED", True)
|
||||
assert grabber.process_message(db, email.message_from_string(mail), "INBOX", str(tmp_path)) is None
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue