diff --git a/Dockerfile.backend b/Dockerfile.backend index e6bbfd3..340f5ff 100644 --- a/Dockerfile.backend +++ b/Dockerfile.backend @@ -13,6 +13,9 @@ COPY backend/requirements.txt . # Install Python dependencies RUN pip install --no-cache-dir -r requirements.txt +# Chromium für den Whop-Roboter +RUN playwright install --with-deps chromium + # Copy application code COPY backend/ ./backend/ COPY .env.example .env diff --git a/backend/fetchers/whop.py b/backend/fetchers/whop.py new file mode 100644 index 0000000..eb45af2 --- /dev/null +++ b/backend/fetchers/whop.py @@ -0,0 +1,142 @@ +"""Whop-Ads-Belege per Browser-Roboter (Whop hat keine Rechnungs-API). + +1. Belegnummern aus den "Order Confirmation"-Mails im Strato-Postfach ("Receipt XYZ" -> pay_XYZ) +2. Login bei Whop per Einmal-Code, den Whop an LOGIN_EMAIL schickt (landet im selben Postfach) +3. PDF über whop.com/api/receipt_pdf mit den Login-Cookies + +Login-Cookies liegen in /.whop-state.json (nur NucBox-Volume, chmod 600), damit nicht jeder Lauf neu einloggt. +""" + +import email +import os +import re +import time +from datetime import date + +from backend.routers.emails import connect_imap, decode_hdr, parse_email_date + +LOGIN_EMAIL = os.getenv("WHOP_LOGIN_EMAIL", "mkruse@c-stone-dev.com") +SENDER_DOMAINS = ("whop.com",) # deren Mails deckt der Roboter ab +FOLDERS = ("INBOX", "Archive", "Archive.2026", "Trash") +PDF_URL = "https://whop.com/api/receipt_pdf/?receiptId={}" + +try: + import playwright # noqa: F401 + ENABLED = os.getenv("WHOP_ROBOT", "1") == "1" +except ImportError: + ENABLED = False + + +def _text(msg) -> str: + parts = [] + for p in msg.walk(): + if p.get_content_type() in ("text/plain", "text/html"): + t = p.get_payload(decode=True).decode(p.get_content_charset() or "utf-8", "replace") + parts.append(re.sub(r"<[^>]+>", " ", re.sub(r"", "", t, flags=re.S))) + return re.sub(r"\s+", " ", " ".join(parts)) + + +def parse_receipt(msg): + """Order-Confirmation-Mail -> {pay_id, date, amount, sender} oder None.""" + t = _text(msg) + r = re.search(r"Receipt\s+([A-Za-z0-9]{8,})", t) + if not r: + return None + a = re.search(r"Amount paid\s+\$?\s?([\d.,]+)", t) + d = parse_email_date(msg.get("Date", "")) + return {"pay_id": "pay_" + r[1], "date": d.replace(tzinfo=None) if d and d.tzinfo else d, + "amount": a[1] if a else "?", "sender": decode_hdr(msg.get("From", ""))} + + +def receipts(cfg: dict, since: date) -> list: + m = connect_imap(cfg["host"], cfg["port"], cfg["user"], cfg["password"]) + out = {} + try: + for f in FOLDERS: + if m.select(f'"{f}"', readonly=True)[0] != "OK": + continue + for i in m.search(None, f'FROM "whop" SUBJECT "Order Confirmation" SINCE {since:%d-%b-%Y}')[1][0].split(): + rec = parse_receipt(email.message_from_bytes(m.fetch(i, "(BODY.PEEK[])")[1][0][1])) + if rec: + out[rec["pay_id"]] = rec + finally: + m.logout() + return list(out.values()) + + +def _code_ids(m) -> set: + m.select('"INBOX"', readonly=True) + return set(m.search(None, f'FROM "whop" SUBJECT "sign-in code" SINCE {date.today():%d-%b-%Y}')[1][0].split()) + + +def _wait_code(cfg: dict, before: set) -> str: + # ponytail: Polling 2 Min; reicht, Whop-Codes kommen in Sekunden + for _ in range(24): + time.sleep(5) + m = connect_imap(cfg["host"], cfg["port"], cfg["user"], cfg["password"]) + try: + new = sorted(_code_ids(m) - before, key=int) + if new: + subj = decode_hdr(email.message_from_bytes( + m.fetch(new[-1], "(BODY.PEEK[HEADER.FIELDS (SUBJECT)])")[1][0][1]).get("Subject", "")) + c = re.search(r"\b(\d{6})\b", subj) + if c: + return c[1] + finally: + m.logout() + raise RuntimeError("Whop: kein Login-Code im Postfach angekommen") + + +def _login(ctx, cfg: dict): + m = connect_imap(cfg["host"], cfg["port"], cfg["user"], cfg["password"]) + try: + before = _code_ids(m) + finally: + m.logout() + pg = ctx.new_page() + pg.goto("https://whop.com/login/", wait_until="networkidle", timeout=60000) + for name in ("Alle ablehnen", "Reject all"): # Cookie-Banner datensparsam + b = pg.get_by_role("button", name=name) + if b.count(): + b.first.click() + break + pg.fill("input[name=email]", LOGIN_EMAIL) + pg.get_by_role("button", name=re.compile("^(Weiter|Continue)$")).click() + pg.wait_for_selector("input[name=otp]", timeout=30000) + pg.fill("input[name=otp]", _wait_code(cfg, before)) + pg.wait_for_url(lambda u: "/login" not in u, timeout=60000) + pg.close() + + +def fetch_pdfs(cfg: dict, pay_ids: list, state_path: str) -> dict: + """-> {pay_id: pdf_bytes} für alle abrufbaren Belege. Loggt nur ein, wenn die gespeicherten Cookies nicht reichen.""" + from playwright.sync_api import sync_playwright + + out = {} + with sync_playwright() as p: + b = p.chromium.launch() + try: + ctx = b.new_context(locale="de-DE", storage_state=state_path if os.path.exists(state_path) else None) + + def get(pid): + r = ctx.request.get(PDF_URL.format(pid), timeout=60000) + body = r.body() if r.ok else b"" + return body if body.startswith(b"%PDF-") else None + + first = get(pay_ids[0]) + if first is None: + _login(ctx, cfg) + first = get(pay_ids[0]) + if first is None: + raise RuntimeError("Whop: Login ok, aber kein PDF erhalten") + ctx.storage_state(path=state_path) + os.chmod(state_path, 0o600) + out[pay_ids[0]] = first + for pid in pay_ids[1:]: + pdf = get(pid) + if pdf: + out[pid] = pdf + time.sleep(1) + finally: + b.close() + return out diff --git a/backend/grabber.py b/backend/grabber.py index 45d86cf..3c0caf4 100644 --- a/backend/grabber.py +++ b/backend/grabber.py @@ -16,7 +16,7 @@ from typing import Optional from sqlalchemy.orm import Session -from backend.fetchers import aws +from backend.fetchers import aws, whop from backend.fetchers.direct import fetch_pdf from backend.models import Outbound from backend.routers.emails import INVOICE_KEYWORDS, classify_mail, connect_imap, decode_hdr, parse_email_date @@ -61,7 +61,8 @@ def process_message(db: Session, msg, folder: str, base: str) -> Optional[str]: if row and row.status != "manual": # "manual" wird bei jedem Lauf neu versucht (neue Adapter greifen rückwirkend) return None d = _domain(sender) - if aws.ENABLED and any(d == x or d.endswith("." + x) for x in aws.SENDER_DOMAINS): # AWS kommt über die API (sync_aws) + robot = (aws.ENABLED and _is(d, aws.SENDER_DOMAINS)) or (whop.ENABLED and _is(d, whop.SENDER_DOMAINS)) + if robot: # AWS kommt über die API (sync_aws), Whop über den Roboter (sync_whop) if row: row.status, row.error = "via_api", None db.commit() @@ -111,6 +112,32 @@ def sync_aws(db: Session, since: date, base: str, counts: dict): counts[row.status] = counts.get(row.status, 0) + 1 +def sync_whop(db: Session, cfg: dict, since: date, counts: dict): + """Whop-Ads-Belege per Roboter (message_id = whop-receipt:).""" + known = {r[0].split(":", 1)[1] for r in db.query(Outbound.message_id) + .filter(Outbound.message_id.like("whop-receipt:%")).all()} + todo = {r["pay_id"]: r for r in whop.receipts(cfg, since) if r["pay_id"] not in known} + if not todo: + return + pdfs = whop.fetch_pdfs(cfg, list(todo), os.path.join(cfg["storage_path"], ".whop-state.json")) + for pid, pdf in pdfs.items(): + r = todo[pid] + row = Outbound(message_id=f"whop-receipt:{pid}", email_folder="whop-portal", email_date=r["date"], + email_from=r["sender"], mail_class="portal", link_url=f"https://whop.com/billing/receipt/{pid}/", + email_subject=f"Whop Ads Rechnung {pid} (${r['amount']})") + db.add(row) + db.flush() + _store(db, row, pdf, cfg["storage_path"]) + db.commit() + counts[row.status] = counts.get(row.status, 0) + 1 + if len(pdfs) < len(todo): + counts["whop_fehlt"] = len(todo) - len(pdfs) + + +def _is(domain: str, domains) -> bool: + return any(domain == x or domain.endswith("." + x) for x in domains) + + def _domain(sender: str) -> str: m = re.search(r"@([\w.-]+)", sender) return m[1].lower() if m else "" @@ -146,6 +173,12 @@ def run(db: Session, since: date, state: Optional[dict] = None) -> dict: _scan(db, cfg, folders, since, state, counts) except Exception as e: state["errors"].append(f"{cfg['host']}: {e}"[:300]) + if whop.ENABLED: + try: + sync_whop(db, get_imap_config(db), since, counts) + except Exception as e: + db.rollback() + state["errors"].append(f"Whop-Roboter: {e}"[:300]) if aws.ENABLED: try: sync_aws(db, since, get_imap_config(db)["storage_path"], counts) diff --git a/backend/requirements.txt b/backend/requirements.txt index 79e4c2a..5050652 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -24,3 +24,6 @@ imaplib2==3.6 # Allgemein python-dotenv==1.0.1 boto3==1.43.36 + +# Whop-Roboter (Browser) +playwright==1.60.0 diff --git a/backend/tests/test_outbound.py b/backend/tests/test_outbound.py index da53249..fa3251c 100644 --- a/backend/tests/test_outbound.py +++ b/backend/tests/test_outbound.py @@ -76,3 +76,24 @@ def test_gmail_nur_rueckblick_und_bericht(monkeypatch): db.commit() txt = mailer.report_text(db, {"counts": {"manual": 1}, "errors": []}, date(2026, 11, 5)) assert "Manuell abholen" in txt and "Rechnung verfügbar" in txt + + +def test_whop(tmp_path, monkeypatch): + from datetime import date + from backend.fetchers import whop + mail = ("Message-ID: \nSubject: Whop Ads Order Confirmation\nFrom: Whop Ads via Whop \n" + "Date: Thu, 12 Nov 2026 10:00:00 +0000\nContent-Type: text/html\n\n" + "

Amount paid $15.01

Receipt NJlZDb5nDSnaFK

") + rec = whop.parse_receipt(email.message_from_string(mail)) + assert rec["pay_id"] == "pay_NJlZDb5nDSnaFK" and rec["amount"] == "15.01" and rec["date"].tzinfo is None + db = sessionmaker(bind=create_engine("sqlite://"))() + Base.metadata.create_all(db.get_bind()) + monkeypatch.setattr(whop, "receipts", lambda cfg, since: [rec]) + calls = [] + monkeypatch.setattr(whop, "fetch_pdfs", lambda cfg, ids, st: calls.append(ids) or {i: b"%PDF-1.4 w" for i in ids}) + cfg, counts = {"storage_path": str(tmp_path)}, {} + grabber.sync_whop(db, cfg, date(2026, 11, 1), counts) + grabber.sync_whop(db, cfg, date(2026, 11, 1), counts) # zweiter Lauf: nichts Neues, kein Browser + assert counts == {"pending_review": 1} and len(calls) == 1 + monkeypatch.setattr(whop, "ENABLED", True) + assert grabber.process_message(db, email.message_from_string(mail), "INBOX", str(tmp_path)) is None