feat(picker): AWS-Adapter über Invoicing-API (ListInvoiceSummaries/GetInvoicePDF), AWS-Hinweismails -> via_api

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Markus Kruse 2026-11-13 23:12:15 +01:00
parent b7840cdcc8
commit e18c174406
6 changed files with 124 additions and 5 deletions

51
backend/fetchers/aws.py Normal file
View file

@ -0,0 +1,51 @@
"""AWS-Rechnungen per Invoicing-API (ListInvoiceSummaries + GetInvoicePDF), kein Konsolen-Login nötig.
IAM-User mit nur: invoicing:ListInvoiceSummaries, invoicing:GetInvoicePDF. Keys im Vault:
INVOICING_AWS_ACCESS_KEY_ID / INVOICING_AWS_SECRET_ACCESS_KEY. Ohne Keys ist der Adapter aus.
Endpoint us-east-1 ist bei AWS fest vorgegeben (globaler Billing-Dienst); übertragen wird nichts, AWS liefert
nur die eigene Rechnung an uns aus.
"""
import os
import urllib.request
from datetime import date
from typing import Iterator, Tuple
ENABLED = bool(os.getenv("INVOICING_AWS_ACCESS_KEY_ID") and os.getenv("INVOICING_AWS_SECRET_ACCESS_KEY"))
SENDER_DOMAINS = ("aws.com", "amazonaws.com", "aws.amazon.com") # deren Hinweis-Mails deckt die API ab
def _months(since: date) -> Iterator[Tuple[int, int]]:
y, m = since.year, since.month
today = date.today()
while (y, m) <= (today.year, today.month):
yield y, m
y, m = (y + 1, 1) if m == 12 else (y, m + 1)
def fetch_invoices(since: date, known: set) -> Iterator[dict]:
"""-> {invoice_id, issued, period, pdf} für jede noch unbekannte Rechnung seit since."""
import boto3
s = boto3.Session(aws_access_key_id=os.environ["INVOICING_AWS_ACCESS_KEY_ID"],
aws_secret_access_key=os.environ["INVOICING_AWS_SECRET_ACCESS_KEY"], region_name="us-east-1")
account = s.client("sts").get_caller_identity()["Account"]
inv = s.client("invoicing")
for y, m in _months(since):
kw = {"Selector": {"ResourceType": "ACCOUNT_ID", "Value": account},
"Filter": {"BillingPeriod": {"Year": y, "Month": m}}}
while True:
page = inv.list_invoice_summaries(**kw)
for summ in page.get("InvoiceSummaries", []):
iid = summ["InvoiceId"]
if iid in known:
continue
url = inv.get_invoice_pdf(InvoiceId=iid)["InvoicePDF"]["DocumentUrl"] # presigned, kurz gültig
with urllib.request.urlopen(url, timeout=60) as r:
pdf = r.read()
if pdf.startswith(b"%PDF-"):
yield {"invoice_id": iid, "issued": summ.get("IssuedDate"), "period": f"{y}-{m:02d}",
"entity": (summ.get("Entity") or {}).get("BillingEntity", "AWS"), "pdf": pdf}
if not page.get("NextToken"):
break
kw["NextToken"] = page["NextToken"]

View file

@ -15,6 +15,7 @@ from typing import Optional
from sqlalchemy.orm import Session
from backend.fetchers import aws
from backend.fetchers.direct import fetch_pdf
from backend.models import Outbound
from backend.routers.emails import INVOICE_KEYWORDS, classify_mail, connect_imap, decode_hdr, parse_email_date
@ -38,7 +39,7 @@ def _status_for(email_date: Optional[datetime]) -> str:
def _save(pdf: bytes, base: str, email_date: Optional[datetime], sender: str) -> tuple:
sha = hashlib.sha256(pdf).hexdigest()
d = email_date or datetime.utcnow()
domain = re.sub(r"[^\w.-]", "", (re.search(r"@([\w.-]+)", sender) or [None, "unbekannt"])[1])
domain = re.sub(r"[^\w.-]", "", _domain(sender)) or "unbekannt"
folder = os.path.join(base, "picker", f"{d:%Y}", f"{d:%m}")
os.makedirs(folder, exist_ok=True)
path = os.path.join(folder, f"{d:%Y-%m-%d}_{domain}_{sha[:8]}.pdf")
@ -58,6 +59,12 @@ def process_message(db: Session, msg, folder: str, base: str) -> Optional[str]:
row = db.query(Outbound).filter(Outbound.message_id == mid).first()
if row and row.status != "manual": # "manual" wird bei jedem Lauf neu versucht (neue Adapter greifen rückwirkend)
return None
d = _domain(sender)
if aws.ENABLED and any(d == x or d.endswith("." + x) for x in aws.SENDER_DOMAINS): # AWS kommt über die API (sync_aws)
if row:
row.status, row.error = "via_api", None
db.commit()
return None
mail_class, url = classify_mail(msg)
if mail_class not in ("link", "portal"): # Anhänge leitet Markus bereits selbst an Accountable weiter
return None
@ -74,9 +81,7 @@ def process_message(db: Session, msg, folder: str, base: str) -> Optional[str]:
else:
row.error = "Portal-Abruf noch nicht automatisiert" # Phase 3: Playwright-Adapter
if pdf:
row.file_path, row.sha256 = _save(pdf, base, email_date, sender)
dup = db.query(Outbound.id).filter(Outbound.sha256 == row.sha256).first()
row.status = "duplicate" if dup else _status_for(email_date)
_store(db, row, pdf, base)
else:
row.status = "manual"
db.add(row)
@ -84,6 +89,32 @@ def process_message(db: Session, msg, folder: str, base: str) -> Optional[str]:
return row.status
def _store(db: Session, row: Outbound, pdf: bytes, base: str):
row.file_path, row.sha256 = _save(pdf, base, row.email_date, row.email_from or "")
dup = db.query(Outbound.id).filter(Outbound.sha256 == row.sha256, Outbound.id != row.id).first()
row.status = "duplicate" if dup else _status_for(row.email_date)
def sync_aws(db: Session, since: date, base: str, counts: dict):
"""AWS-Rechnungen über die Invoicing-API holen (message_id = aws-invoice:<id>)."""
known = {r[0].split(":", 1)[1] for r in db.query(Outbound.message_id)
.filter(Outbound.message_id.like("aws-invoice:%")).all()}
for inv in aws.fetch_invoices(since, known):
issued = inv["issued"].replace(tzinfo=None) if inv["issued"] else None
row = Outbound(message_id=f"aws-invoice:{inv['invoice_id']}", email_folder="api", email_date=issued,
email_from="AWS <invoicing@aws.com>", mail_class="api",
email_subject=f"{inv['entity']} Rechnung {inv['invoice_id']} ({inv['period']})")
_store(db, row, inv["pdf"], base)
db.add(row)
db.commit()
counts[row.status] = counts.get(row.status, 0) + 1
def _domain(sender: str) -> str:
m = re.search(r"@([\w.-]+)", sender)
return m[1].lower() if m else ""
def run(db: Session, since: date, state: Optional[dict] = None) -> dict:
state = state if state is not None else {}
state.setdefault("errors", [])
@ -120,6 +151,12 @@ def run(db: Session, since: date, state: Optional[dict] = None) -> dict:
m.logout()
except Exception:
pass
if aws.ENABLED:
try:
sync_aws(db, since, cfg["storage_path"], counts)
except Exception as e:
db.rollback()
state["errors"].append(f"AWS-API: {e}"[:300])
return state

View file

@ -23,4 +23,4 @@ imaplib2==3.6
# Allgemein
python-dotenv==1.0.1
boto3==1.35.0
boto3==1.43.36

View file

@ -32,3 +32,30 @@ def test_process_message(tmp_path, monkeypatch):
monkeypatch.delenv("GRAB_CUTOFF") # ohne Stichtag nie automatisch senden
monkeypatch.setattr(grabber, "fetch_pdf", lambda url: b"%PDF-1.4 dritte")
assert run(LINK.format(mid="d", date="Mon, 16 Nov 2026 10:00:00 +0000")) == "pending_review"
def test_aws(tmp_path, monkeypatch):
from datetime import date, datetime, timezone
db = sessionmaker(bind=create_engine("sqlite://"))()
Base.metadata.create_all(db.get_bind())
monkeypatch.setattr(grabber.aws, "ENABLED", True)
calls = []
def fake(since, known):
calls.append(set(known))
for iid in ("EUINDE1", "EUINDE2"):
if iid not in known:
yield {"invoice_id": iid, "issued": datetime(2025, 12, 2, tzinfo=timezone.utc), "period": "2025-11",
"entity": "AWS", "pdf": f"%PDF-1.4 {iid}".encode()}
monkeypatch.setattr(grabber.aws, "fetch_invoices", fake)
counts = {}
grabber.sync_aws(db, date(2025, 1, 1), str(tmp_path), counts)
grabber.sync_aws(db, date(2025, 1, 1), str(tmp_path), counts) # zweiter Lauf: nichts Neues
assert counts == {"pending_review": 2} and calls[1] == {"EUINDE1", "EUINDE2"}
# AWS-Hinweismail wird nicht mehr als "manual" geführt
mail = ("Message-ID: <aws1>\nSubject: Amazon Web Services Tax Invoice Available\nFrom: AWS <no-reply@aws.com>\n"
"Content-Type: text/plain\n\nSee https://console.aws.amazon.com/billing")
assert grabber.process_message(db, email.message_from_string(mail), "INBOX", str(tmp_path)) is None
fake_aws = mail.replace("aws.com", "notaws.com").replace("<aws1>", "<aws2>")
assert grabber.process_message(db, email.message_from_string(fake_aws), "INBOX", str(tmp_path)) == "manual"

View file

@ -21,6 +21,8 @@ services:
- GRAB_CUTOFF=${GRAB_CUTOFF:-}
- SMTP_HOST=${SMTP_HOST:-smtp.strato.de}
- SMTP_PORT=${SMTP_PORT:-465}
- INVOICING_AWS_ACCESS_KEY_ID=${INVOICING_AWS_ACCESS_KEY_ID:-}
- INVOICING_AWS_SECRET_ACCESS_KEY=${INVOICING_AWS_SECRET_ACCESS_KEY:-}
depends_on:
- db
networks:

View file

@ -16,6 +16,8 @@ IMAP_USER=$IMAP_USER
IMAP_PASSWORD=$IMAP_PASSWORD
ACCOUNTABLE_INBOX=${ACCOUNTABLE_INBOX:-}
GRAB_CUTOFF=${GRAB_CUTOFF:-}
INVOICING_AWS_ACCESS_KEY_ID=${INVOICING_AWS_ACCESS_KEY_ID:-}
INVOICING_AWS_SECRET_ACCESS_KEY=${INVOICING_AWS_SECRET_ACCESS_KEY:-}
ENV
echo ".env geschrieben"
[ "${1:-}" = "--env-only" ] && exit 0