diff --git a/backend/fetchers/aws.py b/backend/fetchers/aws.py new file mode 100644 index 0000000..7c85524 --- /dev/null +++ b/backend/fetchers/aws.py @@ -0,0 +1,51 @@ +"""AWS-Rechnungen per Invoicing-API (ListInvoiceSummaries + GetInvoicePDF), kein Konsolen-Login nötig. + +IAM-User mit nur: invoicing:ListInvoiceSummaries, invoicing:GetInvoicePDF. Keys im Vault: +INVOICING_AWS_ACCESS_KEY_ID / INVOICING_AWS_SECRET_ACCESS_KEY. Ohne Keys ist der Adapter aus. +Endpoint us-east-1 ist bei AWS fest vorgegeben (globaler Billing-Dienst); übertragen wird nichts, AWS liefert +nur die eigene Rechnung an uns aus. +""" + +import os +import urllib.request +from datetime import date +from typing import Iterator, Tuple + +ENABLED = bool(os.getenv("INVOICING_AWS_ACCESS_KEY_ID") and os.getenv("INVOICING_AWS_SECRET_ACCESS_KEY")) +SENDER_DOMAINS = ("aws.com", "amazonaws.com", "aws.amazon.com") # deren Hinweis-Mails deckt die API ab + + +def _months(since: date) -> Iterator[Tuple[int, int]]: + y, m = since.year, since.month + today = date.today() + while (y, m) <= (today.year, today.month): + yield y, m + y, m = (y + 1, 1) if m == 12 else (y, m + 1) + + +def fetch_invoices(since: date, known: set) -> Iterator[dict]: + """-> {invoice_id, issued, period, pdf} für jede noch unbekannte Rechnung seit since.""" + import boto3 + + s = boto3.Session(aws_access_key_id=os.environ["INVOICING_AWS_ACCESS_KEY_ID"], + aws_secret_access_key=os.environ["INVOICING_AWS_SECRET_ACCESS_KEY"], region_name="us-east-1") + account = s.client("sts").get_caller_identity()["Account"] + inv = s.client("invoicing") + for y, m in _months(since): + kw = {"Selector": {"ResourceType": "ACCOUNT_ID", "Value": account}, + "Filter": {"BillingPeriod": {"Year": y, "Month": m}}} + while True: + page = inv.list_invoice_summaries(**kw) + for summ in page.get("InvoiceSummaries", []): + iid = summ["InvoiceId"] + if iid in known: + continue + url = inv.get_invoice_pdf(InvoiceId=iid)["InvoicePDF"]["DocumentUrl"] # presigned, kurz gültig + with urllib.request.urlopen(url, timeout=60) as r: + pdf = r.read() + if pdf.startswith(b"%PDF-"): + yield {"invoice_id": iid, "issued": summ.get("IssuedDate"), "period": f"{y}-{m:02d}", + "entity": (summ.get("Entity") or {}).get("BillingEntity", "AWS"), "pdf": pdf} + if not page.get("NextToken"): + break + kw["NextToken"] = page["NextToken"] diff --git a/backend/grabber.py b/backend/grabber.py index 71f9841..007a60d 100644 --- a/backend/grabber.py +++ b/backend/grabber.py @@ -15,6 +15,7 @@ from typing import Optional from sqlalchemy.orm import Session +from backend.fetchers import aws from backend.fetchers.direct import fetch_pdf from backend.models import Outbound from backend.routers.emails import INVOICE_KEYWORDS, classify_mail, connect_imap, decode_hdr, parse_email_date @@ -38,7 +39,7 @@ def _status_for(email_date: Optional[datetime]) -> str: def _save(pdf: bytes, base: str, email_date: Optional[datetime], sender: str) -> tuple: sha = hashlib.sha256(pdf).hexdigest() d = email_date or datetime.utcnow() - domain = re.sub(r"[^\w.-]", "", (re.search(r"@([\w.-]+)", sender) or [None, "unbekannt"])[1]) + domain = re.sub(r"[^\w.-]", "", _domain(sender)) or "unbekannt" folder = os.path.join(base, "picker", f"{d:%Y}", f"{d:%m}") os.makedirs(folder, exist_ok=True) path = os.path.join(folder, f"{d:%Y-%m-%d}_{domain}_{sha[:8]}.pdf") @@ -58,6 +59,12 @@ def process_message(db: Session, msg, folder: str, base: str) -> Optional[str]: row = db.query(Outbound).filter(Outbound.message_id == mid).first() if row and row.status != "manual": # "manual" wird bei jedem Lauf neu versucht (neue Adapter greifen rückwirkend) return None + d = _domain(sender) + if aws.ENABLED and any(d == x or d.endswith("." + x) for x in aws.SENDER_DOMAINS): # AWS kommt über die API (sync_aws) + if row: + row.status, row.error = "via_api", None + db.commit() + return None mail_class, url = classify_mail(msg) if mail_class not in ("link", "portal"): # Anhänge leitet Markus bereits selbst an Accountable weiter return None @@ -74,9 +81,7 @@ def process_message(db: Session, msg, folder: str, base: str) -> Optional[str]: else: row.error = "Portal-Abruf noch nicht automatisiert" # Phase 3: Playwright-Adapter if pdf: - row.file_path, row.sha256 = _save(pdf, base, email_date, sender) - dup = db.query(Outbound.id).filter(Outbound.sha256 == row.sha256).first() - row.status = "duplicate" if dup else _status_for(email_date) + _store(db, row, pdf, base) else: row.status = "manual" db.add(row) @@ -84,6 +89,32 @@ def process_message(db: Session, msg, folder: str, base: str) -> Optional[str]: return row.status +def _store(db: Session, row: Outbound, pdf: bytes, base: str): + row.file_path, row.sha256 = _save(pdf, base, row.email_date, row.email_from or "") + dup = db.query(Outbound.id).filter(Outbound.sha256 == row.sha256, Outbound.id != row.id).first() + row.status = "duplicate" if dup else _status_for(row.email_date) + + +def sync_aws(db: Session, since: date, base: str, counts: dict): + """AWS-Rechnungen über die Invoicing-API holen (message_id = aws-invoice:).""" + known = {r[0].split(":", 1)[1] for r in db.query(Outbound.message_id) + .filter(Outbound.message_id.like("aws-invoice:%")).all()} + for inv in aws.fetch_invoices(since, known): + issued = inv["issued"].replace(tzinfo=None) if inv["issued"] else None + row = Outbound(message_id=f"aws-invoice:{inv['invoice_id']}", email_folder="api", email_date=issued, + email_from="AWS ", mail_class="api", + email_subject=f"{inv['entity']} Rechnung {inv['invoice_id']} ({inv['period']})") + _store(db, row, inv["pdf"], base) + db.add(row) + db.commit() + counts[row.status] = counts.get(row.status, 0) + 1 + + +def _domain(sender: str) -> str: + m = re.search(r"@([\w.-]+)", sender) + return m[1].lower() if m else "" + + def run(db: Session, since: date, state: Optional[dict] = None) -> dict: state = state if state is not None else {} state.setdefault("errors", []) @@ -120,6 +151,12 @@ def run(db: Session, since: date, state: Optional[dict] = None) -> dict: m.logout() except Exception: pass + if aws.ENABLED: + try: + sync_aws(db, since, cfg["storage_path"], counts) + except Exception as e: + db.rollback() + state["errors"].append(f"AWS-API: {e}"[:300]) return state diff --git a/backend/requirements.txt b/backend/requirements.txt index cebc275..79e4c2a 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -23,4 +23,4 @@ imaplib2==3.6 # Allgemein python-dotenv==1.0.1 -boto3==1.35.0 +boto3==1.43.36 diff --git a/backend/tests/test_outbound.py b/backend/tests/test_outbound.py index edfe10b..ea40719 100644 --- a/backend/tests/test_outbound.py +++ b/backend/tests/test_outbound.py @@ -32,3 +32,30 @@ def test_process_message(tmp_path, monkeypatch): monkeypatch.delenv("GRAB_CUTOFF") # ohne Stichtag nie automatisch senden monkeypatch.setattr(grabber, "fetch_pdf", lambda url: b"%PDF-1.4 dritte") assert run(LINK.format(mid="d", date="Mon, 16 Nov 2026 10:00:00 +0000")) == "pending_review" + + +def test_aws(tmp_path, monkeypatch): + from datetime import date, datetime, timezone + db = sessionmaker(bind=create_engine("sqlite://"))() + Base.metadata.create_all(db.get_bind()) + monkeypatch.setattr(grabber.aws, "ENABLED", True) + calls = [] + + def fake(since, known): + calls.append(set(known)) + for iid in ("EUINDE1", "EUINDE2"): + if iid not in known: + yield {"invoice_id": iid, "issued": datetime(2025, 12, 2, tzinfo=timezone.utc), "period": "2025-11", + "entity": "AWS", "pdf": f"%PDF-1.4 {iid}".encode()} + + monkeypatch.setattr(grabber.aws, "fetch_invoices", fake) + counts = {} + grabber.sync_aws(db, date(2025, 1, 1), str(tmp_path), counts) + grabber.sync_aws(db, date(2025, 1, 1), str(tmp_path), counts) # zweiter Lauf: nichts Neues + assert counts == {"pending_review": 2} and calls[1] == {"EUINDE1", "EUINDE2"} + # AWS-Hinweismail wird nicht mehr als "manual" geführt + mail = ("Message-ID: \nSubject: Amazon Web Services Tax Invoice Available\nFrom: AWS \n" + "Content-Type: text/plain\n\nSee https://console.aws.amazon.com/billing") + assert grabber.process_message(db, email.message_from_string(mail), "INBOX", str(tmp_path)) is None + fake_aws = mail.replace("aws.com", "notaws.com").replace("", "") + assert grabber.process_message(db, email.message_from_string(fake_aws), "INBOX", str(tmp_path)) == "manual" diff --git a/docker-compose.yml b/docker-compose.yml index cb20afe..65abe22 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,8 @@ services: - GRAB_CUTOFF=${GRAB_CUTOFF:-} - SMTP_HOST=${SMTP_HOST:-smtp.strato.de} - SMTP_PORT=${SMTP_PORT:-465} + - INVOICING_AWS_ACCESS_KEY_ID=${INVOICING_AWS_ACCESS_KEY_ID:-} + - INVOICING_AWS_SECRET_ACCESS_KEY=${INVOICING_AWS_SECRET_ACCESS_KEY:-} depends_on: - db networks: diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 3c40360..e6c2ee5 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -16,6 +16,8 @@ IMAP_USER=$IMAP_USER IMAP_PASSWORD=$IMAP_PASSWORD ACCOUNTABLE_INBOX=${ACCOUNTABLE_INBOX:-} GRAB_CUTOFF=${GRAB_CUTOFF:-} +INVOICING_AWS_ACCESS_KEY_ID=${INVOICING_AWS_ACCESS_KEY_ID:-} +INVOICING_AWS_SECRET_ACCESS_KEY=${INVOICING_AWS_SECRET_ACCESS_KEY:-} ENV echo ".env geschrieben" [ "${1:-}" = "--env-only" ] && exit 0