feat(picker): AWS-Adapter über Invoicing-API (ListInvoiceSummaries/GetInvoicePDF), AWS-Hinweismails -> via_api
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
b7840cdcc8
commit
e18c174406
6 changed files with 124 additions and 5 deletions
51
backend/fetchers/aws.py
Normal file
51
backend/fetchers/aws.py
Normal file
|
|
@ -0,0 +1,51 @@
|
||||||
|
"""AWS-Rechnungen per Invoicing-API (ListInvoiceSummaries + GetInvoicePDF), kein Konsolen-Login nötig.
|
||||||
|
|
||||||
|
IAM-User mit nur: invoicing:ListInvoiceSummaries, invoicing:GetInvoicePDF. Keys im Vault:
|
||||||
|
INVOICING_AWS_ACCESS_KEY_ID / INVOICING_AWS_SECRET_ACCESS_KEY. Ohne Keys ist der Adapter aus.
|
||||||
|
Endpoint us-east-1 ist bei AWS fest vorgegeben (globaler Billing-Dienst); übertragen wird nichts, AWS liefert
|
||||||
|
nur die eigene Rechnung an uns aus.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import urllib.request
|
||||||
|
from datetime import date
|
||||||
|
from typing import Iterator, Tuple
|
||||||
|
|
||||||
|
ENABLED = bool(os.getenv("INVOICING_AWS_ACCESS_KEY_ID") and os.getenv("INVOICING_AWS_SECRET_ACCESS_KEY"))
|
||||||
|
SENDER_DOMAINS = ("aws.com", "amazonaws.com", "aws.amazon.com") # deren Hinweis-Mails deckt die API ab
|
||||||
|
|
||||||
|
|
||||||
|
def _months(since: date) -> Iterator[Tuple[int, int]]:
|
||||||
|
y, m = since.year, since.month
|
||||||
|
today = date.today()
|
||||||
|
while (y, m) <= (today.year, today.month):
|
||||||
|
yield y, m
|
||||||
|
y, m = (y + 1, 1) if m == 12 else (y, m + 1)
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_invoices(since: date, known: set) -> Iterator[dict]:
|
||||||
|
"""-> {invoice_id, issued, period, pdf} für jede noch unbekannte Rechnung seit since."""
|
||||||
|
import boto3
|
||||||
|
|
||||||
|
s = boto3.Session(aws_access_key_id=os.environ["INVOICING_AWS_ACCESS_KEY_ID"],
|
||||||
|
aws_secret_access_key=os.environ["INVOICING_AWS_SECRET_ACCESS_KEY"], region_name="us-east-1")
|
||||||
|
account = s.client("sts").get_caller_identity()["Account"]
|
||||||
|
inv = s.client("invoicing")
|
||||||
|
for y, m in _months(since):
|
||||||
|
kw = {"Selector": {"ResourceType": "ACCOUNT_ID", "Value": account},
|
||||||
|
"Filter": {"BillingPeriod": {"Year": y, "Month": m}}}
|
||||||
|
while True:
|
||||||
|
page = inv.list_invoice_summaries(**kw)
|
||||||
|
for summ in page.get("InvoiceSummaries", []):
|
||||||
|
iid = summ["InvoiceId"]
|
||||||
|
if iid in known:
|
||||||
|
continue
|
||||||
|
url = inv.get_invoice_pdf(InvoiceId=iid)["InvoicePDF"]["DocumentUrl"] # presigned, kurz gültig
|
||||||
|
with urllib.request.urlopen(url, timeout=60) as r:
|
||||||
|
pdf = r.read()
|
||||||
|
if pdf.startswith(b"%PDF-"):
|
||||||
|
yield {"invoice_id": iid, "issued": summ.get("IssuedDate"), "period": f"{y}-{m:02d}",
|
||||||
|
"entity": (summ.get("Entity") or {}).get("BillingEntity", "AWS"), "pdf": pdf}
|
||||||
|
if not page.get("NextToken"):
|
||||||
|
break
|
||||||
|
kw["NextToken"] = page["NextToken"]
|
||||||
|
|
@ -15,6 +15,7 @@ from typing import Optional
|
||||||
|
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from backend.fetchers import aws
|
||||||
from backend.fetchers.direct import fetch_pdf
|
from backend.fetchers.direct import fetch_pdf
|
||||||
from backend.models import Outbound
|
from backend.models import Outbound
|
||||||
from backend.routers.emails import INVOICE_KEYWORDS, classify_mail, connect_imap, decode_hdr, parse_email_date
|
from backend.routers.emails import INVOICE_KEYWORDS, classify_mail, connect_imap, decode_hdr, parse_email_date
|
||||||
|
|
@ -38,7 +39,7 @@ def _status_for(email_date: Optional[datetime]) -> str:
|
||||||
def _save(pdf: bytes, base: str, email_date: Optional[datetime], sender: str) -> tuple:
|
def _save(pdf: bytes, base: str, email_date: Optional[datetime], sender: str) -> tuple:
|
||||||
sha = hashlib.sha256(pdf).hexdigest()
|
sha = hashlib.sha256(pdf).hexdigest()
|
||||||
d = email_date or datetime.utcnow()
|
d = email_date or datetime.utcnow()
|
||||||
domain = re.sub(r"[^\w.-]", "", (re.search(r"@([\w.-]+)", sender) or [None, "unbekannt"])[1])
|
domain = re.sub(r"[^\w.-]", "", _domain(sender)) or "unbekannt"
|
||||||
folder = os.path.join(base, "picker", f"{d:%Y}", f"{d:%m}")
|
folder = os.path.join(base, "picker", f"{d:%Y}", f"{d:%m}")
|
||||||
os.makedirs(folder, exist_ok=True)
|
os.makedirs(folder, exist_ok=True)
|
||||||
path = os.path.join(folder, f"{d:%Y-%m-%d}_{domain}_{sha[:8]}.pdf")
|
path = os.path.join(folder, f"{d:%Y-%m-%d}_{domain}_{sha[:8]}.pdf")
|
||||||
|
|
@ -58,6 +59,12 @@ def process_message(db: Session, msg, folder: str, base: str) -> Optional[str]:
|
||||||
row = db.query(Outbound).filter(Outbound.message_id == mid).first()
|
row = db.query(Outbound).filter(Outbound.message_id == mid).first()
|
||||||
if row and row.status != "manual": # "manual" wird bei jedem Lauf neu versucht (neue Adapter greifen rückwirkend)
|
if row and row.status != "manual": # "manual" wird bei jedem Lauf neu versucht (neue Adapter greifen rückwirkend)
|
||||||
return None
|
return None
|
||||||
|
d = _domain(sender)
|
||||||
|
if aws.ENABLED and any(d == x or d.endswith("." + x) for x in aws.SENDER_DOMAINS): # AWS kommt über die API (sync_aws)
|
||||||
|
if row:
|
||||||
|
row.status, row.error = "via_api", None
|
||||||
|
db.commit()
|
||||||
|
return None
|
||||||
mail_class, url = classify_mail(msg)
|
mail_class, url = classify_mail(msg)
|
||||||
if mail_class not in ("link", "portal"): # Anhänge leitet Markus bereits selbst an Accountable weiter
|
if mail_class not in ("link", "portal"): # Anhänge leitet Markus bereits selbst an Accountable weiter
|
||||||
return None
|
return None
|
||||||
|
|
@ -74,9 +81,7 @@ def process_message(db: Session, msg, folder: str, base: str) -> Optional[str]:
|
||||||
else:
|
else:
|
||||||
row.error = "Portal-Abruf noch nicht automatisiert" # Phase 3: Playwright-Adapter
|
row.error = "Portal-Abruf noch nicht automatisiert" # Phase 3: Playwright-Adapter
|
||||||
if pdf:
|
if pdf:
|
||||||
row.file_path, row.sha256 = _save(pdf, base, email_date, sender)
|
_store(db, row, pdf, base)
|
||||||
dup = db.query(Outbound.id).filter(Outbound.sha256 == row.sha256).first()
|
|
||||||
row.status = "duplicate" if dup else _status_for(email_date)
|
|
||||||
else:
|
else:
|
||||||
row.status = "manual"
|
row.status = "manual"
|
||||||
db.add(row)
|
db.add(row)
|
||||||
|
|
@ -84,6 +89,32 @@ def process_message(db: Session, msg, folder: str, base: str) -> Optional[str]:
|
||||||
return row.status
|
return row.status
|
||||||
|
|
||||||
|
|
||||||
|
def _store(db: Session, row: Outbound, pdf: bytes, base: str):
|
||||||
|
row.file_path, row.sha256 = _save(pdf, base, row.email_date, row.email_from or "")
|
||||||
|
dup = db.query(Outbound.id).filter(Outbound.sha256 == row.sha256, Outbound.id != row.id).first()
|
||||||
|
row.status = "duplicate" if dup else _status_for(row.email_date)
|
||||||
|
|
||||||
|
|
||||||
|
def sync_aws(db: Session, since: date, base: str, counts: dict):
|
||||||
|
"""AWS-Rechnungen über die Invoicing-API holen (message_id = aws-invoice:<id>)."""
|
||||||
|
known = {r[0].split(":", 1)[1] for r in db.query(Outbound.message_id)
|
||||||
|
.filter(Outbound.message_id.like("aws-invoice:%")).all()}
|
||||||
|
for inv in aws.fetch_invoices(since, known):
|
||||||
|
issued = inv["issued"].replace(tzinfo=None) if inv["issued"] else None
|
||||||
|
row = Outbound(message_id=f"aws-invoice:{inv['invoice_id']}", email_folder="api", email_date=issued,
|
||||||
|
email_from="AWS <invoicing@aws.com>", mail_class="api",
|
||||||
|
email_subject=f"{inv['entity']} Rechnung {inv['invoice_id']} ({inv['period']})")
|
||||||
|
_store(db, row, inv["pdf"], base)
|
||||||
|
db.add(row)
|
||||||
|
db.commit()
|
||||||
|
counts[row.status] = counts.get(row.status, 0) + 1
|
||||||
|
|
||||||
|
|
||||||
|
def _domain(sender: str) -> str:
|
||||||
|
m = re.search(r"@([\w.-]+)", sender)
|
||||||
|
return m[1].lower() if m else ""
|
||||||
|
|
||||||
|
|
||||||
def run(db: Session, since: date, state: Optional[dict] = None) -> dict:
|
def run(db: Session, since: date, state: Optional[dict] = None) -> dict:
|
||||||
state = state if state is not None else {}
|
state = state if state is not None else {}
|
||||||
state.setdefault("errors", [])
|
state.setdefault("errors", [])
|
||||||
|
|
@ -120,6 +151,12 @@ def run(db: Session, since: date, state: Optional[dict] = None) -> dict:
|
||||||
m.logout()
|
m.logout()
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
if aws.ENABLED:
|
||||||
|
try:
|
||||||
|
sync_aws(db, since, cfg["storage_path"], counts)
|
||||||
|
except Exception as e:
|
||||||
|
db.rollback()
|
||||||
|
state["errors"].append(f"AWS-API: {e}"[:300])
|
||||||
return state
|
return state
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -23,4 +23,4 @@ imaplib2==3.6
|
||||||
|
|
||||||
# Allgemein
|
# Allgemein
|
||||||
python-dotenv==1.0.1
|
python-dotenv==1.0.1
|
||||||
boto3==1.35.0
|
boto3==1.43.36
|
||||||
|
|
|
||||||
|
|
@ -32,3 +32,30 @@ def test_process_message(tmp_path, monkeypatch):
|
||||||
monkeypatch.delenv("GRAB_CUTOFF") # ohne Stichtag nie automatisch senden
|
monkeypatch.delenv("GRAB_CUTOFF") # ohne Stichtag nie automatisch senden
|
||||||
monkeypatch.setattr(grabber, "fetch_pdf", lambda url: b"%PDF-1.4 dritte")
|
monkeypatch.setattr(grabber, "fetch_pdf", lambda url: b"%PDF-1.4 dritte")
|
||||||
assert run(LINK.format(mid="d", date="Mon, 16 Nov 2026 10:00:00 +0000")) == "pending_review"
|
assert run(LINK.format(mid="d", date="Mon, 16 Nov 2026 10:00:00 +0000")) == "pending_review"
|
||||||
|
|
||||||
|
|
||||||
|
def test_aws(tmp_path, monkeypatch):
|
||||||
|
from datetime import date, datetime, timezone
|
||||||
|
db = sessionmaker(bind=create_engine("sqlite://"))()
|
||||||
|
Base.metadata.create_all(db.get_bind())
|
||||||
|
monkeypatch.setattr(grabber.aws, "ENABLED", True)
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def fake(since, known):
|
||||||
|
calls.append(set(known))
|
||||||
|
for iid in ("EUINDE1", "EUINDE2"):
|
||||||
|
if iid not in known:
|
||||||
|
yield {"invoice_id": iid, "issued": datetime(2025, 12, 2, tzinfo=timezone.utc), "period": "2025-11",
|
||||||
|
"entity": "AWS", "pdf": f"%PDF-1.4 {iid}".encode()}
|
||||||
|
|
||||||
|
monkeypatch.setattr(grabber.aws, "fetch_invoices", fake)
|
||||||
|
counts = {}
|
||||||
|
grabber.sync_aws(db, date(2025, 1, 1), str(tmp_path), counts)
|
||||||
|
grabber.sync_aws(db, date(2025, 1, 1), str(tmp_path), counts) # zweiter Lauf: nichts Neues
|
||||||
|
assert counts == {"pending_review": 2} and calls[1] == {"EUINDE1", "EUINDE2"}
|
||||||
|
# AWS-Hinweismail wird nicht mehr als "manual" geführt
|
||||||
|
mail = ("Message-ID: <aws1>\nSubject: Amazon Web Services Tax Invoice Available\nFrom: AWS <no-reply@aws.com>\n"
|
||||||
|
"Content-Type: text/plain\n\nSee https://console.aws.amazon.com/billing")
|
||||||
|
assert grabber.process_message(db, email.message_from_string(mail), "INBOX", str(tmp_path)) is None
|
||||||
|
fake_aws = mail.replace("aws.com", "notaws.com").replace("<aws1>", "<aws2>")
|
||||||
|
assert grabber.process_message(db, email.message_from_string(fake_aws), "INBOX", str(tmp_path)) == "manual"
|
||||||
|
|
|
||||||
|
|
@ -21,6 +21,8 @@ services:
|
||||||
- GRAB_CUTOFF=${GRAB_CUTOFF:-}
|
- GRAB_CUTOFF=${GRAB_CUTOFF:-}
|
||||||
- SMTP_HOST=${SMTP_HOST:-smtp.strato.de}
|
- SMTP_HOST=${SMTP_HOST:-smtp.strato.de}
|
||||||
- SMTP_PORT=${SMTP_PORT:-465}
|
- SMTP_PORT=${SMTP_PORT:-465}
|
||||||
|
- INVOICING_AWS_ACCESS_KEY_ID=${INVOICING_AWS_ACCESS_KEY_ID:-}
|
||||||
|
- INVOICING_AWS_SECRET_ACCESS_KEY=${INVOICING_AWS_SECRET_ACCESS_KEY:-}
|
||||||
depends_on:
|
depends_on:
|
||||||
- db
|
- db
|
||||||
networks:
|
networks:
|
||||||
|
|
|
||||||
|
|
@ -16,6 +16,8 @@ IMAP_USER=$IMAP_USER
|
||||||
IMAP_PASSWORD=$IMAP_PASSWORD
|
IMAP_PASSWORD=$IMAP_PASSWORD
|
||||||
ACCOUNTABLE_INBOX=${ACCOUNTABLE_INBOX:-}
|
ACCOUNTABLE_INBOX=${ACCOUNTABLE_INBOX:-}
|
||||||
GRAB_CUTOFF=${GRAB_CUTOFF:-}
|
GRAB_CUTOFF=${GRAB_CUTOFF:-}
|
||||||
|
INVOICING_AWS_ACCESS_KEY_ID=${INVOICING_AWS_ACCESS_KEY_ID:-}
|
||||||
|
INVOICING_AWS_SECRET_ACCESS_KEY=${INVOICING_AWS_SECRET_ACCESS_KEY:-}
|
||||||
ENV
|
ENV
|
||||||
echo ".env geschrieben"
|
echo ".env geschrieben"
|
||||||
[ "${1:-}" = "--env-only" ] && exit 0
|
[ "${1:-}" = "--env-only" ] && exit 0
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue