From 937267b5072617c1229334cabc813a51fe9630a6 Mon Sep 17 00:00:00 2001 From: Markus Kruse Date: Fri, 24 Jul 2026 23:45:51 +0200 Subject: [PATCH] feat: Backend complete with IMAP, PDF processing, Alembic migrations - FastAPI Backend mit Router (emails, invoices, auth) - PostgreSQL DB Models (Invoice, EmailLog, InvoiceDetail, User) - PyMuPDF PDF-Verarbeitung mit Steuer-Erkennung - openpyxl Excel-Export - Alembic Migration initialisiert - IMAP-Verbindung zu Strato funktioniert - Docker-Setup mit Traefik-Labels Co-Authored-By: Claude --- Daily Notes/072426.md | 15 ++++ Dockerfile.backend | 2 + alembic.ini | 116 +++++++++++++++++++++++++++++++ backend/alembic/env.py | 123 +++++++++++++++++++++++++++++++++ backend/alembic/script.py.mako | 26 +++++++ backend/auth.py | 4 +- backend/main.py | 3 +- backend/requirements.txt | 2 +- backend/routers/invoices.py | 4 +- backend/tests/test_imap.py | 47 +++++++++++++ backend/tests/test_pdf.py | 54 +++++++++++++++ 11 files changed, 392 insertions(+), 4 deletions(-) create mode 100644 Daily Notes/072426.md create mode 100644 alembic.ini create mode 100644 backend/alembic/env.py create mode 100644 backend/alembic/script.py.mako create mode 100644 backend/tests/test_imap.py create mode 100644 backend/tests/test_pdf.py diff --git a/Daily Notes/072426.md b/Daily Notes/072426.md new file mode 100644 index 0000000..90a714f --- /dev/null +++ b/Daily Notes/072426.md @@ -0,0 +1,15 @@ +# 072426 - Daily Work Log + +## Decisions +- + +## Meetings & Conversations +- + +## Notes +- Session gestartet, AI Lab Verbindung getestet: online +- Memory geladen: Phase 2 Architektur in Bearbeitung +- Task Board geprüft: Backlog mit 15 Tickets, 8 erledigt, 6 offen + +## End of Day Summary +- diff --git a/Dockerfile.backend b/Dockerfile.backend index 0df9f9c..e6bbfd3 100644 --- a/Dockerfile.backend +++ b/Dockerfile.backend @@ -16,6 +16,8 @@ RUN pip install --no-cache-dir -r requirements.txt # Copy application code COPY backend/ ./backend/ COPY .env.example .env +COPY alembic.ini . +COPY backend/alembic ./backend/alembic # Environment ENV PYTHONUNBUFFERED=1 diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..ec69652 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,116 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = backend/alembic + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python>=3.9 or backports.zoneinfo library. +# Any required deps can installed by adding `alembic[tz]` to the pip requirements +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to migrations/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:migrations/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = driver://user:pass@localhost/dbname + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the exec runner, execute a binary +# hooks = ruff +# ruff.type = exec +# ruff.executable = %(here)s/.venv/bin/ruff +# ruff.options = --fix REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/backend/alembic/env.py b/backend/alembic/env.py new file mode 100644 index 0000000..ddaa805 --- /dev/null +++ b/backend/alembic/env.py @@ -0,0 +1,123 @@ +"""Alembic Environment Configuration""" +from logging.config import fileConfig + +from sqlalchemy import pool +from sqlalchemy import create_engine + +from alembic import context + +# Import models directly (avoid env file issues) +from sqlalchemy import Column, Integer, String, DateTime, Float, Boolean, Text, JSON +from sqlalchemy.ext.declarative import declarative_base +from datetime import datetime + +Base = declarative_base() + +# Import models manually +class Invoice(Base): + __tablename__ = "invoices" + id = Column(Integer, primary_key=True) + email_id = Column(String, index=True) + email_subject = Column(String, nullable=True) + email_from = Column(String, nullable=True) + email_date = Column(DateTime, nullable=True) + invoice_number = Column(String, index=True, nullable=True) + invoice_date = Column(DateTime, nullable=True) + invoice_amount = Column(Float, nullable=True) + invoice_tax_amount = Column(Float, nullable=True) + invoice_tax_rate = Column(Float, nullable=True) + invoice_tax_type = Column(String, nullable=True) + issuer_name = Column(String, nullable=True) + issuer_address = Column(Text, nullable=True) + file_path = Column(String, nullable=True) + file_name = Column(String, nullable=True) + storage_year = Column(Integer, nullable=True) + storage_path = Column(String, nullable=True) + processed = Column(Boolean, default=False) + processing_error = Column(Text, nullable=True) + confidence_score = Column(Float, nullable=True) + created_at = Column(DateTime, default=datetime.utcnow) + updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) + +class EmailLog(Base): + __tablename__ = "email_logs" + id = Column(Integer, primary_key=True) + email_id = Column(String, unique=True, index=True) + email_subject = Column(String, nullable=True) + email_from = Column(String, nullable=True) + email_date = Column(DateTime, nullable=True) + email_folder = Column(String, nullable=True) + matched_keywords = Column(JSON, nullable=True) + has_attachment = Column(Boolean, default=False) + attachment_type = Column(String, nullable=True) + processed = Column(Boolean, default=False) + processing_result = Column(String, nullable=True) + invoice_id = Column(Integer, nullable=True) + created_at = Column(DateTime, default=datetime.utcnow) + +class InvoiceDetail(Base): + __tablename__ = "invoice_details" + id = Column(Integer, primary_key=True) + invoice_id = Column(Integer, nullable=False, index=True) + raw_text = Column(Text, nullable=True) + parsed_data = Column(JSON, nullable=True) + confidence_breakdown = Column(JSON, nullable=True) + processing_notes = Column(Text, nullable=True) + created_at = Column(DateTime, default=datetime.utcnow) + +class User(Base): + __tablename__ = "users" + id = Column(Integer, primary_key=True) + username = Column(String, unique=True, index=True) + hashed_password = Column(String) + is_active = Column(Boolean, default=True) + created_at = Column(DateTime, default=datetime.utcnow) + +# Set target metadata for autogenerate +target_metadata = Base.metadata + +# other values from the config +def get_url(): + """Get database URL from environment""" + import os + return os.getenv("DATABASE_URL", "postgresql+psycopg2://postgres:postgres@localhost:5432/invoice_check") + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode.""" + url = get_url() + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + render_as_batch=True, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode.""" + url = get_url() + + connectable = create_engine( + url, + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + render_as_batch=True, + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/backend/alembic/script.py.mako b/backend/alembic/script.py.mako new file mode 100644 index 0000000..fbc4b07 --- /dev/null +++ b/backend/alembic/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/backend/auth.py b/backend/auth.py index 6175d93..3317412 100644 --- a/backend/auth.py +++ b/backend/auth.py @@ -10,11 +10,13 @@ from jose import JWTError, jwt from passlib.context import CryptContext from fastapi import Depends, HTTPException, status from fastapi.security import OAuth2PasswordBearer -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, declarative_base from sqlalchemy import Column, Integer, String, Boolean, DateTime from backend.database import get_db +Base = declarative_base() + # Konfiguration SECRET_KEY = os.getenv("SECRET_KEY", "your-secret-key-change-in-production") ALGORITHM = "HS256" diff --git a/backend/main.py b/backend/main.py index 3b04125..5877a1c 100644 --- a/backend/main.py +++ b/backend/main.py @@ -7,6 +7,7 @@ from fastapi import FastAPI, Depends, HTTPException from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse from sqlalchemy.orm import Session +from sqlalchemy import text from backend.database import get_db, engine, Base @@ -39,7 +40,7 @@ app.add_middleware( async def health_check(db: Session = Depends(get_db)): """Health Check Endpoint""" try: - db.execute("SELECT 1") + db.execute(text("SELECT 1")) return {"status": "healthy", "database": "connected"} except Exception as e: return {"status": "unhealthy", "database": "disconnected", "error": str(e)} diff --git a/backend/requirements.txt b/backend/requirements.txt index 4a636aa..cebc275 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -19,7 +19,7 @@ pymupdf==1.24.10 openpyxl==3.1.5 # Mail (IMAP) -imaplib2==3.5 +imaplib2==3.6 # Allgemein python-dotenv==1.0.1 diff --git a/backend/routers/invoices.py b/backend/routers/invoices.py index 4c5c512..c5cf6ea 100644 --- a/backend/routers/invoices.py +++ b/backend/routers/invoices.py @@ -154,9 +154,11 @@ def extract_pdf_data(file_path: str) -> Dict[str, Any]: try: doc = fitz.open(file_path) text = "" + pages = 0 for page in doc: text += page.get_text() + pages += 1 doc.close() @@ -166,7 +168,7 @@ def extract_pdf_data(file_path: str) -> Dict[str, Any]: "amount": extract_amount(text), "issuer": extract_issuer(text), "tax_info": detect_tax_type(text), - "pages": len(doc) if 'doc' in dir() else 0 + "pages": pages } except Exception as e: diff --git a/backend/tests/test_imap.py b/backend/tests/test_imap.py new file mode 100644 index 0000000..1d6d5c8 --- /dev/null +++ b/backend/tests/test_imap.py @@ -0,0 +1,47 @@ +"""IMAP-Verbindungs-Test""" +import imaplib +import os +import email + +IMAP_HOST = os.getenv("IMAP_HOST", "imap.strato.de") +IMAP_PORT = int(os.getenv("IMAP_PORT", 993)) +IMAP_USER = os.getenv("IMAP_USER", "info@c-stone-dev.com") +IMAP_PASSWORD = os.getenv("IMAP_PASSWORD", "hH22761,018!") + +print(f"Teste IMAP-Verbindung zu {IMAP_HOST}:{IMAP_PORT}") +print(f"Benutzer: {IMAP_USER}") + +try: + mail = imaplib.IMAP4_SSL(IMAP_HOST, IMAP_PORT) + result, data = mail.login(IMAP_USER, IMAP_PASSWORD) + print(f"Login erfolgreich: {result}") + + result, folders = mail.list() + print(f"Ordner:") + for folder in folders[:10]: # Erste 10 Ordner + print(f" {folder.decode()}") + + # INBOX auswählen + result, data = mail.select("INBOX") + if result == "OK": + result, data = mail.search(None, 'ALL') + if result == "OK": + email_ids = data[0].split() + print(f"\nINBOX enthält {len(email_ids)} Emails") + + # Letzte 5 Emails anzeigen + for email_id in email_ids[-5:]: + result, msg_data = mail.fetch(email_id, '(RFC822.HEADER)') + if result == "OK": + msg = email.message_from_bytes(msg_data[0][1]) + subject = msg.get("Subject", "Kein Betreff") + from_addr = msg.get("From", "Unbekannt") + print(f" Email {email_id.decode()}: {from_addr} - {subject}") + + mail.logout() + print("\n✅ IMAP-Verbindung erfolgreich!") + +except imaplib.IMAP4.error as e: + print(f"❌ IMAP-Fehler: {e}") +except Exception as e: + print(f"❌ Fehler: {e}") diff --git a/backend/tests/test_pdf.py b/backend/tests/test_pdf.py new file mode 100644 index 0000000..dd655d4 --- /dev/null +++ b/backend/tests/test_pdf.py @@ -0,0 +1,54 @@ +"""PDF-Verarbeitungs-Test""" +import sys +import os +sys.path.insert(0, '/app') + +import fitz +from backend.routers.invoices import extract_pdf_data, detect_tax_type, extract_amount, extract_invoice_number + +def test_pdf_extraction(file_path: str): + """Testet die Extraktion aus einer PDF-Datei""" + print(f"\n{'='*60}") + print(f"Teste: {file_path}") + print('='*60) + + if not os.path.exists(file_path): + print(f"❌ Datei nicht gefunden: {file_path}") + return + + # Extrahiere Daten + data = extract_pdf_data(file_path) + + if "error" in data: + print(f"❌ Fehler: {data['error']}") + return + + print(f"Seiten: {data.get('pages', 0)}") + print(f"Rechnungsnummer: {data.get('invoice_number') or 'Nicht gefunden'}") + print(f"Betrag: {data.get('amount')} €") + print(f"Austeller: {data.get('issuer') or 'Nicht gefunden'}") + + tax_info = data.get('tax_info', {}) + print(f"Steuerart: {tax_info.get('tax_type')}") + print(f"Steuersatz: {tax_info.get('tax_rate')}") + print(f"Confidence: {tax_info.get('confidence')}") + + # Teste einzelne Funktionen + print(f"\n--- Einzelne Funktionen ---") + text = data.get('text', '')[:500] + print(f"Auszug aus Text:\n{text[:200]}...") + + # Speichere extrahierten Text + with open(f'/tmp/test-invoices/{os.path.basename(file_path)}.txt', 'w') as f: + f.write(data.get('text', '')) + print(f"\nExtrahierter Text gespeichert in: /tmp/test-invoices/{os.path.basename(file_path)}.txt") + +if __name__ == '__main__': + # Teste lokale Dateien + test_files = [ + '/tmp/test-invoices/rechnung_mit_mwst.pdf', + '/tmp/test-invoices/rechnung_ohne_mwst.pdf', + ] + + for test_file in test_files: + test_pdf_extraction(test_file)