lifetrack/apps/api/app/modules/imports/service.py

from typing import Any

from sqlalchemy import select
from sqlalchemy.orm import Session

from app.core.errors import DomainValidationError, NotFoundError
from app.core.importing.base import (
    ImporterParseError,
    RowError,
    UpsertOutcome,
)
from app.core.importing.registry import IMPORTER_REGISTRY, detect_importer
from app.core.ingest.base import IngestRecord
from app.core.ingest.registry import INGEST_REGISTRY
from app.core.timeutils import utcnow
from app.modules.auth.models import User
from app.modules.imports.models import ImportRun
from app.modules.imports.schemas import (
    IngestPayload,
    IngestRecordError,
    IngestResponse,
    SourceInfo,
)

MAX_ERROR_DETAILS = 100


def list_sources() -> list[SourceInfo]:
    return [
        SourceInfo(
            id=cls.id,
            label=cls.label,
            domain=cls.domain,
            accepted_extensions=cls.accepted_extensions,
        )
        for cls in sorted(IMPORTER_REGISTRY.values(), key=lambda c: c.id)
    ]


def resolve_importer_id(source: str, filename: str, head: bytes) -> str:
    """Resolve the form field `source` ("auto" or an importer id) to an id."""
    if source == "auto":
        detected = detect_importer(filename, head)
        if detected is None:
            raise DomainValidationError(
                "Format non reconnu, choisissez un profil de source."
            )
        return detected.id
    if source not in IMPORTER_REGISTRY:
        raise DomainValidationError(
            "Profil de source inconnu.", details={"source": source}
        )
    return source


def run_import(
    db: Session, user: User, importer_id: str, filename: str, data: bytes
) -> ImportRun:
    """Orchestrate one synchronous import run (see architecture §5.3).

    Row-level errors increment counters without stopping the run; a fatal
    parse error marks the run as failed (partial domain rows rolled back).
    A single commit ends the run.
    """
    importer_cls = IMPORTER_REGISTRY[importer_id]
    run = ImportRun(
        user_id=user.id,
        importer_id=importer_id,
        domain=importer_cls.domain,
        filename=filename,
        file_size=len(data),
        status="completed",
        started_at=utcnow(),
    )
    db.add(run)
    db.flush()  # assign run.id so imported rows can reference it

    importer = importer_cls()
    importer.import_run_id = run.id

    counters = {outcome: 0 for outcome in UpsertOutcome}
    error_details: list[dict[str, Any]] = []
    rows_total = 0
    fatal_message: str | None = None

    def record_error(row: int, message: str) -> None:
        counters[UpsertOutcome.ERROR] += 1
        if len(error_details) < MAX_ERROR_DETAILS:
            error_details.append({"row": row, "message": message})

    iterator = importer.parse(data, filename)
    row = 0
    while fatal_message is None:
        row += 1
        try:
            record = next(iterator)
        except StopIteration:
            break
        except RowError as exc:
            rows_total += 1
            record_error(row, str(exc))
            continue
        except ImporterParseError as exc:
            fatal_message = str(exc) or "Fichier illisible pour ce profil de source."
            break
        except Exception:  # noqa: BLE001 — importer bug must fail the run, not the API
            fatal_message = "Erreur inattendue pendant la lecture du fichier."
            break
        rows_total += 1
        try:
            outcome = importer.upsert(db, user.id, record)
        except RowError as exc:
            record_error(row, str(exc))
            continue
        except Exception:  # noqa: BLE001 — a broken row must not stop the run
            record_error(row, "Erreur inattendue sur cette ligne.")
            continue
        if outcome is UpsertOutcome.ERROR:
            record_error(row, "Ligne rejetée par l'importeur.")
        else:
            counters[outcome] += 1

    if fatal_message is not None:
        # Drop partial domain rows AND the flushed run, then persist a failed run.
        db.rollback()
        run = ImportRun(
            user_id=user.id,
            importer_id=importer_id,
            domain=importer_cls.domain,
            filename=filename,
            file_size=len(data),
            status="failed",
            rows_total=rows_total,
            rows_inserted=0,
            rows_updated=0,
            rows_duplicates=0,
            rows_errors=counters[UpsertOutcome.ERROR],
            error_details=(error_details + [{"row": row, "message": fatal_message}])[
                :MAX_ERROR_DETAILS
            ],
            started_at=utcnow(),
            finished_at=utcnow(),
        )
        db.add(run)
        db.commit()
        return run

    run.rows_total = rows_total
    run.rows_inserted = counters[UpsertOutcome.INSERTED]
    run.rows_updated = counters[UpsertOutcome.UPDATED]
    run.rows_duplicates = counters[UpsertOutcome.DUPLICATE]
    run.rows_errors = counters[UpsertOutcome.ERROR]
    run.error_details = error_details
    run.finished_at = utcnow()
    db.commit()
    return run


def get_run(db: Session, user_id: int, run_id: int) -> ImportRun:
    run = db.get(ImportRun, run_id)
    if run is None or run.user_id != user_id:
        raise NotFoundError("Import introuvable.")
    return run


def runs_query(user_id: int, domain: str | None, sort: str | None):
    stmt = select(ImportRun).where(ImportRun.user_id == user_id)
    if domain:
        stmt = stmt.where(ImportRun.domain == domain)
    sort = sort or "-started_at"
    allowed = {"started_at": ImportRun.started_at}
    descending = sort.startswith("-")
    field_name = sort.lstrip("-")
    column = allowed.get(field_name)
    if column is None:
        raise DomainValidationError(
            "Champ de tri non autorisé.", details={"sort": sort}
        )
    order = column.desc() if descending else column.asc()
    return stmt.order_by(order, ImportRun.id.desc())


def rollback_run(db: Session, user_id: int, run_id: int) -> None:
    """Delete a run; imported domain rows disappear via FK ON DELETE CASCADE."""
    run = get_run(db, user_id, run_id)
    db.delete(run)
    db.commit()


def run_ingest(
    db: Session, user: User, domain: str, payload: IngestPayload
) -> IngestResponse:
    handler = INGEST_REGISTRY.get(domain)
    if handler is None:
        raise NotFoundError("Domaine d'ingestion inconnu.", details={"domain": domain})
    counters = {outcome: 0 for outcome in UpsertOutcome}
    errors: list[IngestRecordError] = []
    for index, item in enumerate(payload.records):
        if item.type not in handler.record_types:
            errors.append(
                IngestRecordError(
                    index=index,
                    type=item.type,
                    message="Type d'enregistrement non pris en charge.",
                )
            )
            continue
        record = IngestRecord(
            type=item.type,
            data=item.data,
            external_id=item.external_id,
            source=payload.source,
        )
        try:
            outcome = handler.apply(db, user.id, record)
        except RowError as exc:
            errors.append(
                IngestRecordError(index=index, type=item.type, message=str(exc))
            )
            continue
        except Exception:  # noqa: BLE001 — a broken record must not fail the batch
            errors.append(
                IngestRecordError(
                    index=index,
                    type=item.type,
                    message="Erreur inattendue sur cet enregistrement.",
                )
            )
            continue
        if outcome is UpsertOutcome.ERROR:
            errors.append(
                IngestRecordError(
                    index=index, type=item.type, message="Enregistrement rejeté."
                )
            )
        else:
            counters[outcome] += 1
    db.commit()
    return IngestResponse(
        domain=domain,
        received=len(payload.records),
        inserted=counters[UpsertOutcome.INSERTED],
        updated=counters[UpsertOutcome.UPDATED],
        duplicates=counters[UpsertOutcome.DUPLICATE],
        errors=errors,
    )