lifetrack/apps/api/app/tests/test_imports.py

"""Contract tests for the imports/ingest framework.

They use a self-contained sample domain (model + importer + ingest handler
defined below) so they do not depend on any business module.
"""

from collections.abc import Iterator
from io import BytesIO

from fastapi.testclient import TestClient
from sqlalchemy import ForeignKey, String, UniqueConstraint, select
from sqlalchemy.orm import Mapped, Session, mapped_column

from app.core.database import Base
from app.core.importing.base import (
    BaseImporter,
    ImporterParseError,
    NormalizedRecord,
    RowError,
    UpsertOutcome,
)
from app.core.importing.hashing import content_hash
from app.core.importing.registry import IMPORTER_REGISTRY, register_importer
from app.core.ingest.base import BaseIngestHandler, IngestRecord
from app.core.ingest.registry import INGEST_REGISTRY, register_ingest_handler
from app.core.mixins import SourceMixin, TimestampMixin
from app.modules.auth import service as auth_service
from app.modules.auth.schemas import DeviceKeyCreate


class SampleItem(TimestampMixin, SourceMixin, Base):
    __tablename__ = "test_sample_items"
    __table_args__ = (
        UniqueConstraint(
            "user_id", "source", "external_id", name="uq_test_sample_items_external"
        ),
        UniqueConstraint("user_id", "content_hash", name="uq_test_sample_items_hash"),
    )

    id: Mapped[int] = mapped_column(primary_key=True)
    user_id: Mapped[int] = mapped_column(ForeignKey("users.id"), index=True)
    import_run_id: Mapped[int | None] = mapped_column(
        ForeignKey("import_runs.id", ondelete="CASCADE"), default=None
    )
    label: Mapped[str] = mapped_column(String(100))
    value: Mapped[float] = mapped_column()


def _upsert_sample(
    db: Session,
    user_id: int,
    record: NormalizedRecord,
    source: str,
    import_run_id: int | None,
) -> UpsertOutcome:
    if record.data["value"] < 0:
        raise RowError("Valeur négative interdite.")
    if record.external_id is not None:
        digest = None
        existing = db.scalar(
            select(SampleItem).where(
                SampleItem.user_id == user_id,
                SampleItem.source == source,
                SampleItem.external_id == record.external_id,
            )
        )
    else:
        digest = content_hash(record)
        existing = db.scalar(
            select(SampleItem).where(
                SampleItem.user_id == user_id,
                SampleItem.content_hash == digest,
            )
        )
    if existing is not None:
        return UpsertOutcome.DUPLICATE
    db.add(
        SampleItem(
            user_id=user_id,
            source=source,
            external_id=record.external_id,
            content_hash=digest,
            import_run_id=import_run_id,
            label=record.data["label"],
            value=record.data["value"],
        )
    )
    db.flush()
    return UpsertOutcome.INSERTED


if "test_sample_csv" not in IMPORTER_REGISTRY:

    @register_importer
    class SampleCsvImporter(BaseImporter):
        id = "test_sample_csv"
        label = "Échantillon de test (CSV)"
        domain = "sample"
        accepted_extensions = (".csv",)

        @classmethod
        def sniff(cls, filename: str, head: bytes) -> bool:
            return filename.lower().endswith(".csv") and head.startswith(b"label;value")

        def parse(self, data: bytes, filename: str) -> Iterator[NormalizedRecord]:
            lines = [ln for ln in data.decode("utf-8").splitlines() if ln.strip()]
            if not lines or not lines[0].startswith("label;value"):
                raise ImporterParseError("En-têtes introuvables dans le fichier.")
            for line in lines[1:]:
                label, raw_value, external_id = line.split(";")
                yield NormalizedRecord(
                    kind="sample",
                    data={"label": label, "value": float(raw_value.replace(",", "."))},
                    external_id=external_id or None,
                    dedupe_fields=("label", "value"),
                )

        def upsert(
            self, db: Session, user_id: int, record: NormalizedRecord
        ) -> UpsertOutcome:
            return _upsert_sample(db, user_id, record, self.id, self.import_run_id)


if "sample" not in INGEST_REGISTRY:

    @register_ingest_handler
    class SampleIngestHandler(BaseIngestHandler):
        domain = "sample"
        record_types = ("sample",)

        def apply(
            self, db: Session, user_id: int, record: IngestRecord
        ) -> UpsertOutcome:
            normalized = NormalizedRecord(
                kind=record.type,
                data=record.data,
                external_id=record.external_id,
                dedupe_fields=("label", "value"),
            )
            return _upsert_sample(db, user_id, normalized, record.source, None)


CSV_OK = "label;value;ext\nCafé;1,5;a1\nThé;2;a2\nJus;3;\n"


def _upload(client: TestClient, headers: dict[str, str], content: str, source: str):
    return client.post(
        "/api/imports",
        headers=headers,
        files={"file": ("sample.csv", BytesIO(content.encode("utf-8")), "text/csv")},
        data={"source": source},
    )


def test_sources_endpoint_lists_importers(
    client: TestClient, user, auth_headers
) -> None:
    res = client.get("/api/imports/sources", headers=auth_headers)
    assert res.status_code == 200
    by_id = {src["id"]: src for src in res.json()}
    assert by_id["test_sample_csv"]["label"] == "Échantillon de test (CSV)"
    assert by_id["test_sample_csv"]["domain"] == "sample"
    assert by_id["test_sample_csv"]["accepted_extensions"] == [".csv"]


def test_import_then_reimport_is_idempotent(
    client: TestClient, db: Session, user, auth_headers
) -> None:
    res = _upload(client, auth_headers, CSV_OK, "test_sample_csv")
    assert res.status_code == 201
    run = res.json()
    assert run["status"] == "completed"
    assert run["rows_total"] == 3
    assert run["rows_inserted"] == 3
    assert run["rows_duplicates"] == 0

    # Same file again: everything deduped (external_id + content_hash paths).
    res = _upload(client, auth_headers, CSV_OK, "test_sample_csv")
    assert res.status_code == 201
    rerun = res.json()
    assert rerun["rows_inserted"] == 0
    assert rerun["rows_duplicates"] == 3

    assert db.scalar(select(SampleItem.id).limit(1)) is not None
    rows = db.scalars(select(SampleItem)).all()
    assert len(rows) == 3
    assert all(row.import_run_id == run["id"] for row in rows)


def test_auto_sniff_detects_and_rejects(client: TestClient, user, auth_headers) -> None:
    res = _upload(client, auth_headers, CSV_OK, "auto")
    assert res.status_code == 201
    assert res.json()["importer_id"] == "test_sample_csv"

    res = _upload(client, auth_headers, "something;else\n1;2\n", "auto")
    assert res.status_code == 422
    assert (
        res.json()["error"]["message"]
        == "Format non reconnu, choisissez un profil de source."
    )


def test_unknown_source_rejected(client: TestClient, user, auth_headers) -> None:
    res = _upload(client, auth_headers, CSV_OK, "nope_csv")
    assert res.status_code == 422
    assert res.json()["error"]["code"] == "validation_error"


def test_row_errors_do_not_stop_the_run(client: TestClient, user, auth_headers) -> None:
    csv = "label;value;ext\nCafé;1,5;b1\nMauvais;-4;b2\nThé;2;b3\n"
    res = _upload(client, auth_headers, csv, "test_sample_csv")
    assert res.status_code == 201
    run = res.json()
    assert run["status"] == "completed"
    assert run["rows_total"] == 3
    assert run["rows_inserted"] == 2
    assert run["rows_errors"] == 1
    assert run["error_details"][0]["row"] == 2
    assert "négative" in run["error_details"][0]["message"]


def test_fatal_parse_error_marks_run_failed(
    client: TestClient, db: Session, user, auth_headers
) -> None:
    res = _upload(client, auth_headers, "garbage without headers\n", "test_sample_csv")
    assert res.status_code == 201
    run = res.json()
    assert run["status"] == "failed"
    assert run["error_details"]
    # No partial domain rows are kept for a failed run.
    assert db.scalars(select(SampleItem)).all() == []


def test_history_pagination_and_filters(client: TestClient, user, auth_headers) -> None:
    _upload(client, auth_headers, CSV_OK, "test_sample_csv")
    _upload(client, auth_headers, CSV_OK, "test_sample_csv")

    res = client.get("/api/imports", headers=auth_headers)
    assert res.status_code == 200
    page = res.json()
    assert set(page.keys()) == {"items", "total", "page", "page_size"}
    assert page["total"] == 2

    res = client.get("/api/imports?domain=finance", headers=auth_headers)
    assert res.json()["total"] == 0

    res = client.get("/api/imports?sort=bogus", headers=auth_headers)
    assert res.status_code == 422

    run_id = page["items"][0]["id"]
    res = client.get(f"/api/imports/{run_id}", headers=auth_headers)
    assert res.status_code == 200
    assert res.json()["id"] == run_id


def test_rollback_deletes_run_and_cascades(
    client: TestClient, db: Session, user, auth_headers
) -> None:
    run_id = _upload(client, auth_headers, CSV_OK, "test_sample_csv").json()["id"]
    assert len(db.scalars(select(SampleItem)).all()) == 3

    res = client.delete(f"/api/imports/{run_id}", headers=auth_headers)
    assert res.status_code == 204

    db.expire_all()
    assert db.scalars(select(SampleItem)).all() == []
    res = client.get(f"/api/imports/{run_id}", headers=auth_headers)
    assert res.status_code == 404
    assert res.json()["error"]["code"] == "not_found"


def test_upload_too_large_rejected(
    client: TestClient, user, auth_headers, monkeypatch
) -> None:
    from app.core.config import get_settings

    monkeypatch.setattr(get_settings(), "max_upload_bytes", 10)
    res = _upload(client, auth_headers, CSV_OK, "test_sample_csv")
    assert res.status_code == 413
    assert res.json()["error"]["code"] == "payload_too_large"


def test_ingest_with_device_key_and_dedup(client: TestClient, user, device_key) -> None:
    payload = {
        "source": "android_bridge",
        "records": [
            {"type": "sample", "external_id": "x1", "data": {"label": "A", "value": 1}},
            {"type": "sample", "data": {"label": "B", "value": 2}},
        ],
    }
    res = client.post("/api/ingest/sample", headers=device_key.headers, json=payload)
    assert res.status_code == 200
    body = res.json()
    assert body == {
        "domain": "sample",
        "received": 2,
        "inserted": 2,
        "updated": 0,
        "duplicates": 0,
        "errors": [],
    }

    res = client.post("/api/ingest/sample", headers=device_key.headers, json=payload)
    assert res.json()["duplicates"] == 2


def test_ingest_works_with_jwt_too(client: TestClient, user, auth_headers) -> None:
    payload = {"records": [{"type": "sample", "data": {"label": "J", "value": 9}}]}
    res = client.post("/api/ingest/sample", headers=auth_headers, json=payload)
    assert res.status_code == 200
    assert res.json()["inserted"] == 1


def test_ingest_auth_and_scope_rules(
    client: TestClient, db: Session, user, device_key
) -> None:
    payload = {"records": [{"type": "sample", "data": {"label": "S", "value": 4}}]}

    # No credentials at all -> 401.
    res = client.post("/api/ingest/sample", json=payload)
    assert res.status_code == 401

    # Key scoped to another domain -> 403.
    _, other_plaintext = auth_service.create_device_key(
        db, user.id, DeviceKeyCreate(name="Autre", scopes=["ingest:health"])
    )
    res = client.post(
        "/api/ingest/sample", headers={"X-API-Key": other_plaintext}, json=payload
    )
    assert res.status_code == 403

    # Revoked wildcard key -> 401.
    auth_service.revoke_device_key(db, user.id, device_key.id)
    res = client.post("/api/ingest/sample", headers=device_key.headers, json=payload)
    assert res.status_code == 401


def test_ingest_unknown_domain_and_bad_type(
    client: TestClient, user, device_key
) -> None:
    payload = {"records": [{"type": "sample", "data": {"label": "D", "value": 1}}]}
    res = client.post("/api/ingest/nope", headers=device_key.headers, json=payload)
    assert res.status_code == 404

    payload = {
        "records": [
            {"type": "sample", "data": {"label": "OK", "value": 1}},
            {"type": "unknown_type", "data": {}},
            {"type": "sample", "data": {"label": "KO", "value": -1}},
        ]
    }
    res = client.post("/api/ingest/sample", headers=device_key.headers, json=payload)
    assert res.status_code == 200
    body = res.json()
    assert body["received"] == 3
    assert body["inserted"] == 1
    assert {err["index"] for err in body["errors"]} == {1, 2}


def test_ingest_record_limit(client: TestClient, user, device_key) -> None:
    records = [
        {"type": "sample", "data": {"label": f"r{i}", "value": i}} for i in range(1001)
    ]
    res = client.post(
        "/api/ingest/sample", headers=device_key.headers, json={"records": records}
    )
    assert res.status_code == 422
    assert res.json()["error"]["code"] == "validation_error"