"""Contract tests for the imports/ingest framework. They use a self-contained sample domain (model + importer + ingest handler defined below) so they do not depend on any business module. """ from collections.abc import Iterator from io import BytesIO from fastapi.testclient import TestClient from sqlalchemy import ForeignKey, String, UniqueConstraint, select from sqlalchemy.orm import Mapped, Session, mapped_column from app.core.database import Base from app.core.importing.base import ( BaseImporter, ImporterParseError, NormalizedRecord, RowError, UpsertOutcome, ) from app.core.importing.hashing import content_hash from app.core.importing.registry import IMPORTER_REGISTRY, register_importer from app.core.ingest.base import BaseIngestHandler, IngestRecord from app.core.ingest.registry import INGEST_REGISTRY, register_ingest_handler from app.core.mixins import SourceMixin, TimestampMixin from app.modules.auth import service as auth_service from app.modules.auth.schemas import DeviceKeyCreate class SampleItem(TimestampMixin, SourceMixin, Base): __tablename__ = "test_sample_items" __table_args__ = ( UniqueConstraint( "user_id", "source", "external_id", name="uq_test_sample_items_external" ), UniqueConstraint("user_id", "content_hash", name="uq_test_sample_items_hash"), ) id: Mapped[int] = mapped_column(primary_key=True) user_id: Mapped[int] = mapped_column(ForeignKey("users.id"), index=True) import_run_id: Mapped[int | None] = mapped_column( ForeignKey("import_runs.id", ondelete="CASCADE"), default=None ) label: Mapped[str] = mapped_column(String(100)) value: Mapped[float] = mapped_column() def _upsert_sample( db: Session, user_id: int, record: NormalizedRecord, source: str, import_run_id: int | None, ) -> UpsertOutcome: if record.data["value"] < 0: raise RowError("Valeur négative interdite.") if record.external_id is not None: digest = None existing = db.scalar( select(SampleItem).where( SampleItem.user_id == user_id, SampleItem.source == source, SampleItem.external_id == record.external_id, ) ) else: digest = content_hash(record) existing = db.scalar( select(SampleItem).where( SampleItem.user_id == user_id, SampleItem.content_hash == digest, ) ) if existing is not None: return UpsertOutcome.DUPLICATE db.add( SampleItem( user_id=user_id, source=source, external_id=record.external_id, content_hash=digest, import_run_id=import_run_id, label=record.data["label"], value=record.data["value"], ) ) db.flush() return UpsertOutcome.INSERTED if "test_sample_csv" not in IMPORTER_REGISTRY: @register_importer class SampleCsvImporter(BaseImporter): id = "test_sample_csv" label = "Échantillon de test (CSV)" domain = "sample" accepted_extensions = (".csv",) @classmethod def sniff(cls, filename: str, head: bytes) -> bool: return filename.lower().endswith(".csv") and head.startswith(b"label;value") def parse(self, data: bytes, filename: str) -> Iterator[NormalizedRecord]: lines = [ln for ln in data.decode("utf-8").splitlines() if ln.strip()] if not lines or not lines[0].startswith("label;value"): raise ImporterParseError("En-têtes introuvables dans le fichier.") for line in lines[1:]: label, raw_value, external_id = line.split(";") yield NormalizedRecord( kind="sample", data={"label": label, "value": float(raw_value.replace(",", "."))}, external_id=external_id or None, dedupe_fields=("label", "value"), ) def upsert( self, db: Session, user_id: int, record: NormalizedRecord ) -> UpsertOutcome: return _upsert_sample(db, user_id, record, self.id, self.import_run_id) if "sample" not in INGEST_REGISTRY: @register_ingest_handler class SampleIngestHandler(BaseIngestHandler): domain = "sample" record_types = ("sample",) def apply( self, db: Session, user_id: int, record: IngestRecord ) -> UpsertOutcome: normalized = NormalizedRecord( kind=record.type, data=record.data, external_id=record.external_id, dedupe_fields=("label", "value"), ) return _upsert_sample(db, user_id, normalized, record.source, None) CSV_OK = "label;value;ext\nCafé;1,5;a1\nThé;2;a2\nJus;3;\n" def _upload(client: TestClient, headers: dict[str, str], content: str, source: str): return client.post( "/api/imports", headers=headers, files={"file": ("sample.csv", BytesIO(content.encode("utf-8")), "text/csv")}, data={"source": source}, ) def test_sources_endpoint_lists_importers( client: TestClient, user, auth_headers ) -> None: res = client.get("/api/imports/sources", headers=auth_headers) assert res.status_code == 200 by_id = {src["id"]: src for src in res.json()} assert by_id["test_sample_csv"]["label"] == "Échantillon de test (CSV)" assert by_id["test_sample_csv"]["domain"] == "sample" assert by_id["test_sample_csv"]["accepted_extensions"] == [".csv"] def test_import_then_reimport_is_idempotent( client: TestClient, db: Session, user, auth_headers ) -> None: res = _upload(client, auth_headers, CSV_OK, "test_sample_csv") assert res.status_code == 201 run = res.json() assert run["status"] == "completed" assert run["rows_total"] == 3 assert run["rows_inserted"] == 3 assert run["rows_duplicates"] == 0 # Same file again: everything deduped (external_id + content_hash paths). res = _upload(client, auth_headers, CSV_OK, "test_sample_csv") assert res.status_code == 201 rerun = res.json() assert rerun["rows_inserted"] == 0 assert rerun["rows_duplicates"] == 3 assert db.scalar(select(SampleItem.id).limit(1)) is not None rows = db.scalars(select(SampleItem)).all() assert len(rows) == 3 assert all(row.import_run_id == run["id"] for row in rows) def test_auto_sniff_detects_and_rejects(client: TestClient, user, auth_headers) -> None: res = _upload(client, auth_headers, CSV_OK, "auto") assert res.status_code == 201 assert res.json()["importer_id"] == "test_sample_csv" res = _upload(client, auth_headers, "something;else\n1;2\n", "auto") assert res.status_code == 422 assert ( res.json()["error"]["message"] == "Format non reconnu, choisissez un profil de source." ) def test_unknown_source_rejected(client: TestClient, user, auth_headers) -> None: res = _upload(client, auth_headers, CSV_OK, "nope_csv") assert res.status_code == 422 assert res.json()["error"]["code"] == "validation_error" def test_row_errors_do_not_stop_the_run(client: TestClient, user, auth_headers) -> None: csv = "label;value;ext\nCafé;1,5;b1\nMauvais;-4;b2\nThé;2;b3\n" res = _upload(client, auth_headers, csv, "test_sample_csv") assert res.status_code == 201 run = res.json() assert run["status"] == "completed" assert run["rows_total"] == 3 assert run["rows_inserted"] == 2 assert run["rows_errors"] == 1 assert run["error_details"][0]["row"] == 2 assert "négative" in run["error_details"][0]["message"] def test_fatal_parse_error_marks_run_failed( client: TestClient, db: Session, user, auth_headers ) -> None: res = _upload(client, auth_headers, "garbage without headers\n", "test_sample_csv") assert res.status_code == 201 run = res.json() assert run["status"] == "failed" assert run["error_details"] # No partial domain rows are kept for a failed run. assert db.scalars(select(SampleItem)).all() == [] def test_history_pagination_and_filters(client: TestClient, user, auth_headers) -> None: _upload(client, auth_headers, CSV_OK, "test_sample_csv") _upload(client, auth_headers, CSV_OK, "test_sample_csv") res = client.get("/api/imports", headers=auth_headers) assert res.status_code == 200 page = res.json() assert set(page.keys()) == {"items", "total", "page", "page_size"} assert page["total"] == 2 res = client.get("/api/imports?domain=finance", headers=auth_headers) assert res.json()["total"] == 0 res = client.get("/api/imports?sort=bogus", headers=auth_headers) assert res.status_code == 422 run_id = page["items"][0]["id"] res = client.get(f"/api/imports/{run_id}", headers=auth_headers) assert res.status_code == 200 assert res.json()["id"] == run_id def test_rollback_deletes_run_and_cascades( client: TestClient, db: Session, user, auth_headers ) -> None: run_id = _upload(client, auth_headers, CSV_OK, "test_sample_csv").json()["id"] assert len(db.scalars(select(SampleItem)).all()) == 3 res = client.delete(f"/api/imports/{run_id}", headers=auth_headers) assert res.status_code == 204 db.expire_all() assert db.scalars(select(SampleItem)).all() == [] res = client.get(f"/api/imports/{run_id}", headers=auth_headers) assert res.status_code == 404 assert res.json()["error"]["code"] == "not_found" def test_upload_too_large_rejected( client: TestClient, user, auth_headers, monkeypatch ) -> None: from app.core.config import get_settings monkeypatch.setattr(get_settings(), "max_upload_bytes", 10) res = _upload(client, auth_headers, CSV_OK, "test_sample_csv") assert res.status_code == 413 assert res.json()["error"]["code"] == "payload_too_large" def test_ingest_with_device_key_and_dedup(client: TestClient, user, device_key) -> None: payload = { "source": "android_bridge", "records": [ {"type": "sample", "external_id": "x1", "data": {"label": "A", "value": 1}}, {"type": "sample", "data": {"label": "B", "value": 2}}, ], } res = client.post("/api/ingest/sample", headers=device_key.headers, json=payload) assert res.status_code == 200 body = res.json() assert body == { "domain": "sample", "received": 2, "inserted": 2, "updated": 0, "duplicates": 0, "errors": [], } res = client.post("/api/ingest/sample", headers=device_key.headers, json=payload) assert res.json()["duplicates"] == 2 def test_ingest_works_with_jwt_too(client: TestClient, user, auth_headers) -> None: payload = {"records": [{"type": "sample", "data": {"label": "J", "value": 9}}]} res = client.post("/api/ingest/sample", headers=auth_headers, json=payload) assert res.status_code == 200 assert res.json()["inserted"] == 1 def test_ingest_auth_and_scope_rules( client: TestClient, db: Session, user, device_key ) -> None: payload = {"records": [{"type": "sample", "data": {"label": "S", "value": 4}}]} # No credentials at all -> 401. res = client.post("/api/ingest/sample", json=payload) assert res.status_code == 401 # Key scoped to another domain -> 403. _, other_plaintext = auth_service.create_device_key( db, user.id, DeviceKeyCreate(name="Autre", scopes=["ingest:health"]) ) res = client.post( "/api/ingest/sample", headers={"X-API-Key": other_plaintext}, json=payload ) assert res.status_code == 403 # Revoked wildcard key -> 401. auth_service.revoke_device_key(db, user.id, device_key.id) res = client.post("/api/ingest/sample", headers=device_key.headers, json=payload) assert res.status_code == 401 def test_ingest_unknown_domain_and_bad_type( client: TestClient, user, device_key ) -> None: payload = {"records": [{"type": "sample", "data": {"label": "D", "value": 1}}]} res = client.post("/api/ingest/nope", headers=device_key.headers, json=payload) assert res.status_code == 404 payload = { "records": [ {"type": "sample", "data": {"label": "OK", "value": 1}}, {"type": "unknown_type", "data": {}}, {"type": "sample", "data": {"label": "KO", "value": -1}}, ] } res = client.post("/api/ingest/sample", headers=device_key.headers, json=payload) assert res.status_code == 200 body = res.json() assert body["received"] == 3 assert body["inserted"] == 1 assert {err["index"] for err in body["errors"]} == {1, 2} def test_ingest_record_limit(client: TestClient, user, device_key) -> None: records = [ {"type": "sample", "data": {"label": f"r{i}", "value": i}} for i in range(1001) ] res = client.post( "/api/ingest/sample", headers=device_key.headers, json={"records": records} ) assert res.status_code == 422 assert res.json()["error"]["code"] == "validation_error"