"""File importers registered in the core registry (CONVENTIONS C3). They make the finance formats available through the generic `POST /api/imports` endpoint (source = importer id). The richer, account-aware flow (profile choice, preview, per-run stats) lives in `POST /api/finance/imports` and shares the same parsers/dedup helpers — see `pipeline.py`. Because the generic endpoint carries no account, rows land in a per-format default account created on first use; the user can rename it afterwards. """ from collections.abc import Iterator from typing import Any, ClassVar from sqlalchemy import select from sqlalchemy.orm import Session from app.core.importing.base import ( BaseImporter, ImporterParseError, NormalizedRecord, UpsertOutcome, ) from app.core.importing.registry import register_importer from app.modules.finance import service from app.modules.finance.categorize import ( apply_rules_to_tx, get_transfer_category_id, load_enabled_rules, ) from app.modules.finance.enums import SourceKind from app.modules.finance.models import FinTransaction from app.modules.finance.normalize import compute_dedup_hash, occurrence_key from app.modules.finance.parsers import NormalizedRow from app.modules.finance.pipeline import build_transaction, parse_file from app.modules.finance.presets import BUILTIN_PROFILES # Header signatures used by `sniff()` (lowercased, accent-insensitive enough). _BANK_HEADER_HINTS = ( "dateop;", "date;date valeur", "date;libell", "date de comptabilisation;", "date op", "date_comptabilisation;", "booking date", "completed date", "montant", "d\xe9bit", "debit;", ) _PAYPAL_HINTS = ("transaction id", "num\xe9ro de transaction", "numero de transaction") def _preset_config(slug: str) -> dict[str, Any]: for preset in BUILTIN_PROFILES: if preset["slug"] == slug: return preset["config"] return {} class FinanceImporter(BaseImporter): """Shared upsert logic: dedup by (account, hash) then (account, external_id).""" source_kind: ClassVar[str] profile_slug: ClassVar[str] default_account_name: ClassVar[str] domain: ClassVar[str] = "finance" def __init__(self) -> None: self._occurrences: dict[str, int] = {} def config(self) -> dict[str, Any]: return _preset_config(self.profile_slug) def parse(self, data: bytes, filename: str) -> Iterator[NormalizedRecord]: parsed = parse_file(self.source_kind, self.config(), data) # Only a file where every data row errored is unusable; rows filtered # on purpose (§3.3) are a legitimate empty import. if parsed.rows_total and parsed.rows_error == parsed.rows_total: raise ImporterParseError( "Aucune ligne exploitable — vérifiez le profil de source." ) for row in parsed.rows: yield NormalizedRecord( kind="transaction", data={"row": row}, external_id=row.external_id, dedupe_fields=("booked_date", "amount", "label_raw"), ) def upsert( self, db: Session, user_id: int, record: NormalizedRecord ) -> UpsertOutcome: row: NormalizedRow = record.data["row"] service.ensure_seed(db, user_id) account = service.account_for_source_kind( db, user_id, self.source_kind, self.default_account_name ) # Occurrence numbering of §4.3, kept across the whole file by the # importer instance (one instance per run). key = occurrence_key(row.booked_date, row.amount, row.label_raw) occurrence = self._occurrences.get(key, 0) self._occurrences[key] = occurrence + 1 dedup_hash = compute_dedup_hash( account.id, row.booked_date, row.amount, row.label_raw, occurrence ) if row.external_id: exists = db.scalars( select(FinTransaction.id).where( FinTransaction.account_id == account.id, FinTransaction.external_id == row.external_id, ) ).first() if exists is not None: return UpsertOutcome.DUPLICATE exists = db.scalars( select(FinTransaction.id).where( FinTransaction.account_id == account.id, FinTransaction.dedup_hash == dedup_hash, ) ).first() if exists is not None: return UpsertOutcome.DUPLICATE tx = build_transaction(user_id, account, row, dedup_hash, self.import_run_id) apply_rules_to_tx( tx, load_enabled_rules(db, user_id), get_transfer_category_id(db, user_id) ) db.add(tx) db.flush() return UpsertOutcome.INSERTED @register_importer class BankGenericCsvImporter(FinanceImporter): id = "bank_generic_csv" label = "Relevé bancaire (CSV générique)" accepted_extensions = (".csv", ".txt") source_kind = SourceKind.CSV profile_slug = "generic" default_account_name = "Compte importé (CSV)" @classmethod def sniff(cls, filename: str, head: bytes) -> bool: if not filename.lower().endswith(cls.accepted_extensions): return False try: text = head.decode("utf-8", errors="replace").lower() except Exception: # noqa: BLE001 — sniff must never raise return False first = text.splitlines()[0] if text.splitlines() else "" if any(hint in first for hint in _PAYPAL_HINTS): return False return any(hint in first for hint in _BANK_HEADER_HINTS) @register_importer class BankOfxImporter(FinanceImporter): id = "bank_ofx" label = "Relevé bancaire (OFX)" accepted_extensions = (".ofx", ".qfx") source_kind = SourceKind.OFX profile_slug = "ofx" default_account_name = "Compte importé (OFX)" @classmethod def sniff(cls, filename: str, head: bytes) -> bool: if not filename.lower().endswith(cls.accepted_extensions): return False text = head.decode("latin-1", errors="replace").upper() return "OFXHEADER" in text or "" in text @register_importer class PaypalCsvImporter(FinanceImporter): id = "paypal_csv" label = "PayPal — rapport d'activité (CSV)" accepted_extensions = (".csv",) source_kind = SourceKind.PAYPAL_CSV profile_slug = "paypal" default_account_name = "PayPal" @classmethod def sniff(cls, filename: str, head: bytes) -> bool: if not filename.lower().endswith(cls.accepted_extensions): return False text = head.decode("utf-8", errors="replace").lower() first = text.splitlines()[0] if text.splitlines() else "" return any(hint in first for hint in _PAYPAL_HINTS)