lifetrack/apps/api/app/tests/modules/test_finance_pipeline.py

"""Import pipeline: dedup/idempotence, occurrence, rules, transfers, rollback.

Reference: datamodel-finance.md §4 (pipeline), §5 (rules), §6 (transfers),
§10.3 (same-day twins) and §10.6 (manual categorisation is sacred).
"""

import uuid
from decimal import Decimal
from pathlib import Path

import pytest
from sqlalchemy import func, select
from sqlalchemy.orm import Session

from app.core.errors import DomainValidationError
from app.modules.auth.models import User
from app.modules.finance import categorize, pipeline, service
from app.modules.finance.enums import AccountKind, CategorySource, ImportStatus
from app.modules.finance.models import (
    FinAccount,
    FinCategory,
    FinRule,
    FinSourceProfile,
    FinTransaction,
)

FIXTURES = Path(__file__).resolve().parents[1] / "fixtures" / "finance"


def make_account(
    db: Session, user: User, name: str, kind: AccountKind = AccountKind.CHECKING
) -> FinAccount:
    account = FinAccount(
        id=uuid.uuid4(),
        user_id=user.id,
        name=name,
        kind=kind,
        currency="EUR",
        initial_balance=Decimal("0.00"),
    )
    db.add(account)
    db.commit()
    return account


def builtin(db: Session, name: str) -> FinSourceProfile:
    profile = service.find_profile_by_name(db, name)
    assert profile is not None
    return profile


def import_fixture(
    db: Session, user: User, account: FinAccount, profile_name: str, filename: str
):
    service.ensure_seed(db, user.id)
    return pipeline.run_import(
        db,
        user.id,
        account,
        builtin(db, profile_name),
        filename,
        (FIXTURES / filename).read_bytes(),
    )


def transactions(db: Session, user: User) -> list[FinTransaction]:
    return list(
        db.scalars(
            select(FinTransaction)
            .where(FinTransaction.user_id == user.id)
            .order_by(FinTransaction.booked_date, FinTransaction.amount)
        ).all()
    )


# ---------------------------------------------------------------------------
# Import + idempotence
# ---------------------------------------------------------------------------


def test_import_boursobank_counts(db: Session, user: User) -> None:
    account = make_account(db, user, "BoursoBank")
    run = import_fixture(
        db, user, account, "BoursoBank / Boursorama (CSV)", "boursobank.csv"
    )
    assert run.status == ImportStatus.COMPLETED
    assert run.stats["rows_total"] == 8
    assert run.stats["rows_imported"] == 7
    assert run.stats["rows_error"] == 1
    assert run.stats["rows_skipped_duplicate"] == 0
    assert run.stats["date_min"] == "2026-06-01"
    assert run.stats["date_max"] == "2026-06-12"
    assert len(run.stats["errors"]) == 1

    rows = transactions(db, user)
    assert len(rows) == 7
    assert all(tx.import_run_id == run.import_run_id for tx in rows)
    assert all(tx.currency == "EUR" for tx in rows)
    carrefour = next(tx for tx in rows if "CARREFOUR" in tx.label_raw)
    assert carrefour.amount == Decimal("-42.90")
    assert carrefour.label_clean == "CARREFOUR CITY PARIS"  # technical prefix dropped


def test_reimporting_the_same_file_inserts_nothing(db: Session, user: User) -> None:
    account = make_account(db, user, "BoursoBank")
    first = import_fixture(
        db, user, account, "BoursoBank / Boursorama (CSV)", "boursobank.csv"
    )
    second = import_fixture(
        db, user, account, "BoursoBank / Boursorama (CSV)", "boursobank.csv"
    )
    assert second.stats["rows_imported"] == 0
    assert second.stats["rows_skipped_duplicate"] == 7
    assert second.stats["duplicate_file_of"] == first.import_run_id
    assert len(transactions(db, user)) == 7


def test_same_day_identical_purchases_are_kept(db: Session, user: User) -> None:
    account = make_account(db, user, "BoursoBank")
    import_fixture(db, user, account, "BoursoBank / Boursorama (CSV)", "boursobank.csv")
    twins = [tx for tx in transactions(db, user) if tx.amount == Decimal("-2.50")]
    assert len(twins) == 2
    assert twins[0].dedup_hash != twins[1].dedup_hash


def test_ofx_duplicate_fitid_is_not_a_conflict(db: Session, user: User) -> None:
    account = make_account(db, user, "LCL")
    run = import_fixture(db, user, account, "OFX (toutes banques)", "sample.ofx")
    assert run.stats["rows_imported"] == 4
    again = import_fixture(db, user, account, "OFX (toutes banques)", "sample.ofx")
    assert again.stats["rows_imported"] == 0
    assert again.stats["rows_skipped_duplicate"] == 4


def test_paypal_import_uses_transaction_id(db: Session, user: User) -> None:
    account = make_account(db, user, "PayPal", AccountKind.PAYPAL)
    run = import_fixture(
        db, user, account, "PayPal — rapport d'activité (CSV)", "paypal_fr.csv"
    )
    assert run.stats["rows_imported"] == 3
    assert run.stats["rows_skipped_filtered"] == 4
    external = {tx.external_id for tx in transactions(db, user)}
    assert "5AB12345CD678901E" in external


def test_stats_counters_add_up_to_rows_total(db: Session, user: User) -> None:
    """§2.4: rows_total == imported + skipped_duplicate + skipped_filtered
    + error. Rows dropped on purpose must stay visible in the total, otherwise
    the import summary silently hides them."""
    account = make_account(db, user, "PayPal", AccountKind.PAYPAL)
    for _ in range(2):  # second pass turns the 3 kept rows into duplicates
        run = import_fixture(
            db, user, account, "PayPal — rapport d'activité (CSV)", "paypal_fr.csv"
        )
        stats = run.stats
        assert stats["rows_skipped_filtered"] == 4
        assert stats["rows_total"] == (
            stats["rows_imported"]
            + stats["rows_skipped_duplicate"]
            + stats["rows_skipped_filtered"]
            + stats["rows_error"]
        )
    assert run.stats["rows_skipped_duplicate"] == 3


def test_a_fully_filtered_file_is_an_empty_import_not_a_failure(
    db: Session, user: User
) -> None:
    """A PayPal export made only of authorisations/pending lines is legitimate:
    nothing to import, but the run must not be marked `failed` (§4.2 fails only
    when every row ERRORED)."""
    service.ensure_seed(db, user.id)
    account = make_account(db, user, "PayPal", AccountKind.PAYPAL)
    header = (
        (FIXTURES / "paypal_fr.csv").read_text(encoding="utf-8-sig").splitlines()[0]
    )
    only_filtered = (
        header + "\n"
        "01/06/2026,10:00:00,CEST,Fnac,Autorisation,Autorisation,EUR,"
        '"-120,00","0,00","-120,00",moi@example.com,pay@fnac.com,'
        '9ZZ11111AA222222B,"Casque",,"0,00",Mémo\n'
    )
    run = pipeline.run_import(
        db,
        user.id,
        account,
        builtin(db, "PayPal — rapport d'activité (CSV)"),
        "paypal_vide.csv",
        only_filtered.encode("utf-8-sig"),
    )
    assert run.status == ImportStatus.COMPLETED
    assert run.stats["rows_total"] == 1
    assert run.stats["rows_skipped_filtered"] == 1
    assert run.stats["rows_imported"] == 0
    assert run.stats["rows_error"] == 0


def test_credit_agricole_import(db: Session, user: User) -> None:
    account = make_account(db, user, "Crédit Agricole")
    run = import_fixture(
        db, user, account, "Crédit Agricole (CSV)", "credit_agricole.csv"
    )
    assert run.stats["rows_imported"] == 4
    amounts = sorted(tx.amount for tx in transactions(db, user))
    assert amounts[0] == Decimal("-89.00")
    assert amounts[-1] == Decimal("1500.00")


def test_rollback_removes_the_imported_rows(db: Session, user: User) -> None:
    account = make_account(db, user, "BoursoBank")
    run = import_fixture(
        db, user, account, "BoursoBank / Boursorama (CSV)", "boursobank.csv"
    )
    service.rollback_import(db, user.id, run.id)
    assert transactions(db, user) == []
    assert db.scalar(select(func.count()).select_from(FinTransaction)) == 0


def test_wrong_profile_marks_the_run_as_failed(db: Session, user: User) -> None:
    account = make_account(db, user, "BoursoBank")
    service.ensure_seed(db, user.id)
    run = pipeline.run_import(
        db,
        user.id,
        account,
        builtin(db, "PayPal — rapport d'activité (CSV)"),
        "boursobank.csv",
        (FIXTURES / "boursobank.csv").read_bytes(),
    )
    assert run.status == ImportStatus.FAILED
    assert "PayPal" in run.error_message
    assert transactions(db, user) == []


def test_empty_file_is_rejected(db: Session, user: User) -> None:
    account = make_account(db, user, "BoursoBank")
    service.ensure_seed(db, user.id)
    with pytest.raises(DomainValidationError):
        pipeline.run_import(
            db, user.id, account, builtin(db, "CSV générique"), "vide.csv", b""
        )


# ---------------------------------------------------------------------------
# Rules engine
# ---------------------------------------------------------------------------


def make_rule(
    db: Session,
    user: User,
    name: str,
    matchers: dict,
    actions: dict,
    priority: int = 100,
    stop: bool = True,
) -> FinRule:
    rule = FinRule(
        id=uuid.uuid4(),
        user_id=user.id,
        name=name,
        priority=priority,
        enabled=True,
        stop=stop,
        matchers=matchers,
        actions=actions,
    )
    db.add(rule)
    db.commit()
    return rule


def category_named(db: Session, user: User, name: str) -> FinCategory:
    service.ensure_seed(db, user.id)
    category = db.scalars(
        select(FinCategory).where(
            FinCategory.user_id == user.id, FinCategory.name == name
        )
    ).first()
    assert category is not None, name
    return category


def test_rules_are_applied_before_insert(db: Session, user: User) -> None:
    account = make_account(db, user, "BoursoBank")
    courses = category_named(db, user, "Courses")
    rule = make_rule(
        db,
        user,
        "Courses Carrefour",
        {"label_contains": ["CARREFOUR"], "direction": "debit"},
        {"set_category_id": str(courses.id), "set_counterparty": "Carrefour"},
    )
    run = import_fixture(
        db, user, account, "BoursoBank / Boursorama (CSV)", "boursobank.csv"
    )
    assert run.stats["rules_applied"] == 1
    tx = next(tx for tx in transactions(db, user) if "CARREFOUR" in tx.label_raw)
    assert tx.category_id == courses.id
    assert tx.category_source == CategorySource.RULE
    assert tx.applied_rule_id == rule.id
    assert tx.counterparty == "Carrefour"
    db.refresh(rule)
    assert rule.hit_count == 1
    assert rule.last_applied_at is not None


def test_rule_priority_and_stop_flag(db: Session, user: User) -> None:
    account = make_account(db, user, "BoursoBank")
    loisirs = category_named(db, user, "Abonnements & streaming")
    autres = category_named(db, user, "Autres dépenses")
    make_rule(
        db,
        user,
        "Netflix",
        {"label_contains": ["NETFLIX"]},
        {"set_category_id": str(loisirs.id), "set_label_clean": "Netflix"},
        priority=10,
    )
    make_rule(
        db,
        user,
        "Tout le reste",
        {"direction": "debit"},
        {"set_category_id": str(autres.id)},
        priority=90,
    )
    import_fixture(db, user, account, "BoursoBank / Boursorama (CSV)", "boursobank.csv")
    netflix = next(tx for tx in transactions(db, user) if "NETFLIX" in tx.label_raw)
    assert netflix.category_id == loisirs.id  # priority 10 stopped the chain
    assert netflix.label_clean == "Netflix"
    other = next(tx for tx in transactions(db, user) if "LOYER" in tx.label_raw)
    assert other.category_id == autres.id


def test_amount_bounds_and_regex_matchers(db: Session, user: User) -> None:
    account = make_account(db, user, "BoursoBank")
    loyer = category_named(db, user, "Loyer / Crédit")
    make_rule(
        db,
        user,
        "Loyer",
        {"label_regex": r"^vir\s+sepa\s+loyer", "amount_max": -100},
        {"set_category_id": str(loyer.id)},
    )
    import_fixture(db, user, account, "BoursoBank / Boursorama (CSV)", "boursobank.csv")
    rows = transactions(db, user)
    tagged = [tx for tx in rows if tx.category_id == loyer.id]
    assert len(tagged) == 1
    assert tagged[0].amount == Decimal("-750.00")


def test_manual_categorisation_is_never_overwritten(db: Session, user: User) -> None:
    account = make_account(db, user, "BoursoBank")
    courses = category_named(db, user, "Courses")
    restaurants = category_named(db, user, "Restaurants & bars")
    import_fixture(db, user, account, "BoursoBank / Boursorama (CSV)", "boursobank.csv")
    tx = next(tx for tx in transactions(db, user) if "CARREFOUR" in tx.label_raw)
    tx.category_id = restaurants.id
    tx.category_source = CategorySource.USER
    db.commit()

    make_rule(
        db,
        user,
        "Courses Carrefour",
        {"label_contains": ["CARREFOUR"]},
        {"set_category_id": str(courses.id)},
    )
    result = categorize.apply_rules(db, user.id, scope="all_non_manual")
    db.refresh(tx)
    assert tx.category_id == restaurants.id
    assert result.updated == 0 or tx.category_source == CategorySource.USER

    with pytest.raises(DomainValidationError):
        categorize.apply_rules(db, user.id, scope="all")

    categorize.apply_rules(db, user.id, scope="all", force=True)
    db.refresh(tx)
    assert tx.category_id == courses.id


def test_apply_rules_dry_run_writes_nothing(db: Session, user: User) -> None:
    account = make_account(db, user, "BoursoBank")
    courses = category_named(db, user, "Courses")
    import_fixture(db, user, account, "BoursoBank / Boursorama (CSV)", "boursobank.csv")
    make_rule(
        db,
        user,
        "Courses Carrefour",
        {"label_contains": ["CARREFOUR"]},
        {"set_category_id": str(courses.id)},
    )
    result = categorize.apply_rules(db, user.id, scope="uncategorized", dry_run=True)
    assert result.matched == 1
    assert result.dry_run is True
    assert result.by_rule[0]["matched"] == 1
    tx = next(tx for tx in transactions(db, user) if "CARREFOUR" in tx.label_raw)
    assert tx.category_id is None


# ---------------------------------------------------------------------------
# Internal transfers
# ---------------------------------------------------------------------------


def test_transfer_pairing_between_two_accounts(db: Session, user: User) -> None:
    bank = make_account(db, user, "BoursoBank")
    paypal = make_account(db, user, "PayPal", AccountKind.PAYPAL)
    import_fixture(db, user, bank, "BoursoBank / Boursorama (CSV)", "boursobank.csv")
    run = import_fixture(
        db, user, paypal, "PayPal — rapport d'activité (CSV)", "paypal_fr.csv"
    )
    assert run.stats["transfers_detected"] == 1

    legs = [tx for tx in transactions(db, user) if tx.transfer_group_id is not None]
    assert len(legs) == 2
    assert legs[0].transfer_group_id == legs[1].transfer_group_id
    assert {leg.account_id for leg in legs} == {bank.id, paypal.id}
    assert {leg.amount for leg in legs} == {Decimal("-50.00"), Decimal("50.00")}
    transfer_cat = service.transfer_category(db, user.id)
    assert all(leg.category_id == transfer_cat.id for leg in legs)


def test_manual_link_and_unlink(db: Session, user: User) -> None:
    bank = make_account(db, user, "Compte A")
    other = make_account(db, user, "Compte B")
    service.ensure_seed(db, user.id)
    a = pipeline.build_transaction(
        user.id,
        bank,
        _row("2026-07-01", "-120.00", "VIREMENT VERS LIVRET"),
        "hash-a",
        None,
    )
    b = pipeline.build_transaction(
        user.id,
        other,
        _row("2026-07-02", "120.00", "VIREMENT RECU"),
        "hash-b",
        None,
    )
    db.add_all([a, b])
    db.commit()

    group_id = categorize.link_transfer(db, user.id, a.id, b.id)
    db.refresh(a)
    db.refresh(b)
    assert a.transfer_group_id == group_id == b.transfer_group_id

    categorize.unlink_transfer(db, user.id, group_id)
    db.refresh(a)
    assert a.transfer_group_id is None
    assert a.category_id is None


def test_link_rejects_non_opposite_amounts(db: Session, user: User) -> None:
    bank = make_account(db, user, "Compte A")
    other = make_account(db, user, "Compte B")
    a = pipeline.build_transaction(
        user.id, bank, _row("2026-07-01", "-120.00", "A"), "h1", None
    )
    b = pipeline.build_transaction(
        user.id, other, _row("2026-07-01", "119.00", "B"), "h2", None
    )
    db.add_all([a, b])
    db.commit()
    with pytest.raises(DomainValidationError):
        categorize.link_transfer(db, user.id, a.id, b.id)


def _row(day: str, amount: str, label: str):
    from datetime import date

    from app.modules.finance.parsers import NormalizedRow

    return NormalizedRow(
        booked_date=date.fromisoformat(day),
        value_date=None,
        amount=Decimal(amount),