lifetrack/apps/api/app/modules/finance/stats.py

"""Chart-ready aggregates (datamodel-finance.md §8 and §9.8).

Every expense figure is returned as a POSITIVE number (the semantics carry the
sign, §10.7). The common scope of §8.1 excludes internal transfers, non-EUR
transactions and archived accounts (unless accounts are given explicitly).
"""

import uuid
from collections import defaultdict
from dataclasses import dataclass, field
from datetime import date, timedelta
from decimal import ROUND_HALF_UP, Decimal
from typing import Any

import sqlalchemy as sa
from sqlalchemy import Select, func, or_, select
from sqlalchemy.orm import Session, aliased

from app.modules.finance.categorize import (
    detect_recurring,
    monthly_total_estimate,
)
from app.modules.finance.enums import CategoryKind
from app.modules.finance.models import (
    FinAccount,
    FinBudget,
    FinCategory,
    FinTransaction,
)
from app.modules.finance.normalize import add_months, merchant_key, month_str
from app.modules.finance.presets import (
    DEFICIT_NODE_NAME,
    INCOME_NODE_NAME,
    MUTED_COLOR,
    PALETTE,
    SAVINGS_NODE_NAME,
    UNCATEGORIZED_COLOR,
    UNCATEGORIZED_LABEL,
)

ZERO = Decimal("0.00")
ONE_DAY = timedelta(days=1)
BUDGET_WARNING_RATIO = Decimal(80)
BUDGET_OVER_RATIO = Decimal(100)


def money(value: Any) -> Decimal:
    """Coerce a SQL aggregate (float on SQLite, Decimal on PG) to 2 decimals."""
    if value is None:
        return ZERO
    if not isinstance(value, Decimal):
        value = Decimal(str(value))
    return value.quantize(Decimal("0.01"), rounding=ROUND_HALF_UP)


def month_range(months: int, end: date) -> list[str]:
    """The `months` last month keys ending with `end`'s month (inclusive)."""
    first = add_months(end.replace(day=1), -(months - 1))
    return [month_str(add_months(first, i)) for i in range(months)]


@dataclass
class StatsScope:
    """Aliases + predicates shared by every aggregate (§8.1)."""

    category: Any
    parent: Any
    stmt: Select


def scope(user_id: int, account_ids: list[uuid.UUID] | None) -> StatsScope:
    category = aliased(FinCategory)
    parent = aliased(FinCategory)
    stmt = (
        select()
        .select_from(FinTransaction)
        .join(FinAccount, FinAccount.id == FinTransaction.account_id)
        .outerjoin(category, category.id == FinTransaction.category_id)
        .outerjoin(parent, parent.id == category.parent_id)
        .where(
            FinTransaction.user_id == user_id,
            FinTransaction.currency == "EUR",
            FinTransaction.transfer_group_id.is_(None),
            or_(category.kind.is_(None), category.kind != CategoryKind.TRANSFER),
        )
    )
    if account_ids:
        stmt = stmt.where(FinTransaction.account_id.in_(account_ids))
    else:
        stmt = stmt.where(FinAccount.is_archived.is_(False))
    return StatsScope(category=category, parent=parent, stmt=stmt)


def _year_month() -> tuple[Any, Any]:
    """Portable month extraction (SQLite -> strftime, PostgreSQL -> EXTRACT)."""
    return (
        sa.extract("year", FinTransaction.booked_date),
        sa.extract("month", FinTransaction.booked_date),
    )


def _key(year: Any, month: Any) -> str:
    return f"{int(year):04d}-{int(month):02d}"


def _direction_clause(direction: str) -> Any:
    if direction == "credit":
        return FinTransaction.amount > 0
    return FinTransaction.amount < 0


def _color_for(index: int, color: str | None) -> str:
    return color or PALETTE[index % len(PALETTE)]


# ---------------------------------------------------------------------------
# GET /stats/monthly-by-category
# ---------------------------------------------------------------------------


def monthly_by_category(
    db: Session,
    user_id: int,
    *,
    months: int = 12,
    level: str = "root",
    direction: str = "debit",
    account_ids: list[uuid.UUID] | None = None,
    today: date | None = None,
) -> dict[str, Any]:
    today = today or date.today()  # noqa: DTZ011 — civil day, no tz conversion
    keys = month_range(months, today)
    start = date.fromisoformat(f"{keys[0]}-01")
    end = add_months(date.fromisoformat(f"{keys[-1]}-01"), 1)

    sc = scope(user_id, account_ids)
    year_col, month_col = _year_month()
    if level == "child":
        cat_id = sc.category.id
        cat_name = sc.category.name
        cat_color = func.coalesce(sc.category.color, sc.parent.color)
    else:
        cat_id = func.coalesce(sc.parent.id, sc.category.id)
        cat_name = func.coalesce(sc.parent.name, sc.category.name)
        cat_color = func.coalesce(sc.parent.color, sc.category.color)

    stmt = (
        sc.stmt.add_columns(
            year_col.label("y"),
            month_col.label("m"),
            cat_id.label("cid"),
            cat_name.label("cname"),
            cat_color.label("ccolor"),
            func.sum(FinTransaction.amount).label("total"),
        )
        .where(
            FinTransaction.booked_date >= start,
            FinTransaction.booked_date < end,
            _direction_clause(direction),
        )
        .group_by(year_col, month_col, cat_id, cat_name, cat_color)
    )

    buckets: dict[tuple[uuid.UUID | None, str, str | None], dict[str, Decimal]] = (
        defaultdict(lambda: defaultdict(lambda: ZERO))
    )
    for row in db.execute(stmt):
        key = _key(row.y, row.m)
        ident = (row.cid, row.cname or UNCATEGORIZED_LABEL, row.ccolor)
        buckets[ident][key] = money(abs(money(row.total)))

    series: list[dict[str, Any]] = []
    for index, (ident, per_month) in enumerate(
        sorted(buckets.items(), key=lambda kv: -sum(kv[1].values()))
    ):
        cid, name, color = ident
        series.append(
            {
                "category_id": cid,
                "name": name if cid else UNCATEGORIZED_LABEL,
                "color": UNCATEGORIZED_COLOR if not cid else _color_for(index, color),
                "data": [per_month.get(key, ZERO) for key in keys],
            }
        )
    totals = [
        money(sum((s["data"][i] for s in series), ZERO)) for i in range(len(keys))
    ]
    return {"months": keys, "series": series, "totals": totals}


# ---------------------------------------------------------------------------
# GET /stats/cashflow
# ---------------------------------------------------------------------------


def cashflow(
    db: Session,
    user_id: int,
    *,
    months: int = 12,
    account_ids: list[uuid.UUID] | None = None,
    today: date | None = None,
) -> dict[str, Any]:
    today = today or date.today()  # noqa: DTZ011 — civil day, no tz conversion
    keys = month_range(months, today)
    start = date.fromisoformat(f"{keys[0]}-01")
    end = add_months(date.fromisoformat(f"{keys[-1]}-01"), 1)

    sc = scope(user_id, account_ids)
    year_col, month_col = _year_month()
    positive = func.sum(
        sa.case((FinTransaction.amount > 0, FinTransaction.amount), else_=0)
    )
    negative = func.sum(
        sa.case((FinTransaction.amount < 0, FinTransaction.amount), else_=0)
    )
    stmt = (
        sc.stmt.add_columns(
            year_col.label("y"),
            month_col.label("m"),
            positive.label("income"),
            negative.label("expenses"),
        )
        .where(FinTransaction.booked_date >= start, FinTransaction.booked_date < end)
        .group_by(year_col, month_col)
    )
    income_by_month: dict[str, Decimal] = {}
    expenses_by_month: dict[str, Decimal] = {}
    for row in db.execute(stmt):
        key = _key(row.y, row.m)
        income_by_month[key] = money(row.income)
        expenses_by_month[key] = money(abs(money(row.expenses)))

    income = [income_by_month.get(key, ZERO) for key in keys]
    expenses = [expenses_by_month.get(key, ZERO) for key in keys]
    net = [money(i - e) for i, e in zip(income, expenses, strict=True)]
    cumulative: list[Decimal] = []
    running = ZERO
    for value in net:
        running = money(running + value)
        cumulative.append(running)
    return {
        "months": keys,
        "income": income,
        "expenses": expenses,
        "net": net,
        "cumulative_net": cumulative,
    }


# ---------------------------------------------------------------------------
# GET /stats/top-merchants
# ---------------------------------------------------------------------------


def top_merchants(
    db: Session,
    user_id: int,
    *,
    months: int = 3,
    limit: int = 15,
    direction: str = "debit",
    account_ids: list[uuid.UUID] | None = None,
    today: date | None = None,
) -> dict[str, Any]:
    today = today or date.today()  # noqa: DTZ011 — civil day, no tz conversion
    keys = month_range(months, today)
    start = date.fromisoformat(f"{keys[0]}-01")
    end = add_months(date.fromisoformat(f"{keys[-1]}-01"), 1)

    sc = scope(user_id, account_ids)
    stmt = sc.stmt.add_columns(
        FinTransaction.counterparty,
        FinTransaction.label_clean,
        FinTransaction.label_raw,
        FinTransaction.amount,
        sc.category.name.label("cname"),
        func.coalesce(sc.category.color, sc.parent.color).label("ccolor"),
    ).where(
        FinTransaction.booked_date >= start,
        FinTransaction.booked_date < end,
        _direction_clause(direction),
    )

    groups: dict[str, dict[str, Any]] = {}
    for row in db.execute(stmt):
        name = row.counterparty or merchant_key(row.label_clean or row.label_raw)
        if not name:
            name = UNCATEGORIZED_LABEL
        entry = groups.setdefault(
            name,
            {
                "merchant": name,
                "total": ZERO,
                "count": 0,
                "category_name": row.cname,
                "category_color": row.ccolor,
            },
        )
        entry["total"] = money(entry["total"] + abs(money(row.amount)))
        entry["count"] += 1
        if entry["category_name"] is None and row.cname:
            entry["category_name"] = row.cname
            entry["category_color"] = row.ccolor

    items = sorted(groups.values(), key=lambda e: -e["total"])[:limit]
    for entry in items:
        entry["average"] = money(entry["total"] / entry["count"])
    return {
        "period": {"from": start, "to": end - ONE_DAY},
        "items": items,
    }


# ---------------------------------------------------------------------------
# GET /stats/budget-progress  (§8.3)
# ---------------------------------------------------------------------------


def _children_map(db: Session, user_id: int) -> dict[uuid.UUID, list[uuid.UUID]]:
    rows = db.execute(
        select(FinCategory.id, FinCategory.parent_id).where(
            FinCategory.user_id == user_id
        )
    ).all()
    children: dict[uuid.UUID, list[uuid.UUID]] = defaultdict(list)
    for cid, parent_id in rows:
        if parent_id is not None:
            children[parent_id].append(cid)
    return children


def expenses_by_category(
    db: Session,
    user_id: int,
    start: date,
    end: date,
    account_ids: list[uuid.UUID] | None = None,
) -> dict[uuid.UUID | None, Decimal]:
    """Absolute expense total per (leaf) category over [start, end)."""
    sc = scope(user_id, account_ids)
    stmt = (
        sc.stmt.add_columns(
            FinTransaction.category_id.label("cid"),
            func.sum(FinTransaction.amount).label("total"),
        )
        .where(
            FinTransaction.booked_date >= start,
            FinTransaction.booked_date < end,
            FinTransaction.amount < 0,
        )
        .group_by(FinTransaction.category_id)
    )
    return {row.cid: money(abs(money(row.total))) for row in db.execute(stmt)}


def budget_progress(
    db: Session,
    user_id: int,
    month: date,
    *,
    account_ids: list[uuid.UUID] | None = None,
    today: date | None = None,
) -> dict[str, Any]:
    today = today or date.today()  # noqa: DTZ011 — civil day, no tz conversion
    start = month.replace(day=1)
    end = add_months(start, 1)
    budgets = list(
        db.scalars(
            select(FinBudget)
            .where(
                FinBudget.user_id == user_id,
                FinBudget.start_month <= start,
                or_(FinBudget.end_month.is_(None), FinBudget.end_month >= start),
            )
            .order_by(FinBudget.start_month.asc())
        ).all()
    )
    if not budgets:
        return {
            "month": month_str(start),
            "items": [],
            "totals": {"budget": ZERO, "actual": ZERO, "progress_pct": 0.0},
        }

    actuals = expenses_by_category(db, user_id, start, end, account_ids)
    children = _children_map(db, user_id)
    categories = {
        c.id: c
        for c in db.scalars(
            select(FinCategory).where(FinCategory.user_id == user_id)
        ).all()
    }

    is_current = start <= today < end
    days_elapsed = (today - start).days + 1 if is_current else None
    days_in_month = (end - start).days

    items: list[dict[str, Any]] = []
    total_budget = ZERO
    total_actual = ZERO
    for budget in budgets:
        category = categories.get(budget.category_id)
        subtree = [budget.category_id, *children.get(budget.category_id, [])]
        actual = money(sum((actuals.get(cid, ZERO) for cid in subtree), ZERO))
        amount = money(budget.monthly_amount)
        progress = (
            float((actual / amount * 100).quantize(Decimal("0.1"))) if amount else 0.0
        )
        projected = None
        if is_current and days_elapsed:
            projected = money(actual / Decimal(days_elapsed) * Decimal(days_in_month))
        status = "ok"
        if Decimal(str(progress)) > BUDGET_OVER_RATIO:
            status = "over"
        elif Decimal(str(progress)) >= BUDGET_WARNING_RATIO or (
            projected is not None and projected > amount
        ):
            status = "warning"
        items.append(
            {
                "budget_id": budget.id,
                "category_id": budget.category_id,
                "category_name": category.name if category else UNCATEGORIZED_LABEL,
                "category_color": (category.color if category else None)
                or UNCATEGORIZED_COLOR,
                "budget": amount,
                "actual": actual,
                "remaining": money(amount - actual),
                "progress_pct": progress,
                "projected_eom": projected,
                "status": status,
            }
        )
        total_budget = money(total_budget + amount)
        total_actual = money(total_actual + actual)

    totals_pct = (
        float((total_actual / total_budget * 100).quantize(Decimal("0.1")))
        if total_budget
        else 0.0
    )
    return {
        "month": month_str(start),
        "items": items,
        "totals": {
            "budget": total_budget,
            "actual": total_actual,
            "progress_pct": totals_pct,
        },
    }


# ---------------------------------------------------------------------------
# GET /stats/sankey  (§9.8)
# ---------------------------------------------------------------------------


@dataclass
class _SankeyBuilder:
    nodes: list[dict[str, Any]] = field(default_factory=list)
    links: list[dict[str, Any]] = field(default_factory=list)
    seen: set[str] = field(default_factory=set)

    def node(self, name: str, color: str) -> str:
        if name not in self.seen:
            self.seen.add(name)
            self.nodes.append({"name": name, "color": color})
        return name

    def link(self, source: str, target: str, value: Decimal) -> None:
        if value > 0:
            self.links.append(
                {"source": source, "target": target, "value": money(value)}
            )


def sankey(
    db: Session,
    user_id: int,
    *,
    month: date | None = None,
    months: int = 1,
    account_ids: list[uuid.UUID] | None = None,
    today: date | None = None,
) -> dict[str, Any]:
    today = today or date.today()  # noqa: DTZ011 — civil day, no tz conversion
    if month is not None:
        start = month.replace(day=1)
        end = add_months(start, 1)
    else:
        keys = month_range(months, today)
        start = date.fromisoformat(f"{keys[0]}-01")
        end = add_months(date.fromisoformat(f"{keys[-1]}-01"), 1)

    sc = scope(user_id, account_ids)
    stmt = (
        sc.stmt.add_columns(
            sc.category.id.label("cid"),
            sc.category.name.label("cname"),
            sc.category.kind.label("ckind"),
            sc.category.color.label("ccolor"),
            sc.parent.id.label("pid"),
            sc.parent.name.label("pname"),
            sc.parent.color.label("pcolor"),
            func.sum(FinTransaction.amount).label("total"),
            func.sum(
                sa.case((FinTransaction.amount > 0, FinTransaction.amount), else_=0)
            ).label("credit"),
            func.sum(
                sa.case((FinTransaction.amount < 0, -FinTransaction.amount), else_=0)
            ).label("debit"),
        )
        .where(FinTransaction.booked_date >= start, FinTransaction.booked_date < end)
        .group_by(
            sc.category.id,
            sc.category.name,
            sc.category.kind,
            sc.category.color,
            sc.parent.id,
            sc.parent.name,
            sc.parent.color,
        )
    )

    income_roots: dict[str, dict[str, Any]] = {}
    expense_roots: dict[str, dict[str, Any]] = {}
    children: dict[str, dict[str, Decimal]] = defaultdict(
        lambda: defaultdict(lambda: ZERO)
    )
    child_colors: dict[str, str] = {}
    total_income = ZERO
    total_expense = ZERO

    for row in db.execute(stmt):
        root_name = row.pname or row.cname
        root_color = row.pcolor or row.ccolor
        # Uncategorized rows are TWO independent buckets (§9.8): credits feed
        # "Autres revenus", debits feed "Non catégorisé". Netting them against
        # each other would silently drop the uncategorized expenses AND shrink
        # the income side by the same amount. A real category keeps netting so
        # that a refund simply lowers that category's expense (and a category
        # never ends up on both sides of the hub, which would build a cycle).
        if row.cid is None:
            credit, debit = money(row.credit), money(row.debit)
        else:
            total = money(row.total)
            credit = total if total > 0 else ZERO
            debit = -total if total < 0 else ZERO

        if credit > 0:
            name = root_name or "Autres revenus"
            entry = income_roots.setdefault(
                name, {"total": ZERO, "color": root_color or PALETTE[0]}
            )
            entry["total"] = money(entry["total"] + credit)
            total_income = money(total_income + credit)
        if debit > 0:
            name = root_name or UNCATEGORIZED_LABEL
            entry = expense_roots.setdefault(
                name,
                {
                    "total": ZERO,
                    "color": (root_color or UNCATEGORIZED_COLOR)
                    if row.cid
                    else UNCATEGORIZED_COLOR,
                },
            )
            entry["total"] = money(entry["total"] + debit)
            total_expense = money(total_expense + debit)
            if row.pid is not None and row.cname:
                child_name = row.cname
                if child_name in expense_roots or child_name in income_roots:
                    child_name = f"{row.pname} · {row.cname}"
                children[name][child_name] = money(children[name][child_name] + debit)
                child_colors[child_name] = row.ccolor or row.pcolor or PALETTE[0]

    builder = _SankeyBuilder()
    hub = builder.node(INCOME_NODE_NAME, MUTED_COLOR)
    for index, (name, entry) in enumerate(
        sorted(income_roots.items(), key=lambda kv: -kv[1]["total"])
    ):
        builder.node(name, _color_for(index, entry["color"]))
        builder.link(name, hub, entry["total"])
    if total_expense > total_income:
        builder.node(DEFICIT_NODE_NAME, "#D03B3B")
        builder.link(DEFICIT_NODE_NAME, hub, money(total_expense - total_income))
    for index, (name, entry) in enumerate(
        sorted(expense_roots.items(), key=lambda kv: -kv[1]["total"])
    ):
        builder.node(name, _color_for(index, entry["color"]))
        builder.link(hub, name, entry["total"])
        for child_name, value in sorted(
            children.get(name, {}).items(), key=lambda kv: -kv[1]
        ):
            builder.node(child_name, child_colors.get(child_name, PALETTE[index % 8]))
            builder.link(name, child_name, value)
    if total_income > total_expense:
        builder.node(SAVINGS_NODE_NAME, "#0CA30C")
        builder.link(hub, SAVINGS_NODE_NAME, money(total_income - total_expense))

    return {
        "period": {"from": start, "to": end - ONE_DAY},
        "nodes": builder.nodes,
        "links": builder.links,
    }


# ---------------------------------------------------------------------------
# GET /stats/recurring + GET /dashboard
# ---------------------------------------------------------------------------


def recurring(
    db: Session,
    user_id: int,
    *,
    direction: str = "debit",
    include_inactive: bool = False,
    today: date | None = None,
) -> dict[str, Any]:
    series = detect_recurring(
        db,
        user_id,
        direction=direction,
        include_inactive=include_inactive,
        today=today,
    )
    return {
        "items": series,
        "monthly_total_estimate": monthly_total_estimate(series),
    }


def account_balances(db: Session, user_id: int) -> dict[uuid.UUID, Decimal]:
    rows = db.execute(
        select(
            FinTransaction.account_id,
            func.sum(FinTransaction.amount),
        )
        .where(FinTransaction.user_id == user_id)
        .group_by(FinTransaction.account_id)
    ).all()
    return {account_id: money(total) for account_id, total in rows}


def account_transaction_counts(db: Session, user_id: int) -> dict[uuid.UUID, int]:
    rows = db.execute(
        select(FinTransaction.account_id, func.count())
        .where(FinTransaction.user_id == user_id)
        .group_by(FinTransaction.account_id)
    ).all()
    return {account_id: int(count) for account_id, count in rows}


def dashboard(
    db: Session,
    user_id: int,
    *,
    today: date | None = None,
) -> dict[str, Any]:
    """KPIs of the finance home tab (ux-pages §13.1)."""
    today = today or date.today()  # noqa: DTZ011 — civil day, no tz conversion
    accounts = list(
        db.scalars(
            select(FinAccount).where(
                FinAccount.user_id == user_id, FinAccount.is_archived.is_(False)
            )
        ).all()
    )
    balances = account_balances(db, user_id)
    total_balance = money(
        sum(
            (money(a.initial_balance) + balances.get(a.id, ZERO) for a in accounts),
            ZERO,
        )
    )

    flow = cashflow(db, user_id, months=6, today=today)
    current_key = month_str(today)
    index = flow["months"].index(current_key)
    month_expenses = flow["expenses"][index]
    month_income = flow["income"][index]
    month_net = flow["net"][index]
    past = [value for i, value in enumerate(flow["expenses"]) if i != index]
    average_expenses = money(sum(past, ZERO) / Decimal(len(past))) if past else ZERO

    budgets = budget_progress(db, user_id, today.replace(day=1), today=today)
    uncategorized = (
        db.scalar(
            select(func.count())
            .select_from(FinTransaction)
            .where(
                FinTransaction.user_id == user_id,
                FinTransaction.category_id.is_(None),
                FinTransaction.transfer_group_id.is_(None),
            )
        )
        or 0
    )
    return {
        "month": current_key,
        "total_balance": total_balance,
        "accounts_count": len(accounts),
        "month_expenses": month_expenses,
        "month_income": month_income,
        "month_net": month_net,
        "average_expenses_6m": average_expenses,
        "budget_total": budgets["totals"]["budget"],
        "budget_actual": budgets["totals"]["actual"],
        "budget_progress_pct": budgets["totals"]["progress_pct"],
        "uncategorized_count": int(uncategorized),
    }