MeeJayandClaude Opus 5
93f0689c1e
Tracker de vie auto-hébergé : suivi poids/calories/sport avec planning de pesées, sevrage tabac (vape) avec modèle de coût DIY et économies, et finances personnelles avec import de relevés bancaires. Architecture : FastAPI + SQLAlchemy 2.0 + PostgreSQL 16, React 18 + TS + Vite + Tailwind + ECharts, déploiement Docker Compose. Modules auto-découverts des deux côtés (pkgutil / import.meta.glob) et framework de connecteurs à deux voies (importeurs de fichiers + ingestion JSON) pour brancher de nouvelles sources sans toucher au noyau. Validé : 292 tests pytest, tsc + vite build, contrat API/web vérifié contre le schéma OpenAPI, et déploiement Docker réel sur PostgreSQL 16 (28 tables, SPA servie par nginx, wizard de premier démarrage). Documentation : README.md, docs/GUIDE.md, CONVENTIONS.md, et les documents de conception et de recherche dans docs/. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
578 lines
20 KiB
Python
578 lines
20 KiB
Python
"""Rules engine, internal-transfer detection and recurring-series detection.
|
|
|
|
Reference: datamodel-finance.md §5 (rules), §6 (transfers), §7 (recurring).
|
|
Nothing here ever overwrites a manual categorisation (`category_source =
|
|
'user'`) unless the caller explicitly forces it (§10.6).
|
|
"""
|
|
|
|
import re
|
|
import uuid
|
|
from collections import Counter, defaultdict
|
|
from dataclasses import dataclass, field
|
|
from datetime import date, timedelta
|
|
from decimal import Decimal
|
|
from itertools import pairwise
|
|
from statistics import median
|
|
from typing import Any
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.core.errors import DomainValidationError, NotFoundError
|
|
from app.core.timeutils import utcnow
|
|
from app.modules.finance.enums import CategoryKind, CategorySource
|
|
from app.modules.finance.models import FinCategory, FinRule, FinTransaction
|
|
from app.modules.finance.normalize import merchant_key, normalize_label_for_hash
|
|
|
|
TRANSFER_MAX_DAYS = 3
|
|
TRANSFER_LABEL_RE = re.compile(r"\bVIR(EMENT)?\b|\bVIRT\b|TRANSFERT|PAYPAL")
|
|
|
|
RULE_SCOPES = ("uncategorized", "all_non_manual", "all")
|
|
|
|
# (name, min_interval_days, max_interval_days, nominal_interval_days)
|
|
PERIODICITIES: tuple[tuple[str, int, int, int], ...] = (
|
|
("weekly", 6, 8, 7),
|
|
("monthly", 25, 35, 30),
|
|
("quarterly", 85, 97, 91),
|
|
("yearly", 350, 380, 365),
|
|
)
|
|
# Monthly normalisation factor used by `monthly_total_estimate` (§9.8).
|
|
MONTHLY_FACTOR: dict[str, Decimal] = {
|
|
"weekly": Decimal("4.33"),
|
|
"monthly": Decimal(1),
|
|
"quarterly": Decimal(1) / Decimal(3),
|
|
"yearly": Decimal(1) / Decimal(12),
|
|
}
|
|
MIN_OCCURRENCES = 3
|
|
REGULARITY_RATIO = 0.8
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Rules engine (§5)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def compile_matchers(matchers: dict[str, Any]) -> re.Pattern[str] | None:
|
|
"""Compile `label_regex` once; raises DomainValidationError if invalid."""
|
|
pattern = matchers.get("label_regex")
|
|
if not pattern:
|
|
return None
|
|
try:
|
|
return re.compile(pattern, re.IGNORECASE)
|
|
except re.error as exc:
|
|
raise DomainValidationError(f"Expression régulière invalide : {exc}") from exc
|
|
|
|
|
|
def validate_matchers(matchers: dict[str, Any]) -> None:
|
|
if not isinstance(matchers, dict):
|
|
raise DomainValidationError("Conditions de règle invalides.")
|
|
direction = matchers.get("direction")
|
|
if direction is not None and direction not in ("debit", "credit", "any"):
|
|
raise DomainValidationError(
|
|
"Sens invalide : utilisez « debit », « credit » ou « any »."
|
|
)
|
|
conditions = (
|
|
"label_contains",
|
|
"label_regex",
|
|
"amount_min",
|
|
"amount_max",
|
|
"account_id",
|
|
)
|
|
has_condition = any(
|
|
matchers.get(key) not in (None, [], "") for key in conditions
|
|
) or direction in ("debit", "credit")
|
|
if not has_condition:
|
|
raise DomainValidationError("La règle doit comporter au moins une condition.")
|
|
compile_matchers(matchers)
|
|
|
|
|
|
def validate_actions(actions: dict[str, Any]) -> None:
|
|
if not isinstance(actions, dict) or not any(
|
|
actions.get(key) not in (None, "", False)
|
|
for key in (
|
|
"set_category_id",
|
|
"set_label_clean",
|
|
"set_counterparty",
|
|
"mark_transfer",
|
|
)
|
|
):
|
|
raise DomainValidationError("La règle doit comporter au moins une action.")
|
|
|
|
|
|
@dataclass
|
|
class CompiledRule:
|
|
rule: FinRule
|
|
regex: re.Pattern[str] | None
|
|
contains: tuple[str, ...]
|
|
|
|
@property
|
|
def matchers(self) -> dict[str, Any]:
|
|
return self.rule.matchers or {}
|
|
|
|
@property
|
|
def actions(self) -> dict[str, Any]:
|
|
return self.rule.actions or {}
|
|
|
|
|
|
def compile_rules(rules: list[FinRule]) -> list[CompiledRule]:
|
|
compiled: list[CompiledRule] = []
|
|
for rule in rules:
|
|
matchers = rule.matchers or {}
|
|
try:
|
|
regex = compile_matchers(matchers)
|
|
except DomainValidationError:
|
|
regex = None # stored rule broke: ignore its regex, keep the rest
|
|
contains = tuple(
|
|
normalize_label_for_hash(str(needle))
|
|
for needle in (matchers.get("label_contains") or [])
|
|
if str(needle).strip()
|
|
)
|
|
compiled.append(CompiledRule(rule=rule, regex=regex, contains=contains))
|
|
return compiled
|
|
|
|
|
|
def load_enabled_rules(db: Session, user_id: int) -> list[CompiledRule]:
|
|
stmt = (
|
|
select(FinRule)
|
|
.where(FinRule.user_id == user_id, FinRule.enabled.is_(True))
|
|
.order_by(FinRule.priority.asc(), FinRule.created_at.asc(), FinRule.id.asc())
|
|
)
|
|
return compile_rules(list(db.scalars(stmt).all()))
|
|
|
|
|
|
def rule_matches(compiled: CompiledRule, tx: FinTransaction) -> bool:
|
|
m = compiled.matchers
|
|
account_id = m.get("account_id")
|
|
if account_id and str(tx.account_id) != str(account_id):
|
|
return False
|
|
direction = m.get("direction")
|
|
if direction == "debit" and tx.amount >= 0:
|
|
return False
|
|
if direction == "credit" and tx.amount <= 0:
|
|
return False
|
|
if m.get("amount_min") is not None and tx.amount < Decimal(str(m["amount_min"])):
|
|
return False
|
|
if m.get("amount_max") is not None and tx.amount > Decimal(str(m["amount_max"])):
|
|
return False
|
|
if compiled.contains:
|
|
hay = (
|
|
normalize_label_for_hash(tx.label_raw)
|
|
+ " | "
|
|
+ normalize_label_for_hash(tx.label_clean or "")
|
|
)
|
|
if not any(needle in hay for needle in compiled.contains):
|
|
return False
|
|
return compiled.regex is None or bool(
|
|
compiled.regex.search(tx.label_raw or "")
|
|
or compiled.regex.search(tx.label_clean or "")
|
|
)
|
|
|
|
|
|
def apply_rules_to_tx(
|
|
tx: FinTransaction,
|
|
rules: list[CompiledRule],
|
|
transfer_category_id: uuid.UUID | None,
|
|
*,
|
|
force: bool = False,
|
|
) -> list[uuid.UUID]:
|
|
"""Mutate `tx` in place; returns the ids of the rules that fired (§5.3)."""
|
|
fired: list[uuid.UUID] = []
|
|
for compiled in rules:
|
|
if not rule_matches(compiled, tx):
|
|
continue
|
|
actions = compiled.actions
|
|
protected = tx.category_source == CategorySource.USER and not force
|
|
if actions.get("set_category_id") and not protected:
|
|
tx.category_id = uuid.UUID(str(actions["set_category_id"]))
|
|
tx.category_source = CategorySource.RULE
|
|
tx.applied_rule_id = compiled.rule.id
|
|
if actions.get("set_label_clean"):
|
|
tx.label_clean = str(actions["set_label_clean"])
|
|
if actions.get("set_counterparty"):
|
|
tx.counterparty = str(actions["set_counterparty"])[:150]
|
|
if actions.get("mark_transfer") and not protected and transfer_category_id:
|
|
tx.category_id = transfer_category_id
|
|
tx.category_source = CategorySource.RULE
|
|
tx.applied_rule_id = compiled.rule.id
|
|
fired.append(compiled.rule.id)
|
|
compiled.rule.hit_count += 1
|
|
compiled.rule.last_applied_at = utcnow()
|
|
if compiled.rule.stop:
|
|
break
|
|
return fired
|
|
|
|
|
|
def get_transfer_category_id(db: Session, user_id: int) -> uuid.UUID | None:
|
|
stmt = select(FinCategory.id).where(
|
|
FinCategory.user_id == user_id,
|
|
FinCategory.kind == CategoryKind.TRANSFER,
|
|
)
|
|
return db.scalars(stmt.limit(1)).first()
|
|
|
|
|
|
@dataclass
|
|
class RuleApplyResult:
|
|
scanned: int = 0
|
|
matched: int = 0
|
|
updated: int = 0
|
|
dry_run: bool = False
|
|
by_rule: list[dict[str, Any]] = field(default_factory=list)
|
|
|
|
|
|
def apply_rules(
|
|
db: Session,
|
|
user_id: int,
|
|
*,
|
|
scope: str = "uncategorized",
|
|
date_from: date | None = None,
|
|
date_to: date | None = None,
|
|
account_id: uuid.UUID | None = None,
|
|
rule_id: uuid.UUID | None = None,
|
|
dry_run: bool = False,
|
|
force: bool = False,
|
|
) -> RuleApplyResult:
|
|
"""Re-run the rules over stored transactions (§5.4)."""
|
|
if scope not in RULE_SCOPES:
|
|
raise DomainValidationError(
|
|
"Portée inconnue : utilisez « uncategorized », « all_non_manual » "
|
|
"ou « all »."
|
|
)
|
|
if scope == "all" and not force:
|
|
raise DomainValidationError(
|
|
"La portée « all » écrase les catégories manuelles : "
|
|
"confirmez avec force=true."
|
|
)
|
|
|
|
rules = load_enabled_rules(db, user_id)
|
|
if rule_id is not None:
|
|
rules = [c for c in rules if c.rule.id == rule_id]
|
|
if not rules:
|
|
raise NotFoundError("Règle introuvable.")
|
|
|
|
stmt = select(FinTransaction).where(FinTransaction.user_id == user_id)
|
|
if scope == "uncategorized":
|
|
stmt = stmt.where(FinTransaction.category_id.is_(None))
|
|
elif scope == "all_non_manual":
|
|
stmt = stmt.where(
|
|
(FinTransaction.category_source.is_(None))
|
|
| (FinTransaction.category_source != CategorySource.USER)
|
|
)
|
|
if date_from is not None:
|
|
stmt = stmt.where(FinTransaction.booked_date >= date_from)
|
|
if date_to is not None:
|
|
stmt = stmt.where(FinTransaction.booked_date <= date_to)
|
|
if account_id is not None:
|
|
stmt = stmt.where(FinTransaction.account_id == account_id)
|
|
stmt = stmt.order_by(FinTransaction.booked_date.asc(), FinTransaction.id.asc())
|
|
|
|
transfer_cat = get_transfer_category_id(db, user_id)
|
|
result = RuleApplyResult(dry_run=dry_run)
|
|
per_rule: Counter[uuid.UUID] = Counter()
|
|
|
|
for tx in db.scalars(stmt.execution_options(yield_per=1000)):
|
|
result.scanned += 1
|
|
before = (tx.category_id, tx.category_source, tx.label_clean, tx.counterparty)
|
|
fired = apply_rules_to_tx(tx, rules, transfer_cat, force=force)
|
|
if fired:
|
|
result.matched += 1
|
|
for fired_id in fired:
|
|
per_rule[fired_id] += 1
|
|
after = (tx.category_id, tx.category_source, tx.label_clean, tx.counterparty)
|
|
if before != after:
|
|
result.updated += 1
|
|
|
|
result.by_rule = [
|
|
{
|
|
"rule_id": compiled.rule.id,
|
|
"name": compiled.rule.name,
|
|
"matched": per_rule.get(compiled.rule.id, 0),
|
|
}
|
|
for compiled in rules
|
|
if per_rule.get(compiled.rule.id, 0)
|
|
]
|
|
if dry_run:
|
|
db.rollback()
|
|
else:
|
|
db.commit()
|
|
return result
|
|
|
|
|
|
def preview_rule(
|
|
db: Session,
|
|
user_id: int,
|
|
matchers: dict[str, Any],
|
|
limit: int = 50,
|
|
) -> tuple[list[FinTransaction], int]:
|
|
"""Transactions that an unsaved rule would match (§9.4 `/rules/preview`)."""
|
|
validate_matchers(matchers)
|
|
probe = FinRule(
|
|
id=uuid.uuid4(),
|
|
user_id=user_id,
|
|
name="preview",
|
|
matchers=matchers,
|
|
actions={},
|
|
)
|
|
compiled = compile_rules([probe])[0]
|
|
stmt = (
|
|
select(FinTransaction)
|
|
.where(FinTransaction.user_id == user_id)
|
|
.order_by(FinTransaction.booked_date.desc(), FinTransaction.id.desc())
|
|
)
|
|
matches: list[FinTransaction] = []
|
|
total = 0
|
|
for tx in db.scalars(stmt.execution_options(yield_per=1000)):
|
|
if rule_matches(compiled, tx):
|
|
total += 1
|
|
if len(matches) < limit:
|
|
matches.append(tx)
|
|
return matches, total
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Internal transfers (§6)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def detect_transfers(
|
|
db: Session,
|
|
user_id: int,
|
|
date_min: date | None = None,
|
|
date_max: date | None = None,
|
|
) -> int:
|
|
"""Pair opposite transactions across accounts; returns the pairs created."""
|
|
stmt = select(FinTransaction).where(
|
|
FinTransaction.user_id == user_id,
|
|
FinTransaction.transfer_group_id.is_(None),
|
|
)
|
|
if date_min is not None:
|
|
stmt = stmt.where(FinTransaction.booked_date >= date_min)
|
|
if date_max is not None:
|
|
stmt = stmt.where(FinTransaction.booked_date <= date_max)
|
|
txs = list(db.scalars(stmt).all())
|
|
|
|
debits = [t for t in txs if t.amount < 0]
|
|
credits_by_amount: dict[Decimal, list[FinTransaction]] = defaultdict(list)
|
|
for t in txs:
|
|
if t.amount > 0:
|
|
credits_by_amount[t.amount].append(t)
|
|
|
|
pairs: list[tuple[int, FinTransaction, FinTransaction]] = []
|
|
for debit in debits:
|
|
for credit in credits_by_amount.get(-debit.amount, []):
|
|
if credit.account_id == debit.account_id:
|
|
continue
|
|
if credit.currency != debit.currency:
|
|
continue
|
|
delta = abs((credit.booked_date - debit.booked_date).days)
|
|
if delta > TRANSFER_MAX_DAYS:
|
|
continue
|
|
score = (TRANSFER_MAX_DAYS - delta) * 10
|
|
joined = normalize_label_for_hash(f"{debit.label_raw} {credit.label_raw}")
|
|
if TRANSFER_LABEL_RE.search(joined):
|
|
score += 5
|
|
pairs.append((score, debit, credit))
|
|
|
|
pairs.sort(key=lambda p: (-p[0], p[1].booked_date, str(p[1].id)))
|
|
transfer_cat = get_transfer_category_id(db, user_id)
|
|
used: set[uuid.UUID] = set()
|
|
created = 0
|
|
for _score, debit, credit in pairs:
|
|
if debit.id in used or credit.id in used:
|
|
continue
|
|
group_id = uuid.uuid4()
|
|
for leg in (debit, credit):
|
|
leg.transfer_group_id = group_id
|
|
if leg.category_source != CategorySource.USER:
|
|
leg.category_id = transfer_cat
|
|
leg.category_source = CategorySource.RULE
|
|
used |= {debit.id, credit.id}
|
|
created += 1
|
|
return created
|
|
|
|
|
|
def link_transfer(
|
|
db: Session, user_id: int, id_a: uuid.UUID, id_b: uuid.UUID
|
|
) -> uuid.UUID:
|
|
"""Manual pairing (§6): opposite amounts, different accounts."""
|
|
if id_a == id_b:
|
|
raise DomainValidationError("Sélectionnez deux transactions différentes.")
|
|
legs = list(
|
|
db.scalars(
|
|
select(FinTransaction).where(
|
|
FinTransaction.user_id == user_id,
|
|
FinTransaction.id.in_([id_a, id_b]),
|
|
)
|
|
).all()
|
|
)
|
|
if len(legs) != 2:
|
|
raise NotFoundError("Transaction introuvable.")
|
|
first, second = legs
|
|
if first.amount + second.amount != 0:
|
|
raise DomainValidationError(
|
|
"Les montants doivent être strictement opposés pour former "
|
|
"un virement interne."
|
|
)
|
|
if first.account_id == second.account_id:
|
|
raise DomainValidationError(
|
|
"Les deux jambes d'un virement doivent appartenir à des comptes différents."
|
|
)
|
|
if first.currency != second.currency:
|
|
raise DomainValidationError("Les deux jambes doivent être dans la même devise.")
|
|
group_id = uuid.uuid4()
|
|
transfer_cat = get_transfer_category_id(db, user_id)
|
|
for leg in legs:
|
|
leg.transfer_group_id = group_id
|
|
if leg.category_source != CategorySource.USER:
|
|
leg.category_id = transfer_cat
|
|
leg.category_source = CategorySource.RULE
|
|
db.commit()
|
|
return group_id
|
|
|
|
|
|
def unlink_transfer(db: Session, user_id: int, group_id: uuid.UUID) -> None:
|
|
legs = list(
|
|
db.scalars(
|
|
select(FinTransaction).where(
|
|
FinTransaction.user_id == user_id,
|
|
FinTransaction.transfer_group_id == group_id,
|
|
)
|
|
).all()
|
|
)
|
|
if not legs:
|
|
raise NotFoundError("Virement introuvable.")
|
|
transfer_cat = get_transfer_category_id(db, user_id)
|
|
for leg in legs:
|
|
leg.transfer_group_id = None
|
|
if (
|
|
leg.category_id == transfer_cat
|
|
and leg.category_source == CategorySource.RULE
|
|
):
|
|
leg.category_id = None
|
|
leg.category_source = None
|
|
db.commit()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Recurring series (§7)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@dataclass
|
|
class RecurringSeries:
|
|
merchant_key: str
|
|
label_display: str
|
|
category_id: uuid.UUID | None
|
|
category_name: str | None
|
|
periodicity: str
|
|
occurrences: int
|
|
average_amount: Decimal
|
|
expected_amount: Decimal
|
|
last_date: date
|
|
next_date_predicted: date
|
|
is_active: bool
|
|
|
|
|
|
def _most_common(values: list[str]) -> str:
|
|
counter = Counter(v for v in values if v)
|
|
return counter.most_common(1)[0][0] if counter else ""
|
|
|
|
|
|
def detect_recurring(
|
|
db: Session,
|
|
user_id: int,
|
|
*,
|
|
direction: str = "debit",
|
|
lookback_months: int = 18,
|
|
include_inactive: bool = False,
|
|
today: date | None = None,
|
|
) -> list[RecurringSeries]:
|
|
"""On-the-fly recurring detection (§7.2) — no dedicated table in v1."""
|
|
today = today or utcnow().date()
|
|
since = today - timedelta(days=int(lookback_months * 30.44))
|
|
stmt = select(FinTransaction).where(
|
|
FinTransaction.user_id == user_id,
|
|
FinTransaction.booked_date >= since,
|
|
FinTransaction.transfer_group_id.is_(None),
|
|
)
|
|
stmt = stmt.where(
|
|
FinTransaction.amount < 0 if direction == "debit" else FinTransaction.amount > 0
|
|
)
|
|
txs = list(db.scalars(stmt).all())
|
|
if not txs:
|
|
return []
|
|
|
|
category_names = dict(
|
|
db.execute(
|
|
select(FinCategory.id, FinCategory.name).where(
|
|
FinCategory.user_id == user_id
|
|
)
|
|
).all()
|
|
)
|
|
transfer_cat = get_transfer_category_id(db, user_id)
|
|
|
|
groups: dict[str, list[FinTransaction]] = defaultdict(list)
|
|
for tx in txs:
|
|
if transfer_cat is not None and tx.category_id == transfer_cat:
|
|
continue
|
|
groups[merchant_key(tx.label_raw, tx.counterparty)].append(tx)
|
|
|
|
series: list[RecurringSeries] = []
|
|
for key, items in groups.items():
|
|
if not key or len(items) < MIN_OCCURRENCES:
|
|
continue
|
|
dates = sorted({t.booked_date for t in items})
|
|
if len(dates) < MIN_OCCURRENCES:
|
|
continue
|
|
intervals = [(b - a).days for a, b in pairwise(dates)]
|
|
med_int = median(intervals)
|
|
period = next(
|
|
(p for p in PERIODICITIES if p[1] <= med_int <= p[2]),
|
|
None,
|
|
)
|
|
if period is None:
|
|
continue
|
|
regular = sum(1 for i in intervals if period[1] <= i <= period[2])
|
|
if regular / len(intervals) < REGULARITY_RATIO:
|
|
continue
|
|
amounts = [abs(t.amount) for t in items]
|
|
med_amt = Decimal(str(median(amounts)))
|
|
mad = Decimal(str(median([abs(a - med_amt) for a in amounts])))
|
|
if mad > max(Decimal("1.00"), med_amt * Decimal("0.10")):
|
|
continue
|
|
cat_counter = Counter(t.category_id for t in items if t.category_id)
|
|
category_id = cat_counter.most_common(1)[0][0] if cat_counter else None
|
|
average = (sum(amounts) / Decimal(len(amounts))).quantize(Decimal("0.01"))
|
|
is_active = (today - dates[-1]).days <= period[2] * 2
|
|
if not is_active and not include_inactive:
|
|
continue
|
|
series.append(
|
|
RecurringSeries(
|
|
merchant_key=key,
|
|
label_display=_most_common([t.label_clean for t in items]) or key,
|
|
category_id=category_id,
|
|
category_name=category_names.get(category_id),
|
|
periodicity=period[0],
|
|
occurrences=len(dates),
|
|
average_amount=average,
|
|
expected_amount=med_amt.quantize(Decimal("0.01")),
|
|
last_date=dates[-1],
|
|
next_date_predicted=dates[-1]
|
|
+ timedelta(days=round(median(intervals))),
|
|
is_active=is_active,
|
|
)
|
|
)
|
|
series.sort(key=lambda s: (not s.is_active, s.next_date_predicted))
|
|
return series
|
|
|
|
|
|
def monthly_total_estimate(series: list[RecurringSeries]) -> Decimal:
|
|
total = sum(
|
|
(
|
|
s.expected_amount * MONTHLY_FACTOR.get(s.periodicity, Decimal(1))
|
|
for s in series
|
|
if s.is_active
|
|
),
|
|
Decimal(0),
|
|
)
|
|
return Decimal(total).quantize(Decimal("0.01"))
|