MeeJayandClaude Opus 5
93f0689c1e
Tracker de vie auto-hébergé : suivi poids/calories/sport avec planning de pesées, sevrage tabac (vape) avec modèle de coût DIY et économies, et finances personnelles avec import de relevés bancaires. Architecture : FastAPI + SQLAlchemy 2.0 + PostgreSQL 16, React 18 + TS + Vite + Tailwind + ECharts, déploiement Docker Compose. Modules auto-découverts des deux côtés (pkgutil / import.meta.glob) et framework de connecteurs à deux voies (importeurs de fichiers + ingestion JSON) pour brancher de nouvelles sources sans toucher au noyau. Validé : 292 tests pytest, tsc + vite build, contrat API/web vérifié contre le schéma OpenAPI, et déploiement Docker réel sur PostgreSQL 16 (28 tables, SPA servie par nginx, wizard de premier démarrage). Documentation : README.md, docs/GUIDE.md, CONVENTIONS.md, et les documents de conception et de recherche dans docs/. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
140 lines
4.4 KiB
Python
140 lines
4.4 KiB
Python
"""Label/amount normalisation and dedup hashing (datamodel-finance.md §4.3, §7.1)."""
|
||
|
||
import hashlib
|
||
import re
|
||
import unicodedata
|
||
from datetime import date, timedelta
|
||
from decimal import ROUND_HALF_UP, Decimal, InvalidOperation
|
||
from uuid import UUID
|
||
|
||
TWO_PLACES = Decimal("0.01")
|
||
|
||
# Purely technical bank prefixes stripped from label_clean (§4.3).
|
||
_TECH_PREFIX_RE = re.compile(
|
||
r"^(CARTE \d{2}/\d{2}(/\d{2,4})? |PAIEMENT (PSC |CB )?\d{4} |PRLV SEPA "
|
||
r"|VIR SEPA |VIR INST |ACHAT CB )",
|
||
re.IGNORECASE,
|
||
)
|
||
|
||
# Spaces found in French bank exports: NBSP, narrow NBSP, thin space.
|
||
_SPACES = " "
|
||
|
||
|
||
def normalize_label_for_hash(label: str) -> str:
|
||
"""STABLE and CONSERVATIVE normalisation used by the dedup hash.
|
||
|
||
Uppercase + accents stripped + whitespace collapsed, nothing else — any
|
||
more aggressive cleaning would merge distinct transactions.
|
||
"""
|
||
s = unicodedata.normalize("NFKD", label)
|
||
s = "".join(c for c in s if not unicodedata.combining(c))
|
||
s = s.upper()
|
||
return re.sub(r"\s+", " ", s).strip()
|
||
|
||
|
||
def light_clean(label_raw: str) -> str:
|
||
"""Initial label_clean: trim, collapse whitespace, drop technical prefixes."""
|
||
s = re.sub(r"\s+", " ", label_raw).strip()
|
||
s = _TECH_PREFIX_RE.sub("", s).strip()
|
||
return s or label_raw.strip()
|
||
|
||
|
||
def compute_dedup_hash(
|
||
account_id: UUID,
|
||
booked_date: date,
|
||
amount: Decimal,
|
||
label_raw: str,
|
||
occurrence: int,
|
||
) -> str:
|
||
canonical = "|".join(
|
||
[
|
||
str(account_id),
|
||
booked_date.isoformat(),
|
||
f"{amount:.2f}", # sign included
|
||
normalize_label_for_hash(label_raw),
|
||
str(occurrence), # rank among identical rows of the SAME file
|
||
]
|
||
)
|
||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||
|
||
|
||
def occurrence_key(booked_date: date, amount: Decimal, label_raw: str) -> tuple:
|
||
return (booked_date, f"{amount:.2f}", normalize_label_for_hash(label_raw))
|
||
|
||
|
||
def quantize2(value: Decimal) -> Decimal:
|
||
return value.quantize(TWO_PLACES, rounding=ROUND_HALF_UP)
|
||
|
||
|
||
def parse_amount(raw: str, decimal_separator: str = ",") -> Decimal:
|
||
"""Parse a French/English bank amount string into a signed Decimal.
|
||
|
||
Covers: "-6,4", "+3500,00", "-123 456,78", "1.234,56", "12,50 €",
|
||
"−12,50" (U+2212), "226.68".
|
||
"""
|
||
s = raw.strip()
|
||
if not s:
|
||
raise ValueError("montant vide")
|
||
s = s.replace("−", "-") # unicode minus
|
||
for sp in _SPACES:
|
||
s = s.replace(sp, "")
|
||
s = s.replace(" ", "")
|
||
s = re.sub(r"(?i)(€|EUR)", "", s).strip()
|
||
if decimal_separator == ",":
|
||
# Remove dot/space thousands separators, then turn the comma into a dot.
|
||
if "," in s:
|
||
s = s.replace(".", "").replace(",", ".")
|
||
# else: already dot-decimal (BoursoBank mixes both in one file)
|
||
else:
|
||
s = s.replace(",", "")
|
||
try:
|
||
return Decimal(s)
|
||
except InvalidOperation as exc:
|
||
raise ValueError(f"montant illisible : {raw!r}") from exc
|
||
|
||
|
||
_MERCHANT_NOISE_WORDS = re.compile(
|
||
r"\b(CB|CARTE|PRLV|SEPA|VIR|ECH|PAIEMENT|ACHAT|WEB|FACT(URE)?)\b"
|
||
)
|
||
|
||
|
||
def merchant_key(label_raw: str, counterparty: str | None = None) -> str:
|
||
"""Aggressive grouping key for recurring/merchant detection (§7.1)."""
|
||
if counterparty:
|
||
return normalize_label_for_hash(counterparty)[:60]
|
||
s = normalize_label_for_hash(label_raw)
|
||
s = re.sub(r"\b\d{2}/\d{2}(/\d{2,4})?\b", "", s) # dates inside the label
|
||
s = re.sub(r"\b\d{4,}\b", "", s) # card/reference numbers
|
||
s = _MERCHANT_NOISE_WORDS.sub("", s)
|
||
s = re.sub(r"[^A-Z0-9 ]", " ", s)
|
||
s = re.sub(r"\s+", " ", s).strip()
|
||
return s[:60]
|
||
|
||
|
||
def month_str(d: date) -> str:
|
||
return f"{d.year:04d}-{d.month:02d}"
|
||
|
||
|
||
def month_first_day(year: int, month: int) -> date:
|
||
return date(year, month, 1)
|
||
|
||
|
||
def add_months(d: date, months: int) -> date:
|
||
"""First day of the month `months` after (or before) d's month."""
|
||
total = d.year * 12 + (d.month - 1) + months
|
||
return date(total // 12, total % 12 + 1, 1)
|
||
|
||
|
||
def month_last_day(d: date) -> date:
|
||
return add_months(d, 1) - timedelta(days=1)
|
||
|
||
|
||
def parse_month(value: str) -> date:
|
||
m = re.fullmatch(r"(\d{4})-(\d{2})", value.strip())
|
||
if not m:
|
||
raise ValueError(f"mois invalide : {value!r}")
|
||
year, month = int(m.group(1)), int(m.group(2))
|
||
if not 1 <= month <= 12:
|
||
raise ValueError(f"mois invalide : {value!r}")
|
||
return date(year, month, 1)
|