Étape 3 : import CSV/Excel des bénévoles et disponibilités
- Fichier unique, processus en deux temps : POST /editions/{id}/import/analyze
(colonnes détectées, aperçu, mapping suggéré, demi-journées de l'édition)
puis POST /editions/{id}/import/execute (mapping validé)
- Parser CSV (délimiteur et encodage devinés, BOM) et XLSX (openpyxl),
détection XLSX par nombre magique
- Vocabulaire des valeurs de disponibilité (tri-état, accents/casse) et des
tailles de t-shirt ; compétences créées à la volée dans le référentiel
- Rapport ligne par ligne : codes de traduction + paramètres, jamais de
phrase. Doublons : upsert sur email, signalement sur (nom, prénom) sans
email, doublons intra-fichier. Mode simulation (rollback, rien de persisté)
- 40 tests : parser, valeurs, service (tous les cas), endpoints
- Dépendances : python-multipart, openpyxl
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017uUsRg7XzopkCPWimwg9ZM
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
cb2e84adf0
commit
bb919554ff
@@ -20,3 +20,6 @@ dist/
|
||||
.DS_Store
|
||||
.idea/
|
||||
.vscode/
|
||||
|
||||
# Claude Code (worktrees, agents locaux)
|
||||
.claude/
|
||||
|
||||
@@ -187,7 +187,8 @@ pointage sur site · fonctions d'IA.
|
||||
│ │ ├── config.py # settings (pydantic-settings)
|
||||
│ │ ├── db.py # engine + session
|
||||
│ │ ├── logging_config.py # RGPD : pas de PII dans les logs
|
||||
│ │ ├── api/ # routers
|
||||
│ │ ├── api/ # routers (health, imports)
|
||||
│ │ ├── importing/ # import CSV/Excel (parser, values, service)
|
||||
│ │ ├── models/ # tables SQLModel
|
||||
│ │ ├── validation/ # moteur de règles (cœur du produit)
|
||||
│ │ ├── i18n/
|
||||
@@ -260,4 +261,9 @@ Aucune dépendance locale requise hors Docker.
|
||||
| Étape | Description | État |
|
||||
|-------|-------------------------------------------------------|----------|
|
||||
| 1 | Squelette : monorepo, Docker, CI/CD, backend qui démarre | validée |
|
||||
| 2 | Modèle de données complet + migration Alembic initiale | à valider |
|
||||
| 2 | Modèle de données complet + migration Alembic initiale | validée |
|
||||
| 3 | Import CSV/Excel (analyse + exécution, rapport, doublons, simulation) | à valider |
|
||||
|
||||
> **En attente** : la couche d'authentification (lien magique, `Account`,
|
||||
> JWT, dépendances de rôle) fera l'objet d'une étape dédiée. Les endpoints
|
||||
> actuels ne sont pas encore protégés.
|
||||
|
||||
+2
-2
@@ -35,5 +35,5 @@ docker compose run --rm backend uv run alembic upgrade head
|
||||
|
||||
## Note
|
||||
|
||||
Le `uv.lock` n'est pas encore versionné (pas de `uv` sur le poste de dev).
|
||||
Il sera figé et commité une fois généré dans le conteneur.
|
||||
`uv.lock` est versionné (généré dans le conteneur). Le régénérer après toute
|
||||
modification des dépendances : `docker compose run --rm backend uv lock`.
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
"""Endpoints d'import : analyse puis exécution."""
|
||||
|
||||
import datetime as dt
|
||||
import uuid
|
||||
|
||||
from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile
|
||||
from pydantic import ValidationError
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from app.db import get_session
|
||||
from app.importing.parser import ImportParseError, parse_upload
|
||||
from app.importing.schemas import (
|
||||
AnalyzeResponse,
|
||||
DayPartInfo,
|
||||
ImportMapping,
|
||||
ImportReport,
|
||||
PreviewRow,
|
||||
)
|
||||
from app.importing.service import run_import, suggest_mapping, validate_mapping
|
||||
from app.models import DayPart, Edition
|
||||
|
||||
router = APIRouter(prefix="/editions/{edition_id}/import", tags=["import"])
|
||||
|
||||
_PREVIEW_ROWS = 5
|
||||
_MAX_BYTES = 5 * 1024 * 1024
|
||||
|
||||
|
||||
def _get_edition(edition_id: uuid.UUID, session: Session) -> Edition:
|
||||
edition = session.get(Edition, edition_id)
|
||||
if edition is None:
|
||||
raise HTTPException(status_code=404, detail={"code": "error.not_found"})
|
||||
return edition
|
||||
|
||||
|
||||
async def _read_upload(file: UploadFile) -> bytes:
|
||||
content = await file.read()
|
||||
if not content:
|
||||
raise HTTPException(
|
||||
status_code=400, detail={"code": "import.error.empty_file"}
|
||||
)
|
||||
if len(content) > _MAX_BYTES:
|
||||
raise HTTPException(
|
||||
status_code=413, detail={"code": "import.error.file_too_large"}
|
||||
)
|
||||
return content
|
||||
|
||||
|
||||
def _is_xlsx(filename: str, content: bytes) -> bool:
|
||||
return content[:4] == b"PK\x03\x04" or (filename or "").lower().endswith(
|
||||
(".xlsx", ".xlsm")
|
||||
)
|
||||
|
||||
|
||||
def _edition_dates(edition: Edition) -> list[dt.date]:
|
||||
span = (edition.end_date - edition.start_date).days
|
||||
return [edition.start_date + dt.timedelta(days=i) for i in range(span + 1)]
|
||||
|
||||
|
||||
def _day_parts(session: Session, edition: Edition) -> list[DayPart]:
|
||||
return list(
|
||||
session.exec(
|
||||
select(DayPart)
|
||||
.where(DayPart.edition_id == edition.id)
|
||||
.order_by(DayPart.order)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@router.post("/analyze", response_model=AnalyzeResponse)
|
||||
async def analyze(
|
||||
edition_id: uuid.UUID,
|
||||
file: UploadFile = File(...),
|
||||
session: Session = Depends(get_session),
|
||||
) -> AnalyzeResponse:
|
||||
edition = _get_edition(edition_id, session)
|
||||
content = await _read_upload(file)
|
||||
try:
|
||||
table = parse_upload(file.filename or "", content)
|
||||
except ImportParseError as exc:
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail={"code": exc.code, "params": exc.params},
|
||||
) from exc
|
||||
|
||||
preview = [
|
||||
PreviewRow(line=i + 2, values=values)
|
||||
for i, values in enumerate(table.as_dicts()[:_PREVIEW_ROWS])
|
||||
]
|
||||
return AnalyzeResponse(
|
||||
filename=file.filename or "",
|
||||
format="xlsx" if _is_xlsx(file.filename or "", content) else "csv",
|
||||
headers=table.headers,
|
||||
row_count=len(table.rows),
|
||||
preview=preview,
|
||||
suggested_mapping=suggest_mapping(table.headers),
|
||||
edition_dates=_edition_dates(edition),
|
||||
day_parts=[
|
||||
DayPartInfo(
|
||||
code=dp.code,
|
||||
label=dp.label,
|
||||
start_time=dp.start_time,
|
||||
end_time=dp.end_time,
|
||||
order=dp.order,
|
||||
)
|
||||
for dp in _day_parts(session, edition)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@router.post("/execute", response_model=ImportReport)
|
||||
async def execute(
|
||||
edition_id: uuid.UUID,
|
||||
file: UploadFile = File(...),
|
||||
mapping: str = Form(...),
|
||||
session: Session = Depends(get_session),
|
||||
) -> ImportReport:
|
||||
edition = _get_edition(edition_id, session)
|
||||
content = await _read_upload(file)
|
||||
|
||||
try:
|
||||
parsed_mapping = ImportMapping.model_validate_json(mapping)
|
||||
except ValidationError as exc:
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail={"code": "import.mapping.invalid", "errors": exc.errors()},
|
||||
) from exc
|
||||
|
||||
try:
|
||||
table = parse_upload(file.filename or "", content)
|
||||
except ImportParseError as exc:
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail={"code": exc.code, "params": exc.params},
|
||||
) from exc
|
||||
|
||||
mapping_errors = validate_mapping(
|
||||
table, parsed_mapping, edition, _day_parts(session, edition)
|
||||
)
|
||||
if mapping_errors:
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail={
|
||||
"code": "import.mapping.invalid",
|
||||
"errors": [e.model_dump() for e in mapping_errors],
|
||||
},
|
||||
)
|
||||
|
||||
return run_import(
|
||||
session=session,
|
||||
edition=edition,
|
||||
table=table,
|
||||
mapping=parsed_mapping,
|
||||
)
|
||||
@@ -3,5 +3,27 @@
|
||||
"error.not_found": "Ressource introuvable",
|
||||
"error.forbidden": "Accès refusé",
|
||||
"error.unauthorized": "Authentification requise",
|
||||
"error.validation_failed": "La validation a échoué"
|
||||
"error.validation_failed": "La validation a échoué",
|
||||
|
||||
"import.error.empty_file": "Le fichier est vide.",
|
||||
"import.error.encoding": "L'encodage du fichier n'est pas reconnu.",
|
||||
"import.error.unreadable_file": "Fichier illisible ou format non pris en charge.",
|
||||
"import.error.file_too_large": "Le fichier est trop volumineux.",
|
||||
|
||||
"import.mapping.invalid": "Le paramétrage des colonnes est invalide.",
|
||||
"import.mapping.required_field_missing": "Le champ « {field} » doit être associé à une colonne.",
|
||||
"import.mapping.column_not_found": "La colonne « {column} » est absente du fichier.",
|
||||
"import.mapping.unknown_day_part": "Demi-journée « {day_part_code} » inconnue pour cette édition (colonne « {column} »).",
|
||||
"import.mapping.date_outside_edition": "La date {date} de la colonne « {column} » est hors des dates de l'édition.",
|
||||
"import.mapping.duplicate_availability_column": "La colonne « {column} » est associée plusieurs fois.",
|
||||
|
||||
"import.row.missing_required_field": "Champ obligatoire manquant : {field}.",
|
||||
"import.row.invalid_email": "Adresse email invalide : {value}.",
|
||||
"import.row.duplicate_email_in_file": "Email déjà présent à la ligne {first_line} du fichier.",
|
||||
"import.row.duplicate_name_in_file": "Nom et prénom déjà présents à la ligne {first_line} du fichier.",
|
||||
"import.row.updated_existing": "Bénévole existant mis à jour.",
|
||||
"import.row.possible_duplicate_name": "Un bénévole portant le même nom existe déjà dans l'édition.",
|
||||
"import.row.invalid_tshirt_size": "Taille de t-shirt non reconnue : {value}.",
|
||||
"import.row.skill_created": "Nouvelle compétence créée : {label}.",
|
||||
"import.row.unknown_availability_value": "Valeur de disponibilité non comprise « {value} » (colonne « {column} »)."
|
||||
}
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
"""Import CSV/Excel des bénévoles et de leurs disponibilités.
|
||||
|
||||
Fichier unique (une ligne par bénévole, disponibilités en colonnes). Processus
|
||||
en deux temps : `analyze` (colonnes détectées + aperçu) puis `execute` (mapping
|
||||
validé). Voir CLAUDE.md § 4.2.
|
||||
"""
|
||||
@@ -0,0 +1,111 @@
|
||||
"""Lecture d'un fichier importé (CSV ou XLSX) vers une table normalisée."""
|
||||
|
||||
import csv
|
||||
import io
|
||||
from dataclasses import dataclass
|
||||
|
||||
from openpyxl import load_workbook
|
||||
|
||||
_ENCODINGS = ("utf-8-sig", "utf-8", "cp1252", "latin-1")
|
||||
_DELIMITERS = (";", ",", "\t")
|
||||
_XLSX_MAGIC = b"PK\x03\x04"
|
||||
|
||||
|
||||
class ImportParseError(Exception):
|
||||
"""Fichier illisible. `code` est une clé de traduction."""
|
||||
|
||||
def __init__(self, code: str, **params: object) -> None:
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
self.params = params
|
||||
|
||||
|
||||
@dataclass
|
||||
class ParsedTable:
|
||||
headers: list[str]
|
||||
rows: list[list[str]]
|
||||
|
||||
def as_dicts(self) -> list[dict[str, str]]:
|
||||
return [
|
||||
{h: v for h, v in zip(self.headers, row, strict=False)}
|
||||
for row in self.rows
|
||||
]
|
||||
|
||||
|
||||
def parse_upload(filename: str, content: bytes) -> ParsedTable:
|
||||
if not content:
|
||||
raise ImportParseError("import.error.empty_file")
|
||||
name = (filename or "").lower()
|
||||
if content[:4] == _XLSX_MAGIC or name.endswith((".xlsx", ".xlsm")):
|
||||
return _parse_xlsx(content)
|
||||
return _parse_csv(content)
|
||||
|
||||
|
||||
def _decode(content: bytes) -> str:
|
||||
for enc in _ENCODINGS:
|
||||
try:
|
||||
return content.decode(enc)
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
raise ImportParseError("import.error.encoding")
|
||||
|
||||
|
||||
def _sniff_delimiter(sample: str) -> str:
|
||||
lines = [ln for ln in sample.splitlines() if ln.strip()]
|
||||
header = lines[0] if lines else ""
|
||||
counts = {d: header.count(d) for d in _DELIMITERS}
|
||||
best = max(counts, key=lambda d: counts[d])
|
||||
return best if counts[best] else ","
|
||||
|
||||
|
||||
def _fit(row: list[str], width: int) -> list[str]:
|
||||
if len(row) < width:
|
||||
return row + [""] * (width - len(row))
|
||||
return row[:width]
|
||||
|
||||
|
||||
def _parse_csv(content: bytes) -> ParsedTable:
|
||||
text = _decode(content)
|
||||
delimiter = _sniff_delimiter(text[:4096])
|
||||
reader = csv.reader(io.StringIO(text), delimiter=delimiter)
|
||||
rows = [[(cell or "").strip() for cell in row] for row in reader]
|
||||
rows = [row for row in rows if any(row)]
|
||||
if not rows:
|
||||
raise ImportParseError("import.error.empty_file")
|
||||
headers = rows[0]
|
||||
width = len(headers)
|
||||
return ParsedTable(headers=headers, rows=[_fit(r, width) for r in rows[1:]])
|
||||
|
||||
|
||||
def _cell_to_str(value: object) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, str):
|
||||
return value.strip()
|
||||
if isinstance(value, float) and value.is_integer():
|
||||
return str(int(value))
|
||||
return str(value).strip()
|
||||
|
||||
|
||||
def _parse_xlsx(content: bytes) -> ParsedTable:
|
||||
try:
|
||||
workbook = load_workbook(
|
||||
io.BytesIO(content), read_only=True, data_only=True
|
||||
)
|
||||
except Exception as exc:
|
||||
raise ImportParseError("import.error.unreadable_file") from exc
|
||||
worksheet = workbook.active
|
||||
if worksheet is None:
|
||||
workbook.close()
|
||||
raise ImportParseError("import.error.empty_file")
|
||||
rows = [
|
||||
[_cell_to_str(cell) for cell in row]
|
||||
for row in worksheet.iter_rows(values_only=True)
|
||||
]
|
||||
workbook.close()
|
||||
rows = [row for row in rows if any(row)]
|
||||
if not rows:
|
||||
raise ImportParseError("import.error.empty_file")
|
||||
headers = rows[0]
|
||||
width = len(headers)
|
||||
return ParsedTable(headers=headers, rows=[_fit(r, width) for r in rows[1:]])
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Schémas d'entrée/sortie de l'import.
|
||||
|
||||
Le rapport ne contient que des **codes** de traduction + des paramètres
|
||||
structurés, jamais de phrase (voir `locales/fr.json`).
|
||||
"""
|
||||
|
||||
import datetime as dt
|
||||
import uuid
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
Severity = Literal["ERROR", "WARNING"]
|
||||
RowAction = Literal["CREATED", "UPDATED", "SKIPPED"]
|
||||
|
||||
|
||||
class ColumnMapping(BaseModel):
|
||||
first_name: str
|
||||
last_name: str
|
||||
email: str | None = None
|
||||
phone: str | None = None
|
||||
tshirt_size: str | None = None
|
||||
diet: str | None = None
|
||||
comment: str | None = None
|
||||
skills: str | None = None
|
||||
|
||||
|
||||
class AvailabilityColumnMapping(BaseModel):
|
||||
header: str
|
||||
date: dt.date
|
||||
day_part_code: str
|
||||
|
||||
|
||||
class ImportMapping(BaseModel):
|
||||
dry_run: bool = True
|
||||
columns: ColumnMapping
|
||||
availability_columns: list[AvailabilityColumnMapping] = Field(
|
||||
default_factory=list
|
||||
)
|
||||
|
||||
|
||||
class Issue(BaseModel):
|
||||
code: str
|
||||
severity: Severity
|
||||
field: str | None = None
|
||||
params: dict[str, str | int | None] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class RowReport(BaseModel):
|
||||
line: int
|
||||
action: RowAction
|
||||
volunteer_id: uuid.UUID | None = None
|
||||
issues: list[Issue] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ImportReport(BaseModel):
|
||||
dry_run: bool
|
||||
total_rows: int
|
||||
created: int = 0
|
||||
updated: int = 0
|
||||
skipped: int = 0
|
||||
availabilities_written: int = 0
|
||||
skills_created: list[str] = Field(default_factory=list)
|
||||
rows: list[RowReport] = Field(default_factory=list)
|
||||
|
||||
|
||||
class PreviewRow(BaseModel):
|
||||
line: int
|
||||
values: dict[str, str]
|
||||
|
||||
|
||||
class DayPartInfo(BaseModel):
|
||||
code: str
|
||||
label: str
|
||||
start_time: dt.time
|
||||
end_time: dt.time
|
||||
order: int
|
||||
|
||||
|
||||
class AnalyzeResponse(BaseModel):
|
||||
filename: str
|
||||
format: Literal["csv", "xlsx"]
|
||||
headers: list[str]
|
||||
row_count: int
|
||||
preview: list[PreviewRow]
|
||||
suggested_mapping: dict[str, str]
|
||||
edition_dates: list[dt.date]
|
||||
day_parts: list[DayPartInfo]
|
||||
@@ -0,0 +1,395 @@
|
||||
"""Cœur de l'import : suggestion de mapping, validation, exécution."""
|
||||
|
||||
import re
|
||||
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from app.importing.parser import ParsedTable
|
||||
from app.importing.schemas import (
|
||||
ImportMapping,
|
||||
ImportReport,
|
||||
Issue,
|
||||
RowReport,
|
||||
)
|
||||
from app.importing.text import normalize, slug_code, split_skills
|
||||
from app.importing.values import parse_availability_cell, parse_tshirt_size
|
||||
from app.models import (
|
||||
Availability,
|
||||
DayPart,
|
||||
Edition,
|
||||
Skill,
|
||||
Volunteer,
|
||||
VolunteerSkill,
|
||||
)
|
||||
|
||||
_EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
|
||||
|
||||
_FIELD_HINTS: dict[str, tuple[str, ...]] = {
|
||||
"email": ("mail", "courriel"),
|
||||
"first_name": ("prenom", "first name", "firstname", "given name"),
|
||||
"last_name": ("nom", "last name", "lastname", "surname", "famille"),
|
||||
"phone": ("tel", "phone", "portable", "mobile", "gsm"),
|
||||
"tshirt_size": ("tshirt", "t shirt", "taille", "shirt"),
|
||||
"diet": ("regime", "alimentaire", "diet", "allergie"),
|
||||
"comment": ("commentaire", "remarque", "note", "precision"),
|
||||
"skills": ("competence", "skill", "habilitation", "brevet", "diplome"),
|
||||
}
|
||||
|
||||
|
||||
def suggest_mapping(headers: list[str]) -> dict[str, str]:
|
||||
"""Association approximative en-tête → champ, à confirmer côté client."""
|
||||
result: dict[str, str] = {}
|
||||
used: set[str] = set()
|
||||
for field, hints in _FIELD_HINTS.items():
|
||||
for header in headers:
|
||||
if header in used:
|
||||
continue
|
||||
if any(hint in normalize(header) for hint in hints):
|
||||
result[field] = header
|
||||
used.add(header)
|
||||
break
|
||||
return result
|
||||
|
||||
|
||||
def validate_mapping(
|
||||
table: ParsedTable,
|
||||
mapping: ImportMapping,
|
||||
edition: Edition,
|
||||
day_parts: list[DayPart],
|
||||
) -> list[Issue]:
|
||||
"""Contrôles préalables. Retourne une liste vide si le mapping est correct."""
|
||||
errors: list[Issue] = []
|
||||
headers = set(table.headers)
|
||||
columns = mapping.columns
|
||||
|
||||
for field in ("first_name", "last_name"):
|
||||
header = getattr(columns, field)
|
||||
if not header:
|
||||
errors.append(
|
||||
Issue(
|
||||
code="import.mapping.required_field_missing",
|
||||
severity="ERROR",
|
||||
field=field,
|
||||
)
|
||||
)
|
||||
elif header not in headers:
|
||||
errors.append(
|
||||
Issue(
|
||||
code="import.mapping.column_not_found",
|
||||
severity="ERROR",
|
||||
field=field,
|
||||
params={"column": header},
|
||||
)
|
||||
)
|
||||
|
||||
for field in ("email", "phone", "tshirt_size", "diet", "comment", "skills"):
|
||||
header = getattr(columns, field)
|
||||
if header and header not in headers:
|
||||
errors.append(
|
||||
Issue(
|
||||
code="import.mapping.column_not_found",
|
||||
severity="ERROR",
|
||||
field=field,
|
||||
params={"column": header},
|
||||
)
|
||||
)
|
||||
|
||||
valid_codes = {dp.code for dp in day_parts}
|
||||
seen: set[str] = set()
|
||||
for column in mapping.availability_columns:
|
||||
if column.header not in headers:
|
||||
errors.append(
|
||||
Issue(
|
||||
code="import.mapping.column_not_found",
|
||||
severity="ERROR",
|
||||
field="availability",
|
||||
params={"column": column.header},
|
||||
)
|
||||
)
|
||||
if column.header in seen:
|
||||
errors.append(
|
||||
Issue(
|
||||
code="import.mapping.duplicate_availability_column",
|
||||
severity="ERROR",
|
||||
params={"column": column.header},
|
||||
)
|
||||
)
|
||||
seen.add(column.header)
|
||||
if column.day_part_code not in valid_codes:
|
||||
errors.append(
|
||||
Issue(
|
||||
code="import.mapping.unknown_day_part",
|
||||
severity="ERROR",
|
||||
params={
|
||||
"column": column.header,
|
||||
"day_part_code": column.day_part_code,
|
||||
},
|
||||
)
|
||||
)
|
||||
if not edition.start_date <= column.date <= edition.end_date:
|
||||
errors.append(
|
||||
Issue(
|
||||
code="import.mapping.date_outside_edition",
|
||||
severity="ERROR",
|
||||
params={
|
||||
"column": column.header,
|
||||
"date": column.date.isoformat(),
|
||||
},
|
||||
)
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
def run_import(
|
||||
*,
|
||||
session: Session,
|
||||
edition: Edition,
|
||||
table: ParsedTable,
|
||||
mapping: ImportMapping,
|
||||
) -> ImportReport:
|
||||
"""Exécute l'import. `mapping.dry_run` → aucune persistance (rollback final).
|
||||
|
||||
Le mapping est supposé déjà validé par `validate_mapping`.
|
||||
"""
|
||||
columns = mapping.columns
|
||||
day_part_by_code = {
|
||||
dp.code: dp
|
||||
for dp in session.exec(
|
||||
select(DayPart).where(DayPart.edition_id == edition.id)
|
||||
)
|
||||
}
|
||||
|
||||
existing = list(
|
||||
session.exec(
|
||||
select(Volunteer).where(Volunteer.edition_id == edition.id)
|
||||
)
|
||||
)
|
||||
by_email = {v.email.lower(): v for v in existing if v.email}
|
||||
by_name: dict[tuple[str, str], Volunteer] = {}
|
||||
for volunteer in existing:
|
||||
if not volunteer.email:
|
||||
key = (normalize(volunteer.last_name), normalize(volunteer.first_name))
|
||||
by_name.setdefault(key, volunteer)
|
||||
|
||||
skills_by_code = {
|
||||
s.code: s
|
||||
for s in session.exec(
|
||||
select(Skill).where(Skill.edition_id == edition.id)
|
||||
)
|
||||
}
|
||||
created_skills: set[str] = set()
|
||||
|
||||
report = ImportReport(dry_run=mapping.dry_run, total_rows=len(table.rows))
|
||||
email_seen: dict[str, int] = {}
|
||||
name_seen: dict[tuple[str, str], int] = {}
|
||||
|
||||
for index, row in enumerate(table.as_dicts()):
|
||||
line = index + 2 # +1 en-tête, +1 pour une numérotation à partir de 1
|
||||
entry = RowReport(line=line, action="SKIPPED")
|
||||
|
||||
first_name = (row.get(columns.first_name) or "").strip()
|
||||
last_name = (row.get(columns.last_name) or "").strip()
|
||||
email_raw = (
|
||||
(row.get(columns.email) or "").strip() if columns.email else ""
|
||||
)
|
||||
email = email_raw.lower() or None
|
||||
|
||||
if not first_name or not last_name:
|
||||
entry.issues.append(
|
||||
Issue(
|
||||
code="import.row.missing_required_field",
|
||||
severity="ERROR",
|
||||
field="first_name" if not first_name else "last_name",
|
||||
)
|
||||
)
|
||||
report.rows.append(entry)
|
||||
report.skipped += 1
|
||||
continue
|
||||
|
||||
if email and not _EMAIL_RE.match(email):
|
||||
entry.issues.append(
|
||||
Issue(
|
||||
code="import.row.invalid_email",
|
||||
severity="ERROR",
|
||||
field="email",
|
||||
params={"value": email_raw},
|
||||
)
|
||||
)
|
||||
report.rows.append(entry)
|
||||
report.skipped += 1
|
||||
continue
|
||||
|
||||
if email:
|
||||
if email in email_seen:
|
||||
entry.issues.append(
|
||||
Issue(
|
||||
code="import.row.duplicate_email_in_file",
|
||||
severity="ERROR",
|
||||
field="email",
|
||||
params={"value": email_raw, "first_line": email_seen[email]},
|
||||
)
|
||||
)
|
||||
report.rows.append(entry)
|
||||
report.skipped += 1
|
||||
continue
|
||||
email_seen[email] = line
|
||||
else:
|
||||
key = (normalize(last_name), normalize(first_name))
|
||||
if key in name_seen:
|
||||
entry.issues.append(
|
||||
Issue(
|
||||
code="import.row.duplicate_name_in_file",
|
||||
severity="WARNING",
|
||||
params={"first_line": name_seen[key]},
|
||||
)
|
||||
)
|
||||
else:
|
||||
name_seen[key] = line
|
||||
|
||||
volunteer = by_email.get(email) if email else None
|
||||
if volunteer is not None:
|
||||
entry.action = "UPDATED"
|
||||
entry.issues.append(
|
||||
Issue(
|
||||
code="import.row.updated_existing",
|
||||
severity="WARNING",
|
||||
params={"volunteer_id": str(volunteer.id)},
|
||||
)
|
||||
)
|
||||
else:
|
||||
if not email:
|
||||
match = by_name.get((normalize(last_name), normalize(first_name)))
|
||||
if match is not None:
|
||||
entry.issues.append(
|
||||
Issue(
|
||||
code="import.row.possible_duplicate_name",
|
||||
severity="WARNING",
|
||||
params={"volunteer_id": str(match.id)},
|
||||
)
|
||||
)
|
||||
volunteer = Volunteer(
|
||||
edition_id=edition.id,
|
||||
first_name=first_name,
|
||||
last_name=last_name,
|
||||
)
|
||||
session.add(volunteer)
|
||||
entry.action = "CREATED"
|
||||
|
||||
volunteer.first_name = first_name
|
||||
volunteer.last_name = last_name
|
||||
if columns.email:
|
||||
volunteer.email = email_raw or None
|
||||
if columns.phone:
|
||||
volunteer.phone = (row.get(columns.phone) or "").strip() or None
|
||||
if columns.diet:
|
||||
volunteer.diet = (row.get(columns.diet) or "").strip() or None
|
||||
if columns.comment:
|
||||
volunteer.comment = (row.get(columns.comment) or "").strip() or None
|
||||
if columns.tshirt_size:
|
||||
raw_size = (row.get(columns.tshirt_size) or "").strip()
|
||||
if raw_size:
|
||||
size = parse_tshirt_size(raw_size)
|
||||
if size is None:
|
||||
entry.issues.append(
|
||||
Issue(
|
||||
code="import.row.invalid_tshirt_size",
|
||||
severity="WARNING",
|
||||
field="tshirt_size",
|
||||
params={"value": raw_size},
|
||||
)
|
||||
)
|
||||
else:
|
||||
volunteer.tshirt_size = size
|
||||
|
||||
session.flush()
|
||||
|
||||
if columns.skills:
|
||||
linked = set(
|
||||
session.exec(
|
||||
select(VolunteerSkill.skill_id).where(
|
||||
VolunteerSkill.volunteer_id == volunteer.id
|
||||
)
|
||||
)
|
||||
)
|
||||
for token in split_skills(row.get(columns.skills) or ""):
|
||||
code = slug_code(token)
|
||||
if not code:
|
||||
continue
|
||||
skill = skills_by_code.get(code)
|
||||
if skill is None:
|
||||
skill = Skill(
|
||||
edition_id=edition.id, code=code, label=token.strip()
|
||||
)
|
||||
session.add(skill)
|
||||
session.flush()
|
||||
skills_by_code[code] = skill
|
||||
created_skills.add(code)
|
||||
entry.issues.append(
|
||||
Issue(
|
||||
code="import.row.skill_created",
|
||||
severity="WARNING",
|
||||
params={"code": code, "label": skill.label},
|
||||
)
|
||||
)
|
||||
if skill.id not in linked:
|
||||
session.add(
|
||||
VolunteerSkill(
|
||||
volunteer_id=volunteer.id, skill_id=skill.id
|
||||
)
|
||||
)
|
||||
linked.add(skill.id)
|
||||
|
||||
for column in mapping.availability_columns:
|
||||
status, recognized = parse_availability_cell(
|
||||
row.get(column.header, "") or ""
|
||||
)
|
||||
if not recognized:
|
||||
entry.issues.append(
|
||||
Issue(
|
||||
code="import.row.unknown_availability_value",
|
||||
severity="WARNING",
|
||||
field=column.header,
|
||||
params={
|
||||
"column": column.header,
|
||||
"value": (row.get(column.header, "") or "").strip(),
|
||||
},
|
||||
)
|
||||
)
|
||||
continue
|
||||
if status is None:
|
||||
continue
|
||||
day_part = day_part_by_code[column.day_part_code]
|
||||
current = session.exec(
|
||||
select(Availability)
|
||||
.where(Availability.volunteer_id == volunteer.id)
|
||||
.where(Availability.date == column.date)
|
||||
.where(Availability.day_part_id == day_part.id)
|
||||
).first()
|
||||
if current is not None:
|
||||
current.status = status
|
||||
else:
|
||||
session.add(
|
||||
Availability(
|
||||
edition_id=edition.id,
|
||||
volunteer_id=volunteer.id,
|
||||
date=column.date,
|
||||
day_part_id=day_part.id,
|
||||
status=status,
|
||||
)
|
||||
)
|
||||
report.availabilities_written += 1
|
||||
|
||||
entry.volunteer_id = volunteer.id
|
||||
report.rows.append(entry)
|
||||
if entry.action == "CREATED":
|
||||
report.created += 1
|
||||
else:
|
||||
report.updated += 1
|
||||
|
||||
report.skills_created = sorted(created_skills)
|
||||
|
||||
if mapping.dry_run:
|
||||
session.rollback()
|
||||
else:
|
||||
session.commit()
|
||||
return report
|
||||
@@ -0,0 +1,26 @@
|
||||
"""Normalisation de texte partagée par l'import (accents, casse, découpage)."""
|
||||
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
_SKILL_SEPARATORS = re.compile(r"[;,/\n]+")
|
||||
_MULTI_UNDERSCORE = re.compile(r"_+")
|
||||
|
||||
|
||||
def normalize(value: str) -> str:
|
||||
"""Minuscule, sans accents, espaces réduits."""
|
||||
lowered = unicodedata.normalize("NFKD", (value or "").strip().lower())
|
||||
stripped = "".join(c for c in lowered if not unicodedata.combining(c))
|
||||
return " ".join(stripped.split())
|
||||
|
||||
|
||||
def slug_code(value: str) -> str:
|
||||
"""Code stable pour une compétence : MAJUSCULES, séparateurs en `_`."""
|
||||
base = normalize(value).upper()
|
||||
chars = [c if c.isalnum() else "_" for c in base]
|
||||
return _MULTI_UNDERSCORE.sub("_", "".join(chars)).strip("_")
|
||||
|
||||
|
||||
def split_skills(value: str) -> list[str]:
|
||||
parts = (p.strip() for p in _SKILL_SEPARATORS.split(value or ""))
|
||||
return [p for p in parts if p]
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Interprétation des valeurs de cellule : disponibilités et tailles de t-shirt."""
|
||||
|
||||
from app.importing.text import normalize
|
||||
from app.models import AvailabilityStatus, TshirtSize
|
||||
|
||||
_AVAILABLE = {
|
||||
"x", "o", "oui", "y", "yes", "ok", "1", "vrai", "true",
|
||||
"dispo", "disponible", "present", "presente",
|
||||
"matin", "aprem", "apres-midi", "apres midi", "midi", "soir", "soiree",
|
||||
"journee", "jour",
|
||||
}
|
||||
_IF_NEEDED = {
|
||||
"si besoin", "sib", "peut-etre", "peut etre", "?", "2",
|
||||
"backup", "reserve", "au besoin",
|
||||
}
|
||||
_UNAVAILABLE = {
|
||||
"non", "n", "no", "0", "faux", "false", "indispo", "indisponible", "absent",
|
||||
"-", "x-",
|
||||
}
|
||||
_EMPTY = {"", "na", "n/a", "nc"}
|
||||
|
||||
_TSHIRT = {
|
||||
"XS": TshirtSize.XS,
|
||||
"S": TshirtSize.S,
|
||||
"M": TshirtSize.M,
|
||||
"L": TshirtSize.L,
|
||||
"XL": TshirtSize.XL,
|
||||
"XXL": TshirtSize.XXL,
|
||||
"2XL": TshirtSize.XXL,
|
||||
"XXXL": TshirtSize.XXXL,
|
||||
"3XL": TshirtSize.XXXL,
|
||||
}
|
||||
|
||||
|
||||
def parse_availability_cell(value: str) -> tuple[AvailabilityStatus | None, bool]:
|
||||
"""(statut, reconnu).
|
||||
|
||||
- statut `None` + reconnu `True` → cellule vide, pas de déclaration.
|
||||
- statut `None` + reconnu `False` → valeur non comprise, à signaler.
|
||||
"""
|
||||
norm = normalize(value)
|
||||
if norm in _EMPTY:
|
||||
return None, True
|
||||
if norm in _AVAILABLE:
|
||||
return AvailabilityStatus.AVAILABLE, True
|
||||
if norm in _IF_NEEDED:
|
||||
return AvailabilityStatus.IF_NEEDED, True
|
||||
if norm in _UNAVAILABLE:
|
||||
return AvailabilityStatus.UNAVAILABLE, True
|
||||
return None, False
|
||||
|
||||
|
||||
def parse_tshirt_size(value: str) -> TshirtSize | None:
|
||||
key = normalize(value).upper().replace(" ", "").replace("-", "")
|
||||
return _TSHIRT.get(key)
|
||||
@@ -2,6 +2,7 @@ from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from app.api.health import router as health_router
|
||||
from app.api.imports import router as import_router
|
||||
from app.config import get_settings
|
||||
from app.logging_config import configure_logging
|
||||
|
||||
@@ -20,3 +21,4 @@ app.add_middleware(
|
||||
)
|
||||
|
||||
app.include_router(health_router)
|
||||
app.include_router(import_router)
|
||||
|
||||
@@ -10,6 +10,8 @@ dependencies = [
|
||||
"alembic>=1.14",
|
||||
"psycopg[binary]>=3.2",
|
||||
"pydantic-settings>=2.6",
|
||||
"python-multipart>=0.0.12",
|
||||
"openpyxl>=3.1",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
@@ -34,6 +36,14 @@ target-version = "py312"
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "I", "UP", "B"]
|
||||
|
||||
[tool.ruff.lint.flake8-bugbear]
|
||||
extend-immutable-calls = [
|
||||
"fastapi.Depends",
|
||||
"fastapi.File",
|
||||
"fastapi.Form",
|
||||
"fastapi.Query",
|
||||
]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
addopts = "-ra"
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import datetime as dt
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlmodel import Session, SQLModel, create_engine
|
||||
@@ -6,6 +8,7 @@ from app import models # noqa: F401 (enregistre les tables sur SQLModel.metada
|
||||
from app.config import get_settings
|
||||
from app.db import get_session
|
||||
from app.main import app
|
||||
from app.models import DayPart, Edition
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
@@ -39,3 +42,36 @@ def client(session):
|
||||
with TestClient(app) as c:
|
||||
yield c
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def edition(session) -> Edition:
|
||||
"""Édition de 4 jours avec 3 demi-journées paramétrées."""
|
||||
ed = Edition(
|
||||
year=2026,
|
||||
name="Festival Test",
|
||||
start_date=dt.date(2026, 7, 9),
|
||||
end_date=dt.date(2026, 7, 12),
|
||||
retention_days=365,
|
||||
)
|
||||
session.add(ed)
|
||||
session.commit()
|
||||
session.refresh(ed)
|
||||
session.add_all(
|
||||
[
|
||||
DayPart(
|
||||
edition_id=ed.id, code="MORNING", label="Matin",
|
||||
start_time=dt.time(6, 0), end_time=dt.time(13, 0), order=0,
|
||||
),
|
||||
DayPart(
|
||||
edition_id=ed.id, code="AFTERNOON", label="Après-midi",
|
||||
start_time=dt.time(13, 0), end_time=dt.time(19, 0), order=1,
|
||||
),
|
||||
DayPart(
|
||||
edition_id=ed.id, code="EVENING", label="Soirée",
|
||||
start_time=dt.time(19, 0), end_time=dt.time(6, 0), order=2,
|
||||
),
|
||||
]
|
||||
)
|
||||
session.commit()
|
||||
return ed
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
import json
|
||||
import uuid
|
||||
|
||||
from sqlmodel import select
|
||||
|
||||
from app.models import Volunteer
|
||||
|
||||
_CSV = (
|
||||
"Prénom;Nom;Email;Compétences;Ven matin\n"
|
||||
"Alex;Martin;alex@ex.fr;SST, Permis B;x\n"
|
||||
"Sam;Nguyen;sam@ex.fr;SST;si besoin\n"
|
||||
).encode()
|
||||
|
||||
|
||||
def _mapping(dry_run: bool) -> str:
|
||||
return json.dumps(
|
||||
{
|
||||
"dry_run": dry_run,
|
||||
"columns": {
|
||||
"first_name": "Prénom",
|
||||
"last_name": "Nom",
|
||||
"email": "Email",
|
||||
"skills": "Compétences",
|
||||
},
|
||||
"availability_columns": [
|
||||
{
|
||||
"header": "Ven matin",
|
||||
"date": "2026-07-10",
|
||||
"day_part_code": "MORNING",
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_analyze_returns_headers_preview_and_day_parts(client, edition):
|
||||
resp = client.post(
|
||||
f"/editions/{edition.id}/import/analyze",
|
||||
files={"file": ("benevoles.csv", _CSV, "text/csv")},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["format"] == "csv"
|
||||
assert body["headers"] == ["Prénom", "Nom", "Email", "Compétences", "Ven matin"]
|
||||
assert body["row_count"] == 2
|
||||
assert body["preview"][0]["values"]["Nom"] == "Martin"
|
||||
assert body["suggested_mapping"]["first_name"] == "Prénom"
|
||||
assert {d["code"] for d in body["day_parts"]} == {
|
||||
"MORNING",
|
||||
"AFTERNOON",
|
||||
"EVENING",
|
||||
}
|
||||
assert body["edition_dates"][0] == "2026-07-09"
|
||||
|
||||
|
||||
def test_execute_dry_run_then_real(client, session, edition):
|
||||
dry = client.post(
|
||||
f"/editions/{edition.id}/import/execute",
|
||||
files={"file": ("benevoles.csv", _CSV, "text/csv")},
|
||||
data={"mapping": _mapping(dry_run=True)},
|
||||
)
|
||||
assert dry.status_code == 200
|
||||
assert dry.json()["created"] == 2
|
||||
assert session.exec(select(Volunteer)).all() == []
|
||||
|
||||
real = client.post(
|
||||
f"/editions/{edition.id}/import/execute",
|
||||
files={"file": ("benevoles.csv", _CSV, "text/csv")},
|
||||
data={"mapping": _mapping(dry_run=False)},
|
||||
)
|
||||
assert real.status_code == 200
|
||||
report = real.json()
|
||||
assert report["created"] == 2
|
||||
assert sorted(report["skills_created"]) == ["PERMIS_B", "SST"]
|
||||
assert len(session.exec(select(Volunteer)).all()) == 2
|
||||
|
||||
|
||||
def test_execute_rejects_invalid_mapping(client, edition):
|
||||
bad_mapping = json.dumps(
|
||||
{
|
||||
"dry_run": True,
|
||||
"columns": {"first_name": "Prénom", "last_name": "Nom"},
|
||||
"availability_columns": [
|
||||
{
|
||||
"header": "Ven matin",
|
||||
"date": "2026-07-10",
|
||||
"day_part_code": "NIGHT",
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
resp = client.post(
|
||||
f"/editions/{edition.id}/import/execute",
|
||||
files={"file": ("benevoles.csv", _CSV, "text/csv")},
|
||||
data={"mapping": bad_mapping},
|
||||
)
|
||||
assert resp.status_code == 422
|
||||
detail = resp.json()["detail"]
|
||||
assert detail["code"] == "import.mapping.invalid"
|
||||
assert any(
|
||||
e["code"] == "import.mapping.unknown_day_part" for e in detail["errors"]
|
||||
)
|
||||
|
||||
|
||||
def test_analyze_unknown_edition_is_404(client):
|
||||
resp = client.post(
|
||||
f"/editions/{uuid.uuid4()}/import/analyze",
|
||||
files={"file": ("benevoles.csv", _CSV, "text/csv")},
|
||||
)
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
def test_analyze_rejects_empty_file(client, edition):
|
||||
resp = client.post(
|
||||
f"/editions/{edition.id}/import/analyze",
|
||||
files={"file": ("benevoles.csv", b"", "text/csv")},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
assert resp.json()["detail"]["code"] == "import.error.empty_file"
|
||||
@@ -0,0 +1,63 @@
|
||||
import io
|
||||
|
||||
import pytest
|
||||
from openpyxl import Workbook
|
||||
|
||||
from app.importing.parser import ImportParseError, parse_upload
|
||||
|
||||
|
||||
def _xlsx_bytes(rows: list[list[object]]) -> bytes:
|
||||
wb = Workbook()
|
||||
ws = wb.active
|
||||
for row in rows:
|
||||
ws.append(row)
|
||||
buffer = io.BytesIO()
|
||||
wb.save(buffer)
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def test_csv_semicolon_with_bom():
|
||||
raw = "Prénom;Nom;Email\nAlex;Martin;a@b.fr\n".encode()
|
||||
table = parse_upload("benevoles.csv", raw)
|
||||
assert table.headers == ["Prénom", "Nom", "Email"]
|
||||
assert table.as_dicts() == [
|
||||
{"Prénom": "Alex", "Nom": "Martin", "Email": "a@b.fr"}
|
||||
]
|
||||
|
||||
|
||||
def test_csv_comma_delimiter():
|
||||
raw = b"first,last\nSam,Nguyen\n"
|
||||
table = parse_upload("x.csv", raw)
|
||||
assert table.headers == ["first", "last"]
|
||||
assert table.rows == [["Sam", "Nguyen"]]
|
||||
|
||||
|
||||
def test_csv_blank_lines_dropped_and_short_rows_padded():
|
||||
raw = b"a;b;c\n\n1;2\n3;4;5;6\n"
|
||||
table = parse_upload("x.csv", raw)
|
||||
assert table.rows == [["1", "2", ""], ["3", "4", "5"]]
|
||||
|
||||
|
||||
def test_xlsx_detected_by_magic_without_extension():
|
||||
data = _xlsx_bytes([["Prénom", "Nom"], ["Alex", "Martin"], [None, None]])
|
||||
table = parse_upload("upload.dat", data)
|
||||
assert table.headers == ["Prénom", "Nom"]
|
||||
assert table.rows == [["Alex", "Martin"]]
|
||||
|
||||
|
||||
def test_xlsx_numeric_cells_become_plain_strings():
|
||||
data = _xlsx_bytes([["Nom", "Tel"], ["Martin", 601020304]])
|
||||
table = parse_upload("x.xlsx", data)
|
||||
assert table.rows == [["Martin", "601020304"]]
|
||||
|
||||
|
||||
def test_empty_file_raises():
|
||||
with pytest.raises(ImportParseError) as err:
|
||||
parse_upload("x.csv", b"")
|
||||
assert err.value.code == "import.error.empty_file"
|
||||
|
||||
|
||||
def test_unreadable_xlsx_raises():
|
||||
with pytest.raises(ImportParseError) as err:
|
||||
parse_upload("x.xlsx", b"not a real workbook")
|
||||
assert err.value.code == "import.error.unreadable_file"
|
||||
@@ -0,0 +1,243 @@
|
||||
import datetime as dt
|
||||
|
||||
from sqlmodel import select
|
||||
|
||||
from app.importing.parser import ParsedTable
|
||||
from app.importing.schemas import (
|
||||
AvailabilityColumnMapping,
|
||||
ColumnMapping,
|
||||
ImportMapping,
|
||||
)
|
||||
from app.importing.service import run_import, suggest_mapping, validate_mapping
|
||||
from app.models import Availability, Skill, Volunteer
|
||||
|
||||
HEADERS = ["Prénom", "Nom", "Email", "Taille", "Compétences", "Ven matin", "Ven aprem"]
|
||||
|
||||
|
||||
def _mapping(dry_run: bool = False) -> ImportMapping:
|
||||
return ImportMapping(
|
||||
dry_run=dry_run,
|
||||
columns=ColumnMapping(
|
||||
first_name="Prénom",
|
||||
last_name="Nom",
|
||||
email="Email",
|
||||
tshirt_size="Taille",
|
||||
skills="Compétences",
|
||||
),
|
||||
availability_columns=[
|
||||
AvailabilityColumnMapping(
|
||||
header="Ven matin", date=dt.date(2026, 7, 10),
|
||||
day_part_code="MORNING",
|
||||
),
|
||||
AvailabilityColumnMapping(
|
||||
header="Ven aprem", date=dt.date(2026, 7, 10),
|
||||
day_part_code="AFTERNOON",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _table(rows: list[list[str]]) -> ParsedTable:
|
||||
return ParsedTable(headers=HEADERS, rows=rows)
|
||||
|
||||
|
||||
def _volunteers(session, edition):
|
||||
return list(
|
||||
session.exec(
|
||||
select(Volunteer).where(Volunteer.edition_id == edition.id)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_suggest_mapping_matches_common_headers():
|
||||
suggestion = suggest_mapping(
|
||||
["Prénom", "Nom de famille", "Adresse e-mail", "Numéro de portable"]
|
||||
)
|
||||
assert suggestion["first_name"] == "Prénom"
|
||||
assert suggestion["last_name"] == "Nom de famille"
|
||||
assert suggestion["email"] == "Adresse e-mail"
|
||||
assert suggestion["phone"] == "Numéro de portable"
|
||||
|
||||
|
||||
def test_happy_path_creates_volunteers_skills_and_availabilities(session, edition):
|
||||
report = run_import(
|
||||
session=session,
|
||||
edition=edition,
|
||||
table=_table(
|
||||
[
|
||||
["Alex", "Martin", "alex@ex.fr", "M", "SST, Permis B", "x", "non"],
|
||||
["Sam", "Nguyen", "sam@ex.fr", "L", "SST", "si besoin", ""],
|
||||
]
|
||||
),
|
||||
mapping=_mapping(),
|
||||
)
|
||||
|
||||
assert (report.created, report.updated, report.skipped) == (2, 0, 0)
|
||||
assert report.skills_created == ["PERMIS_B", "SST"]
|
||||
assert report.availabilities_written == 3 # x, non, si besoin
|
||||
|
||||
volunteers = {v.email: v for v in _volunteers(session, edition)}
|
||||
assert set(volunteers) == {"alex@ex.fr", "sam@ex.fr"}
|
||||
assert volunteers["alex@ex.fr"].tshirt_size.value == "M"
|
||||
assert {s.code for s in volunteers["alex@ex.fr"].skills} == {"SST", "PERMIS_B"}
|
||||
|
||||
avails = session.exec(
|
||||
select(Availability).where(
|
||||
Availability.volunteer_id == volunteers["alex@ex.fr"].id
|
||||
)
|
||||
).all()
|
||||
assert {a.status.value for a in avails} == {"AVAILABLE", "UNAVAILABLE"}
|
||||
|
||||
|
||||
def test_dry_run_persists_nothing(session, edition):
|
||||
report = run_import(
|
||||
session=session,
|
||||
edition=edition,
|
||||
table=_table([["Alex", "Martin", "alex@ex.fr", "M", "SST", "x", "x"]]),
|
||||
mapping=_mapping(dry_run=True),
|
||||
)
|
||||
assert report.created == 1
|
||||
assert report.dry_run is True
|
||||
assert _volunteers(session, edition) == []
|
||||
assert session.exec(select(Skill)).all() == []
|
||||
|
||||
|
||||
def test_existing_email_is_updated_not_duplicated(session, edition):
|
||||
session.add(
|
||||
Volunteer(
|
||||
edition_id=edition.id,
|
||||
email="alex@ex.fr",
|
||||
first_name="Alexandre",
|
||||
last_name="M.",
|
||||
phone="0600000000",
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
report = run_import(
|
||||
session=session,
|
||||
edition=edition,
|
||||
table=_table([["Alex", "Martin", "ALEX@ex.fr", "M", "SST", "x", ""]]),
|
||||
mapping=_mapping(),
|
||||
)
|
||||
|
||||
assert (report.created, report.updated) == (0, 1)
|
||||
assert report.rows[0].issues[0].code == "import.row.updated_existing"
|
||||
volunteers = _volunteers(session, edition)
|
||||
assert len(volunteers) == 1
|
||||
assert volunteers[0].first_name == "Alex"
|
||||
assert volunteers[0].last_name == "Martin"
|
||||
|
||||
|
||||
def test_duplicate_email_within_file_is_skipped(session, edition):
|
||||
report = run_import(
|
||||
session=session,
|
||||
edition=edition,
|
||||
table=_table(
|
||||
[
|
||||
["Alex", "Martin", "dup@ex.fr", "M", "", "x", ""],
|
||||
["Alex", "Doublon", "dup@ex.fr", "L", "", "x", ""],
|
||||
]
|
||||
),
|
||||
mapping=_mapping(),
|
||||
)
|
||||
assert (report.created, report.skipped) == (1, 1)
|
||||
skipped = report.rows[1]
|
||||
assert skipped.action == "SKIPPED"
|
||||
assert skipped.issues[0].code == "import.row.duplicate_email_in_file"
|
||||
assert skipped.issues[0].params["first_line"] == 2
|
||||
|
||||
|
||||
def test_row_without_email_matching_existing_name_warns_but_creates(
|
||||
session, edition
|
||||
):
|
||||
session.add(
|
||||
Volunteer(
|
||||
edition_id=edition.id,
|
||||
first_name="Jean",
|
||||
last_name="Dupont",
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
report = run_import(
|
||||
session=session,
|
||||
edition=edition,
|
||||
table=_table([["jean", "DUPONT", "", "", "", "x", ""]]),
|
||||
mapping=_mapping(),
|
||||
)
|
||||
assert report.created == 1
|
||||
assert report.rows[0].issues[0].code == "import.row.possible_duplicate_name"
|
||||
assert len(_volunteers(session, edition)) == 2
|
||||
|
||||
|
||||
def test_missing_required_field_is_skipped(session, edition):
|
||||
report = run_import(
|
||||
session=session,
|
||||
edition=edition,
|
||||
table=_table([["", "Martin", "x@ex.fr", "", "", "x", ""]]),
|
||||
mapping=_mapping(),
|
||||
)
|
||||
assert report.skipped == 1
|
||||
assert report.rows[0].issues[0].code == "import.row.missing_required_field"
|
||||
assert _volunteers(session, edition) == []
|
||||
|
||||
|
||||
def test_invalid_email_is_skipped(session, edition):
|
||||
report = run_import(
|
||||
session=session,
|
||||
edition=edition,
|
||||
table=_table([["Alex", "Martin", "pas-un-email", "", "", "x", ""]]),
|
||||
mapping=_mapping(),
|
||||
)
|
||||
assert report.skipped == 1
|
||||
assert report.rows[0].issues[0].code == "import.row.invalid_email"
|
||||
|
||||
|
||||
def test_unknown_tshirt_size_warns_and_still_imports(session, edition):
|
||||
report = run_import(
|
||||
session=session,
|
||||
edition=edition,
|
||||
table=_table([["Alex", "Martin", "a@ex.fr", "Large", "", "x", ""]]),
|
||||
mapping=_mapping(),
|
||||
)
|
||||
assert report.created == 1
|
||||
assert report.rows[0].issues[0].code == "import.row.invalid_tshirt_size"
|
||||
assert _volunteers(session, edition)[0].tshirt_size is None
|
||||
|
||||
|
||||
def test_unknown_availability_value_is_flagged(session, edition):
|
||||
report = run_import(
|
||||
session=session,
|
||||
edition=edition,
|
||||
table=_table([["Alex", "Martin", "a@ex.fr", "", "", "bof", "x"]]),
|
||||
mapping=_mapping(),
|
||||
)
|
||||
codes = [i.code for i in report.rows[0].issues]
|
||||
assert "import.row.unknown_availability_value" in codes
|
||||
assert report.availabilities_written == 1 # seule "Ven aprem" a été retenue
|
||||
|
||||
|
||||
def test_validate_mapping_reports_bad_day_part_and_date(session, edition):
|
||||
from app.models import DayPart
|
||||
|
||||
day_parts = list(
|
||||
session.exec(select(DayPart).where(DayPart.edition_id == edition.id))
|
||||
)
|
||||
mapping = ImportMapping(
|
||||
columns=ColumnMapping(first_name="Prénom", last_name="Nom"),
|
||||
availability_columns=[
|
||||
AvailabilityColumnMapping(
|
||||
header="Ven matin", date=dt.date(2026, 7, 10),
|
||||
day_part_code="NIGHT",
|
||||
),
|
||||
AvailabilityColumnMapping(
|
||||
header="Ven aprem", date=dt.date(2030, 1, 1),
|
||||
day_part_code="MORNING",
|
||||
),
|
||||
],
|
||||
)
|
||||
errors = validate_mapping(_table([]), mapping, edition, day_parts)
|
||||
codes = {e.code for e in errors}
|
||||
assert "import.mapping.unknown_day_part" in codes
|
||||
assert "import.mapping.date_outside_edition" in codes
|
||||
@@ -0,0 +1,66 @@
|
||||
import pytest
|
||||
|
||||
from app.importing.text import slug_code, split_skills
|
||||
from app.importing.values import parse_availability_cell, parse_tshirt_size
|
||||
from app.models import AvailabilityStatus, TshirtSize
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("cell", "expected"),
|
||||
[
|
||||
("x", AvailabilityStatus.AVAILABLE),
|
||||
("OUI", AvailabilityStatus.AVAILABLE),
|
||||
(" Disponible ", AvailabilityStatus.AVAILABLE),
|
||||
("si besoin", AvailabilityStatus.IF_NEEDED),
|
||||
("Peut-être", AvailabilityStatus.IF_NEEDED),
|
||||
("non", AvailabilityStatus.UNAVAILABLE),
|
||||
("0", AvailabilityStatus.UNAVAILABLE),
|
||||
],
|
||||
)
|
||||
def test_recognized_availability_values(cell, expected):
|
||||
status, recognized = parse_availability_cell(cell)
|
||||
assert recognized is True
|
||||
assert status is expected
|
||||
|
||||
|
||||
def test_empty_cell_is_recognized_but_no_declaration():
|
||||
assert parse_availability_cell(" ") == (None, True)
|
||||
assert parse_availability_cell("N/A") == (None, True)
|
||||
|
||||
|
||||
def test_unknown_cell_is_flagged():
|
||||
assert parse_availability_cell("bof") == (None, False)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("raw", "expected"),
|
||||
[
|
||||
("M", TshirtSize.M),
|
||||
(" m ", TshirtSize.M),
|
||||
("xl", TshirtSize.XL),
|
||||
("2XL", TshirtSize.XXL),
|
||||
("3 XL", TshirtSize.XXXL),
|
||||
],
|
||||
)
|
||||
def test_tshirt_sizes(raw, expected):
|
||||
assert parse_tshirt_size(raw) is expected
|
||||
|
||||
|
||||
def test_unknown_tshirt_size_is_none():
|
||||
assert parse_tshirt_size("large") is None
|
||||
|
||||
|
||||
def test_slug_code_is_stable_across_accents_and_case():
|
||||
assert slug_code("Régie son") == "REGIE_SON"
|
||||
assert slug_code(" permis B ") == "PERMIS_B"
|
||||
assert slug_code("SST") == "SST"
|
||||
|
||||
|
||||
def test_split_skills():
|
||||
assert split_skills("SST, Permis B / régie son\nlangues") == [
|
||||
"SST",
|
||||
"Permis B",
|
||||
"régie son",
|
||||
"langues",
|
||||
]
|
||||
assert split_skills("") == []
|
||||
Generated
+34
@@ -53,8 +53,10 @@ source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "alembic" },
|
||||
{ name = "fastapi" },
|
||||
{ name = "openpyxl" },
|
||||
{ name = "psycopg", extra = ["binary"] },
|
||||
{ name = "pydantic-settings" },
|
||||
{ name = "python-multipart" },
|
||||
{ name = "sqlmodel" },
|
||||
{ name = "uvicorn", extra = ["standard"] },
|
||||
]
|
||||
@@ -71,8 +73,10 @@ dev = [
|
||||
requires-dist = [
|
||||
{ name = "alembic", specifier = ">=1.14" },
|
||||
{ name = "fastapi", specifier = ">=0.115" },
|
||||
{ name = "openpyxl", specifier = ">=3.1" },
|
||||
{ name = "psycopg", extras = ["binary"], specifier = ">=3.2" },
|
||||
{ name = "pydantic-settings", specifier = ">=2.6" },
|
||||
{ name = "python-multipart", specifier = ">=0.0.12" },
|
||||
{ name = "sqlmodel", specifier = ">=0.0.22" },
|
||||
{ name = "uvicorn", extras = ["standard"], specifier = ">=0.32" },
|
||||
]
|
||||
@@ -136,6 +140,15 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/b1/5a/234e8fadf85c3cc48cb31c247b9e8e0c7f06ece80f5b29f9b8c241f9da4c/coverage-7.16.0-py3-none-any.whl", hash = "sha256:245f7de6d023a5bba375dbec9f2e0869bfa26ac0cc639bbb7b4c814884000b73", size = 214977 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "et-xmlfile"
|
||||
version = "2.0.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/d3/38/af70d7ab1ae9d4da450eeec1fa3918940a5fafb9055e934af8d6eb0c2313/et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54", size = 17234 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fastapi"
|
||||
version = "0.141.1"
|
||||
@@ -271,6 +284,18 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openpyxl"
|
||||
version = "3.1.5"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "et-xmlfile" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/3d/f9/88d94a75de065ea32619465d2f77b29a0469500e99012523b91cc4141cd1/openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050", size = 186464 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", size = 250910 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "packaging"
|
||||
version = "26.3"
|
||||
@@ -436,6 +461,15 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/0d/17/c5c6b53ddc18f297992099b3d9ec16c855c0ccc83263a21fe4d1c625ec6c/python_dotenv-1.2.3-py3-none-any.whl", hash = "sha256:904552145e8bfed22162c09dab1c2b9b54fefa7b23ba780f4f26ca0316b0f0d9", size = 22780 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "python-multipart"
|
||||
version = "0.0.32"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/5b/42/55c32bb9b12693c092ad250a0e82edb5b31ddeda6eb772de5f308b3804ad/python_multipart-0.0.32.tar.gz", hash = "sha256:be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e", size = 46881 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23", size = 30042 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pyyaml"
|
||||
version = "6.0.3"
|
||||
|
||||
Reference in New Issue
Block a user