Étape 3 : import CSV/Excel des bénévoles et disponibilités
- Fichier unique, processus en deux temps : POST /editions/{id}/import/analyze
(colonnes détectées, aperçu, mapping suggéré, demi-journées de l'édition)
puis POST /editions/{id}/import/execute (mapping validé)
- Parser CSV (délimiteur et encodage devinés, BOM) et XLSX (openpyxl),
détection XLSX par nombre magique
- Vocabulaire des valeurs de disponibilité (tri-état, accents/casse) et des
tailles de t-shirt ; compétences créées à la volée dans le référentiel
- Rapport ligne par ligne : codes de traduction + paramètres, jamais de
phrase. Doublons : upsert sur email, signalement sur (nom, prénom) sans
email, doublons intra-fichier. Mode simulation (rollback, rien de persisté)
- 40 tests : parser, valeurs, service (tous les cas), endpoints
- Dépendances : python-multipart, openpyxl
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017uUsRg7XzopkCPWimwg9ZM
This commit is contained in:
AdamThomas
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
cb2e84adf0
commit
bb919554ff
@@ -0,0 +1,395 @@
|
||||
"""Cœur de l'import : suggestion de mapping, validation, exécution."""
|
||||
|
||||
import re
|
||||
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from app.importing.parser import ParsedTable
|
||||
from app.importing.schemas import (
|
||||
ImportMapping,
|
||||
ImportReport,
|
||||
Issue,
|
||||
RowReport,
|
||||
)
|
||||
from app.importing.text import normalize, slug_code, split_skills
|
||||
from app.importing.values import parse_availability_cell, parse_tshirt_size
|
||||
from app.models import (
|
||||
Availability,
|
||||
DayPart,
|
||||
Edition,
|
||||
Skill,
|
||||
Volunteer,
|
||||
VolunteerSkill,
|
||||
)
|
||||
|
||||
_EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
|
||||
|
||||
_FIELD_HINTS: dict[str, tuple[str, ...]] = {
|
||||
"email": ("mail", "courriel"),
|
||||
"first_name": ("prenom", "first name", "firstname", "given name"),
|
||||
"last_name": ("nom", "last name", "lastname", "surname", "famille"),
|
||||
"phone": ("tel", "phone", "portable", "mobile", "gsm"),
|
||||
"tshirt_size": ("tshirt", "t shirt", "taille", "shirt"),
|
||||
"diet": ("regime", "alimentaire", "diet", "allergie"),
|
||||
"comment": ("commentaire", "remarque", "note", "precision"),
|
||||
"skills": ("competence", "skill", "habilitation", "brevet", "diplome"),
|
||||
}
|
||||
|
||||
|
||||
def suggest_mapping(headers: list[str]) -> dict[str, str]:
|
||||
"""Association approximative en-tête → champ, à confirmer côté client."""
|
||||
result: dict[str, str] = {}
|
||||
used: set[str] = set()
|
||||
for field, hints in _FIELD_HINTS.items():
|
||||
for header in headers:
|
||||
if header in used:
|
||||
continue
|
||||
if any(hint in normalize(header) for hint in hints):
|
||||
result[field] = header
|
||||
used.add(header)
|
||||
break
|
||||
return result
|
||||
|
||||
|
||||
def validate_mapping(
|
||||
table: ParsedTable,
|
||||
mapping: ImportMapping,
|
||||
edition: Edition,
|
||||
day_parts: list[DayPart],
|
||||
) -> list[Issue]:
|
||||
"""Contrôles préalables. Retourne une liste vide si le mapping est correct."""
|
||||
errors: list[Issue] = []
|
||||
headers = set(table.headers)
|
||||
columns = mapping.columns
|
||||
|
||||
for field in ("first_name", "last_name"):
|
||||
header = getattr(columns, field)
|
||||
if not header:
|
||||
errors.append(
|
||||
Issue(
|
||||
code="import.mapping.required_field_missing",
|
||||
severity="ERROR",
|
||||
field=field,
|
||||
)
|
||||
)
|
||||
elif header not in headers:
|
||||
errors.append(
|
||||
Issue(
|
||||
code="import.mapping.column_not_found",
|
||||
severity="ERROR",
|
||||
field=field,
|
||||
params={"column": header},
|
||||
)
|
||||
)
|
||||
|
||||
for field in ("email", "phone", "tshirt_size", "diet", "comment", "skills"):
|
||||
header = getattr(columns, field)
|
||||
if header and header not in headers:
|
||||
errors.append(
|
||||
Issue(
|
||||
code="import.mapping.column_not_found",
|
||||
severity="ERROR",
|
||||
field=field,
|
||||
params={"column": header},
|
||||
)
|
||||
)
|
||||
|
||||
valid_codes = {dp.code for dp in day_parts}
|
||||
seen: set[str] = set()
|
||||
for column in mapping.availability_columns:
|
||||
if column.header not in headers:
|
||||
errors.append(
|
||||
Issue(
|
||||
code="import.mapping.column_not_found",
|
||||
severity="ERROR",
|
||||
field="availability",
|
||||
params={"column": column.header},
|
||||
)
|
||||
)
|
||||
if column.header in seen:
|
||||
errors.append(
|
||||
Issue(
|
||||
code="import.mapping.duplicate_availability_column",
|
||||
severity="ERROR",
|
||||
params={"column": column.header},
|
||||
)
|
||||
)
|
||||
seen.add(column.header)
|
||||
if column.day_part_code not in valid_codes:
|
||||
errors.append(
|
||||
Issue(
|
||||
code="import.mapping.unknown_day_part",
|
||||
severity="ERROR",
|
||||
params={
|
||||
"column": column.header,
|
||||
"day_part_code": column.day_part_code,
|
||||
},
|
||||
)
|
||||
)
|
||||
if not edition.start_date <= column.date <= edition.end_date:
|
||||
errors.append(
|
||||
Issue(
|
||||
code="import.mapping.date_outside_edition",
|
||||
severity="ERROR",
|
||||
params={
|
||||
"column": column.header,
|
||||
"date": column.date.isoformat(),
|
||||
},
|
||||
)
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
def run_import(
|
||||
*,
|
||||
session: Session,
|
||||
edition: Edition,
|
||||
table: ParsedTable,
|
||||
mapping: ImportMapping,
|
||||
) -> ImportReport:
|
||||
"""Exécute l'import. `mapping.dry_run` → aucune persistance (rollback final).
|
||||
|
||||
Le mapping est supposé déjà validé par `validate_mapping`.
|
||||
"""
|
||||
columns = mapping.columns
|
||||
day_part_by_code = {
|
||||
dp.code: dp
|
||||
for dp in session.exec(
|
||||
select(DayPart).where(DayPart.edition_id == edition.id)
|
||||
)
|
||||
}
|
||||
|
||||
existing = list(
|
||||
session.exec(
|
||||
select(Volunteer).where(Volunteer.edition_id == edition.id)
|
||||
)
|
||||
)
|
||||
by_email = {v.email.lower(): v for v in existing if v.email}
|
||||
by_name: dict[tuple[str, str], Volunteer] = {}
|
||||
for volunteer in existing:
|
||||
if not volunteer.email:
|
||||
key = (normalize(volunteer.last_name), normalize(volunteer.first_name))
|
||||
by_name.setdefault(key, volunteer)
|
||||
|
||||
skills_by_code = {
|
||||
s.code: s
|
||||
for s in session.exec(
|
||||
select(Skill).where(Skill.edition_id == edition.id)
|
||||
)
|
||||
}
|
||||
created_skills: set[str] = set()
|
||||
|
||||
report = ImportReport(dry_run=mapping.dry_run, total_rows=len(table.rows))
|
||||
email_seen: dict[str, int] = {}
|
||||
name_seen: dict[tuple[str, str], int] = {}
|
||||
|
||||
for index, row in enumerate(table.as_dicts()):
|
||||
line = index + 2 # +1 en-tête, +1 pour une numérotation à partir de 1
|
||||
entry = RowReport(line=line, action="SKIPPED")
|
||||
|
||||
first_name = (row.get(columns.first_name) or "").strip()
|
||||
last_name = (row.get(columns.last_name) or "").strip()
|
||||
email_raw = (
|
||||
(row.get(columns.email) or "").strip() if columns.email else ""
|
||||
)
|
||||
email = email_raw.lower() or None
|
||||
|
||||
if not first_name or not last_name:
|
||||
entry.issues.append(
|
||||
Issue(
|
||||
code="import.row.missing_required_field",
|
||||
severity="ERROR",
|
||||
field="first_name" if not first_name else "last_name",
|
||||
)
|
||||
)
|
||||
report.rows.append(entry)
|
||||
report.skipped += 1
|
||||
continue
|
||||
|
||||
if email and not _EMAIL_RE.match(email):
|
||||
entry.issues.append(
|
||||
Issue(
|
||||
code="import.row.invalid_email",
|
||||
severity="ERROR",
|
||||
field="email",
|
||||
params={"value": email_raw},
|
||||
)
|
||||
)
|
||||
report.rows.append(entry)
|
||||
report.skipped += 1
|
||||
continue
|
||||
|
||||
if email:
|
||||
if email in email_seen:
|
||||
entry.issues.append(
|
||||
Issue(
|
||||
code="import.row.duplicate_email_in_file",
|
||||
severity="ERROR",
|
||||
field="email",
|
||||
params={"value": email_raw, "first_line": email_seen[email]},
|
||||
)
|
||||
)
|
||||
report.rows.append(entry)
|
||||
report.skipped += 1
|
||||
continue
|
||||
email_seen[email] = line
|
||||
else:
|
||||
key = (normalize(last_name), normalize(first_name))
|
||||
if key in name_seen:
|
||||
entry.issues.append(
|
||||
Issue(
|
||||
code="import.row.duplicate_name_in_file",
|
||||
severity="WARNING",
|
||||
params={"first_line": name_seen[key]},
|
||||
)
|
||||
)
|
||||
else:
|
||||
name_seen[key] = line
|
||||
|
||||
volunteer = by_email.get(email) if email else None
|
||||
if volunteer is not None:
|
||||
entry.action = "UPDATED"
|
||||
entry.issues.append(
|
||||
Issue(
|
||||
code="import.row.updated_existing",
|
||||
severity="WARNING",
|
||||
params={"volunteer_id": str(volunteer.id)},
|
||||
)
|
||||
)
|
||||
else:
|
||||
if not email:
|
||||
match = by_name.get((normalize(last_name), normalize(first_name)))
|
||||
if match is not None:
|
||||
entry.issues.append(
|
||||
Issue(
|
||||
code="import.row.possible_duplicate_name",
|
||||
severity="WARNING",
|
||||
params={"volunteer_id": str(match.id)},
|
||||
)
|
||||
)
|
||||
volunteer = Volunteer(
|
||||
edition_id=edition.id,
|
||||
first_name=first_name,
|
||||
last_name=last_name,
|
||||
)
|
||||
session.add(volunteer)
|
||||
entry.action = "CREATED"
|
||||
|
||||
volunteer.first_name = first_name
|
||||
volunteer.last_name = last_name
|
||||
if columns.email:
|
||||
volunteer.email = email_raw or None
|
||||
if columns.phone:
|
||||
volunteer.phone = (row.get(columns.phone) or "").strip() or None
|
||||
if columns.diet:
|
||||
volunteer.diet = (row.get(columns.diet) or "").strip() or None
|
||||
if columns.comment:
|
||||
volunteer.comment = (row.get(columns.comment) or "").strip() or None
|
||||
if columns.tshirt_size:
|
||||
raw_size = (row.get(columns.tshirt_size) or "").strip()
|
||||
if raw_size:
|
||||
size = parse_tshirt_size(raw_size)
|
||||
if size is None:
|
||||
entry.issues.append(
|
||||
Issue(
|
||||
code="import.row.invalid_tshirt_size",
|
||||
severity="WARNING",
|
||||
field="tshirt_size",
|
||||
params={"value": raw_size},
|
||||
)
|
||||
)
|
||||
else:
|
||||
volunteer.tshirt_size = size
|
||||
|
||||
session.flush()
|
||||
|
||||
if columns.skills:
|
||||
linked = set(
|
||||
session.exec(
|
||||
select(VolunteerSkill.skill_id).where(
|
||||
VolunteerSkill.volunteer_id == volunteer.id
|
||||
)
|
||||
)
|
||||
)
|
||||
for token in split_skills(row.get(columns.skills) or ""):
|
||||
code = slug_code(token)
|
||||
if not code:
|
||||
continue
|
||||
skill = skills_by_code.get(code)
|
||||
if skill is None:
|
||||
skill = Skill(
|
||||
edition_id=edition.id, code=code, label=token.strip()
|
||||
)
|
||||
session.add(skill)
|
||||
session.flush()
|
||||
skills_by_code[code] = skill
|
||||
created_skills.add(code)
|
||||
entry.issues.append(
|
||||
Issue(
|
||||
code="import.row.skill_created",
|
||||
severity="WARNING",
|
||||
params={"code": code, "label": skill.label},
|
||||
)
|
||||
)
|
||||
if skill.id not in linked:
|
||||
session.add(
|
||||
VolunteerSkill(
|
||||
volunteer_id=volunteer.id, skill_id=skill.id
|
||||
)
|
||||
)
|
||||
linked.add(skill.id)
|
||||
|
||||
for column in mapping.availability_columns:
|
||||
status, recognized = parse_availability_cell(
|
||||
row.get(column.header, "") or ""
|
||||
)
|
||||
if not recognized:
|
||||
entry.issues.append(
|
||||
Issue(
|
||||
code="import.row.unknown_availability_value",
|
||||
severity="WARNING",
|
||||
field=column.header,
|
||||
params={
|
||||
"column": column.header,
|
||||
"value": (row.get(column.header, "") or "").strip(),
|
||||
},
|
||||
)
|
||||
)
|
||||
continue
|
||||
if status is None:
|
||||
continue
|
||||
day_part = day_part_by_code[column.day_part_code]
|
||||
current = session.exec(
|
||||
select(Availability)
|
||||
.where(Availability.volunteer_id == volunteer.id)
|
||||
.where(Availability.date == column.date)
|
||||
.where(Availability.day_part_id == day_part.id)
|
||||
).first()
|
||||
if current is not None:
|
||||
current.status = status
|
||||
else:
|
||||
session.add(
|
||||
Availability(
|
||||
edition_id=edition.id,
|
||||
volunteer_id=volunteer.id,
|
||||
date=column.date,
|
||||
day_part_id=day_part.id,
|
||||
status=status,
|
||||
)
|
||||
)
|
||||
report.availabilities_written += 1
|
||||
|
||||
entry.volunteer_id = volunteer.id
|
||||
report.rows.append(entry)
|
||||
if entry.action == "CREATED":
|
||||
report.created += 1
|
||||
else:
|
||||
report.updated += 1
|
||||
|
||||
report.skills_created = sorted(created_skills)
|
||||
|
||||
if mapping.dry_run:
|
||||
session.rollback()
|
||||
else:
|
||||
session.commit()
|
||||
return report
|
||||
Reference in New Issue
Block a user