Étape 3 : import CSV/Excel des bénévoles et disponibilités
- Fichier unique, processus en deux temps : POST /editions/{id}/import/analyze
(colonnes détectées, aperçu, mapping suggéré, demi-journées de l'édition)
puis POST /editions/{id}/import/execute (mapping validé)
- Parser CSV (délimiteur et encodage devinés, BOM) et XLSX (openpyxl),
détection XLSX par nombre magique
- Vocabulaire des valeurs de disponibilité (tri-état, accents/casse) et des
tailles de t-shirt ; compétences créées à la volée dans le référentiel
- Rapport ligne par ligne : codes de traduction + paramètres, jamais de
phrase. Doublons : upsert sur email, signalement sur (nom, prénom) sans
email, doublons intra-fichier. Mode simulation (rollback, rien de persisté)
- 40 tests : parser, valeurs, service (tous les cas), endpoints
- Dépendances : python-multipart, openpyxl
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017uUsRg7XzopkCPWimwg9ZM
This commit is contained in:
AdamThomas
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
cb2e84adf0
commit
bb919554ff
@@ -1,3 +1,5 @@
|
||||
import datetime as dt
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlmodel import Session, SQLModel, create_engine
|
||||
@@ -6,6 +8,7 @@ from app import models # noqa: F401 (enregistre les tables sur SQLModel.metada
|
||||
from app.config import get_settings
|
||||
from app.db import get_session
|
||||
from app.main import app
|
||||
from app.models import DayPart, Edition
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
@@ -39,3 +42,36 @@ def client(session):
|
||||
with TestClient(app) as c:
|
||||
yield c
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def edition(session) -> Edition:
|
||||
"""Édition de 4 jours avec 3 demi-journées paramétrées."""
|
||||
ed = Edition(
|
||||
year=2026,
|
||||
name="Festival Test",
|
||||
start_date=dt.date(2026, 7, 9),
|
||||
end_date=dt.date(2026, 7, 12),
|
||||
retention_days=365,
|
||||
)
|
||||
session.add(ed)
|
||||
session.commit()
|
||||
session.refresh(ed)
|
||||
session.add_all(
|
||||
[
|
||||
DayPart(
|
||||
edition_id=ed.id, code="MORNING", label="Matin",
|
||||
start_time=dt.time(6, 0), end_time=dt.time(13, 0), order=0,
|
||||
),
|
||||
DayPart(
|
||||
edition_id=ed.id, code="AFTERNOON", label="Après-midi",
|
||||
start_time=dt.time(13, 0), end_time=dt.time(19, 0), order=1,
|
||||
),
|
||||
DayPart(
|
||||
edition_id=ed.id, code="EVENING", label="Soirée",
|
||||
start_time=dt.time(19, 0), end_time=dt.time(6, 0), order=2,
|
||||
),
|
||||
]
|
||||
)
|
||||
session.commit()
|
||||
return ed
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
import json
|
||||
import uuid
|
||||
|
||||
from sqlmodel import select
|
||||
|
||||
from app.models import Volunteer
|
||||
|
||||
_CSV = (
|
||||
"Prénom;Nom;Email;Compétences;Ven matin\n"
|
||||
"Alex;Martin;alex@ex.fr;SST, Permis B;x\n"
|
||||
"Sam;Nguyen;sam@ex.fr;SST;si besoin\n"
|
||||
).encode()
|
||||
|
||||
|
||||
def _mapping(dry_run: bool) -> str:
|
||||
return json.dumps(
|
||||
{
|
||||
"dry_run": dry_run,
|
||||
"columns": {
|
||||
"first_name": "Prénom",
|
||||
"last_name": "Nom",
|
||||
"email": "Email",
|
||||
"skills": "Compétences",
|
||||
},
|
||||
"availability_columns": [
|
||||
{
|
||||
"header": "Ven matin",
|
||||
"date": "2026-07-10",
|
||||
"day_part_code": "MORNING",
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_analyze_returns_headers_preview_and_day_parts(client, edition):
|
||||
resp = client.post(
|
||||
f"/editions/{edition.id}/import/analyze",
|
||||
files={"file": ("benevoles.csv", _CSV, "text/csv")},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["format"] == "csv"
|
||||
assert body["headers"] == ["Prénom", "Nom", "Email", "Compétences", "Ven matin"]
|
||||
assert body["row_count"] == 2
|
||||
assert body["preview"][0]["values"]["Nom"] == "Martin"
|
||||
assert body["suggested_mapping"]["first_name"] == "Prénom"
|
||||
assert {d["code"] for d in body["day_parts"]} == {
|
||||
"MORNING",
|
||||
"AFTERNOON",
|
||||
"EVENING",
|
||||
}
|
||||
assert body["edition_dates"][0] == "2026-07-09"
|
||||
|
||||
|
||||
def test_execute_dry_run_then_real(client, session, edition):
|
||||
dry = client.post(
|
||||
f"/editions/{edition.id}/import/execute",
|
||||
files={"file": ("benevoles.csv", _CSV, "text/csv")},
|
||||
data={"mapping": _mapping(dry_run=True)},
|
||||
)
|
||||
assert dry.status_code == 200
|
||||
assert dry.json()["created"] == 2
|
||||
assert session.exec(select(Volunteer)).all() == []
|
||||
|
||||
real = client.post(
|
||||
f"/editions/{edition.id}/import/execute",
|
||||
files={"file": ("benevoles.csv", _CSV, "text/csv")},
|
||||
data={"mapping": _mapping(dry_run=False)},
|
||||
)
|
||||
assert real.status_code == 200
|
||||
report = real.json()
|
||||
assert report["created"] == 2
|
||||
assert sorted(report["skills_created"]) == ["PERMIS_B", "SST"]
|
||||
assert len(session.exec(select(Volunteer)).all()) == 2
|
||||
|
||||
|
||||
def test_execute_rejects_invalid_mapping(client, edition):
|
||||
bad_mapping = json.dumps(
|
||||
{
|
||||
"dry_run": True,
|
||||
"columns": {"first_name": "Prénom", "last_name": "Nom"},
|
||||
"availability_columns": [
|
||||
{
|
||||
"header": "Ven matin",
|
||||
"date": "2026-07-10",
|
||||
"day_part_code": "NIGHT",
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
resp = client.post(
|
||||
f"/editions/{edition.id}/import/execute",
|
||||
files={"file": ("benevoles.csv", _CSV, "text/csv")},
|
||||
data={"mapping": bad_mapping},
|
||||
)
|
||||
assert resp.status_code == 422
|
||||
detail = resp.json()["detail"]
|
||||
assert detail["code"] == "import.mapping.invalid"
|
||||
assert any(
|
||||
e["code"] == "import.mapping.unknown_day_part" for e in detail["errors"]
|
||||
)
|
||||
|
||||
|
||||
def test_analyze_unknown_edition_is_404(client):
|
||||
resp = client.post(
|
||||
f"/editions/{uuid.uuid4()}/import/analyze",
|
||||
files={"file": ("benevoles.csv", _CSV, "text/csv")},
|
||||
)
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
def test_analyze_rejects_empty_file(client, edition):
|
||||
resp = client.post(
|
||||
f"/editions/{edition.id}/import/analyze",
|
||||
files={"file": ("benevoles.csv", b"", "text/csv")},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
assert resp.json()["detail"]["code"] == "import.error.empty_file"
|
||||
@@ -0,0 +1,63 @@
|
||||
import io
|
||||
|
||||
import pytest
|
||||
from openpyxl import Workbook
|
||||
|
||||
from app.importing.parser import ImportParseError, parse_upload
|
||||
|
||||
|
||||
def _xlsx_bytes(rows: list[list[object]]) -> bytes:
|
||||
wb = Workbook()
|
||||
ws = wb.active
|
||||
for row in rows:
|
||||
ws.append(row)
|
||||
buffer = io.BytesIO()
|
||||
wb.save(buffer)
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def test_csv_semicolon_with_bom():
|
||||
raw = "Prénom;Nom;Email\nAlex;Martin;a@b.fr\n".encode()
|
||||
table = parse_upload("benevoles.csv", raw)
|
||||
assert table.headers == ["Prénom", "Nom", "Email"]
|
||||
assert table.as_dicts() == [
|
||||
{"Prénom": "Alex", "Nom": "Martin", "Email": "a@b.fr"}
|
||||
]
|
||||
|
||||
|
||||
def test_csv_comma_delimiter():
|
||||
raw = b"first,last\nSam,Nguyen\n"
|
||||
table = parse_upload("x.csv", raw)
|
||||
assert table.headers == ["first", "last"]
|
||||
assert table.rows == [["Sam", "Nguyen"]]
|
||||
|
||||
|
||||
def test_csv_blank_lines_dropped_and_short_rows_padded():
|
||||
raw = b"a;b;c\n\n1;2\n3;4;5;6\n"
|
||||
table = parse_upload("x.csv", raw)
|
||||
assert table.rows == [["1", "2", ""], ["3", "4", "5"]]
|
||||
|
||||
|
||||
def test_xlsx_detected_by_magic_without_extension():
|
||||
data = _xlsx_bytes([["Prénom", "Nom"], ["Alex", "Martin"], [None, None]])
|
||||
table = parse_upload("upload.dat", data)
|
||||
assert table.headers == ["Prénom", "Nom"]
|
||||
assert table.rows == [["Alex", "Martin"]]
|
||||
|
||||
|
||||
def test_xlsx_numeric_cells_become_plain_strings():
|
||||
data = _xlsx_bytes([["Nom", "Tel"], ["Martin", 601020304]])
|
||||
table = parse_upload("x.xlsx", data)
|
||||
assert table.rows == [["Martin", "601020304"]]
|
||||
|
||||
|
||||
def test_empty_file_raises():
|
||||
with pytest.raises(ImportParseError) as err:
|
||||
parse_upload("x.csv", b"")
|
||||
assert err.value.code == "import.error.empty_file"
|
||||
|
||||
|
||||
def test_unreadable_xlsx_raises():
|
||||
with pytest.raises(ImportParseError) as err:
|
||||
parse_upload("x.xlsx", b"not a real workbook")
|
||||
assert err.value.code == "import.error.unreadable_file"
|
||||
@@ -0,0 +1,243 @@
|
||||
import datetime as dt
|
||||
|
||||
from sqlmodel import select
|
||||
|
||||
from app.importing.parser import ParsedTable
|
||||
from app.importing.schemas import (
|
||||
AvailabilityColumnMapping,
|
||||
ColumnMapping,
|
||||
ImportMapping,
|
||||
)
|
||||
from app.importing.service import run_import, suggest_mapping, validate_mapping
|
||||
from app.models import Availability, Skill, Volunteer
|
||||
|
||||
HEADERS = ["Prénom", "Nom", "Email", "Taille", "Compétences", "Ven matin", "Ven aprem"]
|
||||
|
||||
|
||||
def _mapping(dry_run: bool = False) -> ImportMapping:
|
||||
return ImportMapping(
|
||||
dry_run=dry_run,
|
||||
columns=ColumnMapping(
|
||||
first_name="Prénom",
|
||||
last_name="Nom",
|
||||
email="Email",
|
||||
tshirt_size="Taille",
|
||||
skills="Compétences",
|
||||
),
|
||||
availability_columns=[
|
||||
AvailabilityColumnMapping(
|
||||
header="Ven matin", date=dt.date(2026, 7, 10),
|
||||
day_part_code="MORNING",
|
||||
),
|
||||
AvailabilityColumnMapping(
|
||||
header="Ven aprem", date=dt.date(2026, 7, 10),
|
||||
day_part_code="AFTERNOON",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _table(rows: list[list[str]]) -> ParsedTable:
|
||||
return ParsedTable(headers=HEADERS, rows=rows)
|
||||
|
||||
|
||||
def _volunteers(session, edition):
|
||||
return list(
|
||||
session.exec(
|
||||
select(Volunteer).where(Volunteer.edition_id == edition.id)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_suggest_mapping_matches_common_headers():
|
||||
suggestion = suggest_mapping(
|
||||
["Prénom", "Nom de famille", "Adresse e-mail", "Numéro de portable"]
|
||||
)
|
||||
assert suggestion["first_name"] == "Prénom"
|
||||
assert suggestion["last_name"] == "Nom de famille"
|
||||
assert suggestion["email"] == "Adresse e-mail"
|
||||
assert suggestion["phone"] == "Numéro de portable"
|
||||
|
||||
|
||||
def test_happy_path_creates_volunteers_skills_and_availabilities(session, edition):
|
||||
report = run_import(
|
||||
session=session,
|
||||
edition=edition,
|
||||
table=_table(
|
||||
[
|
||||
["Alex", "Martin", "alex@ex.fr", "M", "SST, Permis B", "x", "non"],
|
||||
["Sam", "Nguyen", "sam@ex.fr", "L", "SST", "si besoin", ""],
|
||||
]
|
||||
),
|
||||
mapping=_mapping(),
|
||||
)
|
||||
|
||||
assert (report.created, report.updated, report.skipped) == (2, 0, 0)
|
||||
assert report.skills_created == ["PERMIS_B", "SST"]
|
||||
assert report.availabilities_written == 3 # x, non, si besoin
|
||||
|
||||
volunteers = {v.email: v for v in _volunteers(session, edition)}
|
||||
assert set(volunteers) == {"alex@ex.fr", "sam@ex.fr"}
|
||||
assert volunteers["alex@ex.fr"].tshirt_size.value == "M"
|
||||
assert {s.code for s in volunteers["alex@ex.fr"].skills} == {"SST", "PERMIS_B"}
|
||||
|
||||
avails = session.exec(
|
||||
select(Availability).where(
|
||||
Availability.volunteer_id == volunteers["alex@ex.fr"].id
|
||||
)
|
||||
).all()
|
||||
assert {a.status.value for a in avails} == {"AVAILABLE", "UNAVAILABLE"}
|
||||
|
||||
|
||||
def test_dry_run_persists_nothing(session, edition):
|
||||
report = run_import(
|
||||
session=session,
|
||||
edition=edition,
|
||||
table=_table([["Alex", "Martin", "alex@ex.fr", "M", "SST", "x", "x"]]),
|
||||
mapping=_mapping(dry_run=True),
|
||||
)
|
||||
assert report.created == 1
|
||||
assert report.dry_run is True
|
||||
assert _volunteers(session, edition) == []
|
||||
assert session.exec(select(Skill)).all() == []
|
||||
|
||||
|
||||
def test_existing_email_is_updated_not_duplicated(session, edition):
|
||||
session.add(
|
||||
Volunteer(
|
||||
edition_id=edition.id,
|
||||
email="alex@ex.fr",
|
||||
first_name="Alexandre",
|
||||
last_name="M.",
|
||||
phone="0600000000",
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
report = run_import(
|
||||
session=session,
|
||||
edition=edition,
|
||||
table=_table([["Alex", "Martin", "ALEX@ex.fr", "M", "SST", "x", ""]]),
|
||||
mapping=_mapping(),
|
||||
)
|
||||
|
||||
assert (report.created, report.updated) == (0, 1)
|
||||
assert report.rows[0].issues[0].code == "import.row.updated_existing"
|
||||
volunteers = _volunteers(session, edition)
|
||||
assert len(volunteers) == 1
|
||||
assert volunteers[0].first_name == "Alex"
|
||||
assert volunteers[0].last_name == "Martin"
|
||||
|
||||
|
||||
def test_duplicate_email_within_file_is_skipped(session, edition):
|
||||
report = run_import(
|
||||
session=session,
|
||||
edition=edition,
|
||||
table=_table(
|
||||
[
|
||||
["Alex", "Martin", "dup@ex.fr", "M", "", "x", ""],
|
||||
["Alex", "Doublon", "dup@ex.fr", "L", "", "x", ""],
|
||||
]
|
||||
),
|
||||
mapping=_mapping(),
|
||||
)
|
||||
assert (report.created, report.skipped) == (1, 1)
|
||||
skipped = report.rows[1]
|
||||
assert skipped.action == "SKIPPED"
|
||||
assert skipped.issues[0].code == "import.row.duplicate_email_in_file"
|
||||
assert skipped.issues[0].params["first_line"] == 2
|
||||
|
||||
|
||||
def test_row_without_email_matching_existing_name_warns_but_creates(
|
||||
session, edition
|
||||
):
|
||||
session.add(
|
||||
Volunteer(
|
||||
edition_id=edition.id,
|
||||
first_name="Jean",
|
||||
last_name="Dupont",
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
report = run_import(
|
||||
session=session,
|
||||
edition=edition,
|
||||
table=_table([["jean", "DUPONT", "", "", "", "x", ""]]),
|
||||
mapping=_mapping(),
|
||||
)
|
||||
assert report.created == 1
|
||||
assert report.rows[0].issues[0].code == "import.row.possible_duplicate_name"
|
||||
assert len(_volunteers(session, edition)) == 2
|
||||
|
||||
|
||||
def test_missing_required_field_is_skipped(session, edition):
|
||||
report = run_import(
|
||||
session=session,
|
||||
edition=edition,
|
||||
table=_table([["", "Martin", "x@ex.fr", "", "", "x", ""]]),
|
||||
mapping=_mapping(),
|
||||
)
|
||||
assert report.skipped == 1
|
||||
assert report.rows[0].issues[0].code == "import.row.missing_required_field"
|
||||
assert _volunteers(session, edition) == []
|
||||
|
||||
|
||||
def test_invalid_email_is_skipped(session, edition):
|
||||
report = run_import(
|
||||
session=session,
|
||||
edition=edition,
|
||||
table=_table([["Alex", "Martin", "pas-un-email", "", "", "x", ""]]),
|
||||
mapping=_mapping(),
|
||||
)
|
||||
assert report.skipped == 1
|
||||
assert report.rows[0].issues[0].code == "import.row.invalid_email"
|
||||
|
||||
|
||||
def test_unknown_tshirt_size_warns_and_still_imports(session, edition):
|
||||
report = run_import(
|
||||
session=session,
|
||||
edition=edition,
|
||||
table=_table([["Alex", "Martin", "a@ex.fr", "Large", "", "x", ""]]),
|
||||
mapping=_mapping(),
|
||||
)
|
||||
assert report.created == 1
|
||||
assert report.rows[0].issues[0].code == "import.row.invalid_tshirt_size"
|
||||
assert _volunteers(session, edition)[0].tshirt_size is None
|
||||
|
||||
|
||||
def test_unknown_availability_value_is_flagged(session, edition):
|
||||
report = run_import(
|
||||
session=session,
|
||||
edition=edition,
|
||||
table=_table([["Alex", "Martin", "a@ex.fr", "", "", "bof", "x"]]),
|
||||
mapping=_mapping(),
|
||||
)
|
||||
codes = [i.code for i in report.rows[0].issues]
|
||||
assert "import.row.unknown_availability_value" in codes
|
||||
assert report.availabilities_written == 1 # seule "Ven aprem" a été retenue
|
||||
|
||||
|
||||
def test_validate_mapping_reports_bad_day_part_and_date(session, edition):
|
||||
from app.models import DayPart
|
||||
|
||||
day_parts = list(
|
||||
session.exec(select(DayPart).where(DayPart.edition_id == edition.id))
|
||||
)
|
||||
mapping = ImportMapping(
|
||||
columns=ColumnMapping(first_name="Prénom", last_name="Nom"),
|
||||
availability_columns=[
|
||||
AvailabilityColumnMapping(
|
||||
header="Ven matin", date=dt.date(2026, 7, 10),
|
||||
day_part_code="NIGHT",
|
||||
),
|
||||
AvailabilityColumnMapping(
|
||||
header="Ven aprem", date=dt.date(2030, 1, 1),
|
||||
day_part_code="MORNING",
|
||||
),
|
||||
],
|
||||
)
|
||||
errors = validate_mapping(_table([]), mapping, edition, day_parts)
|
||||
codes = {e.code for e in errors}
|
||||
assert "import.mapping.unknown_day_part" in codes
|
||||
assert "import.mapping.date_outside_edition" in codes
|
||||
@@ -0,0 +1,66 @@
|
||||
import pytest
|
||||
|
||||
from app.importing.text import slug_code, split_skills
|
||||
from app.importing.values import parse_availability_cell, parse_tshirt_size
|
||||
from app.models import AvailabilityStatus, TshirtSize
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("cell", "expected"),
|
||||
[
|
||||
("x", AvailabilityStatus.AVAILABLE),
|
||||
("OUI", AvailabilityStatus.AVAILABLE),
|
||||
(" Disponible ", AvailabilityStatus.AVAILABLE),
|
||||
("si besoin", AvailabilityStatus.IF_NEEDED),
|
||||
("Peut-être", AvailabilityStatus.IF_NEEDED),
|
||||
("non", AvailabilityStatus.UNAVAILABLE),
|
||||
("0", AvailabilityStatus.UNAVAILABLE),
|
||||
],
|
||||
)
|
||||
def test_recognized_availability_values(cell, expected):
|
||||
status, recognized = parse_availability_cell(cell)
|
||||
assert recognized is True
|
||||
assert status is expected
|
||||
|
||||
|
||||
def test_empty_cell_is_recognized_but_no_declaration():
|
||||
assert parse_availability_cell(" ") == (None, True)
|
||||
assert parse_availability_cell("N/A") == (None, True)
|
||||
|
||||
|
||||
def test_unknown_cell_is_flagged():
|
||||
assert parse_availability_cell("bof") == (None, False)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("raw", "expected"),
|
||||
[
|
||||
("M", TshirtSize.M),
|
||||
(" m ", TshirtSize.M),
|
||||
("xl", TshirtSize.XL),
|
||||
("2XL", TshirtSize.XXL),
|
||||
("3 XL", TshirtSize.XXXL),
|
||||
],
|
||||
)
|
||||
def test_tshirt_sizes(raw, expected):
|
||||
assert parse_tshirt_size(raw) is expected
|
||||
|
||||
|
||||
def test_unknown_tshirt_size_is_none():
|
||||
assert parse_tshirt_size("large") is None
|
||||
|
||||
|
||||
def test_slug_code_is_stable_across_accents_and_case():
|
||||
assert slug_code("Régie son") == "REGIE_SON"
|
||||
assert slug_code(" permis B ") == "PERMIS_B"
|
||||
assert slug_code("SST") == "SST"
|
||||
|
||||
|
||||
def test_split_skills():
|
||||
assert split_skills("SST, Permis B / régie son\nlangues") == [
|
||||
"SST",
|
||||
"Permis B",
|
||||
"régie son",
|
||||
"langues",
|
||||
]
|
||||
assert split_skills("") == []
|
||||
Reference in New Issue
Block a user