PM-pinou
9f06e40a41
Task A: Replace _run_clustering_pipeline (533L->228L) — remove dead
docstring code, delegate to analysis.services.clustering.
Task B: Split analysis/data_loader.py (746L) into package:
analysis/data_loader/
__init__.py — re-exports all public symbols
_csv.py — load_csv_directory, BOM/encoding detection
_schema.py — schema merging and validation
_clean.py — numeric coercion helper
_sqlite.py — save_to_db, load_from_db, load_from_db_lazy, etc.
Keep old data_loader.py as thin re-export wrapper.
Verified: manage.py check (0 issues), run_pipeline (3 clusters, OK)
58 lines
2.0 KiB
Python
58 lines
2.0 KiB
Python
"""Schema detection, validation, and merging across CSV files."""
|
|
import logging
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def _merge_schema_entries(
|
|
schemas: list[dict],
|
|
) -> tuple[dict[str, str], list[str]]:
|
|
"""Merge multiple schema dicts, raising on type conflicts.
|
|
|
|
Returns a single {name: dtype} dict. If a column appears in multiple
|
|
files with different dtypes the *last* non-null dtype wins (with a
|
|
warning embedded in the result).
|
|
"""
|
|
merged: dict[str, str] = {}
|
|
conflicts: list[str] = []
|
|
for s in schemas:
|
|
for name, dtype in s.items():
|
|
if name in merged and merged[name] != dtype:
|
|
conflicts.append(name)
|
|
merged[name] = dtype # last wins
|
|
return merged, conflicts
|
|
|
|
|
|
def validate_schemas(schemas: list[dict], strict: bool = True) -> None:
|
|
"""Compare schemas across files and raise or warn on column mismatch.
|
|
|
|
Checks that every file has the same set of column names. Type
|
|
differences across files are **not** considered an error (Polars can
|
|
cast), but differences in *column sets* are.
|
|
|
|
Args:
|
|
schemas: List of ``{column_name: dtype_string}`` dicts, one per file.
|
|
strict: If ``True`` (default), raise :class:`ValueError` on any
|
|
column mismatch. If ``False``, log a warning instead.
|
|
"""
|
|
if not schemas:
|
|
return
|
|
|
|
col_sets = [set(s.keys()) for s in schemas]
|
|
reference = col_sets[0]
|
|
|
|
for i, cs in enumerate(col_sets[1:], start=1):
|
|
only_in_ref = reference - cs
|
|
only_in_current = cs - reference
|
|
if only_in_ref or only_in_current:
|
|
msg_parts = []
|
|
if only_in_ref:
|
|
msg_parts.append(f"File #{i} missing columns: {sorted(only_in_ref)}")
|
|
if only_in_current:
|
|
msg_parts.append(f"File #{i} has extra columns: {sorted(only_in_current)}")
|
|
full_msg = f"Schema mismatch: {'; '.join(msg_parts)}"
|
|
if strict:
|
|
raise ValueError(full_msg)
|
|
else:
|
|
logger.warning(full_msg)
|