PM-pinou
9f06e40a41
Task A: Replace _run_clustering_pipeline (533L->228L) — remove dead
docstring code, delegate to analysis.services.clustering.
Task B: Split analysis/data_loader.py (746L) into package:
analysis/data_loader/
__init__.py — re-exports all public symbols
_csv.py — load_csv_directory, BOM/encoding detection
_schema.py — schema merging and validation
_clean.py — numeric coercion helper
_sqlite.py — save_to_db, load_from_db, load_from_db_lazy, etc.
Keep old data_loader.py as thin re-export wrapper.
Verified: manage.py check (0 issues), run_pipeline (3 clusters, OK)
33 lines
1.3 KiB
Python
33 lines
1.3 KiB
Python
"""Data cleaning helpers: numeric coercion for string columns."""
|
|
import polars as pl
|
|
|
|
|
|
def _coerce_to_float(series: pl.Series) -> pl.Series:
|
|
"""Convert string series to Float64.
|
|
|
|
Every entry that doesn't parse as a valid float becomes null.
|
|
Handles blank strings, standalone '+'/'-', 'N/A', 'null', 'None', etc.
|
|
"""
|
|
series = series.cast(pl.Utf8).str.strip_chars()
|
|
artifacts = ('', '+', '-', '.', '..', 'N/A', 'n/a', 'NA', 'null', 'None', 'NULL', '?', '---', '--', '/')
|
|
is_artifact = series.is_in(pl.Series(artifacts).implode())
|
|
result = []
|
|
for v, bad in zip(series.to_list(), is_artifact.to_list()):
|
|
if v is None or bad:
|
|
result.append(None)
|
|
else:
|
|
# Check for "+"/"-" prefix artifacts (e.g., "+abc", "-xyz")
|
|
# where the value isn't a genuine numeric string. Values like
|
|
# "+5" or "-3.14" are still accepted as valid floats below.
|
|
if len(v) > 1 and v[0] in '+-':
|
|
try:
|
|
float(v[1:]) # validate suffix minus prefix
|
|
except ValueError:
|
|
result.append(None)
|
|
continue
|
|
try:
|
|
result.append(float(v))
|
|
except (ValueError, TypeError):
|
|
result.append(None)
|
|
return pl.Series(result, dtype=pl.Float64)
|