PM-pinou
393 lines
16 KiB
Python
393 lines
16 KiB
Python
"""
|
||
数据预处理校验阶段。在 load_csv_directory 之后、build_entity_profiles 之前自动运行。
|
||
不阻塞流程，只记录 warning 到日志。
|
||
"""
|
||
import logging
|
||
import json
|
||
from typing import Any, Optional
|
||
|
||
import polars as pl
|
||
import numpy as np
|
||
|
||
from analysis.session_store import SessionStore
|
||
from analysis.type_classifier import classify_column, DataType
|
||
from analysis.ip_clustering import ip_to_int
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# Mapping from Polars dtype strings to DataType for schema comparison
|
||
_DTYPE_TO_DATATYPE: dict[str, DataType] = {
|
||
'Int8': DataType.INT, 'Int16': DataType.INT, 'Int32': DataType.INT, 'Int64': DataType.INT,
|
||
'UInt8': DataType.INT, 'UInt16': DataType.INT, 'UInt32': DataType.INT, 'UInt64': DataType.INT,
|
||
'Float32': DataType.FLOAT, 'Float64': DataType.FLOAT,
|
||
'Boolean': DataType.BOOL_ENUM, 'Bool': DataType.BOOL_ENUM,
|
||
'String': DataType.STRING, 'Utf8': DataType.STRING, 'Categorical': DataType.STRING,
|
||
'Date': DataType.TIMESTAMP, 'Datetime': DataType.TIMESTAMP,
|
||
'Time': DataType.TIMESTAMP, 'Duration': DataType.TIMESTAMP,
|
||
}
|
||
|
||
_MAX_OUTLIER_SAMPLE = 1000
|
||
"""Max rows to sample for Z-score outlier detection."""
|
||
|
||
_MAX_IPV4_INVALID_SAMPLES = 10
|
||
"""Max invalid IPv4 samples to include in the report."""
|
||
|
||
|
||
def _dtype_to_datatype(dtype_str: str) -> Optional[DataType]:
|
||
"""Map a Polars dtype string (e.g. ``'String'``, ``'Int64'``) to :class:`DataType`.
|
||
|
||
Strips any type parameters (e.g. ``Datetime(time_unit='us', time_zone=None)``)
|
||
before lookup.
|
||
"""
|
||
base = dtype_str.split('(')[0].strip()
|
||
return _DTYPE_TO_DATATYPE.get(base)
|
||
|
||
|
||
def _safe_float(val: Any) -> Optional[float]:
|
||
"""Convert *val* to float or return *None* on failure."""
|
||
if val is None:
|
||
return None
|
||
try:
|
||
f = float(val)
|
||
return None if np.isnan(f) or np.isinf(f) else round(f, 6)
|
||
except (ValueError, TypeError):
|
||
return None
|
||
|
||
|
||
def _is_acceptable_mismatch(schema_type: DataType, inferred_type: DataType) -> bool:
|
||
"""Return *True* if a schema vs. inferred type mismatch is benign.
|
||
|
||
Acceptable mismatches:
|
||
- Schema says ``STRING`` but classifier infers a more specific type
|
||
(``IPv4``, ``URL``, ``HEX``, etc.) — this is expected because
|
||
``data_loader`` only sees raw Polars dtypes while ``classify_column``
|
||
applies value-based heuristics.
|
||
- ``INT`` ↔ ``FLOAT`` — Polars can cast between numeric types freely.
|
||
- ``ENUM`` ↔ ``BOOL_ENUM`` — both are low-cardinality categorical types.
|
||
"""
|
||
if schema_type == DataType.STRING:
|
||
return True
|
||
if {schema_type, inferred_type} == {DataType.INT, DataType.FLOAT}:
|
||
return True
|
||
if {schema_type, inferred_type} == {DataType.ENUM, DataType.BOOL_ENUM}:
|
||
return True
|
||
return False
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Public API
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def validate(dataset_id: str, strict: bool = False) -> dict:
|
||
"""运行完整校验，返回 JSON 报告。
|
||
|
||
校验项
|
||
------
|
||
1. **列类型** — 用 :func:`~analysis.type_classifier.classify_column`
|
||
判定每列类型，与 *schema* 对比矛盾。
|
||
2. **缺失率** — 每列 ``null_count / len``，>50% 标记为高风险。
|
||
3. **异常值** — 数值列的 ``min`` / ``max`` / ``std``，Z-score > 5 比例。
|
||
4. **枚举分布** — ``ENUM`` / ``BOOL_ENUM`` 的值频率，特殊符号
|
||
（``+``、空白）标记。
|
||
5. **IPv4 有效性** — 对 ``IPv4`` 类型列的每个值调用
|
||
:func:`~analysis.ip_clustering.ip_to_int` 验证。
|
||
|
||
输出
|
||
----
|
||
每条校验结果通过 :data:`logger` 以 ``[VALIDATE]`` 标记写入
|
||
``info`` 或 ``warning`` 级别。
|
||
|
||
参数
|
||
----------
|
||
dataset_id:
|
||
SessionStore 中的数据集 ID。
|
||
strict:
|
||
为 *True* 时遇到高风险项（缺失率 >50%、无效 IPv4 比例 >50%）
|
||
抛出 :class:`RuntimeError`。默认为 *False*（仅记录 warning）。
|
||
|
||
返回
|
||
-------
|
||
dict
|
||
包含 ``valid``、``columns``、``warnings``、``risks`` 等键的
|
||
校验报告。
|
||
"""
|
||
store = SessionStore()
|
||
entry = store.get_dataset(dataset_id)
|
||
if entry is None:
|
||
msg = f"Dataset '{dataset_id}' not found in SessionStore"
|
||
logger.error(f'[VALIDATE] {msg}')
|
||
return {'valid': False, 'columns': {}, 'warnings': [], 'risks': [msg]}
|
||
|
||
lf: pl.LazyFrame = entry['lazyframe']
|
||
schema: dict = entry.get('schema', {})
|
||
|
||
# Materialise the LazyFrame
|
||
try:
|
||
df = lf.collect(streaming=True)
|
||
except Exception as e:
|
||
msg = f"Failed to collect LazyFrame for dataset '{dataset_id}': {e}"
|
||
logger.error(f'[VALIDATE] {msg}')
|
||
return {'valid': False, 'columns': {}, 'warnings': [], 'risks': [msg]}
|
||
|
||
total_rows = len(df)
|
||
logger.info(f'[VALIDATE] dataset={dataset_id} rows={total_rows} columns={len(df.columns)}')
|
||
|
||
columns_report: dict[str, dict] = {}
|
||
all_warnings: list[str] = []
|
||
all_risks: list[str] = []
|
||
|
||
for col_name in df.columns:
|
||
series = df[col_name]
|
||
col_info: dict[str, Any] = {
|
||
'name': col_name,
|
||
'dtype': str(series.dtype),
|
||
}
|
||
col_warnings: list[str] = []
|
||
|
||
# ── 1. Column type classification ────────────────────────────────
|
||
inferred_type = classify_column(col_name, series, config_type=None)
|
||
col_info['inferred_type'] = inferred_type.name
|
||
|
||
schema_dtype_str = schema.get(col_name)
|
||
if schema_dtype_str:
|
||
schema_type = _dtype_to_datatype(schema_dtype_str)
|
||
col_info['schema_type'] = schema_type.name if schema_type else schema_dtype_str
|
||
if schema_type and schema_type != inferred_type:
|
||
if not _is_acceptable_mismatch(schema_type, inferred_type):
|
||
msg = (
|
||
f"Column '{col_name}': schema type ({schema_type.name}) "
|
||
f"differs from inferred type ({inferred_type.name})"
|
||
)
|
||
col_warnings.append(msg)
|
||
all_warnings.append(msg)
|
||
else:
|
||
col_info['schema_type'] = None
|
||
|
||
# ── 2. Missing rate ──────────────────────────────────────────────
|
||
null_count = int(series.null_count())
|
||
null_rate = null_count / total_rows if total_rows > 0 else 0.0
|
||
col_info['null_count'] = null_count
|
||
col_info['null_rate'] = round(null_rate, 4)
|
||
|
||
if null_rate > 0.5:
|
||
msg = (
|
||
f"Column '{col_name}': null rate {null_rate:.1%} "
|
||
f"exceeds 50% (high risk)"
|
||
)
|
||
col_warnings.append(msg)
|
||
all_risks.append(msg)
|
||
elif null_rate > 0.3:
|
||
msg = (
|
||
f"Column '{col_name}': null rate {null_rate:.1%} "
|
||
f"exceeds 30%"
|
||
)
|
||
col_warnings.append(msg)
|
||
all_warnings.append(msg)
|
||
|
||
# ── 3. Outlier detection (numeric columns) ───────────────────────
|
||
if inferred_type in (DataType.INT, DataType.FLOAT):
|
||
non_null = series.drop_nulls()
|
||
if len(non_null) > 1:
|
||
try:
|
||
col_info['min'] = _safe_float(non_null.min())
|
||
col_info['max'] = _safe_float(non_null.max())
|
||
col_info['std'] = _safe_float(non_null.std())
|
||
|
||
# Sample for Z-score computation
|
||
if len(non_null) > _MAX_OUTLIER_SAMPLE:
|
||
sample = non_null.sample(n=_MAX_OUTLIER_SAMPLE, seed=42)
|
||
else:
|
||
sample = non_null
|
||
|
||
num_values = sample.cast(pl.Float64).to_numpy()
|
||
mean = float(np.nanmean(num_values))
|
||
std = float(np.nanstd(num_values))
|
||
if std > 0:
|
||
z_scores = np.abs((num_values - mean) / std)
|
||
outlier_ratio = float(np.mean(z_scores > 5))
|
||
col_info['z_score_gt_5_ratio'] = round(outlier_ratio, 4)
|
||
if outlier_ratio > 0.05:
|
||
msg = (
|
||
f"Column '{col_name}': {outlier_ratio:.1%} "
|
||
f"values have |Z-score| > 5 (high outlier ratio)"
|
||
)
|
||
col_warnings.append(msg)
|
||
all_warnings.append(msg)
|
||
else:
|
||
col_info['z_score_gt_5_ratio'] = 0.0
|
||
except Exception as e:
|
||
col_info['numeric_error'] = str(e)
|
||
col_info['z_score_gt_5_ratio'] = 0.0
|
||
else:
|
||
col_info['z_score_gt_5_ratio'] = 0.0
|
||
|
||
# ── 4. Enum distribution ─────────────────────────────────────────
|
||
if inferred_type in (DataType.ENUM, DataType.BOOL_ENUM):
|
||
non_null = series.drop_nulls()
|
||
if len(non_null) > 0:
|
||
vc = non_null.value_counts()
|
||
val_col = vc.columns[0]
|
||
cnt_col = vc.columns[1]
|
||
total_non_null = len(non_null)
|
||
distribution: dict[str, dict] = {}
|
||
for row in vc.iter_rows(named=True):
|
||
val = str(row[val_col])
|
||
count = int(row[cnt_col])
|
||
distribution[val] = {
|
||
'count': count,
|
||
'ratio': round(count / total_non_null, 4),
|
||
}
|
||
col_info['distribution'] = distribution
|
||
|
||
# Flag special symbols (empty string, whitespace-only, +)
|
||
for val, info in distribution.items():
|
||
stripped = val.strip()
|
||
if stripped == '' or stripped == '+':
|
||
msg = (
|
||
f"Column '{col_name}': "
|
||
f"value {repr(val)} appears {info['count']} times"
|
||
)
|
||
col_warnings.append(msg)
|
||
all_warnings.append(msg)
|
||
break # one warning per column
|
||
|
||
# ── 5. IPv4 validation ───────────────────────────────────────────
|
||
if inferred_type == DataType.IPv4:
|
||
non_null = series.drop_nulls()
|
||
total_valid = 0
|
||
total_invalid = 0
|
||
invalid_samples: list[str] = []
|
||
for v in non_null:
|
||
ip_str = str(v).strip()
|
||
if ip_to_int(ip_str) is not None:
|
||
total_valid += 1
|
||
else:
|
||
total_invalid += 1
|
||
if len(invalid_samples) < _MAX_IPV4_INVALID_SAMPLES:
|
||
invalid_samples.append(ip_str)
|
||
|
||
col_info['ipv4_valid_count'] = total_valid
|
||
col_info['ipv4_invalid_count'] = total_invalid
|
||
col_info['ipv4_invalid_samples'] = invalid_samples
|
||
|
||
if total_invalid > 0:
|
||
total_ipv4 = total_valid + total_invalid
|
||
invalid_ratio = total_invalid / total_ipv4 if total_ipv4 > 0 else 1.0
|
||
sample_str = str(invalid_samples[:5])
|
||
msg = (
|
||
f"Column '{col_name}': {total_invalid}/{total_ipv4} "
|
||
f"({invalid_ratio:.1%}) values are invalid IPv4. "
|
||
f"Samples: {sample_str}"
|
||
)
|
||
col_warnings.append(msg)
|
||
if invalid_ratio > 0.5:
|
||
all_risks.append(msg)
|
||
else:
|
||
all_warnings.append(msg)
|
||
|
||
col_info['warnings'] = col_warnings
|
||
columns_report[col_name] = col_info
|
||
|
||
# ── Log all findings ─────────────────────────────────────────────────
|
||
for col_name, col_info in columns_report.items():
|
||
for w in col_info.get('warnings', []):
|
||
logger.warning(f'[VALIDATE] {w}')
|
||
|
||
report_data: dict[str, Any] = {
|
||
'valid': len(all_risks) == 0,
|
||
'dataset_id': dataset_id,
|
||
'total_rows': total_rows,
|
||
'total_columns': len(df.columns),
|
||
'columns': columns_report,
|
||
'warnings': all_warnings,
|
||
'risks': all_risks,
|
||
}
|
||
|
||
n_warn = len(all_warnings)
|
||
n_risk = len(all_risks)
|
||
logger.info(
|
||
f'[VALIDATE] Complete: valid={report_data["valid"]} '
|
||
f'warnings={n_warn} risks={n_risk}'
|
||
)
|
||
|
||
if strict and all_risks:
|
||
raise RuntimeError(
|
||
f'Validation strict mode: {len(all_risks)} risk(s) found:\n' +
|
||
'\n'.join(f' - {r}' for r in all_risks)
|
||
)
|
||
|
||
return report_data
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Formatted report
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def report(dataset_id: str) -> str:
|
||
"""返回格式化的校验报告字符串。
|
||
|
||
参数
|
||
----------
|
||
dataset_id:
|
||
SessionStore 中的数据集 ID。
|
||
|
||
返回
|
||
-------
|
||
str
|
||
人类可读的格式化报告。
|
||
"""
|
||
result = validate(dataset_id, strict=False)
|
||
|
||
lines: list[str] = []
|
||
lines.append('=' * 60)
|
||
lines.append(f'Data Validation Report — dataset: {dataset_id}')
|
||
lines.append('=' * 60)
|
||
lines.append(f' Valid: {result["valid"]}')
|
||
lines.append(f' Total rows: {result.get("total_rows", "?")}')
|
||
lines.append(f' Total columns: {result.get("total_columns", "?")}')
|
||
lines.append(f' Warnings: {len(result.get("warnings", []))}')
|
||
lines.append(f' Risks: {len(result.get("risks", []))}')
|
||
lines.append('')
|
||
|
||
columns = result.get('columns', {})
|
||
for col_name, col_info in sorted(columns.items()):
|
||
lines.append(f' --- {col_name} ---')
|
||
lines.append(f' dtype: {col_info.get("dtype", "?")}')
|
||
lines.append(f' inferred: {col_info.get("inferred_type", "?")}')
|
||
lines.append(f' null_rate: {col_info.get("null_rate", "?")}')
|
||
if 'min' in col_info:
|
||
cmin = col_info.get('min')
|
||
cmax = col_info.get('max')
|
||
lines.append(f' range: [{cmin}, {cmax}]')
|
||
if 'z_score_gt_5_ratio' in col_info:
|
||
lines.append(f' |Z|>5 ratio: {col_info["z_score_gt_5_ratio"]}')
|
||
if 'distribution' in col_info:
|
||
dist = col_info['distribution']
|
||
top5 = sorted(dist.items(), key=lambda kv: kv[1]['count'], reverse=True)[:5]
|
||
top5_str = ', '.join(f'{k}={v["count"]}' for k, v in top5)
|
||
lines.append(f' top values: {top5_str}')
|
||
if 'ipv4_invalid_count' in col_info and col_info['ipv4_invalid_count'] > 0:
|
||
lines.append(
|
||
f' invalid IPv4: {col_info["ipv4_invalid_count"]} '
|
||
f'(samples: {col_info.get("ipv4_invalid_samples", [])})'
|
||
)
|
||
for w in col_info.get('warnings', []):
|
||
lines.append(f' ! {w}')
|
||
lines.append('')
|
||
|
||
if result.get('warnings'):
|
||
lines.append('Warnings:')
|
||
for w in result['warnings']:
|
||
lines.append(f' - {w}')
|
||
lines.append('')
|
||
if result.get('risks'):
|
||
lines.append('Risks:')
|
||
for r in result['risks']:
|
||
lines.append(f' # {r}')
|
||
lines.append('')
|
||
|
||
lines.append('=' * 60)
|
||
return '\n'.join(lines)
|