""" 数据预处理校验阶段。在 load_csv_directory 之后、build_entity_profiles 之前自动运行。 不阻塞流程，只记录 warning 到日志。 """ import logging import json from typing import Any, Optional import polars as pl import numpy as np from analysis.session_store import SessionStore from analysis.type_classifier import classify_column, DataType from analysis.ip_clustering import ip_to_int logger = logging.getLogger(__name__) # Mapping from Polars dtype strings to DataType for schema comparison _DTYPE_TO_DATATYPE: dict[str, DataType] = { 'Int8': DataType.INT, 'Int16': DataType.INT, 'Int32': DataType.INT, 'Int64': DataType.INT, 'UInt8': DataType.INT, 'UInt16': DataType.INT, 'UInt32': DataType.INT, 'UInt64': DataType.INT, 'Float32': DataType.FLOAT, 'Float64': DataType.FLOAT, 'Boolean': DataType.BOOL_ENUM, 'Bool': DataType.BOOL_ENUM, 'String': DataType.STRING, 'Utf8': DataType.STRING, 'Categorical': DataType.STRING, 'Date': DataType.TIMESTAMP, 'Datetime': DataType.TIMESTAMP, 'Time': DataType.TIMESTAMP, 'Duration': DataType.TIMESTAMP, } _MAX_OUTLIER_SAMPLE = 1000 """Max rows to sample for Z-score outlier detection.""" _MAX_IPV4_INVALID_SAMPLES = 10 """Max invalid IPv4 samples to include in the report.""" def _dtype_to_datatype(dtype_str: str) -> Optional[DataType]: """Map a Polars dtype string (e.g. ``'String'``, ``'Int64'``) to :class:`DataType`. Strips any type parameters (e.g. ``Datetime(time_unit='us', time_zone=None)``) before lookup. """ base = dtype_str.split('(')[0].strip() return _DTYPE_TO_DATATYPE.get(base) def _safe_float(val: Any) -> Optional[float]: """Convert *val* to float or return *None* on failure.""" if val is None: return None try: f = float(val) return None if np.isnan(f) or np.isinf(f) else round(f, 6) except (ValueError, TypeError): return None def _is_acceptable_mismatch(schema_type: DataType, inferred_type: DataType) -> bool: """Return *True* if a schema vs. inferred type mismatch is benign. Acceptable mismatches: - Schema says ``STRING`` but classifier infers a more specific type (``IPv4``, ``URL``, ``HEX``, etc.) — this is expected because ``data_loader`` only sees raw Polars dtypes while ``classify_column`` applies value-based heuristics. - ``INT`` ↔ ``FLOAT`` — Polars can cast between numeric types freely. - ``ENUM`` ↔ ``BOOL_ENUM`` — both are low-cardinality categorical types. """ if schema_type == DataType.STRING: return True if {schema_type, inferred_type} == {DataType.INT, DataType.FLOAT}: return True if {schema_type, inferred_type} == {DataType.ENUM, DataType.BOOL_ENUM}: return True return False # --------------------------------------------------------------------------- # Public API # --------------------------------------------------------------------------- def validate(dataset_id: str, strict: bool = False) -> dict: """运行完整校验，返回 JSON 报告。 校验项 ------ 1. **列类型** — 用 :func:`~analysis.type_classifier.classify_column` 判定每列类型，与 *schema* 对比矛盾。 2. **缺失率** — 每列 ``null_count / len``，>50% 标记为高风险。 3. **异常值** — 数值列的 ``min`` / ``max`` / ``std``，Z-score > 5 比例。 4. **枚举分布** — ``ENUM`` / ``BOOL_ENUM`` 的值频率，特殊符号 （``+``、空白）标记。 5. **IPv4 有效性** — 对 ``IPv4`` 类型列的每个值调用 :func:`~analysis.ip_clustering.ip_to_int` 验证。 输出 ---- 每条校验结果通过 :data:`logger` 以 ``[VALIDATE]`` 标记写入 ``info`` 或 ``warning`` 级别。 参数 ---------- dataset_id: SessionStore 中的数据集 ID。 strict: 为 *True* 时遇到高风险项（缺失率 >50%、无效 IPv4 比例 >50%） 抛出 :class:`RuntimeError`。默认为 *False*（仅记录 warning）。 返回 ------- dict 包含 ``valid``、``columns``、``warnings``、``risks`` 等键的 校验报告。 """ store = SessionStore() entry = store.get_dataset(dataset_id) if entry is None: msg = f"Dataset '{dataset_id}' not found in SessionStore" logger.error(f'[VALIDATE] {msg}') return {'valid': False, 'columns': {}, 'warnings': [], 'risks': [msg]} lf: pl.LazyFrame = entry['lazyframe'] schema: dict = entry.get('schema', {}) # Materialise the LazyFrame try: df = lf.collect(streaming=True) except Exception as e: msg = f"Failed to collect LazyFrame for dataset '{dataset_id}': {e}" logger.error(f'[VALIDATE] {msg}') return {'valid': False, 'columns': {}, 'warnings': [], 'risks': [msg]} total_rows = len(df) logger.info(f'[VALIDATE] dataset={dataset_id} rows={total_rows} columns={len(df.columns)}') columns_report: dict[str, dict] = {} all_warnings: list[str] = [] all_risks: list[str] = [] for col_name in df.columns: series = df[col_name] col_info: dict[str, Any] = { 'name': col_name, 'dtype': str(series.dtype), } col_warnings: list[str] = [] # ── 1. Column type classification ──────────────────────────────── inferred_type = classify_column(col_name, series, config_type=None) col_info['inferred_type'] = inferred_type.name schema_dtype_str = schema.get(col_name) if schema_dtype_str: schema_type = _dtype_to_datatype(schema_dtype_str) col_info['schema_type'] = schema_type.name if schema_type else schema_dtype_str if schema_type and schema_type != inferred_type: if not _is_acceptable_mismatch(schema_type, inferred_type): msg = ( f"Column '{col_name}': schema type ({schema_type.name}) " f"differs from inferred type ({inferred_type.name})" ) col_warnings.append(msg) all_warnings.append(msg) else: col_info['schema_type'] = None # ── 2. Missing rate ────────────────────────────────────────────── null_count = int(series.null_count()) null_rate = null_count / total_rows if total_rows > 0 else 0.0 col_info['null_count'] = null_count col_info['null_rate'] = round(null_rate, 4) if null_rate > 0.5: msg = ( f"Column '{col_name}': null rate {null_rate:.1%} " f"exceeds 50% (high risk)" ) col_warnings.append(msg) all_risks.append(msg) elif null_rate > 0.3: msg = ( f"Column '{col_name}': null rate {null_rate:.1%} " f"exceeds 30%" ) col_warnings.append(msg) all_warnings.append(msg) # ── 3. Outlier detection (numeric columns) ─────────────────────── if inferred_type in (DataType.INT, DataType.FLOAT): non_null = series.drop_nulls() if len(non_null) > 1: try: col_info['min'] = _safe_float(non_null.min()) col_info['max'] = _safe_float(non_null.max()) col_info['std'] = _safe_float(non_null.std()) # Sample for Z-score computation if len(non_null) > _MAX_OUTLIER_SAMPLE: sample = non_null.sample(n=_MAX_OUTLIER_SAMPLE, seed=42) else: sample = non_null num_values = sample.cast(pl.Float64).to_numpy() mean = float(np.nanmean(num_values)) std = float(np.nanstd(num_values)) if std > 0: z_scores = np.abs((num_values - mean) / std) outlier_ratio = float(np.mean(z_scores > 5)) col_info['z_score_gt_5_ratio'] = round(outlier_ratio, 4) if outlier_ratio > 0.05: msg = ( f"Column '{col_name}': {outlier_ratio:.1%} " f"values have |Z-score| > 5 (high outlier ratio)" ) col_warnings.append(msg) all_warnings.append(msg) else: col_info['z_score_gt_5_ratio'] = 0.0 except Exception as e: col_info['numeric_error'] = str(e) col_info['z_score_gt_5_ratio'] = 0.0 else: col_info['z_score_gt_5_ratio'] = 0.0 # ── 4. Enum distribution ───────────────────────────────────────── if inferred_type in (DataType.ENUM, DataType.BOOL_ENUM): non_null = series.drop_nulls() if len(non_null) > 0: vc = non_null.value_counts() val_col = vc.columns[0] cnt_col = vc.columns[1] total_non_null = len(non_null) distribution: dict[str, dict] = {} for row in vc.iter_rows(named=True): val = str(row[val_col]) count = int(row[cnt_col]) distribution[val] = { 'count': count, 'ratio': round(count / total_non_null, 4), } col_info['distribution'] = distribution # Flag special symbols (empty string, whitespace-only, +) for val, info in distribution.items(): stripped = val.strip() if stripped == '' or stripped == '+': msg = ( f"Column '{col_name}': " f"value {repr(val)} appears {info['count']} times" ) col_warnings.append(msg) all_warnings.append(msg) break # one warning per column # ── 5. IPv4 validation ─────────────────────────────────────────── if inferred_type == DataType.IPv4: non_null = series.drop_nulls() total_valid = 0 total_invalid = 0 invalid_samples: list[str] = [] for v in non_null: ip_str = str(v).strip() if ip_to_int(ip_str) is not None: total_valid += 1 else: total_invalid += 1 if len(invalid_samples) < _MAX_IPV4_INVALID_SAMPLES: invalid_samples.append(ip_str) col_info['ipv4_valid_count'] = total_valid col_info['ipv4_invalid_count'] = total_invalid col_info['ipv4_invalid_samples'] = invalid_samples if total_invalid > 0: total_ipv4 = total_valid + total_invalid invalid_ratio = total_invalid / total_ipv4 if total_ipv4 > 0 else 1.0 sample_str = str(invalid_samples[:5]) msg = ( f"Column '{col_name}': {total_invalid}/{total_ipv4} " f"({invalid_ratio:.1%}) values are invalid IPv4. " f"Samples: {sample_str}" ) col_warnings.append(msg) if invalid_ratio > 0.5: all_risks.append(msg) else: all_warnings.append(msg) col_info['warnings'] = col_warnings columns_report[col_name] = col_info # ── Log all findings ───────────────────────────────────────────────── for col_name, col_info in columns_report.items(): for w in col_info.get('warnings', []): logger.warning(f'[VALIDATE] {w}') report_data: dict[str, Any] = { 'valid': len(all_risks) == 0, 'dataset_id': dataset_id, 'total_rows': total_rows, 'total_columns': len(df.columns), 'columns': columns_report, 'warnings': all_warnings, 'risks': all_risks, } n_warn = len(all_warnings) n_risk = len(all_risks) logger.info( f'[VALIDATE] Complete: valid={report_data["valid"]} ' f'warnings={n_warn} risks={n_risk}' ) if strict and all_risks: raise RuntimeError( f'Validation strict mode: {len(all_risks)} risk(s) found:\n' + '\n'.join(f' - {r}' for r in all_risks) ) return report_data # --------------------------------------------------------------------------- # Formatted report # --------------------------------------------------------------------------- def report(dataset_id: str) -> str: """返回格式化的校验报告字符串。 参数 ---------- dataset_id: SessionStore 中的数据集 ID。 返回 ------- str 人类可读的格式化报告。 """ result = validate(dataset_id, strict=False) lines: list[str] = [] lines.append('=' * 60) lines.append(f'Data Validation Report — dataset: {dataset_id}') lines.append('=' * 60) lines.append(f' Valid: {result["valid"]}') lines.append(f' Total rows: {result.get("total_rows", "?")}') lines.append(f' Total columns: {result.get("total_columns", "?")}') lines.append(f' Warnings: {len(result.get("warnings", []))}') lines.append(f' Risks: {len(result.get("risks", []))}') lines.append('') columns = result.get('columns', {}) for col_name, col_info in sorted(columns.items()): lines.append(f' --- {col_name} ---') lines.append(f' dtype: {col_info.get("dtype", "?")}') lines.append(f' inferred: {col_info.get("inferred_type", "?")}') lines.append(f' null_rate: {col_info.get("null_rate", "?")}') if 'min' in col_info: cmin = col_info.get('min') cmax = col_info.get('max') lines.append(f' range: [{cmin}, {cmax}]') if 'z_score_gt_5_ratio' in col_info: lines.append(f' |Z|>5 ratio: {col_info["z_score_gt_5_ratio"]}') if 'distribution' in col_info: dist = col_info['distribution'] top5 = sorted(dist.items(), key=lambda kv: kv[1]['count'], reverse=True)[:5] top5_str = ', '.join(f'{k}={v["count"]}' for k, v in top5) lines.append(f' top values: {top5_str}') if 'ipv4_invalid_count' in col_info and col_info['ipv4_invalid_count'] > 0: lines.append( f' invalid IPv4: {col_info["ipv4_invalid_count"]} ' f'(samples: {col_info.get("ipv4_invalid_samples", [])})' ) for w in col_info.get('warnings', []): lines.append(f' ! {w}') lines.append('') if result.get('warnings'): lines.append('Warnings:') for w in result['warnings']: lines.append(f' - {w}') lines.append('') if result.get('risks'): lines.append('Risks:') for r in result['risks']: lines.append(f' # {r}') lines.append('') lines.append('=' * 60) return '\n'.join(lines)