"""
数据预处理校验阶段。在 load_csv_directory 之后、build_entity_profiles 之前自动运行。
不阻塞流程，只记录 warning 到日志。
"""
import logging
import json
from typing import Any, Optional

import polars as pl
import numpy as np

from analysis.session_store import SessionStore
from analysis.type_classifier import classify_column, DataType
from analysis.ip_clustering import ip_to_int

logger = logging.getLogger(__name__)

# Mapping from Polars dtype strings to DataType for schema comparison
_DTYPE_TO_DATATYPE: dict[str, DataType] = {
    'Int8': DataType.INT, 'Int16': DataType.INT, 'Int32': DataType.INT, 'Int64': DataType.INT,
    'UInt8': DataType.INT, 'UInt16': DataType.INT, 'UInt32': DataType.INT, 'UInt64': DataType.INT,
    'Float32': DataType.FLOAT, 'Float64': DataType.FLOAT,
    'Boolean': DataType.BOOL_ENUM, 'Bool': DataType.BOOL_ENUM,
    'String': DataType.STRING, 'Utf8': DataType.STRING, 'Categorical': DataType.STRING,
    'Date': DataType.TIMESTAMP, 'Datetime': DataType.TIMESTAMP,
    'Time': DataType.TIMESTAMP, 'Duration': DataType.TIMESTAMP,
}

_MAX_OUTLIER_SAMPLE = 1000
"""Max rows to sample for Z-score outlier detection."""

_MAX_IPV4_INVALID_SAMPLES = 10
"""Max invalid IPv4 samples to include in the report."""


def _dtype_to_datatype(dtype_str: str) -> Optional[DataType]:
    """Map a Polars dtype string (e.g. ``'String'``, ``'Int64'``) to :class:`DataType`.

    Strips any type parameters (e.g. ``Datetime(time_unit='us', time_zone=None)``)
    before lookup.
    """
    base = dtype_str.split('(')[0].strip()
    return _DTYPE_TO_DATATYPE.get(base)


def _safe_float(val: Any) -> Optional[float]:
    """Convert *val* to float or return *None* on failure."""
    if val is None:
        return None
    try:
        f = float(val)
        return None if np.isnan(f) or np.isinf(f) else round(f, 6)
    except (ValueError, TypeError):
        return None


def _is_acceptable_mismatch(schema_type: DataType, inferred_type: DataType) -> bool:
    """Return *True* if a schema vs. inferred type mismatch is benign.

    Acceptable mismatches:
    - Schema says ``STRING`` but classifier infers a more specific type
      (``IPv4``, ``URL``, ``HEX``, etc.) — this is expected because
      ``data_loader`` only sees raw Polars dtypes while ``classify_column``
      applies value-based heuristics.
    - ``INT`` ↔ ``FLOAT`` — Polars can cast between numeric types freely.
    - ``ENUM`` ↔ ``BOOL_ENUM`` — both are low-cardinality categorical types.
    """
    if schema_type == DataType.STRING:
        return True
    if {schema_type, inferred_type} == {DataType.INT, DataType.FLOAT}:
        return True
    if {schema_type, inferred_type} == {DataType.ENUM, DataType.BOOL_ENUM}:
        return True
    return False


# ---------------------------------------------------------------------------
#  Public API
# ---------------------------------------------------------------------------


def validate(dataset_id: str, strict: bool = False) -> dict:
    """运行完整校验，返回 JSON 报告。

    校验项
    ------
    1. **列类型** — 用 :func:`~analysis.type_classifier.classify_column`
       判定每列类型，与 *schema* 对比矛盾。
    2. **缺失率** — 每列 ``null_count / len``，>50% 标记为高风险。
    3. **异常值** — 数值列的 ``min`` / ``max`` / ``std``，Z-score > 5 比例。
    4. **枚举分布** — ``ENUM`` / ``BOOL_ENUM`` 的值频率，特殊符号
       （``+``、空白）标记。
    5. **IPv4 有效性** — 对 ``IPv4`` 类型列的每个值调用
       :func:`~analysis.ip_clustering.ip_to_int` 验证。

    输出
    ----
    每条校验结果通过 :data:`logger` 以 ``[VALIDATE]`` 标记写入
    ``info`` 或 ``warning`` 级别。

    参数
    ----------
    dataset_id:
        SessionStore 中的数据集 ID。
    strict:
        为 *True* 时遇到高风险项（缺失率 >50%、无效 IPv4 比例 >50%）
        抛出 :class:`RuntimeError`。默认为 *False*（仅记录 warning）。

    返回
    -------
    dict
        包含 ``valid``、``columns``、``warnings``、``risks`` 等键的
        校验报告。
    """
    store = SessionStore()
    entry = store.get_dataset(dataset_id)
    if entry is None:
        msg = f"Dataset '{dataset_id}' not found in SessionStore"
        logger.error(f'[VALIDATE] {msg}')
        return {'valid': False, 'columns': {}, 'warnings': [], 'risks': [msg]}

    lf: pl.LazyFrame = entry['lazyframe']
    schema: dict = entry.get('schema', {})

    # Materialise the LazyFrame
    try:
        df = lf.collect(streaming=True)
    except Exception as e:
        msg = f"Failed to collect LazyFrame for dataset '{dataset_id}': {e}"
        logger.error(f'[VALIDATE] {msg}')
        return {'valid': False, 'columns': {}, 'warnings': [], 'risks': [msg]}

    total_rows = len(df)
    logger.info(f'[VALIDATE] dataset={dataset_id} rows={total_rows} columns={len(df.columns)}')

    columns_report: dict[str, dict] = {}
    all_warnings: list[str] = []
    all_risks: list[str] = []

    for col_name in df.columns:
        series = df[col_name]
        col_info: dict[str, Any] = {
            'name': col_name,
            'dtype': str(series.dtype),
        }
        col_warnings: list[str] = []

        # ── 1. Column type classification ────────────────────────────────
        inferred_type = classify_column(col_name, series, config_type=None)
        col_info['inferred_type'] = inferred_type.name

        schema_dtype_str = schema.get(col_name)
        if schema_dtype_str:
            schema_type = _dtype_to_datatype(schema_dtype_str)
            col_info['schema_type'] = schema_type.name if schema_type else schema_dtype_str
            if schema_type and schema_type != inferred_type:
                if not _is_acceptable_mismatch(schema_type, inferred_type):
                    msg = (
                        f"Column '{col_name}': schema type ({schema_type.name}) "
                        f"differs from inferred type ({inferred_type.name})"
                    )
                    col_warnings.append(msg)
                    all_warnings.append(msg)
        else:
            col_info['schema_type'] = None

        # ── 2. Missing rate ──────────────────────────────────────────────
        null_count = int(series.null_count())
        null_rate = null_count / total_rows if total_rows > 0 else 0.0
        col_info['null_count'] = null_count
        col_info['null_rate'] = round(null_rate, 4)

        if null_rate > 0.5:
            msg = (
                f"Column '{col_name}': null rate {null_rate:.1%} "
                f"exceeds 50% (high risk)"
            )
            col_warnings.append(msg)
            all_risks.append(msg)
        elif null_rate > 0.3:
            msg = (
                f"Column '{col_name}': null rate {null_rate:.1%} "
                f"exceeds 30%"
            )
            col_warnings.append(msg)
            all_warnings.append(msg)

        # ── 3. Outlier detection (numeric columns) ───────────────────────
        if inferred_type in (DataType.INT, DataType.FLOAT):
            non_null = series.drop_nulls()
            if len(non_null) > 1:
                try:
                    col_info['min'] = _safe_float(non_null.min())
                    col_info['max'] = _safe_float(non_null.max())
                    col_info['std'] = _safe_float(non_null.std())

                    # Sample for Z-score computation
                    if len(non_null) > _MAX_OUTLIER_SAMPLE:
                        sample = non_null.sample(n=_MAX_OUTLIER_SAMPLE, seed=42)
                    else:
                        sample = non_null

                    num_values = sample.cast(pl.Float64).to_numpy()
                    mean = float(np.nanmean(num_values))
                    std = float(np.nanstd(num_values))
                    if std > 0:
                        z_scores = np.abs((num_values - mean) / std)
                        outlier_ratio = float(np.mean(z_scores > 5))
                        col_info['z_score_gt_5_ratio'] = round(outlier_ratio, 4)
                        if outlier_ratio > 0.05:
                            msg = (
                                f"Column '{col_name}': {outlier_ratio:.1%} "
                                f"values have |Z-score| > 5 (high outlier ratio)"
                            )
                            col_warnings.append(msg)
                            all_warnings.append(msg)
                    else:
                        col_info['z_score_gt_5_ratio'] = 0.0
                except Exception as e:
                    col_info['numeric_error'] = str(e)
                    col_info['z_score_gt_5_ratio'] = 0.0
            else:
                col_info['z_score_gt_5_ratio'] = 0.0

        # ── 4. Enum distribution ─────────────────────────────────────────
        if inferred_type in (DataType.ENUM, DataType.BOOL_ENUM):
            non_null = series.drop_nulls()
            if len(non_null) > 0:
                vc = non_null.value_counts()
                val_col = vc.columns[0]
                cnt_col = vc.columns[1]
                total_non_null = len(non_null)
                distribution: dict[str, dict] = {}
                for row in vc.iter_rows(named=True):
                    val = str(row[val_col])
                    count = int(row[cnt_col])
                    distribution[val] = {
                        'count': count,
                        'ratio': round(count / total_non_null, 4),
                    }
                col_info['distribution'] = distribution

                # Flag special symbols (empty string, whitespace-only, +)
                for val, info in distribution.items():
                    stripped = val.strip()
                    if stripped == '' or stripped == '+':
                        msg = (
                            f"Column '{col_name}': "
                            f"value {repr(val)} appears {info['count']} times"
                        )
                        col_warnings.append(msg)
                        all_warnings.append(msg)
                        break  # one warning per column

        # ── 5. IPv4 validation ───────────────────────────────────────────
        if inferred_type == DataType.IPv4:
            non_null = series.drop_nulls()
            total_valid = 0
            total_invalid = 0
            invalid_samples: list[str] = []
            for v in non_null:
                ip_str = str(v).strip()
                if ip_to_int(ip_str) is not None:
                    total_valid += 1
                else:
                    total_invalid += 1
                    if len(invalid_samples) < _MAX_IPV4_INVALID_SAMPLES:
                        invalid_samples.append(ip_str)

            col_info['ipv4_valid_count'] = total_valid
            col_info['ipv4_invalid_count'] = total_invalid
            col_info['ipv4_invalid_samples'] = invalid_samples

            if total_invalid > 0:
                total_ipv4 = total_valid + total_invalid
                invalid_ratio = total_invalid / total_ipv4 if total_ipv4 > 0 else 1.0
                sample_str = str(invalid_samples[:5])
                msg = (
                    f"Column '{col_name}': {total_invalid}/{total_ipv4} "
                    f"({invalid_ratio:.1%}) values are invalid IPv4. "
                    f"Samples: {sample_str}"
                )
                col_warnings.append(msg)
                if invalid_ratio > 0.5:
                    all_risks.append(msg)
                else:
                    all_warnings.append(msg)

        col_info['warnings'] = col_warnings
        columns_report[col_name] = col_info

    # ── Log all findings ─────────────────────────────────────────────────
    for col_name, col_info in columns_report.items():
        for w in col_info.get('warnings', []):
            logger.warning(f'[VALIDATE] {w}')

    report_data: dict[str, Any] = {
        'valid': len(all_risks) == 0,
        'dataset_id': dataset_id,
        'total_rows': total_rows,
        'total_columns': len(df.columns),
        'columns': columns_report,
        'warnings': all_warnings,
        'risks': all_risks,
    }

    n_warn = len(all_warnings)
    n_risk = len(all_risks)
    logger.info(
        f'[VALIDATE] Complete: valid={report_data["valid"]} '
        f'warnings={n_warn} risks={n_risk}'
    )

    if strict and all_risks:
        raise RuntimeError(
            f'Validation strict mode: {len(all_risks)} risk(s) found:\n' +
            '\n'.join(f'  - {r}' for r in all_risks)
        )

    return report_data


# ---------------------------------------------------------------------------
#  Formatted report
# ---------------------------------------------------------------------------


def report(dataset_id: str) -> str:
    """返回格式化的校验报告字符串。

    参数
    ----------
    dataset_id:
        SessionStore 中的数据集 ID。

    返回
    -------
    str
        人类可读的格式化报告。
    """
    result = validate(dataset_id, strict=False)

    lines: list[str] = []
    lines.append('=' * 60)
    lines.append(f'Data Validation Report  —  dataset: {dataset_id}')
    lines.append('=' * 60)
    lines.append(f'  Valid:           {result["valid"]}')
    lines.append(f'  Total rows:      {result.get("total_rows", "?")}')
    lines.append(f'  Total columns:   {result.get("total_columns", "?")}')
    lines.append(f'  Warnings:        {len(result.get("warnings", []))}')
    lines.append(f'  Risks:           {len(result.get("risks", []))}')
    lines.append('')

    columns = result.get('columns', {})
    for col_name, col_info in sorted(columns.items()):
        lines.append(f'  --- {col_name} ---')
        lines.append(f'    dtype:         {col_info.get("dtype", "?")}')
        lines.append(f'    inferred:      {col_info.get("inferred_type", "?")}')
        lines.append(f'    null_rate:     {col_info.get("null_rate", "?")}')
        if 'min' in col_info:
            cmin = col_info.get('min')
            cmax = col_info.get('max')
            lines.append(f'    range:         [{cmin}, {cmax}]')
        if 'z_score_gt_5_ratio' in col_info:
            lines.append(f'    |Z|>5 ratio:   {col_info["z_score_gt_5_ratio"]}')
        if 'distribution' in col_info:
            dist = col_info['distribution']
            top5 = sorted(dist.items(), key=lambda kv: kv[1]['count'], reverse=True)[:5]
            top5_str = ', '.join(f'{k}={v["count"]}' for k, v in top5)
            lines.append(f'    top values:    {top5_str}')
        if 'ipv4_invalid_count' in col_info and col_info['ipv4_invalid_count'] > 0:
            lines.append(
                f'    invalid IPv4:  {col_info["ipv4_invalid_count"]} '
                f'(samples: {col_info.get("ipv4_invalid_samples", [])})'
            )
        for w in col_info.get('warnings', []):
            lines.append(f'    ! {w}')
        lines.append('')

    if result.get('warnings'):
        lines.append('Warnings:')
        for w in result['warnings']:
            lines.append(f'  - {w}')
        lines.append('')
    if result.get('risks'):
        lines.append('Risks:')
        for r in result['risks']:
            lines.append(f'  # {r}')
        lines.append('')

    lines.append('=' * 60)
    return '\n'.join(lines)
