refactor: clean views/clustering.py and split data_loader.py into package

Task A: Replace _run_clustering_pipeline (533L->228L) — remove dead
docstring code, delegate to analysis.services.clustering.

Task B: Split analysis/data_loader.py (746L) into package:
  analysis/data_loader/
    __init__.py  — re-exports all public symbols
    _csv.py      — load_csv_directory, BOM/encoding detection
    _schema.py   — schema merging and validation
    _clean.py    — numeric coercion helper
    _sqlite.py   — save_to_db, load_from_db, load_from_db_lazy, etc.
  Keep old data_loader.py as thin re-export wrapper.

Verified: manage.py check (0 issues), run_pipeline (3 clusters, OK)

.gitignore

@@ -43,3 +43,12 @@ data/multi_upload/
 
 # User config (contains API keys)
 config/config.yaml
+
+data/test_csvs/
+
+data/geoip_chunks/
+*.mmdb.gz
+*.mmdb
+
+# Playwright MCP sessions
+.playwright-mcp/

AGENTS.md

@@ -354,9 +354,17 @@ runtime\python\python.exe scripts\column_survey.py --csv "data/*.csv"
 
 ## 近期变更 (beta-clean 分支)
 
-- views.py (2000+ 行单体) → `analysis/views/` 包 (12 模块)
-- tool_registry.py → `analysis/tools/` 包 (17 模块)
-- 删除 `simple_analysis/` 模块 (已移至 master 分支独立维护)
-- 工具数量从 27 升至 30 (新增 export_entities_json, diagnostic toolbox 扩展)
-- 模板目录重组: 6 个分析页面在 `analysis/`, 6 个功能页面在 `tianxuan/`
-- 新增 `analysis/distance.py` (距离计算) 和 `analysis/nl_describe.py` (自然语言描述)
+- views.py (2000+ 行单体) → `analysis/views/` 包 (13 模块)
+- tool_registry.py (3000+ 行) → `analysis/tools/` 包 (18 模块)
+- 默认聚类算法改为 AgglomerativeClustering
+- 移除实体概念，直接对原始行聚类 (entity_value = row_N)
+- cluster_detail 显示原始数据行而非实体画像
+- run_detail 新增全域 SVD 特征分析 + LLM 工作流时间线
+- 3D 地球侧栏复用 `/globe/?embed=1`（iframe + postMessage 联动）
+- UI 全面中文化
+- 非阻塞启动 (run.bat 打印 PID 后退出)
+
+## 待办
+
+- [ ] **前端 UI 验证**: 打开 `prototype_cluster_ui.html` 查看交互设计是否符合要求
+- [ ] **更新前端原型**后，将验证通过的交互逻辑合并到正式模板

analysis/column_processor.py

@@ -0,0 +1,162 @@
+"""Unified column processor — dispatches per-type distance computation.
+
+Replaces the if-elif chain in :func:`distance.run_preprocessing_pipeline`
+with :data:`~analysis.types.TYPE_REGISTRY` dispatch.
+
+Usage::
+
+    from analysis.types import classify_schema
+    from analysis.column_processor import ColumnProcessor
+
+    column_types = classify_schema(lf)
+    proc = ColumnProcessor(lf, column_types)
+    lf = proc.compute_distances()   # Applies each type's .distance()
+    lf = proc.normalize()           # StandardScaler on numeric dist cols
+    lf, noise = proc.svd_denoise()  # SVD with noise profile
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import polars as pl
+
+from sklearn.preprocessing import StandardScaler
+from sklearn.decomposition import TruncatedSVD
+
+
+# ── Prefixes produced by each type's .distance() ──
+_DIST_PREFIXES: tuple[str, ...] = (
+    '_ip_dist_', '_str_dist_', '_bool_dist_', '_bytes_dist_',
+    '_ts_dist_', '_enum_dist_', '_raw_',
+)
+
+# ── Types that produce numeric distance/raw output (all 8 do) ──
+_ALL_TYPES: set[str] = {
+    "枚举", "字节", "浮点", "整数", "布尔", "字符串", "IPv4", "时间戳",
+}
+
+
+class ColumnProcessor:
+    """Processes all columns in a dataset by their registered types.
+
+    Groups columns by type, dispatches each group to the corresponding
+    :class:`~analysis.types.ColumnType` instance in
+    :data:`~analysis.types.TYPE_REGISTRY`, then chains normalisation
+    and SVD denoising.
+
+    Parameters
+    ----------
+    lf:
+        Input LazyFrame with raw data columns.
+    column_types:
+        ``{col_name: ColumnType}`` mapping from
+        :func:`~analysis.types.classify_schema`.
+    """
+
+    def __init__(
+        self,
+        lf: pl.LazyFrame,
+        column_types: dict,
+    ) -> None:
+        self._lf = lf
+        self._column_types = column_types
+        self._feature_cols: list[str] = []
+        self._noise_profile: dict = {}
+
+    # ── public API ──────────────────────────────────────────────────────
+
+    @property
+    def feature_cols(self) -> list[str]:
+        """Numeric feature columns available after :meth:`compute_distances`."""
+        return list(self._feature_cols)
+
+    @property
+    def noise_profile(self) -> dict:
+        """SVD noise profile populated by :meth:`svd_denoise`."""
+        return dict(self._noise_profile)
+
+    @property
+    def lf(self) -> pl.LazyFrame:
+        """Current LazyFrame state."""
+        return self._lf
+
+    def compute_distances(self) -> pl.LazyFrame:
+        """Group columns by type and dispatch to TYPE_REGISTRY.
+
+        Each :class:`~analysis.types.ColumnType` subclass handles all
+        columns of that type in one call.  IP columns with fewer than
+        2 entries are skipped (they need pairs).
+
+        Returns the LazyFrame with distance/raw columns appended.
+        """
+        from analysis.types import TYPE_REGISTRY
+
+        # Group columns by type name
+        groups: dict[str, list[str]] = {}
+        for col_name, ct in self._column_types.items():
+            groups.setdefault(ct.name, []).append(col_name)
+
+        result = self._lf
+
+        # Dispatch each group to its type's distance method
+        for type_name, cols in groups.items():
+            type_obj = TYPE_REGISTRY.get(type_name)
+            if type_obj is None:
+                continue
+            # IP distance needs at least 2 columns (pairs)
+            if type_name == "IPv4" and len(cols) < 2:
+                continue
+            result = type_obj.distance(result, cols)
+
+        # ── Collect feature columns produced by distance computation ──
+        schema = result.collect_schema()
+        self._feature_cols = [
+            c for c in schema.names()
+            if any(c.startswith(p) for p in _DIST_PREFIXES)
+        ]
+
+        self._lf = result
+        return result
+
+    def normalize(self) -> pl.LazyFrame:
+        """Apply StandardScaler normalisation to feature columns.
+
+        Uses :func:`analysis.distance.normalize_features` internally.
+        Requires :meth:`compute_distances` to have been called first.
+        """
+        if not self._feature_cols:
+            return self._lf
+
+        from analysis.distance import normalize_features
+        self._lf = normalize_features(self._lf, self._feature_cols)
+        return self._lf
+
+    def svd_denoise(self, variance_threshold: float = 0.95) -> tuple[pl.LazyFrame, dict]:
+        """SVD noise removal on normalised feature columns.
+
+        Uses :func:`analysis.distance.svd_denoise` internally.
+        Requires :meth:`normalize` to have been called first.
+
+        Parameters
+        ----------
+        variance_threshold:
+            Fraction of total variance to preserve (0.0–1.0).
+
+        Returns
+        -------
+        tuple[pl.LazyFrame, dict]
+            ``(denoised_lf, noise_profile)``.
+        """
+        if not self._feature_cols:
+            self._noise_profile = {
+                'kept_components': 0, 'total_components': 0,
+                'kept_variance': 0.0, 'noise_components': [],
+            }
+            return self._lf, self._noise_profile
+
+        from analysis.distance import svd_denoise as _svd_denoise
+        norm_cols = [f'_norm_{c}' for c in self._feature_cols]
+        self._lf, self._noise_profile = _svd_denoise(
+            self._lf, norm_cols, variance_threshold,
+        )
+        return self._lf, self._noise_profile

analysis/constants.py

@@ -9,36 +9,40 @@ so they can be updated in one place. Import with::
 import polars as pl
 
 # ---------------------------------------------------------------------------
-#  Clustering / UMAP
+#  Clustering / UMAP — tuned for 25M rows × 50 cols on 8GB RAM
 # ---------------------------------------------------------------------------
 
-MAX_SAMPLE_ROWS = 50000
-"""Maximum rows for clustering sample (downsampling threshold)."""
+MAX_SAMPLE_ROWS = 200000
+"""Maximum rows for clustering sample.  200K ≈ 0.8% of 25M rows, fits 8GB RAM."""
 
-MAX_DISTRIBUTION_SAMPLE = 10000
-"""Default sample size for distribution exploration and scoring."""
+MAX_DISTRIBUTION_SAMPLE = 50000
+"""Default sample for distribution exploration — 50K provides stable statistics."""
 
-UMAP_TRAIN_SAMPLE = 10000
-"""Max rows used to train a UMAP reducer in extract_features."""
+UMAP_TRAIN_SAMPLE = 30000
+"""Max rows for UMAP training.  30K is a balance: UMAP O(n²) for neighbors."""
+# Note: UMAP neighbors search is O(n * n_neighbors * log n).  30K × 15 ≈ fine.
+# Going above 50K causes UMAP to allocate several GB for the nearest-neighbor index.
 
-UMAP_BATCH_SIZE = 1000
+UMAP_BATCH_SIZE = 2000
 """Batch size for UMAP transform when dataset exceeds UMAP_TRAIN_SAMPLE."""
 
 LOW_MEMORY_THRESHOLD_GB = 2
 """If available memory < this (in GiB), enable low-memory clustering path."""
 
-LOW_MEMORY_THRESHOLD_ROWS = 100000
-"""Row count above which low-memory downsampling is triggered."""
+LOW_MEMORY_THRESHOLD_ROWS = 500000
+"""Row count above which low-memory downsampling is triggered.
+500K gives 2% coverage of 25M rows while keeping numpy matrix < 200 MB."""
 
 RANDOM_SEED = 42
 """Default random seed for all sklearn/numpy operations."""
 
 # ---------------------------------------------------------------------------
-#  Globe
+#  Globe — 8GB RAM / 25M rows constraints
 # ---------------------------------------------------------------------------
 
-GLOBE_MAX_ROWS = 300
-"""Max rows to load per run for 3D globe visualization."""
+GLOBE_MAX_ROWS = 500
+"""Max rows to load per run for 3D globe visualization.  300→500 for better
+coverage on the globe, still < 1 MB of flow data."""
 
 # ---------------------------------------------------------------------------
 #  Filter
@@ -76,11 +80,11 @@ entity aggregation, anomaly detection, and feature extraction."""
 #  Diagnostics / Validation
 # ---------------------------------------------------------------------------
 
-MAX_VALIDATE_SAMPLE = 1000
+MAX_VALIDATE_SAMPLE = 5000
 """Max rows sampled during validate_data."""
 
-MAX_DIAGNOSE_SAMPLE = 5000
-"""Max rows sampled for clustering diagnosis (SVD)."""
+MAX_DIAGNOSE_SAMPLE = 50000
+"""Max rows sampled for clustering diagnosis (SVD).  50K for stable SVD."""
 
 MAX_CLUSTER_FEATURES = 50
 """Feature count threshold above which TruncatedSVD is applied."""
@@ -92,5 +96,5 @@ CLUSTER_MAX_FEATURES_TOP = 10
 #  Globe / Flow extraction
 # ---------------------------------------------------------------------------
 
-GLOBE_FLOW_MAX_ROWS = 300
+GLOBE_FLOW_MAX_ROWS = 500
 """Alias for GLOBE_MAX_ROWS — max rows loaded for flow extraction."""

analysis/data_loader.py

@@ -1,773 +1,18 @@
-"""CSV loading with per-file schema validation, BOM detection, and multi-file merge.
+"""Thin re-export wrapper — implementation moved to analysis/data_loader/ package.
 
-Supports multiple encodings and delimiter-separated files.  The main entry
-point is :func:`load_csv_directory`, which globs files, detects BOM per file,
-validates schema consistency, and returns a merged :class:`pl.LazyFrame`.
+When both ``analysis/data_loader.py`` and ``analysis/data_loader/`` exist,
+Python prefers the package directory, so this file is effectively dead code.
+Kept for git history continuity — all real code lives in the package.
+
+If you need to restore the monolithic version, see git history.
 """
-import glob
-import io
-import logging
-import os
-import sqlite3
-from pathlib import Path
-from typing import Optional
 
-import polars as pl
-import yaml
-
-from analysis.type_classifier import classify_schema, DataType
-
-logger = logging.getLogger(__name__)
-
-
-SUPPORTED_ENCODINGS = ['utf-8', 'utf-8-sig', 'utf-16le', 'utf-16be', 'latin-1']
-
-
-# ── numeric coercion helper ──────────────────────────────────────────────
-
-def _coerce_to_float(series: pl.Series) -> pl.Series:
-    """Convert string series to Float64.
-
-    Every entry that doesn't parse as a valid float becomes null.
-    Handles blank strings, standalone '+'/'-', 'N/A', 'null', 'None', etc.
-    """
-    series = series.cast(pl.Utf8).str.strip_chars()
-    artifacts = ('', '+', '-', '.', '..', 'N/A', 'n/a', 'NA', 'null', 'None', 'NULL', '?', '---', '--', '/')
-    is_artifact = series.is_in(pl.Series(artifacts).implode())
-    result = []
-    for v, bad in zip(series.to_list(), is_artifact.to_list()):
-        if v is None or bad:
-            result.append(None)
-        else:
-            # Check for "+"/"-" prefix artifacts (e.g., "+abc", "-xyz")
-            # where the value isn't a genuine numeric string.  Values like
-            # "+5" or "-3.14" are still accepted as valid floats below.
-            if len(v) > 1 and v[0] in '+-':
-                try:
-                    float(v[1:])  # validate suffix minus prefix
-                except ValueError:
-                    result.append(None)
-                    continue
-            try:
-                result.append(float(v))
-            except (ValueError, TypeError):
-                result.append(None)
-    return pl.Series(result, dtype=pl.Float64)
-
-
-# ── BOM detection ───────────────────────────────────────────────────────
-
-def detect_bom(path: str) -> str:
-    """Read the first 4 bytes of *path* and return the detected encoding.
-
-    Returns ``'utf-8'`` when no BOM is found.
-
-    Note: Python 3 ``open()`` fully supports Unicode/Chinese paths on all
-    platforms (no encoding dance needed).
-    """
-    with open(path, 'rb') as f:
-        header = f.read(4)
-
-    if header[:3] == b'\xef\xbb\xbf':
-        return 'utf-8-sig'
-    if header[:2] == b'\xff\xfe':
-        return 'utf-16le'
-    if header[:2] == b'\xfe\xff':
-        return 'utf-16be'
-    return 'utf-8'
-
-
-# ── schema utilities ────────────────────────────────────────────────────
-
-def collect_schema(
-    path: str,
-    encoding: str = 'utf-8',
-    delimiter: str = ',',
-    sample_rows: int = 10000,
-) -> dict:
-    """Return {column_name: dtype_string} for a single CSV file.
-
-    Uses :func:`pl.scan_csv` with ``infer_schema_length=0`` to skip type
-    inference.  All columns are forced to ``Utf8`` for cross-file compatibility,
-    avoiding type-inference failures when one file's column is ``Int64`` and
-    another's is ``String``.
-    """
-    lf = pl.scan_csv(
-        str(path),  # str() ensures Unicode/Chinese paths resolve correctly
-        encoding=encoding,
-        separator=delimiter,
-        infer_schema_length=0,
+from analysis.data_loader._csv import load_csv_directory
+from analysis.data_loader._sqlite import (
+    load_from_db,
+    load_from_db_lazy,
+    save_to_db,
+    drop_sqlite_table,
 )
-    schema = lf.collect_schema()
-    result = {}
-    for name, dtype in zip(schema.names(), schema.dtypes()):
-        result[name] = 'Utf8'  # force Utf8 for cross-file compatibility
-    return result
-
-
-def _merge_schema_entries(
-    schemas: list[dict],
-) -> dict:
-    """Merge multiple schema dicts, raising on type conflicts.
-
-    Returns a single {name: dtype} dict.  If a column appears in multiple
-    files with different dtypes the *last* non-null dtype wins (with a
-    warning embedded in the result).
-    """
-    merged: dict[str, str] = {}
-    conflicts: list[str] = []
-    for s in schemas:
-        for name, dtype in s.items():
-            if name in merged and merged[name] != dtype:
-                conflicts.append(name)
-            merged[name] = dtype  # last wins
-    return merged, conflicts
-
-
-def validate_schemas(schemas: list[dict], strict: bool = True) -> None:
-    """Compare schemas across files and raise or warn on column mismatch.
-
-    Checks that every file has the same set of column names.  Type
-    differences across files are **not** considered an error (Polars can
-    cast), but differences in *column sets* are.
-
-    Args:
-        schemas: List of ``{column_name: dtype_string}`` dicts, one per file.
-        strict: If ``True`` (default), raise :class:`ValueError` on any
-            column mismatch.  If ``False``, log a warning instead.
-    """
-    if not schemas:
-        return
-
-    col_sets = [set(s.keys()) for s in schemas]
-    reference = col_sets[0]
-
-    for i, cs in enumerate(col_sets[1:], start=1):
-        only_in_ref = reference - cs
-        only_in_current = cs - reference
-        if only_in_ref or only_in_current:
-            msg_parts = []
-            if only_in_ref:
-                msg_parts.append(f"File #{i} missing columns: {sorted(only_in_ref)}")
-            if only_in_current:
-                msg_parts.append(f"File #{i} has extra columns: {sorted(only_in_current)}")
-            full_msg = f"Schema mismatch: {'; '.join(msg_parts)}"
-            if strict:
-                raise ValueError(full_msg)
-            else:
-                logger.warning(full_msg)
-
-
-# ── dtype inference from schema ─────────────────────────────────────────
-
-def detect_schema_dtypes(
-    lf: pl.LazyFrame,
-    sample_rows: int = 10000,
-) -> dict:
-    """Infer Polars dtype strings from a LazyFrame's schema.
-
-    The schema is read without collecting the full data (the scan scans
-    a small sample).
-    """
-    schema = lf.collect_schema()
-    return {name: str(dtype) for name, dtype in zip(schema.names(), schema.dtypes())}
-
-
-# ── config loading ──────────────────────────────────────────────────────
-
-def load_config(config_path: Optional[str]) -> dict:
-    """Load a YAML config file; returns an empty dict if *config_path* is None."""
-    if config_path is None:
-        return {}
-    path = Path(config_path)
-    if not path.exists():
-        raise FileNotFoundError(f"Config file not found: {config_path}")
-    with open(path, 'r', encoding='utf-8') as f:
-        return yaml.safe_load(f) or {}
-
-
-# ── SQLite persistence ──────────────────────────────────────────────────
-
-def _dtype_to_sqlite(dtype: pl.DataType) -> str:
-    """Map a Polars dtype to a SQLite column type affinity."""
-    if dtype in (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
-                 pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64):
-        return 'INTEGER'
-    if dtype in (pl.Float32, pl.Float64):
-        return 'REAL'
-    if dtype == pl.Boolean:
-        return 'INTEGER'
-    return 'TEXT'  # String, Date, Datetime, etc. all map to TEXT
-
-
-def save_to_db(run_id: int, lf: pl.LazyFrame, mode: str = 'replace') -> str:
-    """Write a LazyFrame to a SQLite table ``_data_{run_id}``.
-
-    Collects the LazyFrame (streaming if possible), creates or replaces the
-    table, bulk-inserts rows in chunks, and returns the table name.
-
-    Args:
-        run_id: Analysis run ID (used as ``_data_{run_id}``).
-        lf: LazyFrame to persist.
-        mode: ``'replace'`` (default) → drop+create+insert.
-              ``'append'`` → skip DDL, only INSERT (table must exist).
-
-    This is a *persistence copy* — the original CSV files remain untouched.
-    """
-    from django.conf import settings
-
-    table_name = f"_data_{run_id}"
-    db_path = settings.DATABASES['default']['NAME']
-
-    df = lf.collect(streaming=True)
-    conn = sqlite3.connect(db_path)
-
-    try:
-        if mode == 'replace':
-            # Drop existing table for retry/idempotency safety
-            conn.execute(f'DROP TABLE IF EXISTS "{table_name}"')
-
-            # Build CREATE TABLE from Polars schema
-            col_defs = [f'"{name}" {_dtype_to_sqlite(dtype)}'
-                        for name, dtype in zip(df.columns, df.dtypes)]
-            conn.execute(f'CREATE TABLE "{table_name}" ({", ".join(col_defs)})')
-
-        # Bulk insert in chunks
-        col_names = [f'"{c}"' for c in df.columns]
-        placeholders = ','.join(['?' for _ in df.columns])
-        insert_sql = f'INSERT INTO "{table_name}" ({", ".join(col_names)}) VALUES ({placeholders})'
-
-        total_rows = len(df)
-        chunk_size = 10000
-        for i in range(0, total_rows, chunk_size):
-            chunk = df.slice(i, chunk_size)
-            conn.executemany(insert_sql, chunk.rows())
-
-        conn.commit()
-    finally:
-        conn.close()
-
-    logger.info('[SAVE_TO_DB] table=%s rows=%d mode=%s', table_name, total_rows, mode)
-    return table_name
-
-
-_STR_TO_DTYPE = {
-    'Float64': pl.Float64, 'Float32': pl.Float32,
-    'Int64': pl.Int64, 'Int32': pl.Int32, 'Int16': pl.Int16, 'Int8': pl.Int8,
-    'UInt64': pl.UInt64, 'UInt32': pl.UInt32, 'UInt16': pl.UInt16, 'UInt8': pl.UInt8,
-    'Utf8': pl.Utf8, 'String': pl.Utf8, 'Categorical': pl.Categorical,
-    'Boolean': pl.Boolean, 'Bool': pl.Boolean,
-    'Date': pl.Date, 'Datetime': pl.Datetime, 'Time': pl.Time,
-}
-
-
-def _str_to_polars_dtype(s: str) -> pl.DataType:
-    """Convert a Polars dtype string (e.g. ``'Float64'``, ``'Int32'``) to a ``pl.DataType``.
-
-    Strips trailing metadata like ``Datetime(time_unit='us', time_zone=None)``.
-    """
-    base = s.split('(')[0].strip()
-    return _STR_TO_DTYPE.get(base, pl.Utf8)
-
-
-def _rows_to_csv_df(rows: list[sqlite3.Row]) -> pl.DataFrame:
-    """Convert a batch of sqlite3.Row objects to a DataFrame via CSV fallback.
-
-    Loads all columns as Utf8 to handle mixed-type SQLite columns.
-    """
-    import csv as _csv_mod
-
-    buf = io.StringIO()
-    writer = _csv_mod.writer(buf)
-    writer.writerow(rows[0].keys())
-    for r in rows:
-        writer.writerow(str(v) if v is not None else '' for v in r)
-    buf.seek(0)
-    return pl.read_csv(buf, infer_schema_length=0)
-
-
-def load_from_db(
-    table_name: str,
-    schema_overrides: dict[str, pl.DataType | str] | None = None,
-) -> Optional[pl.LazyFrame]:
-    """Load a SQLite table into a LazyFrame.
-
-    Args:
-        table_name: SQLite table name.
-        schema_overrides: Optional dict of ``{column_name: dtype}`` to cast
-            columns after loading.  Values can be ``pl.DataType`` instances
-            (e.g. ``pl.Float64``) or dtype strings (e.g. ``'Float64'``).
-            Use when SQLite ``TEXT`` columns need to be restored to
-            ``Float64`` / ``Int64`` etc.
-
-    Returns ``None`` if the table does not exist.
-    """
-    from django.conf import settings
-
-    db_path = settings.DATABASES['default']['NAME']
-    conn = sqlite3.connect(db_path)
-
-    try:
-        # Check table exists
-        cursor = conn.execute(
-            "SELECT name FROM sqlite_master WHERE type='table' AND name=?",
-            (table_name,),
-        )
-        if cursor.fetchone() is None:
-            return None
-
-        conn.row_factory = sqlite3.Row
-        cursor = conn.execute(f'SELECT * FROM "{table_name}"')
-
-        # Peek first row to detect if table is empty
-        first_row = cursor.fetchone()
-        if first_row is None:
-            return pl.DataFrame().lazy()
-
-        batch_rows = [first_row]
-        dfs = []
-        _use_csv_fallback = False
-
-        while True:
-            chunk = cursor.fetchmany(10000)
-            if not chunk:
-                # Flush remaining rows
-                if batch_rows:
-                    if _use_csv_fallback:
-                        dfs.append(_rows_to_csv_df(batch_rows))
-                    else:
-                        try:
-                            dfs.append(pl.from_dicts([dict(r) for r in batch_rows]))
-                        except Exception:
-                            _use_csv_fallback = True
-                            dfs.append(_rows_to_csv_df(batch_rows))
-                break
-
-            batch_rows.extend(chunk)
-            if len(batch_rows) >= 10000:
-                if _use_csv_fallback:
-                    dfs.append(_rows_to_csv_df(batch_rows))
-                else:
-                    try:
-                        dfs.append(pl.from_dicts([dict(r) for r in batch_rows]))
-                    except Exception:
-                        _use_csv_fallback = True
-                        # Re-process current batch with CSV fallback
-                        dfs.append(_rows_to_csv_df(batch_rows))
-                batch_rows = []
-
-        # Concat all batches
-        if len(dfs) == 1:
-            df = dfs[0]
-        else:
-            df = pl.concat(dfs, how='vertical')
-
-        # Apply type overrides to restore original column types
-        if schema_overrides:
-            casts = {}
-            for col, dtype in schema_overrides.items():
-                if col not in df.columns:
-                    continue
-                dtype_obj = dtype if isinstance(dtype, pl.DataType) else _str_to_polars_dtype(dtype)
-                if df[col].dtype != dtype_obj:
-                    try:
-                        casts[col] = pl.col(col).cast(dtype_obj)
-                    except Exception:
-                        # If a cast fails (e.g. "A" → Int64), keep the column as-is
-                        pass
-            if casts:
-                df = df.with_columns(list(casts.values()))
-
-        return df.lazy()
-    finally:
-        conn.close()
-
-
-def load_from_db_lazy(
-    table_name: str,
-    schema_overrides: dict[str, pl.DataType | str] | None = None,
-) -> Optional[pl.LazyFrame]:
-    """Load a SQLite table into a LazyFrame using Polars native DB reader.
-
-    Uses ``pl.read_database`` for efficient bulk reading — an order of
-    magnitude faster than the manual batch-processing path in
-    :func:`load_from_db`.  Returns a :class:`pl.LazyFrame` so downstream
-    operations remain lazy (query-plan, not materialised).
-
-    Args:
-        table_name: SQLite table name.
-        schema_overrides: Optional ``{column_name: dtype}`` dict to cast
-            columns after loading.  Values can be ``pl.DataType`` instances
-            (e.g. ``pl.Float64``) or dtype strings (e.g. ``'Float64'``).
-
-    Returns:
-        ``None`` if the table does not exist or is empty.
-    """
-    from django.conf import settings
-
-    db_path = settings.DATABASES['default']['NAME']
-    conn = sqlite3.connect(db_path)
-
-    try:
-        # Check table existence (fast metadata query)
-        cursor = conn.execute(
-            "SELECT name FROM sqlite_master WHERE type='table' AND name=?",
-            (table_name,),
-        )
-        if cursor.fetchone() is None:
-            return None
-
-        # Bulk-read with Polars native engine — leverages the DBAPI2
-        # connection directly for efficient C-level data transfer.
-        query = f'SELECT * FROM "{table_name}"'
-        df = pl.read_database(connection=conn, query=query)
-    finally:
-        conn.close()
-
-    if df.height == 0:
-        return None
-
-    # Apply type overrides to restore original column types
-    # (SQLite stores everything as TEXT/INTEGER/REAL/BLOB;
-    #  we need to cast back to Float64 / Int64 / etc.)
-    if schema_overrides:
-        casts = {}
-        for col, dtype in schema_overrides.items():
-            if col not in df.columns:
-                continue
-            dtype_obj = (
-                dtype if isinstance(dtype, pl.DataType)
-                else _str_to_polars_dtype(dtype)
-            )
-            if df[col].dtype != dtype_obj:
-                try:
-                    casts[col] = pl.col(col).cast(dtype_obj)
-                except Exception:
-                    pass  # uncastable column — keep as-is
-        if casts:
-            df = df.with_columns(list(casts.values()))
-
-    rows = df.height
-    cols = df.width
-    logger.info(
-        '[LOAD_FROM_DB_LAZY] table=%s rows=%d cols=%d',
-        table_name, rows, cols,
-    )
-    return df.lazy()
-
-
-def drop_sqlite_table(table_name: str) -> None:
-    """Drop a SQLite data table silently if it exists."""
-    from django.conf import settings
-
-    db_path = settings.DATABASES['default']['NAME']
-    conn = sqlite3.connect(db_path)
-    try:
-        conn.execute(f'DROP TABLE IF EXISTS "{table_name}"')
-        conn.commit()
-    finally:
-        conn.close()
-
-
-# ── helpers ─────────────────────────────────────────────────────────────
-
-def _file_count(paths: list[str]) -> int:
-    return len(paths)
-
-
-def _resolve_encoding(encoding: str, file_encoding: str) -> str:
-    """BOM-detected encoding overrides user-supplied encoding.
-
-    Polars >= 1.0 requires ``'utf8'`` (not ``'utf-8'``).
-    """
-    resolved = file_encoding if file_encoding != 'utf-8' else encoding
-    return resolved.replace('-', '') if resolved in ('utf-8', 'utf-8-sig') else resolved
-
-
-# ── main entry ──────────────────────────────────────────────────────────
-
-def load_csv_directory(
-    glob_pattern: str,
-    encoding: str = 'utf-8',
-    delimiter: str = ',',
-    config_path: Optional[str] = None,
-    schema_override: Optional[dict] = None,
-    sample_rows: int = 10000,
-    schema_strict: bool = False,
-    recursive: bool = False,
-    head: Optional[int] = None,
-) -> tuple[pl.LazyFrame, dict, int, int, float]:
-    """Load, validate, and merge CSV files matching *glob_pattern*.
-
-    Args:
-        glob_pattern: Glob pattern for CSV files (e.g. ``data/*.csv``).
-        encoding: Fallback encoding when no BOM is detected.
-        delimiter: Column delimiter character.
-        config_path: Optional path to a YAML config (merged into metadata).
-        schema_override: If given, force these {column: dtype} overrides.
-        sample_rows: Rows to scan for dtype inference per file.
-        schema_strict: When True, raise ValueError on column mismatch across
-            files. When False (default), log warnings and merge with
-            ``diagonal_relaxed`` (missing columns get null).
-        recursive: When True, use ``glob.glob(..., recursive=True)`` to
-            support ``**/*.csv`` patterns.
-        head: Optional row limit for preview scenarios. When set, applies
-            ``lf.head(head)`` before returning.
-
-    Returns:
-        A tuple of ``(lazyframe, schema_dict, total_row_count, file_count,
-        memory_estimate_mb)``.
-
-    Raises:
-        FileNotFoundError: If no files match the glob pattern.
-        ValueError: If schemas are inconsistent across files and
-            *schema_strict* is True.
-    """
-    # Normalise path separators for cross-platform Unicode support
-    glob_pattern = os.path.normpath(glob_pattern)
-
-    # Resolve file list
-    paths = sorted(glob.glob(glob_pattern, recursive=recursive))
-    if not paths and not recursive:
-        # Fallback: try recursive pattern
-        paths = sorted(glob.glob(glob_pattern, recursive=True))
-    if not paths:
-        raise FileNotFoundError(f"No files match glob pattern: {glob_pattern}")
-
-    # Load config (user-provided or fall back to project default)
-    config = load_config(config_path)
-    if not config:
-        default_path = Path(__file__).parent.parent / 'config' / 'config.yaml'
-        if default_path.exists():
-            config = load_config(str(default_path))
-
-    # Collect per-file schema
-    schemas: list[dict] = []
-    lazyframes: list[pl.LazyFrame] = []
-    for path in paths:
-        file_encoding = detect_bom(path)
-        actual_encoding = _resolve_encoding(encoding, file_encoding)
-
-        lf = pl.scan_csv(
-            str(path),  # str() ensures Unicode/Chinese paths resolve correctly
-            encoding=actual_encoding,
-            separator=delimiter,
-            infer_schema_length=max(sample_rows, 1000),
-        )
-        schema = detect_schema_dtypes(lf, sample_rows=sample_rows)
-
-        # Apply schema override for this file
-        if schema_override:
-            for col, dtype in schema_override.items():
-                if col in schema:
-                    schema[col] = dtype
-
-        schemas.append(schema)
-        lazyframes.append(lf)
-
-    if schema_strict:
-        # Raise on column mismatch
-        validate_schemas(schemas)
-    else:
-        # Lenient mode: log warnings for column mismatches
-        col_sets = [set(s.keys()) for s in schemas]
-        reference = col_sets[0]
-        for i, cs in enumerate(col_sets[1:], start=1):
-            only_in_ref = reference - cs
-            only_in_current = cs - reference
-            if only_in_ref:
-                logger.warning(
-                    "File #%d missing columns: %s (will be null)",
-                    i + 1, sorted(only_in_ref),
-                )
-            if only_in_current:
-                logger.warning(
-                    "File #%d has extra columns: %s (will be null in other files)",
-                    i + 1, sorted(only_in_current),
-                )
-
-    # Merge schemas (unify column ordering — first file wins order)
-    merged_schema, _ = _merge_schema_entries(schemas)
-
-    # Concatenate all LazyFrames (union vertically)
-    merged_lf: pl.LazyFrame = pl.concat(lazyframes, how='diagonal_relaxed')
-
-    # Log schema and size estimate
-    total_row_count_est = len(paths) * (lazyframes[0].collect(streaming=True).height if lazyframes else 0)
-    logger.info(f'[LOAD] files={len(paths)} total_rows_est={total_row_count_est}')
-    for col_name, dtype in sorted(merged_schema.items()):
-        logger.info(f'[LOAD_SCHEMA] col={col_name} dtype={dtype}')
-
-    # Apply schema override on merged frame (cast columns)
-    if schema_override:
-        casts = {}
-        for col, dtype_str in schema_override.items():
-            if col in merged_schema:
-                try:
-                    polars_dtype = getattr(pl, dtype_str.split('(')[0], None)
-                    if polars_dtype:
-                        casts[col] = pl.col(col).cast(polars_dtype)
-                except Exception:
-                    pass  # skip uncastable overrides silently
-        if casts:
-            merged_lf = merged_lf.with_columns(list(casts.values()))
-
-    # ── Cleaning: drop outdated columns ──────────────────────────────────
-    geoip_cfg = config.get('data', {}).get('geoip', {})
-    drop_cols = geoip_cfg.get(
-        'drop_columns',
-        [],  # Empty by default — lat/lon columns are now cleaned, not dropped
-    )
-    existing_drop = [c for c in drop_cols if c in merged_schema]
-    if existing_drop:
-        merged_lf = merged_lf.drop(existing_drop)
-        merged_schema = {k: v for k, v in merged_schema.items() if k not in existing_drop}
-        logger.info('[CLEAN] dropped columns: %s', existing_drop)
-
-    # ── Cleaning: BOOL_ENUM normalisation ────────────────────────────────
-    try:
-        type_map = classify_schema(
-            merged_lf.head(1000),
-            config_overrides=config.get('columns', {}),
-        )
-    except Exception:
-        type_map = {}
-
-    bool_enum_cols = [
-        c for c, t in type_map.items()
-        if t == DataType.BOOL_ENUM and c in merged_schema
-    ]
-    if bool_enum_cols:
-        # Only clean string-typed BOOL_ENUM columns; native Boolean is already clean
-        schema_dtypes = dict(zip(
-            merged_lf.collect_schema().names(),
-            merged_lf.collect_schema().dtypes(),
-        ))
-        bool_exprs = []
-        cleaned_names: list[str] = []
-        for c in bool_enum_cols:
-            if schema_dtypes.get(c) not in (pl.Utf8, pl.String):
-                continue  # native Boolean / numeric — skip
-            c_str = pl.col(c).cast(pl.Utf8).str.strip_chars()
-            cleaned = (
-                pl.when(c_str == "")
-                .then(pl.lit("False"))
-                .when(c_str.str.to_lowercase() == "+")
-                .then(pl.lit("True"))
-                .when(c_str.str.to_lowercase() == "-")
-                .then(pl.lit("False"))
-                .otherwise(c_str)
-            )
-            bool_exprs.append(cleaned.alias(c))
-            cleaned_names.append(c)
-        if bool_exprs:
-            merged_lf = merged_lf.with_columns(bool_exprs)
-            for c in cleaned_names:
-                merged_schema[c] = 'Utf8'
-            logger.info('[CLEAN] BOOL_ENUM columns normalised: %s', cleaned_names)
-
-    # ── Cleaning: LAT_LON normalisation ────────────────────────────────
-    latlon_cols = [
-        c for c, t in type_map.items()
-        if t == DataType.LAT_LON and c in merged_schema
-    ]
-    if latlon_cols:
-        schema_dtypes_ll = dict(zip(
-            merged_lf.collect_schema().names(),
-            merged_lf.collect_schema().dtypes(),
-        ))
-        latlon_exprs = []
-        for c in latlon_cols:
-            dtype = schema_dtypes_ll.get(c)
-            if dtype in (pl.Utf8, pl.String):
-                latlon_exprs.append(
-                    pl.col(c).map_batches(_coerce_to_float, return_dtype=pl.Float64).alias(c)
-                )
-                merged_schema[c] = 'Float64'
-        if latlon_exprs:
-            merged_lf = merged_lf.with_columns(latlon_exprs)
-            logger.info('[CLEAN] LAT_LON columns normalised to Float64: %s', latlon_cols)
-
-    # ── Cleaning: Generic numeric coercion ────────────────────────────
-    # For any remaining string column (not IPv4/URL/HEX/ENUM/etc.),
-    # try to coerce to Float64. Whatever can't parse becomes null.
-    # This prevents "mean on str column" crashes for ANY column.
-    _SKIP_NUM_COERCE = (DataType.IPv4, DataType.URL, DataType.HEX,
-                         DataType.ENUM, DataType.LAT_LON, DataType.BOOL_ENUM)
-    # Also protect columns with these name patterns (MAC addresses, etc.)
-    _SKIP_NAME_PATTERNS = ('mac', 'mac_', '_mac', 'addr', 'address')
-    schema_dtypes_gn = dict(zip(
-        merged_lf.collect_schema().names(),
-        merged_lf.collect_schema().dtypes(),
-    ))
-    generic_numeric_cols = []
-    for c, t in type_map.items():
-        if t in _SKIP_NUM_COERCE:
-            continue
-        if c not in merged_schema:
-            continue
-        dtype = schema_dtypes_gn.get(c)
-        if dtype not in (pl.Utf8, pl.String):
-            continue  # already numeric
-        # Skip columns with protected name patterns
-        cl = c.lower().replace('-', '_').replace(' ', '_')
-        if any(p in cl for p in _SKIP_NAME_PATTERNS):
-            continue
-        generic_numeric_cols.append(c)
-
-    if generic_numeric_cols:
-        num_exprs = []
-        for c in generic_numeric_cols:
-            num_exprs.append(
-                pl.col(c).map_batches(_coerce_to_float, return_dtype=pl.Float64).alias(c)
-            )
-        if num_exprs:
-            merged_lf = merged_lf.with_columns(num_exprs)
-            for c in generic_numeric_cols:
-                merged_schema[c] = 'Float64'
-            logger.info('[CLEAN] Generic numeric coercion applied: %s', generic_numeric_cols)
-
-    # ── Cleaning: HEX preprocessing (hex_pairs_count) ────────────────────
-    hex_cols = [
-        c for c, t in type_map.items()
-        if t == DataType.HEX and c in merged_schema
-    ]
-    if hex_cols:
-        hex_exprs = []
-        for c in hex_cols:
-            hex_exprs.append(
-                pl.col(c).str.split(" ").list.len().alias(f"{c}_hex_pairs_count")
-            )
-        merged_lf = merged_lf.with_columns(hex_exprs)
-        for c in hex_cols:
-            merged_schema[f"{c}_hex_pairs_count"] = 'UInt32'
-        logger.info('[CLEAN] HEX columns added hex_pairs_count: %s', hex_cols)
-
-    # Row count estimate (fast path: use first file row count × files)
-    # For accurate count we'd need to collect, but that defeats lazy.
-    # We estimate from the first file's scanned schema info.
-    first_rows = lazyframes[0].collect(streaming=True).height if lazyframes else 0
-    total_row_count = first_rows * len(paths)
-
-    # Memory estimate
-    bytes_per_row = 0
-    for col_name, dtype_str in merged_schema.items():
-        numeric_part = dtype_str.split('(')[0].lower()
-        size_map = {
-            'int8': 1, 'int16': 2, 'int32': 4, 'int64': 8,
-            'uint8': 1, 'uint16': 2, 'uint32': 4, 'uint64': 8,
-            'float32': 4, 'float64': 8,
-            'boolean': 1, 'bool': 1,
-            'utf8': 50, 'string': 50, 'str': 50,
-            'categorical': 8, 'cat': 8,
-            'date': 4, 'datetime': 8, 'time': 8, 'duration': 8,
-        }
-        bytes_per_row += size_map.get(numeric_part, 8)
-    memory_mb = (bytes_per_row * total_row_count) / (1024 * 1024)
-
-    # Apply head limit for preview (keeps LazyFrame lazy — rows are limited on collect)
-    if head is not None:
-        merged_lf = merged_lf.head(head)
-        total_row_count = min(total_row_count, head)
-
-    return merged_lf, merged_schema, total_row_count, len(paths), memory_mb
+from analysis.data_loader._clean import _coerce_to_float
+from analysis.data_loader._sqlite import _dtype_to_sqlite

analysis/data_loader/__init__.py

@@ -0,0 +1,30 @@
+"""CSV data loading, schema handling, cleaning, and SQLite persistence.
+
+Package layout:
+    _csv.py    — CSV scanning, BOM/encoding detection, :func:`load_csv_directory`
+    _schema.py — Schema merging and cross-file validation
+    _clean.py  — Data cleaning helpers (:func:`_coerce_to_float`)
+    _sqlite.py — SQLite persistence (save/load/drop tables)
+"""
+
+from ._csv import (
+    SUPPORTED_ENCODINGS,
+    detect_bom,
+    collect_schema,
+    detect_schema_dtypes,
+    load_config,
+    _file_count,
+    _resolve_encoding,
+    load_csv_directory,
+)
+from ._schema import _merge_schema_entries, validate_schemas
+from ._clean import _coerce_to_float
+from ._sqlite import (
+    _dtype_to_sqlite,
+    _str_to_polars_dtype,
+    _rows_to_csv_df,
+    save_to_db,
+    load_from_db,
+    load_from_db_lazy,
+    drop_sqlite_table,
+)

analysis/data_loader/_clean.py

@@ -0,0 +1,32 @@
+"""Data cleaning helpers: numeric coercion for string columns."""
+import polars as pl
+
+
+def _coerce_to_float(series: pl.Series) -> pl.Series:
+    """Convert string series to Float64.
+
+    Every entry that doesn't parse as a valid float becomes null.
+    Handles blank strings, standalone '+'/'-', 'N/A', 'null', 'None', etc.
+    """
+    series = series.cast(pl.Utf8).str.strip_chars()
+    artifacts = ('', '+', '-', '.', '..', 'N/A', 'n/a', 'NA', 'null', 'None', 'NULL', '?', '---', '--', '/')
+    is_artifact = series.is_in(pl.Series(artifacts).implode())
+    result = []
+    for v, bad in zip(series.to_list(), is_artifact.to_list()):
+        if v is None or bad:
+            result.append(None)
+        else:
+            # Check for "+"/"-" prefix artifacts (e.g., "+abc", "-xyz")
+            # where the value isn't a genuine numeric string.  Values like
+            # "+5" or "-3.14" are still accepted as valid floats below.
+            if len(v) > 1 and v[0] in '+-':
+                try:
+                    float(v[1:])  # validate suffix minus prefix
+                except ValueError:
+                    result.append(None)
+                    continue
+            try:
+                result.append(float(v))
+            except (ValueError, TypeError):
+                result.append(None)
+    return pl.Series(result, dtype=pl.Float64)

analysis/data_loader/_csv.py

@@ -0,0 +1,375 @@
+"""CSV loading: glob, BOM detection, encoding, schema collection, multi-file merge.
+
+Main entry: :func:`load_csv_directory`.
+"""
+import glob
+import logging
+import os
+from pathlib import Path
+from typing import Optional
+
+import polars as pl
+import yaml
+
+from analysis.types import classify_schema, TYPE_REGISTRY
+from ._clean import _coerce_to_float
+from ._schema import _merge_schema_entries, validate_schemas
+
+logger = logging.getLogger(__name__)
+
+SUPPORTED_ENCODINGS = ['utf-8', 'utf-8-sig', 'utf-16le', 'utf-16be', 'latin-1']
+
+
+# ── BOM detection ───────────────────────────────────────────────────────
+
+def detect_bom(path: str) -> str:
+    """Read the first 4 bytes of *path* and return the detected encoding.
+
+    Returns ``'utf-8'`` when no BOM is found.
+
+    Note: Python 3 ``open()`` fully supports Unicode/Chinese paths on all
+    platforms (no encoding dance needed).
+    """
+    with open(path, 'rb') as f:
+        header = f.read(4)
+
+    if header[:3] == b'\xef\xbb\xbf':
+        return 'utf-8-sig'
+    if header[:2] == b'\xff\xfe':
+        return 'utf-16le'
+    if header[:2] == b'\xfe\xff':
+        return 'utf-16be'
+    return 'utf-8'
+
+
+# ── schema utilities ────────────────────────────────────────────────────
+
+def collect_schema(
+    path: str,
+    encoding: str = 'utf-8',
+    delimiter: str = ',',
+    sample_rows: int = 10000,
+) -> dict:
+    """Return {column_name: dtype_string} for a single CSV file.
+
+    Uses :func:`pl.scan_csv` with ``infer_schema_length=0`` to skip type
+    inference.  All columns are forced to ``Utf8`` for cross-file compatibility,
+    avoiding type-inference failures when one file's column is ``Int64`` and
+    another's is ``String``.
+    """
+    lf = pl.scan_csv(
+        str(path),  # str() ensures Unicode/Chinese paths resolve correctly
+        encoding=encoding,
+        separator=delimiter,
+        infer_schema_length=0,
+    )
+    schema = lf.collect_schema()
+    result = {}
+    for name, dtype in zip(schema.names(), schema.dtypes()):
+        result[name] = 'Utf8'  # force Utf8 for cross-file compatibility
+    return result
+
+
+# ── dtype inference from schema ─────────────────────────────────────────
+
+def detect_schema_dtypes(
+    lf: pl.LazyFrame,
+    sample_rows: int = 10000,
+) -> dict:
+    """Infer Polars dtype strings from a LazyFrame's schema.
+
+    The schema is read without collecting the full data (the scan scans
+    a small sample).
+    """
+    schema = lf.collect_schema()
+    return {name: str(dtype) for name, dtype in zip(schema.names(), schema.dtypes())}
+
+
+# ── config loading ──────────────────────────────────────────────────────
+
+def load_config(config_path: Optional[str]) -> dict:
+    """Load a YAML config file; returns an empty dict if *config_path* is None."""
+    if config_path is None:
+        return {}
+    path = Path(config_path)
+    if not path.exists():
+        raise FileNotFoundError(f"Config file not found: {config_path}")
+    with open(path, 'r', encoding='utf-8') as f:
+        return yaml.safe_load(f) or {}
+
+
+# ── helpers ─────────────────────────────────────────────────────────────
+
+def _file_count(paths: list[str]) -> int:
+    return len(paths)
+
+
+def _resolve_encoding(encoding: str, file_encoding: str) -> str:
+    """BOM-detected encoding overrides user-supplied encoding.
+
+    Polars >= 1.0 requires ``'utf8'`` (not ``'utf-8'``).
+    """
+    resolved = file_encoding if file_encoding != 'utf-8' else encoding
+    return resolved.replace('-', '') if resolved in ('utf-8', 'utf-8-sig') else resolved
+
+
+# ── main entry ──────────────────────────────────────────────────────────
+
+def load_csv_directory(
+    glob_pattern: str,
+    encoding: str = 'utf-8',
+    delimiter: str = ',',
+    config_path: Optional[str] = None,
+    schema_override: Optional[dict] = None,
+    sample_rows: int = 10000,
+    schema_strict: bool = False,
+    recursive: bool = False,
+    head: Optional[int] = None,
+) -> tuple[pl.LazyFrame, dict, int, int, float]:
+    """Load, validate, and merge CSV files matching *glob_pattern*.
+
+    Args:
+        glob_pattern: Glob pattern for CSV files (e.g. ``data/*.csv``).
+        encoding: Fallback encoding when no BOM is detected.
+        delimiter: Column delimiter character.
+        config_path: Optional path to a YAML config (merged into metadata).
+        schema_override: If given, force these {column: dtype} overrides.
+        sample_rows: Rows to scan for dtype inference per file.
+        schema_strict: When True, raise ValueError on column mismatch across
+            files. When False (default), log warnings and merge with
+            ``diagonal_relaxed`` (missing columns get null).
+        recursive: When True, use ``glob.glob(..., recursive=True)`` to
+            support ``**/*.csv`` patterns.
+        head: Optional row limit for preview scenarios. When set, applies
+            ``lf.head(head)`` before returning.
+
+    Returns:
+        A tuple of ``(lazyframe, schema_dict, total_row_count, file_count,
+        memory_estimate_mb)``.
+
+    Raises:
+        FileNotFoundError: If no files match the glob pattern.
+        ValueError: If schemas are inconsistent across files and
+            *schema_strict* is True.
+    """
+    # Normalise path separators for cross-platform Unicode support
+    glob_pattern = os.path.normpath(glob_pattern)
+
+    # Resolve file list
+    paths = sorted(glob.glob(glob_pattern, recursive=recursive))
+    if not paths and not recursive:
+        # Fallback: try recursive pattern
+        paths = sorted(glob.glob(glob_pattern, recursive=True))
+    if not paths:
+        raise FileNotFoundError(f"No files match glob pattern: {glob_pattern}")
+
+    # Load config (user-provided or fall back to project default)
+    config = load_config(config_path)
+    if not config:
+        default_path = Path(__file__).parent.parent.parent / 'config' / 'config.yaml'
+        if default_path.exists():
+            config = load_config(str(default_path))
+
+    # Collect per-file schema
+    schemas: list[dict] = []
+    lazyframes: list[pl.LazyFrame] = []
+    for path in paths:
+        file_encoding = detect_bom(path)
+        actual_encoding = _resolve_encoding(encoding, file_encoding)
+
+        lf = pl.scan_csv(
+            str(path),  # str() ensures Unicode/Chinese paths resolve correctly
+            encoding=actual_encoding,
+            separator=delimiter,
+            infer_schema_length=max(sample_rows, 1000),
+        )
+        schema = detect_schema_dtypes(lf, sample_rows=sample_rows)
+
+        # Apply schema override for this file
+        if schema_override:
+            for col, dtype in schema_override.items():
+                if col in schema:
+                    schema[col] = dtype
+
+        schemas.append(schema)
+        lazyframes.append(lf)
+
+    if schema_strict:
+        # Raise on column mismatch
+        validate_schemas(schemas)
+    else:
+        # Lenient mode: log warnings for column mismatches
+        col_sets = [set(s.keys()) for s in schemas]
+        reference = col_sets[0]
+        for i, cs in enumerate(col_sets[1:], start=1):
+            only_in_ref = reference - cs
+            only_in_current = cs - reference
+            if only_in_ref:
+                logger.warning(
+                    "File #%d missing columns: %s (will be null)",
+                    i + 1, sorted(only_in_ref),
+                )
+            if only_in_current:
+                logger.warning(
+                    "File #%d has extra columns: %s (will be null in other files)",
+                    i + 1, sorted(only_in_current),
+                )
+
+    # Merge schemas (unify column ordering — first file wins order)
+    merged_schema, _ = _merge_schema_entries(schemas)
+
+    # Concatenate all LazyFrames (union vertically)
+    merged_lf: pl.LazyFrame = pl.concat(lazyframes, how='diagonal_relaxed')
+
+    # Log schema and size estimate
+    total_row_count_est = len(paths) * (lazyframes[0].collect(streaming=True).height if lazyframes else 0)
+    logger.info(f'[LOAD] files={len(paths)} total_rows_est={total_row_count_est}')
+    for col_name, dtype in sorted(merged_schema.items()):
+        logger.info(f'[LOAD_SCHEMA] col={col_name} dtype={dtype}')
+
+    # Apply schema override on merged frame (cast columns)
+    if schema_override:
+        casts = {}
+        for col, dtype_str in schema_override.items():
+            if col in merged_schema:
+                try:
+                    polars_dtype = getattr(pl, dtype_str.split('(')[0], None)
+                    if polars_dtype:
+                        casts[col] = pl.col(col).cast(polars_dtype)
+                except Exception:
+                    pass  # skip uncastable overrides silently
+        if casts:
+            merged_lf = merged_lf.with_columns(list(casts.values()))
+
+    # ── Cleaning: drop outdated columns ──────────────────────────────────
+    geoip_cfg = config.get('data', {}).get('geoip', {})
+    drop_cols = geoip_cfg.get(
+        'drop_columns',
+        [],  # Empty by default — lat/lon columns are now cleaned, not dropped
+    )
+    existing_drop = [c for c in drop_cols if c in merged_schema]
+    if existing_drop:
+        merged_lf = merged_lf.drop(existing_drop)
+        merged_schema = {k: v for k, v in merged_schema.items() if k not in existing_drop}
+        logger.info('[CLEAN] dropped columns: %s', existing_drop)
+
+    # ── Cleaning: BOOL normalisation ─────────────────────────────────────
+    try:
+        type_map = classify_schema(
+            merged_lf.head(1000),
+            config_overrides=config.get('columns', {}),
+        )
+    except Exception:
+        type_map = {}
+
+    bool_cols = [
+        c for c, t in type_map.items()
+        if t.name == "布尔" and c in merged_schema
+    ]
+    if bool_cols:
+        schema_dtypes = dict(zip(
+            merged_lf.collect_schema().names(),
+            merged_lf.collect_schema().dtypes(),
+        ))
+        bool_exprs = []
+        cleaned_names: list[str] = []
+        for c in bool_cols:
+            if schema_dtypes.get(c) not in (pl.Utf8, pl.String):
+                continue
+            c_str = pl.col(c).cast(pl.Utf8).str.strip_chars()
+            cleaned = (
+                pl.when(c_str == "")
+                .then(pl.lit("False"))
+                .when(c_str.str.to_lowercase() == "+")
+                .then(pl.lit("True"))
+                .when(c_str.str.to_lowercase() == "-")
+                .then(pl.lit("False"))
+                .otherwise(c_str)
+            )
+            bool_exprs.append(cleaned.alias(c))
+            cleaned_names.append(c)
+        if bool_exprs:
+            merged_lf = merged_lf.with_columns(bool_exprs)
+            for c in cleaned_names:
+                merged_schema[c] = 'Utf8'
+            logger.info('[CLEAN] BOOL columns normalised: %s', cleaned_names)
+
+    # ── Generic numeric coercion ────────────────────────────────────────
+    # For remaining string columns (not IPv4/BYTES/ENUM/BOOL etc.),
+    # try to coerce to Float64.
+    _SKIP_NUM_COERCE = {"IPv4", "字节", "枚举", "布尔"}
+    _SKIP_NAME_PATTERNS = ('mac', 'mac_', '_mac', 'addr', 'address')
+    schema_dtypes_gn = dict(zip(
+        merged_lf.collect_schema().names(),
+        merged_lf.collect_schema().dtypes(),
+    ))
+    generic_numeric_cols = []
+    for c, t in type_map.items():
+        if t.name in _SKIP_NUM_COERCE:
+            continue
+        if c not in merged_schema:
+            continue
+        dtype = schema_dtypes_gn.get(c)
+        if dtype not in (pl.Utf8, pl.String):
+            continue
+        cl = c.lower().replace('-', '_').replace(' ', '_')
+        if any(p in cl for p in _SKIP_NAME_PATTERNS):
+            continue
+        generic_numeric_cols.append(c)
+
+    if generic_numeric_cols:
+        num_exprs = []
+        for c in generic_numeric_cols:
+            num_exprs.append(
+                pl.col(c).map_batches(_coerce_to_float, return_dtype=pl.Float64).alias(c)
+            )
+        if num_exprs:
+            merged_lf = merged_lf.with_columns(num_exprs)
+            for c in generic_numeric_cols:
+                merged_schema[c] = 'Float64'
+            logger.info('[CLEAN] Generic numeric coercion applied: %s', generic_numeric_cols)
+
+    # ── Cleaning: BYTES (hex) preprocessing ─────────────────────────────
+    hex_cols = [
+        c for c, t in type_map.items()
+        if t.name == "字节" and c in merged_schema
+    ]
+    if hex_cols:
+        hex_exprs = []
+        for c in hex_cols:
+            hex_exprs.append(
+                pl.col(c).str.split(" ").list.len().alias(f"{c}_hex_pairs_count")
+            )
+        merged_lf = merged_lf.with_columns(hex_exprs)
+        for c in hex_cols:
+            merged_schema[f"{c}_hex_pairs_count"] = 'UInt32'
+        logger.info('[CLEAN] BYTES columns added hex_pairs_count: %s', hex_cols)
+
+    # Row count estimate (fast path: use first file row count × files)
+    # For accurate count we'd need to collect, but that defeats lazy.
+    # We estimate from the first file's scanned schema info.
+    first_rows = lazyframes[0].collect(streaming=True).height if lazyframes else 0
+    total_row_count = first_rows * len(paths)
+
+    # Memory estimate
+    bytes_per_row = 0
+    for col_name, dtype_str in merged_schema.items():
+        numeric_part = dtype_str.split('(')[0].lower()
+        size_map = {
+            'int8': 1, 'int16': 2, 'int32': 4, 'int64': 8,
+            'uint8': 1, 'uint16': 2, 'uint32': 4, 'uint64': 8,
+            'float32': 4, 'float64': 8,
+            'boolean': 1, 'bool': 1,
+            'utf8': 50, 'string': 50, 'str': 50,
+            'categorical': 8, 'cat': 8,
+            'date': 4, 'datetime': 8, 'time': 8, 'duration': 8,
+        }
+        bytes_per_row += size_map.get(numeric_part, 8)
+    memory_mb = (bytes_per_row * total_row_count) / (1024 * 1024)
+
+    # Apply head limit for preview (keeps LazyFrame lazy — rows are limited on collect)
+    if head is not None:
+        merged_lf = merged_lf.head(head)
+        total_row_count = min(total_row_count, head)
+
+    return merged_lf, merged_schema, total_row_count, len(paths), memory_mb

analysis/data_loader/_schema.py

@@ -0,0 +1,57 @@
+"""Schema detection, validation, and merging across CSV files."""
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def _merge_schema_entries(
+    schemas: list[dict],
+) -> tuple[dict[str, str], list[str]]:
+    """Merge multiple schema dicts, raising on type conflicts.
+
+    Returns a single {name: dtype} dict.  If a column appears in multiple
+    files with different dtypes the *last* non-null dtype wins (with a
+    warning embedded in the result).
+    """
+    merged: dict[str, str] = {}
+    conflicts: list[str] = []
+    for s in schemas:
+        for name, dtype in s.items():
+            if name in merged and merged[name] != dtype:
+                conflicts.append(name)
+            merged[name] = dtype  # last wins
+    return merged, conflicts
+
+
+def validate_schemas(schemas: list[dict], strict: bool = True) -> None:
+    """Compare schemas across files and raise or warn on column mismatch.
+
+    Checks that every file has the same set of column names.  Type
+    differences across files are **not** considered an error (Polars can
+    cast), but differences in *column sets* are.
+
+    Args:
+        schemas: List of ``{column_name: dtype_string}`` dicts, one per file.
+        strict: If ``True`` (default), raise :class:`ValueError` on any
+            column mismatch.  If ``False``, log a warning instead.
+    """
+    if not schemas:
+        return
+
+    col_sets = [set(s.keys()) for s in schemas]
+    reference = col_sets[0]
+
+    for i, cs in enumerate(col_sets[1:], start=1):
+        only_in_ref = reference - cs
+        only_in_current = cs - reference
+        if only_in_ref or only_in_current:
+            msg_parts = []
+            if only_in_ref:
+                msg_parts.append(f"File #{i} missing columns: {sorted(only_in_ref)}")
+            if only_in_current:
+                msg_parts.append(f"File #{i} has extra columns: {sorted(only_in_current)}")
+            full_msg = f"Schema mismatch: {'; '.join(msg_parts)}"
+            if strict:
+                raise ValueError(full_msg)
+            else:
+                logger.warning(full_msg)

analysis/data_loader/_sqlite.py

@@ -0,0 +1,292 @@
+"""SQLite persistence layer for LazyFrame data."""
+import csv as _csv_mod
+import io
+import logging
+import sqlite3
+from typing import Optional
+
+import polars as pl
+
+logger = logging.getLogger(__name__)
+
+# ── dtype mapping ─────────────────────────────────────────────────────────
+
+_STR_TO_DTYPE = {
+    'Float64': pl.Float64, 'Float32': pl.Float32,
+    'Int64': pl.Int64, 'Int32': pl.Int32, 'Int16': pl.Int16, 'Int8': pl.Int8,
+    'UInt64': pl.UInt64, 'UInt32': pl.UInt32, 'UInt16': pl.UInt16, 'UInt8': pl.UInt8,
+    'Utf8': pl.Utf8, 'String': pl.Utf8, 'Categorical': pl.Categorical,
+    'Boolean': pl.Boolean, 'Bool': pl.Boolean,
+    'Date': pl.Date, 'Datetime': pl.Datetime, 'Time': pl.Time,
+}
+
+
+def _dtype_to_sqlite(dtype: pl.DataType) -> str:
+    """Map a Polars dtype to a SQLite column type affinity."""
+    if dtype in (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
+                 pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64):
+        return 'INTEGER'
+    if dtype in (pl.Float32, pl.Float64):
+        return 'REAL'
+    if dtype == pl.Boolean:
+        return 'INTEGER'
+    return 'TEXT'  # String, Date, Datetime, etc. all map to TEXT
+
+
+def _str_to_polars_dtype(s: str) -> pl.DataType:
+    """Convert a Polars dtype string (e.g. ``'Float64'``, ``'Int32'``) to a ``pl.DataType``.
+
+    Strips trailing metadata like ``Datetime(time_unit='us', time_zone=None)``.
+    """
+    base = s.split('(')[0].strip()
+    return _STR_TO_DTYPE.get(base, pl.Utf8)
+
+
+def _rows_to_csv_df(rows: list[sqlite3.Row]) -> pl.DataFrame:
+    """Convert a batch of sqlite3.Row objects to a DataFrame via CSV fallback.
+
+    Loads all columns as Utf8 to handle mixed-type SQLite columns.
+    """
+    buf = io.StringIO()
+    writer = _csv_mod.writer(buf)
+    writer.writerow(rows[0].keys())
+    for r in rows:
+        writer.writerow(str(v) if v is not None else '' for v in r)
+    buf.seek(0)
+    return pl.read_csv(buf, infer_schema_length=0)
+
+
+# ── public API ───────────────────────────────────────────────────────────
+
+def save_to_db(run_id: int, lf: pl.LazyFrame, mode: str = 'replace') -> str:
+    """Write a LazyFrame to a SQLite table ``_data_{run_id}``.
+
+    Collects the LazyFrame (streaming if possible), creates or replaces the
+    table, bulk-inserts rows in chunks, and returns the table name.
+
+    Args:
+        run_id: Analysis run ID (used as ``_data_{run_id}``).
+        lf: LazyFrame to persist.
+        mode: ``'replace'`` (default) → drop+create+insert.
+              ``'append'`` → skip DDL, only INSERT (table must exist).
+
+    This is a *persistence copy* — the original CSV files remain untouched.
+    """
+    from django.conf import settings
+
+    table_name = f"_data_{run_id}"
+    db_path = settings.DATABASES['default']['NAME']
+
+    df = lf.collect(streaming=True)
+    conn = sqlite3.connect(db_path)
+
+    try:
+        if mode == 'replace':
+            # Drop existing table for retry/idempotency safety
+            conn.execute(f'DROP TABLE IF EXISTS "{table_name}"')
+
+            # Build CREATE TABLE from Polars schema
+            col_defs = [f'"{name}" {_dtype_to_sqlite(dtype)}'
+                        for name, dtype in zip(df.columns, df.dtypes)]
+            conn.execute(f'CREATE TABLE "{table_name}" ({", ".join(col_defs)})')
+
+        # Bulk insert in chunks
+        col_names = [f'"{c}"' for c in df.columns]
+        placeholders = ','.join(['?' for _ in df.columns])
+        insert_sql = f'INSERT INTO "{table_name}" ({", ".join(col_names)}) VALUES ({placeholders})'
+
+        total_rows = len(df)
+        chunk_size = 10000
+        for i in range(0, total_rows, chunk_size):
+            chunk = df.slice(i, chunk_size)
+            conn.executemany(insert_sql, chunk.rows())
+
+        conn.commit()
+    finally:
+        conn.close()
+
+    logger.info('[SAVE_TO_DB] table=%s rows=%d mode=%s', table_name, total_rows, mode)
+    return table_name
+
+
+def load_from_db(
+    table_name: str,
+    schema_overrides: dict[str, pl.DataType | str] | None = None,
+) -> Optional[pl.LazyFrame]:
+    """Load a SQLite table into a LazyFrame.
+
+    Args:
+        table_name: SQLite table name.
+        schema_overrides: Optional dict of ``{column_name: dtype}`` to cast
+            columns after loading.  Values can be ``pl.DataType`` instances
+            (e.g. ``pl.Float64``) or dtype strings (e.g. ``'Float64'``).
+            Use when SQLite ``TEXT`` columns need to be restored to
+            ``Float64`` / ``Int64`` etc.
+
+    Returns ``None`` if the table does not exist.
+    """
+    from django.conf import settings
+
+    db_path = settings.DATABASES['default']['NAME']
+    conn = sqlite3.connect(db_path)
+
+    try:
+        # Check table exists
+        cursor = conn.execute(
+            "SELECT name FROM sqlite_master WHERE type='table' AND name=?",
+            (table_name,),
+        )
+        if cursor.fetchone() is None:
+            return None
+
+        conn.row_factory = sqlite3.Row
+        cursor = conn.execute(f'SELECT * FROM "{table_name}"')
+
+        # Peek first row to detect if table is empty
+        first_row = cursor.fetchone()
+        if first_row is None:
+            return pl.DataFrame().lazy()
+
+        batch_rows = [first_row]
+        dfs = []
+        _use_csv_fallback = False
+
+        while True:
+            chunk = cursor.fetchmany(10000)
+            if not chunk:
+                # Flush remaining rows
+                if batch_rows:
+                    if _use_csv_fallback:
+                        dfs.append(_rows_to_csv_df(batch_rows))
+                    else:
+                        try:
+                            dfs.append(pl.from_dicts([dict(r) for r in batch_rows]))
+                        except Exception:
+                            _use_csv_fallback = True
+                            dfs.append(_rows_to_csv_df(batch_rows))
+                break
+
+            batch_rows.extend(chunk)
+            if len(batch_rows) >= 10000:
+                if _use_csv_fallback:
+                    dfs.append(_rows_to_csv_df(batch_rows))
+                else:
+                    try:
+                        dfs.append(pl.from_dicts([dict(r) for r in batch_rows]))
+                    except Exception:
+                        _use_csv_fallback = True
+                        # Re-process current batch with CSV fallback
+                        dfs.append(_rows_to_csv_df(batch_rows))
+                batch_rows = []
+
+        # Concat all batches
+        if len(dfs) == 1:
+            df = dfs[0]
+        else:
+            df = pl.concat(dfs, how='vertical')
+
+        # Apply type overrides to restore original column types
+        if schema_overrides:
+            casts = {}
+            for col, dtype in schema_overrides.items():
+                if col not in df.columns:
+                    continue
+                dtype_obj = dtype if isinstance(dtype, pl.DataType) else _str_to_polars_dtype(dtype)
+                if df[col].dtype != dtype_obj:
+                    try:
+                        casts[col] = pl.col(col).cast(dtype_obj)
+                    except Exception:
+                        # If a cast fails (e.g. "A" → Int64), keep the column as-is
+                        pass
+            if casts:
+                df = df.with_columns(list(casts.values()))
+
+        return df.lazy()
+    finally:
+        conn.close()
+
+
+def load_from_db_lazy(
+    table_name: str,
+    schema_overrides: dict[str, pl.DataType | str] | None = None,
+) -> Optional[pl.LazyFrame]:
+    """Load a SQLite table into a LazyFrame using Polars native DB reader.
+
+    Uses ``pl.read_database`` for efficient bulk reading — an order of
+    magnitude faster than the manual batch-processing path in
+    :func:`load_from_db`.  Returns a :class:`pl.LazyFrame` so downstream
+    operations remain lazy (query-plan, not materialised).
+
+    Args:
+        table_name: SQLite table name.
+        schema_overrides: Optional ``{column_name: dtype}`` dict to cast
+            columns after loading.  Values can be ``pl.DataType`` instances
+            (e.g. ``pl.Float64``) or dtype strings (e.g. ``'Float64'``).
+
+    Returns:
+        ``None`` if the table does not exist or is empty.
+    """
+    from django.conf import settings
+
+    db_path = settings.DATABASES['default']['NAME']
+    conn = sqlite3.connect(db_path)
+
+    try:
+        # Check table existence (fast metadata query)
+        cursor = conn.execute(
+            "SELECT name FROM sqlite_master WHERE type='table' AND name=?",
+            (table_name,),
+        )
+        if cursor.fetchone() is None:
+            return None
+
+        # Bulk-read with Polars native engine — leverages the DBAPI2
+        # connection directly for efficient C-level data transfer.
+        query = f'SELECT * FROM "{table_name}"'
+        df = pl.read_database(connection=conn, query=query)
+    finally:
+        conn.close()
+
+    if df.height == 0:
+        return None
+
+    # Apply type overrides to restore original column types
+    # (SQLite stores everything as TEXT/INTEGER/REAL/BLOB;
+    #  we need to cast back to Float64 / Int64 / etc.)
+    if schema_overrides:
+        casts = {}
+        for col, dtype in schema_overrides.items():
+            if col not in df.columns:
+                continue
+            dtype_obj = (
+                dtype if isinstance(dtype, pl.DataType)
+                else _str_to_polars_dtype(dtype)
+            )
+            if df[col].dtype != dtype_obj:
+                try:
+                    casts[col] = pl.col(col).cast(dtype_obj)
+                except Exception:
+                    pass  # uncastable column — keep as-is
+        if casts:
+            df = df.with_columns(list(casts.values()))
+
+    rows = df.height
+    cols = df.width
+    logger.info(
+        '[LOAD_FROM_DB_LAZY] table=%s rows=%d cols=%d',
+        table_name, rows, cols,
+    )
+    return df.lazy()
+
+
+def drop_sqlite_table(table_name: str) -> None:
+    """Drop a SQLite data table silently if it exists."""
+    from django.conf import settings
+
+    db_path = settings.DATABASES['default']['NAME']
+    conn = sqlite3.connect(db_path)
+    try:
+        conn.execute(f'DROP TABLE IF EXISTS "{table_name}"')
+        conn.commit()
+    finally:
+        conn.close()

analysis/distance/__init__.py

@@ -0,0 +1,37 @@
+"""Distance computation package for TianXuan analysis pipeline.
+
+Provides distance functions for various data types (IP, string, boolean,
+bytes, timestamp), plus normalisation, SVD denoising, UMAP reduction,
+and per-cluster SVD feature extraction.
+
+Public API:
+    ip_distance          — subnet-mask + Haversine geographic distance for IP pairs
+    string_distance      — Levenshtein / enum-like distance for string columns
+    bool_distance        — 0/1 match-to-mode distance for boolean columns
+    bytes_distance       — Hamming (popcount) distance for hex byte columns
+    timestamp_fft_distance — FFT phase distance for timestamp columns
+    normalize_features   — StandardScaler normalisation (μ=0, σ=1)
+    svd_denoise          — TruncatedSVD noise removal (returns noise_profile)
+    umap_reduce          — UMAP dimensionality reduction (3D→2D fallback)
+    cluster_svd_extract  — Per-cluster SVD distinguishing feature extraction
+"""
+
+from analysis.distance._geo import ip_distance
+from analysis.distance._text import bool_distance, string_distance
+from analysis.distance._numeric import bytes_distance, timestamp_fft_distance
+from analysis.distance._normalize import normalize_features
+from analysis.distance._svd import svd_denoise
+from analysis.distance._umap import umap_reduce
+from analysis.distance._cluster_svd import cluster_svd_extract
+
+__all__ = [
+    "ip_distance",
+    "string_distance",
+    "bool_distance",
+    "bytes_distance",
+    "timestamp_fft_distance",
+    "normalize_features",
+    "svd_denoise",
+    "umap_reduce",
+    "cluster_svd_extract",
+]

analysis/distance/_cluster_svd.py

@@ -0,0 +1,87 @@
+"""Per-cluster SVD feature extraction."""
+
+import numpy as np
+import polars as pl
+from sklearn.decomposition import TruncatedSVD
+
+
+def cluster_svd_extract(
+    lf: pl.LazyFrame,
+    labels: list[int],
+    feature_cols: list[str],
+) -> dict[int, dict]:
+    """Per-cluster SVD-based distinguishing feature extraction.
+
+    For each cluster label:
+    1. Extracts rows belonging to that cluster.
+    2. Fits ``TruncatedSVD(n_components=min(N, len(feature_cols)))``.
+    3. The absolute loadings on the first SVD component serve as
+       *feature_strength* — higher values mean more distinguishing.
+
+    Returns a mapping ``{cluster_label: {...}}`` where each value is::
+
+        {
+            'features': [...],              # top-10 feature names by strength
+            'explained_variance_ratio': [...],  # per-component ratios
+            'feature_strength': [...],      # per-feature loading magnitude
+        }
+
+    Parameters
+    ----------
+    lf:
+        LazyFrame with rows aligned to *labels*.
+    labels:
+        Cluster labels (same length as *lf* row count, ``-1`` = noise).
+    feature_cols:
+        Numeric column names to analyse.
+
+    Returns
+    -------
+    dict[int, dict]
+        Per-cluster SVD extraction results.
+    """
+    available = [c for c in feature_cols if c in lf.collect_schema().names()]
+    if not available:
+        return {}
+
+    df = lf.select(available).collect(streaming=True)
+    mat = df.to_numpy().astype(np.float64)
+    mat = np.nan_to_num(mat, nan=0.0, posinf=0.0, neginf=0.0)
+
+    if len(labels) != len(mat):
+        labels = labels[:len(mat)]
+
+    unique_labels = sorted(set(labels))
+    result: dict[int, dict] = {}
+
+    for label in unique_labels:
+        mask = np.array(labels) == label
+        n = int(mask.sum())
+        if n < 2:
+            continue
+
+        cluster_mat = mat[mask]
+        n_comp = min(n, len(available))
+        svd = TruncatedSVD(n_components=n_comp, random_state=42)
+        svd.fit(cluster_mat)
+
+        # First component loadings → feature strength
+        if n_comp > 0 and svd.components_.shape[0] > 0:
+            feature_strength = np.abs(svd.components_[0]).astype(np.float64)
+        else:
+            feature_strength = np.zeros(len(available), dtype=np.float64)
+
+        total_strength = float(feature_strength.sum())
+        if total_strength > 1e-10:
+            feature_strength = feature_strength / total_strength
+
+        top_indices = np.argsort(-feature_strength)
+        top_k = min(10, len(available))
+
+        result[int(label)] = {
+            'features': [available[i] for i in top_indices[:top_k]],
+            'explained_variance_ratio': svd.explained_variance_ratio_.tolist(),
+            'feature_strength': feature_strength.tolist(),
+        }
+
+    return result

analysis/distance/_geo.py

@@ -0,0 +1,170 @@
+"""IP distance functions for Polars LazyFrames.
+
+Provides subnet-mask distance and Haversine geographic distance
+between pairs of IP columns.  All distances are normalised to [0, 1];
+invalid / non-geolocatable IPs produce ``NaN``.
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Optional
+
+import polars as pl
+
+from analysis.geoip import ip_to_int, lookup
+
+# ---------------------------------------------------------------------------
+#  Earth constant
+# ---------------------------------------------------------------------------
+
+_HALF_EARTH_CIRCUMFERENCE_KM = 20_037.508  # equatorial / 2
+
+
+# ---------------------------------------------------------------------------
+#  Core distance helpers
+# ---------------------------------------------------------------------------
+
+def _subnet_mask_distance(ip1: Optional[str], ip2: Optional[str], mask: int) -> float:
+    """Normalised subnet-mask distance between two IP strings.
+
+    Computes ``abs(int(ip1)/2**mask - int(ip2)/2**mask) / 2**(32-mask)``.
+    Returns ``NaN`` when either IP is ``None`` or unparseable.
+    """
+    if ip1 is None or ip2 is None:
+        return float("nan")
+    if not isinstance(ip1, str):
+        ip1 = str(ip1)
+    if not isinstance(ip2, str):
+        ip2 = str(ip2)
+
+    int1 = ip_to_int(ip1)
+    int2 = ip_to_int(ip2)
+    if int1 is None or int2 is None:
+        return float("nan")
+
+    divisor = 2.0 ** mask
+    max_val = 2.0 ** (32 - mask)
+    return abs(int1 / divisor - int2 / divisor) / max_val
+
+
+def _haversine_km(lat1: float, lon1: float, lat2: float, lon2: float) -> float:
+    """Great-circle distance (km) via the Haversine formula."""
+    dlat = math.radians(lat2 - lat1)
+    dlon = math.radians(lon2 - lon1)
+    a = (
+        math.sin(dlat / 2) ** 2
+        + math.cos(math.radians(lat1))
+        * math.cos(math.radians(lat2))
+        * math.sin(dlon / 2) ** 2
+    )
+    return 6371.0 * 2.0 * math.asin(math.sqrt(a))
+
+
+def _geo_distance(ip1: Optional[str], ip2: Optional[str]) -> float:
+    """Normalised [0, 1] geographic distance between two IP strings.
+
+    Uses ``analysis.geoip.lookup()`` to obtain lat/lon, then Haversine.
+    Returns ``NaN`` when either IP cannot be geolocated.
+    """
+    if ip1 is None or ip2 is None:
+        return float("nan")
+    if not isinstance(ip1, str):
+        ip1 = str(ip1)
+    if not isinstance(ip2, str):
+        ip2 = str(ip2)
+
+    geo1 = lookup(ip1)
+    geo2 = lookup(ip2)
+    if geo1 is None or geo2 is None:
+        return float("nan")
+
+    km = _haversine_km(geo1["lat"], geo1["lon"], geo2["lat"], geo2["lon"])
+    return km / _HALF_EARTH_CIRCUMFERENCE_KM
+
+
+# ---------------------------------------------------------------------------
+#  Public API
+# ---------------------------------------------------------------------------
+
+def ip_distance(
+    lf: pl.LazyFrame,
+    ip_columns: list[str],
+    subnet_masks: Optional[list[int]] = None,
+) -> pl.LazyFrame:
+    """Append IP distance columns to a LazyFrame.
+
+    For each **consecutive pair** of *ip_columns*
+    (``ip_columns[0]`` vs ``ip_columns[1]``,
+    ``ip_columns[2]`` vs ``ip_columns[3]``, …) computes:
+
+    * one subnet-mask distance per entry in *subnet_masks*
+    * one Haversine geographic distance
+
+    All distances are normalised to [0, 1].  Invalid IPs or IPs not
+    found in the GeoIP database produce ``NaN``.
+
+    Parameters
+    ----------
+    lf:
+        LazyFrame containing the IP columns.
+    ip_columns:
+        List of IP column names (used in consecutive pairs).
+    subnet_masks:
+        Subnet masks for network distance (default ``[24, 28]``).
+
+    Returns
+    -------
+    pl.LazyFrame
+        Input frame with ``_ip_dist_0``, ``_ip_dist_1``, …
+        columns appended.
+
+    Examples
+    --------
+    >>> lf = pl.LazyFrame({
+    ...     "src_ip": ["8.8.8.8", "1.1.1.1"],
+    ...     "dst_ip": ["8.8.4.4", "1.0.0.1"],
+    ... })
+    >>> result = ip_distance(lf, ["src_ip", "dst_ip"], subnet_masks=[24])
+    >>> result.collect().columns
+    ['src_ip', 'dst_ip', '_ip_dist_0', '_ip_dist_1']
+    """
+    if subnet_masks is None:
+        subnet_masks = [24, 28]
+
+    result = lf
+    dist_idx = 0
+
+    # Walk consecutive pairs
+    for i in range(0, len(ip_columns) - 1, 2):
+        col_a = ip_columns[i]
+        col_b = ip_columns[i + 1]
+
+        # -- subnet-mask distances --
+        for mask in subnet_masks:
+            name = f"_ip_dist_{dist_idx}"
+            result = result.with_columns(
+                pl.struct([col_a, col_b])
+                .map_elements(
+                    lambda s, _mask=mask, _ca=col_a, _cb=col_b: (
+                        _subnet_mask_distance(s[_ca], s[_cb], _mask)
+                    ),
+                    return_dtype=pl.Float64,
+                )
+                .alias(name)
+            )
+            dist_idx += 1
+
+        # -- geographic distance --
+        name = f"_ip_dist_{dist_idx}"
+        result = result.with_columns(
+            pl.struct([col_a, col_b])
+            .map_elements(
+                lambda s, _ca=col_a, _cb=col_b: _geo_distance(s[_ca], s[_cb]),
+                return_dtype=pl.Float64,
+            )
+            .alias(name)
+        )
+        dist_idx += 1
+
+    return result

analysis/distance/_normalize.py

@@ -0,0 +1,52 @@
+"""StandardScaler normalisation (step 2 of preprocessing pipeline)."""
+
+import numpy as np
+import polars as pl
+from sklearn.preprocessing import StandardScaler
+
+
+def normalize_features(lf: pl.LazyFrame, feature_cols: list[str]) -> pl.LazyFrame:
+    """Apply StandardScaler normalisation to numeric feature columns.
+
+    Collects *feature_cols* into a numpy array, replaces ``NaN`` with
+    zero, centres to μ=0 σ=1 via ``sklearn.preprocessing.StandardScaler``,
+    then returns the result as *lf* extended with ``_norm_*`` prefixed
+    columns.
+
+    Parameters
+    ----------
+    lf:
+        LazyFrame containing *feature_cols*.
+    feature_cols:
+        Numeric column names to normalise.  Columns not present in
+        *lf* are silently skipped; non-numeric columns raise an error.
+
+    Returns
+    -------
+    pl.LazyFrame
+        Input frame with additional ``_norm_{col}`` columns.
+
+    Memory
+    ------
+    Peak memory ≈ collected numpy matrix + scaler workspace
+    (well under the 800 MB budget for typical inputs).
+    """
+    available = [c for c in feature_cols if c in lf.collect_schema().names()]
+    if not available:
+        return lf
+
+    df = lf.select(available).collect(streaming=True)
+    mat = df.to_numpy().astype(np.float64)
+    mat = np.nan_to_num(mat, nan=0.0)
+
+    scaler = StandardScaler()
+    scaled = scaler.fit_transform(mat)
+
+    # Build new columns with _norm_ prefix
+    df_result = lf.collect(streaming=True)
+    for i, col in enumerate(available):
+        df_result = df_result.with_columns(
+            pl.Series(f'_norm_{col}', scaled[:, i])
+        )
+
+    return df_result.lazy()

analysis/distance/_numeric.py

@@ -0,0 +1,278 @@
+"""Bytes/hex distance (Hamming/popcount) and timestamp FFT phase distance."""
+
+import numpy as np
+import polars as pl
+
+
+# ---------------------------------------------------------------------------
+#  Bytes Hamming distance
+# ---------------------------------------------------------------------------
+
+_HEX_POPCOUNT: dict[str, int] = {
+    '0': 0, '1': 1, '2': 1, '3': 2,
+    '4': 1, '5': 2, '6': 2, '7': 3,
+    '8': 1, '9': 2,
+    'a': 2, 'b': 3, 'c': 2, 'd': 3, 'e': 3, 'f': 4,
+    'A': 2, 'B': 3, 'C': 2, 'D': 3, 'E': 3, 'F': 4,
+}
+
+
+def _hex_popcount_normalized(val: str | None) -> float:
+    """Return the proportion of 1-bits in a hex byte string (0.0–1.0).
+
+    This is the Hamming distance from the all-zero bit string,
+    normalised by total bits.  Returns ``NaN`` for null or
+    non-hex input.
+    """
+    if val is None:
+        return float("nan")
+    s = str(val).replace(' ', '').replace('\t', '').replace('\n', '')
+    if not s:
+        return 0.0
+
+    total_ones = 0
+    for ch in s:
+        ones = _HEX_POPCOUNT.get(ch)
+        if ones is None:
+            return float("nan")  # non-hex character
+        total_ones += ones
+
+    total_bits = len(s) * 4
+    return total_ones / total_bits
+
+
+def bytes_distance(
+    lf: pl.LazyFrame,
+    byte_columns: list[str],
+) -> pl.LazyFrame:
+    """Append normalised Hamming-distance columns for hex byte columns.
+
+    Each byte column is converted from hex to binary, and the
+    proportion of 1-bits (Hamming distance from the all-zero
+    bit vector) is computed as a ``_bytes_dist_N`` Float64 column
+    normalised to [0, 1].  Non-hex / null values produce ``NaN``.
+
+    Parameters
+    ----------
+    lf:
+        LazyFrame containing the byte columns.
+    byte_columns:
+        Column names whose values are hex byte strings
+        (with optional whitespace separators, e.g. ``"A3 F2"``).
+
+    Returns
+    -------
+    pl.LazyFrame
+        Input frame with ``_bytes_dist_0``, ``_bytes_dist_1``, …
+        appended.  Columns not present in *lf* are silently skipped.
+
+    Examples
+    --------
+    >>> df = pl.DataFrame({"payload": ["A3F2", "FFFF", None]})
+    >>> result = bytes_distance(df.lazy(), ["payload"])
+    >>> result.collect()
+    shape: (3, 2)
+    ┌─────────┬────────────────┐
+    │ payload ┆ _bytes_dist_0  │
+    │ ---     ┆ ---            │
+    │ str     ┆ f64            │
+    ╞═════════╪════════════════╡
+    │ A3F2    ┆ 0.625          │
+    │ FFFF    ┆ 1.0            │
+    │ null    ┆ NaN            │
+    └─────────┴────────────────┘
+    """
+    existing = set(lf.collect_schema().names())
+    result = lf
+
+    for i, col_name in enumerate(byte_columns):
+        if col_name not in existing:
+            continue
+        result = result.with_columns(
+            pl.col(col_name)
+            .cast(pl.Utf8)
+            .map_elements(_hex_popcount_normalized, return_dtype=pl.Float64)
+            .alias(f'_bytes_dist_{i}')
+        )
+
+    return result
+
+
+# ---------------------------------------------------------------------------
+#  Timestamp FFT phase distance
+# ---------------------------------------------------------------------------
+
+# ── threshold: minimum fraction of repeated intervals to use FFT path ──
+_FFT_PERIODICITY_THRESHOLD = 0.05
+# ── minimum samples before FFT is attempted ──
+_FFT_MIN_SAMPLES = 4
+
+
+def _parse_timestamp_to_epoch(series: pl.Series) -> pl.Series:
+    """Convert a timestamp series to Unix epoch seconds (float64).
+
+    Handles int/float (assumed already epoch), pl.Datetime/pl.Date
+    (ns→seconds), and str/pl.Utf8 (strptime→ns→seconds, with direct
+    float-cast fallback on parse failure).
+    """
+    dtype = series.dtype
+    if dtype in (pl.Utf8, pl.String):
+        try:
+            parsed = series.str.strptime(pl.Datetime, format=None, strict=False)
+        except Exception:
+            return series.cast(pl.Float64)
+        return parsed.cast(pl.Int64).cast(pl.Float64) / 1_000_000_000.0
+    elif dtype in (pl.Datetime, pl.Date):
+        return series.cast(pl.Int64).cast(pl.Float64) / 1_000_000_000.0
+    else:
+        return series.cast(pl.Float64)
+
+
+def _compute_fft_distances(values: np.ndarray) -> np.ndarray:
+    """Compute per-row phase distances for a 1-D array.
+
+    Sorts *values*, checks for periodicity, then either:
+    * FFT path: ``np.fft.fft()`` → ``np.angle()``, mapped back to
+      original (unsorted) row order via rank lookup.
+    * Statistical fallback: interval z-scores mapped to row order.
+
+    Returns an ``(n,)`` float64 array, normalised to zero-mean
+    unit-variance.
+    """
+    n = len(values)
+    if n < _FFT_MIN_SAMPLES:
+        return np.full(n, np.nan, dtype=np.float64)
+
+    # Sort once; record rank→position mapping for unsorting later
+    sorted_order = np.argsort(values)
+    ranks = np.empty(n, dtype=np.int64)
+    ranks[sorted_order] = np.arange(n)
+    sorted_vals = values[sorted_order]
+
+    # ── periodicity check ──────────────────────────────────────────
+    intervals = np.diff(sorted_vals)
+    unique_int, counts = np.unique(
+        np.round(intervals, 6), return_counts=True
+    )
+    repeat_ratio = float(np.sum(counts[counts > 1])) / max(len(intervals), 1)
+
+    if repeat_ratio >= _FFT_PERIODICITY_THRESHOLD:
+        # ── FFT phase path ─────────────────────────────────────────
+        centered = sorted_vals - np.mean(sorted_vals)
+        fft_result = np.fft.fft(centered)
+        phases = np.angle(fft_result)
+        dist_values = phases[ranks]
+    else:
+        # ── statistical fallback ───────────────────────────────────
+        mean_int = float(np.mean(intervals))
+        std_int = float(np.std(intervals))
+        if std_int > 1e-10:
+            z_scores = (intervals - mean_int) / std_int
+        else:
+            z_scores = np.zeros_like(intervals)
+
+        # Row at sorted position r gets the z-score of interval r-1
+        # (preceding interval); row at position 0 reuses last interval.
+        dist_values = np.empty(n, dtype=np.float64)
+        for r in range(n):
+            if r > 0:
+                dist_values[ranks[r]] = z_scores[r - 1]
+            else:
+                dist_values[ranks[0]] = (
+                    z_scores[-1] if n > 1 else 0.0
+                )
+
+    # Normalise to zero-mean unit-variance
+    d_std = float(np.std(dist_values))
+    d_mean = float(np.mean(dist_values))
+    if d_std > 1e-10:
+        dist_values = (dist_values - d_mean) / d_std
+    else:
+        dist_values = dist_values - d_mean
+
+    return dist_values
+
+
+def timestamp_fft_distance(
+    lf: pl.LazyFrame, ts_columns: list[str]
+) -> pl.LazyFrame:
+    """Append FFT-based phase-distance columns for timestamp columns.
+
+    For each column in *ts_columns* that exists in the schema:
+
+    1. Parse values to Unix epoch float seconds (handles int, float,
+       datetime strings, and ``pl.Datetime``).
+    2. Collect the column to numpy, sort, then run ``np.fft.fft``
+       followed by ``np.angle`` to extract per-row phase values.
+    3. If fewer than 5 % of timestamp intervals repeat (weak
+       periodicity), fall back to statistical features derived
+       from interval mean, standard deviation, and z-scores.
+
+    Columns are appended as ``_ts_dist_0``, ``_ts_dist_1``, … in the
+    order *ts_columns* are supplied.  Rows whose timestamp is null
+    receive ``NaN``.
+
+    Parameters
+    ----------
+    lf : pl.LazyFrame
+        The input LazyFrame with at least one timestamp column.
+    ts_columns : list[str]
+        Column names to process.
+
+    Returns
+    -------
+    pl.LazyFrame
+        A new LazyFrame with ``_ts_dist_N`` columns appended.
+
+    Examples
+    --------
+    >>> import polars as pl
+    >>> from analysis.distance import timestamp_fft_distance
+    >>> df = pl.DataFrame({
+    ...     'ts': [0, 60, 120, 180, 240, 300],
+    ... })
+    >>> result = timestamp_fft_distance(df.lazy(), ['ts'])
+    >>> result.collect().columns
+    ['ts', '_ts_dist_0']
+    """
+    schema = lf.collect_schema()
+    available = [c for c in ts_columns if c in schema.names()]
+    if not available:
+        return lf
+
+    # Collect all needed columns once
+    df = lf.select(available).collect(streaming=True)
+    n_rows = len(df)
+
+    dist_arrays: list[np.ndarray] = []
+
+    for col in available:
+        series = df[col]
+        ts_numeric = _parse_timestamp_to_epoch(series)
+
+        # Identify valid (non-null) rows
+        valid = ~ts_numeric.is_null()
+        valid_indices = np.where(valid.to_numpy())[0]
+        values = ts_numeric.filter(valid).to_numpy().astype(np.float64)
+
+        if len(values) < _FFT_MIN_SAMPLES:
+            continue
+
+        phase_values = _compute_fft_distances(values)
+
+        # Build full-length array; invalid rows stay NaN
+        full_dist = np.full(n_rows, np.nan, dtype=np.float64)
+        full_dist[valid_indices] = phase_values
+        dist_arrays.append(full_dist)
+
+    if not dist_arrays:
+        return lf
+
+    # Re-materialise the original frame and bolt on distance columns
+    result_df = lf.collect(streaming=True)
+    for i, arr in enumerate(dist_arrays):
+        result_df = result_df.with_columns(
+            pl.Series(f'_ts_dist_{i}', arr)
+        )
+
+    return result_df.lazy()

analysis/distance/_svd.py

@@ -0,0 +1,116 @@
+"""SVD noise removal (step 3 of preprocessing pipeline)."""
+
+import numpy as np
+import polars as pl
+
+
+def svd_denoise(
+    lf: pl.LazyFrame,
+    feature_cols: list[str],
+    variance_threshold: float = 0.95,
+) -> tuple[pl.LazyFrame, dict]:
+    """Remove noise from numeric features via TruncatedSVD.
+
+    1. Fits TruncatedSVD on *feature_cols*.
+    2. Keeps *k* components where cumulative explained variance
+       ``trace(S[:k]) / trace(S) ≥ variance_threshold`` (default 0.95).
+    3. Inverse-transforms to reconstruct the denoised matrix.
+    4. **Replaces** the original *feature_cols* with denoised values.
+
+    The noise components (removed by SVD) are returned as a dict for
+    display, NOT as new feature columns.
+
+    Parameters
+    ----------
+    lf:
+        LazyFrame containing *feature_cols*.
+    feature_cols:
+        Numeric column names to denoise.  These columns are replaced
+        in-place by their denoised reconstructions.
+    variance_threshold:
+        Fraction of total variance to preserve (0.0–1.0).
+        Default 0.95 keeps 95 % of signal and treats 5 % as noise.
+
+    Returns
+    -------
+    tuple[pl.LazyFrame, dict]
+        ``(denoised_lf, noise_profile)`` where *noise_profile* contains:
+        - ``kept_components`` : int — number of components kept
+        - ``total_components`` : int — max possible components
+        - ``kept_variance`` : float — fraction of variance kept
+        - ``noise_components`` : list[dict] — per-component noise info
+          (component index, explained variance, top-5 contributing features)
+    """
+    noise_profile = {
+        'kept_components': 0,
+        'total_components': 0,
+        'kept_variance': 0.0,
+        'noise_components': [],
+    }
+
+    available = [c for c in feature_cols if c in lf.collect_schema().names()]
+    if not available:
+        return lf, noise_profile
+
+    df = lf.select(available).collect(streaming=True)
+    mat = df.to_numpy().astype(np.float64)
+    mat = np.nan_to_num(mat, nan=0.0)
+
+    n, d = mat.shape
+    max_components = min(n, d - 1)
+    if max_components < 1:
+        return lf, noise_profile
+
+    from sklearn.decomposition import TruncatedSVD
+    svd = TruncatedSVD(n_components=max_components, random_state=42)
+    _ = svd.fit_transform(mat)
+
+    total_variance = float(np.sum(svd.explained_variance_))
+    if total_variance <= 1e-10:
+        return lf, noise_profile
+
+    cumsum = np.cumsum(svd.explained_variance_)
+    for k in range(1, max_components + 1):
+        if cumsum[k - 1] / total_variance >= variance_threshold:
+            break
+    else:
+        k = max_components
+
+    # Re-fit with chosen k
+    svd_k = TruncatedSVD(n_components=k, random_state=42)
+    reduced = svd_k.fit_transform(mat)
+    reconstructed = svd_k.inverse_transform(reduced)
+
+    # --- Build noise profile (removed components) ---
+    # The "noise" is the variance in components k..max_components
+    kept_var_ratio = cumsum[k - 1] / total_variance if k > 0 else 0.0
+    noise_profile = {
+        'kept_components': k,
+        'total_components': max_components,
+        'kept_variance': round(float(kept_var_ratio), 4),
+    }
+
+    # Per-component noise info for removed components (k and beyond)
+    noise_info = []
+    for comp_idx in range(k, min(k + 10, max_components)):
+        loading = np.abs(svd.components_[comp_idx])
+        top5_idx = np.argsort(loading)[-5:][::-1]
+        top5 = [
+            {'feature': available[i], 'strength': round(float(loading[i]), 4)}
+            for i in top5_idx
+        ]
+        noise_info.append({
+            'component': int(comp_idx + 1),
+            'explained_var_ratio': round(float(svd.explained_variance_ratio_[comp_idx]) * 100, 3),
+            'top_features': top5,
+        })
+    noise_profile['noise_components'] = noise_info
+
+    # --- Replace original columns with denoised values ---
+    df_result = lf.collect(streaming=True)
+    for i, col in enumerate(available):
+        df_result = df_result.with_columns(
+            pl.Series(col, reconstructed[:, i])
+        )
+
+    return df_result.lazy(), noise_profile

analysis/distance/_text.py

@@ -0,0 +1,164 @@
+"""String distance, enum detection, and boolean distance functions."""
+
+import polars as pl
+
+
+# ---------------------------------------------------------------------------
+#  Enum-like detection helper
+# ---------------------------------------------------------------------------
+
+def _is_enum_like(s: pl.Series) -> bool:
+    """Check if ≥50% of rows have values appearing ≥2 times."""
+    total = len(s)
+    if total == 0:
+        return False
+    non_null = s.drop_nulls()
+    if len(non_null) == 0:
+        return False
+    vc = non_null.value_counts()
+    if 'count' not in vc.columns:
+        return False
+    duplicated = vc.filter(pl.col('count') >= 2)
+    if len(duplicated) == 0:
+        return False
+    duplicated_count: int = duplicated['count'].sum()  # type: ignore[union-attr]
+    return duplicated_count >= 0.5 * total
+
+
+def _levenshtein_normalized(s1: str | None, s2: str) -> float:
+    """Normalised Levenshtein distance between *s1* and *s2*.
+
+    Empty or ``None`` values return 1.0 (maximum distance).
+    """
+    if s1 is None or s1 == '':
+        return 1.0
+    if s2 is None or s2 == '':
+        return 1.0
+    s1_str = str(s1)
+    s2_str = str(s2)
+    max_len = max(len(s1_str), len(s2_str))
+    if max_len == 0:
+        return 0.0
+    import Levenshtein
+    return Levenshtein.distance(s1_str, s2_str) / max_len
+
+
+# ---------------------------------------------------------------------------
+#  String distance
+# ---------------------------------------------------------------------------
+
+def string_distance(lf: pl.LazyFrame, str_columns: list[str]) -> pl.LazyFrame:
+    """Compute string distance columns.
+
+    For each column in *str_columns*:
+
+    * If **≥50%** of rows have values appearing ≥2 times, the column is
+      treated as an enum: 0 when the value equals the mode, 1 otherwise.
+    * Otherwise, the normalised Levenshtein distance to the mode is
+      computed for each row.
+
+    Empty or ``None`` values always produce the maximum distance (1.0).
+
+    Parameters
+    ----------
+    lf:
+        Input LazyFrame.
+    str_columns:
+        List of string column names to compute distances for.
+
+    Returns
+    -------
+    pl.LazyFrame
+        Input LazyFrame extended with ``_str_dist_0``, ``_str_dist_1``,
+        ... columns (one per entry in *str_columns*).
+    """
+    df = lf.collect(streaming=True)
+    dist_exprs: list[pl.Expr] = []
+
+    for idx, col in enumerate(str_columns):
+        if col not in df.columns:
+            continue
+
+        series = df[col].cast(pl.Utf8)
+        enum_like = _is_enum_like(series)
+
+        non_null = series.drop_nulls()
+        if len(non_null) > 0:
+            mode_val = non_null.value_counts().sort('count', descending=True).row(0)[0]
+            mode_str = str(mode_val)
+        else:
+            mode_str = ''
+
+        col_name = f'_str_dist_{idx}'
+
+        if enum_like:
+            dist_exprs.append(
+                pl.when(pl.col(col).is_null())
+                .then(pl.lit(1.0))
+                .when(pl.col(col).cast(pl.Utf8) == pl.lit(mode_str))
+                .then(pl.lit(0.0))
+                .otherwise(pl.lit(1.0))
+                .alias(col_name)
+            )
+        else:
+            # Capture in closure for map_elements
+            _mode = mode_str
+            dist_exprs.append(
+                pl.col(col).map_elements(
+                    lambda v: _levenshtein_normalized(v, _mode),
+                    return_dtype=pl.Float64,
+                ).alias(col_name)
+            )
+
+    return lf.with_columns(dist_exprs)
+
+
+# ---------------------------------------------------------------------------
+#  Boolean distance
+# ---------------------------------------------------------------------------
+
+def bool_distance(lf: pl.LazyFrame, bool_columns: list[str]) -> pl.LazyFrame:
+    """Compute boolean distance columns.
+
+    For each column in *bool_columns*, each row gets:
+    0 if its value matches the column mode, 1 otherwise.
+    ``None`` values always produce 1.0 (maximum distance).
+
+    Parameters
+    ----------
+    lf:
+        Input LazyFrame.
+    bool_columns:
+        List of boolean column names to compute distances for.
+
+    Returns
+    -------
+    pl.LazyFrame
+        Input LazyFrame extended with ``_bool_dist_0``, ``_bool_dist_1``,
+        ... columns (one per entry in *bool_columns*).
+    """
+    df = lf.collect(streaming=True)
+    dist_exprs: list[pl.Expr] = []
+
+    for idx, col in enumerate(bool_columns):
+        if col not in df.columns:
+            continue
+
+        series = df[col]
+        non_null = series.drop_nulls()
+        col_name = f'_bool_dist_{idx}'
+
+        if len(non_null) == 0:
+            dist_exprs.append(pl.lit(0.0).alias(col_name))
+        else:
+            mode_val = non_null.value_counts().sort('count', descending=True).row(0)[0]
+            dist_exprs.append(
+                pl.when(pl.col(col).is_null())
+                .then(pl.lit(1.0))
+                .when(pl.col(col) == pl.lit(mode_val))
+                .then(pl.lit(0.0))
+                .otherwise(pl.lit(1.0))
+                .alias(col_name)
+            )
+
+    return lf.with_columns(dist_exprs)

analysis/distance/_umap.py

@@ -0,0 +1,122 @@
+"""UMAP dimensionality reduction (3D → 2D fallback)."""
+
+import logging
+import numpy as np
+import polars as pl
+from sklearn.preprocessing import StandardScaler
+import umap
+
+
+def umap_reduce(
+    lf: pl.LazyFrame,
+    feature_cols: list[str],
+    n_components: int = 3,
+) -> tuple[pl.LazyFrame, dict]:
+    """UMAP dimensionality reduction with automatic 3D→2D fallback.
+
+    Trains on a ≤10K-row sample (when *lf* exceeds that limit), then
+    batch-transforms the remaining rows in 1K-row batches.
+
+    If ``n_components=3`` fails (e.g. too few rows or numerical issues),
+    falls back to ``n_components=2`` automatically.
+
+    Parameters
+    ----------
+    lf:
+        LazyFrame containing the numeric feature columns.
+    feature_cols:
+        Numeric column names to use for UMAP projection.  Columns not
+        present in *lf* are silently skipped.
+    n_components:
+        Target dimensionality (2 or 3).  Default 3 with fallback to 2.
+
+    Returns
+    -------
+    tuple[pl.LazyFrame, dict]
+        - LazyFrame: input data extended with ``_umap_x``, ``_umap_y``
+          (and ``_umap_z`` when ``n_components ≥ 3``).
+        - dict: metadata with keys ``n_components`` and ``reducer``.
+    """
+    logger = logging.getLogger(__name__)
+    MAX_TRAIN = 10000
+    BATCH_SIZE = 1000
+
+    available = [c for c in feature_cols if c in lf.collect_schema().names()]
+    if not available:
+        return lf, {'n_components': 0, 'reducer': 'umap', 'error': 'no feature columns available'}
+
+    df = lf.collect(streaming=True)
+    n_total = len(df)
+
+    if n_total < 2:
+        return lf, {'n_components': 0, 'reducer': 'umap', 'error': 'too few rows'}
+
+    mat = df.select(available).to_numpy().astype(np.float64)
+    mat = np.nan_to_num(mat, nan=0.0, posinf=0.0, neginf=0.0)
+
+    scaler = StandardScaler()
+    if n_total > MAX_TRAIN:
+        rng = np.random.RandomState(42)
+        idx_sample = rng.choice(n_total, size=MAX_TRAIN, replace=False)
+        mat_sample = mat[idx_sample]
+        scaler.fit(mat_sample)
+        mat_sample_scaled = np.nan_to_num(scaler.transform(mat_sample), nan=0.0)
+    else:
+        scaler.fit(mat)
+        mat_sample_scaled = np.nan_to_num(scaler.transform(mat), nan=0.0)
+
+    # ── Train UMAP; fallback 3→2 on failure ──
+    reducer = None
+    coords = None
+    effective_n = n_components
+
+    # Try requested n_components first, then fall back to 2 (no duplicates)
+    attempts = sorted(set((n_components, 2)), reverse=True)
+    for attempt_n in attempts:
+        if attempt_n != n_components:
+            logger.warning(f'UMAP {n_components}D failed, falling back to 2D')
+        effective_n = attempt_n
+        try:
+            reducer = umap.UMAP(
+                n_components=effective_n, random_state=42,
+                n_neighbors=15, min_dist=0.1, metric='euclidean',
+            )
+            if n_total > MAX_TRAIN:
+                reducer.fit(mat_sample_scaled)
+            else:
+                coords = reducer.fit_transform(mat_sample_scaled)
+            break
+        except Exception as e:
+            if attempt_n == 2:
+                raise
+            logger.warning(f'UMAP {attempt_n}D failed: {e}')
+
+    # ── Batch transform if dataset was larger than training sample ──
+    if n_total > MAX_TRAIN and reducer is not None:
+        coords_list: list[np.ndarray] = []
+        for start in range(0, n_total, BATCH_SIZE):
+            end = min(start + BATCH_SIZE, n_total)
+            mat_batch = scaler.transform(mat[start:end])
+            mat_batch = np.nan_to_num(mat_batch, nan=0.0, posinf=0.0, neginf=0.0)
+            coords_list.append(reducer.transform(mat_batch))
+        coords = np.vstack(coords_list)
+
+    if coords is None:
+        return lf, {'n_components': 0, 'reducer': 'umap', 'error': 'UMAP produced no output'}
+
+    # ── Build result LazyFrame with coordinate columns ──
+    coord_series = [
+        pl.Series('_umap_x', coords[:, 0]),
+        pl.Series('_umap_y', coords[:, 1]),
+    ]
+    if effective_n >= 3 and coords.shape[1] >= 3:
+        coord_series.append(pl.Series('_umap_z', coords[:, 2]))
+
+    df_result = df.with_columns(coord_series)
+
+    metadata: dict = {
+        'n_components': effective_n,
+        'reducer': 'umap',
+    }
+
+    return df_result.lazy(), metadata

analysis/management/commands/run_pipeline.py

@@ -67,8 +67,8 @@ class Command(BaseCommand):
             _self.stdout.write(f'  → 聚类特征: {feature_cols}')
             run.total_flows = row_count
             run.save(update_fields=['total_flows'])
-            from analysis.views import _run_clustering_pipeline
-            _run_clustering_pipeline(
+            from analysis.services.clustering import run_clustering_pipeline
+            run_clustering_pipeline(
                 run=run, store=store, entity_ds_id=ds_id,
                 feature_columns=feature_cols,
                 algorithm=algo, min_cluster_size=5,

analysis/nl_describe.py

@@ -285,6 +285,7 @@ def describe_feature(
     feature_name: str,
     value: float,
     std_dev: float | None = None,
+    cluster_std: float | None = None,
 ) -> str:
     """Generate a Chinese natural-language description for a single feature.
 
@@ -296,14 +297,15 @@ def describe_feature(
         The feature's mean value in the current context (cluster mean).
     std_dev : float | None
         Optional Z-score magnitude (``abs(distinguishing_score)``).  When
-        provided and >= 1.0, appends a deviation suffix like
-        "比全局均值高2.1个标准差".
+        provided and >= 1.0, appends a deviation suffix.
+    cluster_std : float | None
+        Optional within-cluster standard deviation. Describes how spread
+        the values are within this cluster (high = scattered, low = tight).
 
     Returns
     -------
     str
-        Natural-language description string, e.g.
-        ``"中等数据包 (720 bytes)，比全局均值高2.1个标准差"``.
+        Natural-language description string.
     """
     meaning = _lookup_meaning(feature_name)
     category = _resolve_category(feature_name)
@@ -322,7 +324,6 @@ def describe_feature(
                                        meaning=meaning)
                 break
         except Exception:
-            # Fall through to next template on format errors
             continue
 
     if not text:
@@ -332,6 +333,13 @@ def describe_feature(
     if std_dev is not None and std_dev >= 1.0:
         text += f"，比全局均值高{std_dev:.1f}个标准差"
 
+    # Append within-cluster variance context
+    if cluster_std is not None and cluster_std > 0:
+        if cluster_std > 2.0:
+            text += "（簇内离散度高）"
+        elif cluster_std < 0.3:
+            text += "（簇内高度集中）"
+
     return text
 
 
@@ -380,11 +388,13 @@ def describe_cluster(
         name = f.get("feature_name", "")
         mean = f.get("mean")
         score = abs(f.get("distinguishing_score", 0.0))
+        cstd = f.get("std")
 
         if mean is None or name == "":
             continue
 
-        desc = describe_feature(name, float(mean), std_dev=score if score >= 1.0 else None)
+        desc = describe_feature(name, float(mean), std_dev=score if score >= 1.0 else None,
+                                cluster_std=float(cstd) if cstd is not None else None)
         sentences.append(desc)
 
     if not sentences:

analysis/services/__init__.py

@@ -0,0 +1,2 @@
+"""Analysis services package — pure business logic, no request/response/rendering."""
+from .clustering import run_clustering_pipeline, compute_umap_embedding, save_entity_profiles

analysis/services/clustering.py

@@ -0,0 +1,361 @@
+"""Clustering service layer: pure business logic for the clustering pipeline.
+
+Extracted from analysis/views/clustering.py — no request/response, no rendering.
+"""
+import asyncio
+import traceback
+import logging
+import warnings
+
+import polars as pl
+import numpy as np
+
+from analysis.constants import RANDOM_SEED, UMAP_BATCH_SIZE, UMAP_TRAIN_SAMPLE, NUMERIC_DTYPES
+
+logger = logging.getLogger(__name__)
+
+
+def run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorithm,
+                             min_cluster_size, run_umap=True, head=None):
+    """Pure business logic — no request/response, no rendering.
+
+    Unified clustering pipeline: clustering → feature extraction → UMAP → ORM save.
+
+    Args:
+        run: AnalysisRun ORM object.
+        store: SessionStore instance.
+        entity_ds_id: dataset ID in SessionStore pointing to entity-aggregated data.
+        feature_columns: list of feature column names (None = auto-detect).
+        algorithm: 'hdbscan' or 'kmeans'.
+        min_cluster_size: int for HDBSCAN.
+        run_umap: bool, whether to compute and save UMAP-2D embeddings.
+        head: optional int, downsample to at most this many rows (min 100).
+    """
+    warnings.filterwarnings('ignore', category=RuntimeWarning, module='sklearn')
+
+    try:
+        from analysis.tool_registry import _handle_run_clustering, _handle_extract_features
+
+        entry = store.get_dataset(entity_ds_id)
+        if entry is None:
+            run.status = 'failed'
+            run.error_message = 'Entity dataset not found in session store'
+            run.save(update_fields=['status', 'error_message'])
+            return
+
+        schema = entry.get('schema', {})
+
+        # ── Downsampling ──
+        if head is not None and head > 0:
+            lf = entry['lazyframe']
+            try:
+                row_count = lf.select(pl.len()).collect(streaming=True).item()
+            except Exception:
+                logger.error("unknown failed: {}".format(traceback.format_exc()))
+                row_count = 0
+
+            if row_count > head:
+                head = max(head, 100)  # ensure minimum rows for clustering
+                run.progress_msg = f'正在采样 {head}/{row_count} 行...'
+                run.save(update_fields=['progress_msg'])
+                lf = lf.head(head)
+                store.store_dataset(
+                    entity_ds_id, lf,
+                    schema=schema,
+                    metadata=entry.get('metadata', {}),
+                )
+                entry = store.get_dataset(entity_ds_id)  # refresh
+                row_count = head
+
+        # Resolve feature columns — use actual LazyFrame dtypes (not stored schema dict
+        # which may be stale after Utf8 coercion in _background_process).
+        numeric_types = (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
+                         pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
+                         pl.Float32, pl.Float64)
+
+        lf = entry['lazyframe']
+        live_schema = lf.collect_schema()
+        live_names = live_schema.names()
+        live_dtypes = list(live_schema.dtypes())
+
+        if feature_columns:
+            feature_cols = [
+                c for c in feature_columns
+                if c in live_names and live_dtypes[live_names.index(c)] in numeric_types
+            ]
+        else:
+            feature_cols = [
+                live_names[i] for i, dt in enumerate(live_dtypes)
+                if dt in numeric_types and not live_names[i].startswith('_')
+            ][:10]
+
+        # ── Step 1: Clustering ────────────────────────────────────────────────
+        # If no numeric columns detected, pass empty list to let
+        # _handle_run_clustering apply its Utf8→Float64 coercion fallback.
+        run.status = 'clustering'
+        run.progress_pct = 70
+        run.progress_msg = '正在进行聚类分析...'
+        run.save(update_fields=['status', 'progress_pct', 'progress_msg'])
+
+        result = asyncio.run(_handle_run_clustering(
+            dataset_id=entity_ds_id,
+            cluster_columns=feature_cols,
+            algorithm=algorithm,
+            params={'min_cluster_size': min_cluster_size},
+            random_state=RANDOM_SEED,
+        ))
+
+        if 'error' in result:
+            err = result['error']
+            if 'numeric' in err.lower():
+                available = [
+                    f"{k}({v})" for k, v in schema.items()
+                    if v.split('(')[0].strip() in numeric_types and not k.startswith('_')
+                ]
+                err += f"。可用数值列: {available}"
+            raise Exception(err)
+
+        cluster_id = result.get('cluster_result_id', '')
+        run.cluster_count = result.get('n_clusters', 0)
+        run.save(update_fields=['cluster_count'])
+
+        # ── Step 2: Feature extraction ──────────────────────────────────────
+        run.status = 'extracting'
+        run.progress_pct = 90
+        run.progress_msg = '正在提取聚类特征...'
+        run.save(update_fields=['status', 'progress_pct', 'progress_msg'])
+
+        feat_result = asyncio.run(_handle_extract_features(
+            dataset_id=entity_ds_id,
+            cluster_result_id=cluster_id,
+            top_k=10, method='zscore', save_to_db=True,
+            run_id=run.id,
+        ))
+
+        if 'error' in feat_result:
+            raise Exception(feat_result['error'])
+
+        # ── Step 2b: Cluster SVD feature extraction ──────────────────────
+        run.progress_msg = '正在进行聚类SVD特征提取...'
+        run.save(update_fields=['progress_msg'])
+        try:
+            from analysis.distance import cluster_svd_extract
+            lf = entry['lazyframe']
+            cluster_entry = store.get_cluster_result(cluster_id)
+            labels_list = cluster_entry.get('labels', []) if cluster_entry else []
+
+            if labels_list and feature_cols:
+                svd_results = cluster_svd_extract(lf, labels_list, feature_cols)
+
+                # Persist SVD features to ClusterFeature (method='cluster_svd')
+                from analysis.models import ClusterFeature as CF
+                from analysis.models import ClusterResult as CR
+                for label, svd_info in svd_results.items():
+                    cr = CR.objects.filter(run=run, cluster_label=label).first()
+                    if cr is None:
+                        continue
+                    # Delete old zscore entries for this cluster, replace with SVD
+                    CF.objects.filter(cluster=cr, distinguishing_method='zscore').delete()
+                    for feat_name in svd_info.get('features', [])[:10]:
+                        idx = svd_info['features'].index(feat_name) if feat_name in svd_info['features'] else -1
+                        strength = (svd_info.get('feature_strength', [])[idx]
+                                    if idx >= 0 and idx < len(svd_info.get('feature_strength', []))
+                                    else None)
+                        CF.objects.get_or_create(
+                            cluster=cr,
+                            feature_name=feat_name,
+                            defaults={
+                                'mean': None, 'std': None, 'median': None,
+                                'p25': None, 'p75': None, 'missing_rate': None,
+                                'distinguishing_score': float(strength) if strength is not None else None,
+                                'distinguishing_method': 'cluster_svd',
+                            },
+                        )
+        except Exception as svd_err:
+            logger.warning(f'Cluster SVD extraction skipped: {svd_err}')
+
+        # ── Step 3: UMAP-3D embedding (2D fallback) ───────────────────────
+        if run_umap:
+            try:
+                lf = entry['lazyframe']
+                # UMAP: sample max 10K for training, batch-transform rest in 1K batches
+                MAX_UMAP_TRAIN = UMAP_TRAIN_SAMPLE
+                BATCH_SIZE = UMAP_BATCH_SIZE
+                try:
+                    n_total = lf.select(pl.len()).collect(streaming=True).item()
+                except Exception:
+                    logger.error("unknown failed: {}".format(traceback.format_exc()))
+                    n_total = 0
+                from sklearn.preprocessing import StandardScaler
+                import umap
+
+                live_schema = lf.collect_schema()
+                num_cols = [name for name, dt in zip(live_schema.names(), live_schema.dtypes())
+                            if dt in (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
+                                      pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
+                                      pl.Float32, pl.Float64) and not name.startswith('_')]
+
+                if len(num_cols) >= 2:
+                    if n_total > MAX_UMAP_TRAIN:
+                        run.progress_msg = f'UMAP 采样 {MAX_UMAP_TRAIN}/{n_total} 实体训练...'
+                        run.save(update_fields=['progress_msg'])
+                        df_sample = lf.sample(n=MAX_UMAP_TRAIN, seed=RANDOM_SEED).collect(streaming=True)
+                        df_umap = lf.collect(streaming=True)
+                    else:
+                        df_umap = lf.collect(streaming=True)
+
+                    coords, umap_components = compute_umap_embedding(
+                        df_umap, num_cols, n_total,
+                        max_train=MAX_UMAP_TRAIN,
+                        batch_size=BATCH_SIZE,
+                        seed=RANDOM_SEED,
+                    )
+
+                    cluster_entry = store.get_cluster_result(cluster_id)
+                    labels_list = cluster_entry.get('labels', []) if cluster_entry else []
+
+                    save_entity_profiles(run, df_umap, labels_list, coords, umap_components)
+            except Exception as umap_err:
+                logger.warning(f'UMAP embedding skipped: {umap_err}')
+
+        # Fallback: if entity_count not set (UMAP skipped), derive from cluster sizes
+        if run.entity_count is None:
+            from django.db.models import Sum
+            total = run.clusters.aggregate(total=Sum('size'))['total'] or 0
+            run.entity_count = total
+            run.save(update_fields=['entity_count'])
+
+        # ── Done ──
+        run.status = 'completed'
+        run.progress_pct = 100
+        run.progress_msg = '分析完成'
+        run.save(update_fields=['status', 'progress_pct', 'progress_msg', 'entity_count'])
+
+    except Exception:
+        tb = traceback.format_exc()
+        logger.error(tb)
+        try:
+            run.status = 'failed'
+            run.error_message = tb
+            run.progress_msg = f'失败: {tb}'
+            run.run_log += f'\n[ERROR] {tb}'
+            run.save(update_fields=['status', 'error_message', 'progress_msg', 'run_log'])
+        except Exception as e:
+            logger.warning('save failed after pipeline error: %s', e)
+
+
+def compute_umap_embedding(df, num_cols, n_total, max_train, batch_size, seed):
+    """UMAP 3D→2D fallback, returns (coords, umap_components).
+
+    Args:
+        df: Collected Polars DataFrame with entity-level data.
+        num_cols: list of numeric column names to use for UMAP.
+        n_total: total row count.
+        max_train: max rows for UMAP training sample.
+        batch_size: batch size for transform when dataset exceeds max_train.
+        seed: random seed.
+
+    Returns:
+        (coords, umap_components): numpy array of shape (n_total, umap_components),
+        and umap_components is 2 or 3.
+    """
+    from sklearn.preprocessing import StandardScaler
+    import umap
+
+    if len(df) <= 2 or len(num_cols) < 2:
+        return None, 0
+
+    if n_total > max_train:
+        df_sample = df.sample(n=max_train, seed=seed)
+    else:
+        df_sample = df
+
+    umap_components = 3  # try 3D first
+    coords = None
+
+    for attempt_n in (3, 2):
+        try:
+            if n_total > max_train:
+                # Train UMAP on sample, batch-transform full dataset
+                mat_sample = df_sample.select(num_cols).to_numpy()
+                scaler = StandardScaler().fit(mat_sample)
+                mat_sample_scaled = np.nan_to_num(
+                    scaler.transform(mat_sample), nan=0.0)
+                reducer = umap.UMAP(n_components=attempt_n,
+                                    random_state=seed,
+                                    n_neighbors=15, min_dist=0.1,
+                                    metric='euclidean')
+                reducer.fit(mat_sample_scaled)
+                # Batch transform in batch_size-row batches
+                coords_list = []
+                for start in range(0, len(df), batch_size):
+                    end = min(start + batch_size, len(df))
+                    mat_batch = scaler.transform(
+                        df[start:end].select(num_cols).to_numpy())
+                    mat_batch = np.nan_to_num(mat_batch, nan=0.0)
+                    coords_list.append(reducer.transform(mat_batch))
+                coords = np.vstack(coords_list)
+            else:
+                mat = np.nan_to_num(StandardScaler().fit_transform(
+                    df.select(num_cols).to_numpy()), nan=0.0)
+                reducer = umap.UMAP(n_components=attempt_n,
+                                    random_state=seed,
+                                    n_neighbors=15, min_dist=0.1,
+                                    metric='euclidean')
+                coords = reducer.fit_transform(mat)
+            umap_components = attempt_n
+            break
+        except Exception as e_umap:
+            if attempt_n == 2:
+                raise
+            logger.warning(f'UMAP 3D failed ({e_umap}), falling back to 2D')
+
+    if coords is None:
+        raise Exception('UMAP produced no output')
+
+    return coords, umap_components
+
+
+def save_entity_profiles(run, df_umap, labels_list, coords, umap_components):
+    """Persist per-row cluster labels + UMAP coords to EntityProfile.
+
+    Args:
+        run: AnalysisRun ORM object.
+        df_umap: Collected Polars DataFrame (used for entity column detection + row iteration).
+        labels_list: list of cluster labels per row.
+        coords: numpy array of UMAP coordinates (must not be None).
+        umap_components: 2 or 3.
+    """
+    from analysis.models import EntityProfile as EP
+    from analysis.models import ClusterResult as CR
+
+    # Determine entity column (first non-numeric, non-underscore column)
+    df_num_cols = [c for c, dt in zip(df_umap.columns, df_umap.dtypes)
+                   if dt in NUMERIC_DTYPES]
+    non_num = [c for c in df_umap.columns
+               if c not in df_num_cols and not c.startswith('_')]
+    ent_col = non_num[0] if non_num else df_umap.columns[0]
+
+    profiles = []
+    cr_cache = {}
+    for i, row in enumerate(df_umap.iter_rows(named=True)):
+        ev = str(row.get(ent_col, '')) or 'entity'
+        ev = f'{ev}_{i}'
+        lbl = labels_list[i] if i < len(labels_list) else -1
+        cache_key = f'{run.id}_{lbl}'
+        if cache_key not in cr_cache:
+            cr_cache[cache_key] = CR.objects.filter(
+                run=run, cluster_label=lbl).first()
+        has_z = umap_components >= 3 and coords.shape[1] >= 3
+        profiles.append(EP(
+            entity_value=ev, run=run, cluster_label=lbl,
+            cluster=cr_cache[cache_key],
+            embedding_x=float(coords[i, 0]) if i < len(coords) else None,
+            embedding_y=float(coords[i, 1]) if i < len(coords) else None,
+            embedding_z=float(coords[i, 2]) if has_z
+                and i < len(coords) else None,
+        ))
+    if profiles:
+        EP.objects.bulk_create(profiles, ignore_conflicts=True, batch_size=1000)
+        run.entity_count = len(profiles)
+        run.save(update_fields=['entity_count'])

analysis/tools/__init__.py

@@ -60,3 +60,44 @@ from .data_mgmt import (
     _handle_clone_dataset,
 )
 from .distance_matrix import _handle_compute_distance_matrix
+
+__all__ = [
+    # Helpers
+    '_json_size', '_truncate_response', '_build_filter_expr', '_resolve_dataset',
+    '_count_rows', '_get_numeric_columns', 'NUMERIC_DTYPES', 'NUMERIC_TYPE_NAMES',
+    # Registry + dispatch
+    'get_tools_meta', 'handle_call',
+    # Load
+    '_handle_load_data',
+    # Profile
+    '_handle_profile_data',
+    # Filter
+    '_handle_filter_data',
+    # Preprocess
+    '_handle_preprocess_data',
+    # Clustering
+    '_handle_run_clustering', '_handle_filter_and_cluster',
+    # Evaluate
+    '_handle_evaluate_clustering',
+    # Features
+    '_handle_extract_features', '_save_features_to_db',
+    # Export
+    '_handle_export_results',
+    # Entities
+    '_handle_build_entity_profiles', '_handle_compute_scores', '_add_adaptive_scores',
+    '_ENTITY_KEYWORDS', '_NUMERIC_TYPES_FOR_AGG', '_ANOMALY_FEATURE_COLS',
+    # Anomalies
+    '_handle_detect_anomalies', '_handle_visualize_anomalies',
+    # Diagnostics
+    '_handle_validate_data', '_handle_explore_distributions', '_handle_find_outliers',
+    '_handle_diagnose_clustering', '_handle_compare_datasets',
+    '_handle_export_debug_sample', '_handle_repair_schema',
+    # Analysis
+    '_handle_analyze_patterns', '_handle_analyze_temporal', '_handle_analyze_fft',
+    '_handle_analyze_tls_health', '_handle_analyze_geo_distribution',
+    '_handle_analyze_entity_detail',
+    # Data management
+    '_handle_list_datasets', '_handle_drop_dataset', '_handle_clone_dataset',
+    # Distance matrix
+    '_handle_compute_distance_matrix',
+]

analysis/tools/_dispatch.py

@@ -1,4 +1,5 @@
 """Tool call dispatcher — maps tool names to handler functions."""
+import asyncio
 from ._helpers import _truncate_response
 from .load_data import _handle_load_data
 from .profile import _handle_profile_data
@@ -27,9 +28,12 @@ from .distance_matrix import _handle_compute_distance_matrix
 async def handle_call(name: str, arguments: dict) -> dict:
     """Dispatch a tool call to the appropriate handler.
 
+    Extracts the mandatory ``_timeout`` parameter (set by the LLM) from
+    *arguments* and enforces it as an ``asyncio.wait_for`` deadline.
+
     Args:
         name: Tool name (must match one of the 30 tools).
-        arguments: Tool-specific parameters.
+        arguments: Tool-specific parameters, must include ``_timeout``.
 
     Returns:
         A JSON-serialisable dict.
@@ -72,7 +76,23 @@ async def handle_call(name: str, arguments: dict) -> dict:
     handler = _handlers.get(name)
     if handler is None:
         raise ValueError(f"Unknown tool: '{name}'")
-    result = await handler(**arguments)
+
+    # Extract mandatory _timeout from the LLM-provided arguments
+    timeout = arguments.pop('_timeout', 0)
+    cleaned_args = arguments  # _timeout already removed by pop
+
+    # Execute the handler with optional timeout
+    if timeout and timeout > 0:
+        try:
+            result = await asyncio.wait_for(
+                handler(**cleaned_args),
+                timeout=timeout,
+            )
+        except asyncio.TimeoutError:
+            result = {'error': f'工具 {name} 执行超时 ({timeout}秒)'}
+    else:
+        result = await handler(**cleaned_args)
+
     # Always ensure truncated key exists
     if 'truncated' not in result:
         result['truncated'] = False

analysis/tools/_registry.py

@@ -8,9 +8,10 @@ from mcp.types import Tool
 def get_tools_meta() -> list[Tool]:
     """Return the list of Tool metadata objects for MCP server registration.
 
-    Call this from :meth:`TLSAnalyzerMCPServer._setup_tools`.
+    Each tool gets a mandatory ``_timeout`` parameter so the LLM must specify
+    how many seconds to allow for each tool invocation (0 = no limit).
     """
-    return [
+    raw_tools = [
         Tool(
             name="load_data",
             description=(
@@ -751,4 +752,26 @@ def get_tools_meta() -> list[Tool]:
                 "required": ["dataset_id", "python_function"],
             },
         ),
-    ]
+    ]  # end of raw_tools
+
+    # Inject mandatory _timeout parameter into every tool
+    result = []
+    for tool in raw_tools:
+        schema = dict(tool.inputSchema)
+        props = dict(schema.get("properties", {}))
+        req = list(schema.get("required", []))
+        # Add _timeout as the first required property
+        props["_timeout"] = {
+            "type": "integer",
+            "description": "工具调用超时秒数（必填，LLM决定每次调用的最长等待时间；0表示不限制）",
+            "default": 120,
+        }
+        req.insert(0, "_timeout")
+        schema["properties"] = props
+        schema["required"] = req
+        result.append(Tool(
+            name=tool.name,
+            description=tool.description,
+            inputSchema=schema,
+        ))
+    return result

analysis/tools/anomalies.py

@@ -1,4 +1,5 @@
 """Anomaly detection + visualization handlers."""
+import traceback
 import numpy as np
 from analysis.constants import MAX_DISTRIBUTION_SAMPLE, RANDOM_SEED
 import polars as pl
@@ -12,7 +13,6 @@ logger = logging.getLogger(__name__)
 async def _handle_detect_anomalies(dataset_id: str, contamination: float = 0.05) -> dict:
     """Run Isolation Forest on scored entity data."""
     from sklearn.ensemble import IsolationForest
-import traceback
     entry, err = _resolve_dataset(dataset_id)
     if err:
         return err

analysis/tools/clustering.py

@@ -6,6 +6,7 @@ Contains:
 - _cluster_core: shared clustering pipeline extracted from both handlers
 """
 import logging
+import traceback
 from analysis.constants import LOW_MEMORY_THRESHOLD_GB, LOW_MEMORY_THRESHOLD_ROWS, MAX_CLUSTER_FEATURES, MAX_SAMPLE_ROWS, RANDOM_SEED
 from typing import Optional
 
@@ -14,13 +15,13 @@ import polars as pl
 
 from ..session_store import SessionStore, _generate_id
 from ._helpers import (
-logger = logging.getLogger(__name__)
     NUMERIC_DTYPES,
     _build_filter_expr,
     _resolve_dataset,
     _count_rows,
 )
 
+logger = logging.getLogger(__name__)
 _clust_logger = logging.getLogger(__name__)
 
 
@@ -172,7 +173,6 @@ def _cluster_core(
             quality['davies_bouldin_score'] = None
         try:
             from sklearn.metrics import calinski_harabasz_score
-import traceback
             chs = calinski_harabasz_score(data_scaled, labels_array)
             quality['calinski_harabasz_score'] = round(float(chs), 4)
         except Exception:

analysis/tools/entities.py

@@ -116,7 +116,45 @@ async def _handle_build_entity_profiles(
                 agg_exprs.append(pl.col(c).n_unique().alias(alias_name))
 
     # ── Execute grouping ───────────────────────────────────────────────
+    row_count = _count_rows(lf)
+    ENTITY_CHUNK_THRESHOLD = 1_000_000   # 1M rows
+    ENTITY_CHUNK_SIZE = 100_000          # 100K rows per chunk
+
+    if row_count >= ENTITY_CHUNK_THRESHOLD:
+        # Chunked path: process in 100K batches → group → merge across chunks
+        all_chunk_dfs = []
+        for offset in range(0, row_count, ENTITY_CHUNK_SIZE):
+            chunk_df = lf.slice(offset, ENTITY_CHUNK_SIZE).collect(streaming=True)
+            chunk_grouped = chunk_df.group_by(group_cols).agg(agg_exprs)
+            all_chunk_dfs.append(chunk_grouped)
+        merged_df = pl.concat(all_chunk_dfs)
+
+        # Re-aggregate across chunks (same entity may appear in multiple chunks)
+        re_agg_exprs: list[pl.Expr] = []
+        for col_name in merged_df.columns:
+            if col_name in group_cols:
+                continue
+            if col_name == 'flow_count':
+                re_agg_exprs.append(pl.sum('flow_count').alias('flow_count'))
+            elif col_name.startswith('total_'):
+                re_agg_exprs.append(pl.sum(col_name).alias(col_name))
+            elif col_name.startswith('avg_'):
+                # Weighted average: sum(total) / sum(flow_count)
+                base = col_name[4:]
+                total_col = f'total_{base}'
+                if total_col in merged_df.columns:
+                    re_agg_exprs.append(
+                        (pl.sum(total_col) / pl.sum('flow_count')).alias(col_name))
+            elif col_name.startswith('unique_'):
+                # Approximate: max of per-chunk distinct counts
+                re_agg_exprs.append(pl.max(col_name).alias(col_name))
+
+        entity_lf = merged_df.lazy().group_by(group_cols).agg(re_agg_exprs)
+        logger.info('[ENTITY] chunked aggregation: %d rows → %d chunks → %d merged rows',
+                    row_count, len(all_chunk_dfs), len(merged_df))
+    else:
         entity_lf = lf.group_by(group_cols).agg(agg_exprs)
+
     n_entities = _count_rows(entity_lf)
 
     # ── Store result ───────────────────────────────────────────────────

analysis/type_classifier.py

@@ -1,415 +1,3 @@
-"""Data type classifier for column-level type inference.
-
-Provides :class:`DataType` enumeration and functions to classify Polars
-columns based on name heuristics and value sampling.
-
-Typical usage::
-
-    from analysis.type_classifier import classify_column, classify_schema, DataType
-
-    # Single column
-    dtype = classify_column('src_ip', series, config_type='ipv4')
-
-    # Full schema
-    type_map = classify_schema(lazyframe, config_overrides={'src_ip': 'ipv4'})
-"""
-from __future__ import annotations
-
-import re
-from enum import Enum
-
-import polars as pl
-
-from analysis.tls_ref import get_tls_ref_dict
-
-# ---------------------------------------------------------------------------
-#  TLS version hex-to-label mapping
-#  (Duplicated in views.py for globe visualization — keep both in sync.)
-# ---------------------------------------------------------------------------
-
-TLS_HEX_MAP: dict[str, str] = {
-    '0303': 'TLSv1.2',
-    '0304': 'TLSv1.3',
-    '0302': 'TLSv1.1',
-    '0301': 'TLSv1.0',
-}
-
-# ---------------------------------------------------------------------------
-#  DataType enumeration
-# ---------------------------------------------------------------------------
-
-
-class DataType(Enum):
-    """Enumerated column types recognised by the classifier engine."""
-
-    INT = 1
-    FLOAT = 2
-    ENUM = 3
-    HEX = 4
-    URL = 5
-    IPv4 = 6
-    LAT_LON = 7
-    TIMESTAMP = 8
-    BOOL_ENUM = 9
-    STRING = 10
-
-
-# ---------------------------------------------------------------------------
-#  Detection patterns (raw strings for regex)
-# ---------------------------------------------------------------------------
-
-HEX_PATTERN = r'^[0-9a-f]{2}(?:\s[0-9a-f]{2})*$'
-"""Matches ``"0e a6 3f"``, ``"ab cd ef"`` etc. (space-separated hex pairs)."""
-
-URL_PATTERN = r'^https?://'
-"""Matches URLs starting with ``http://`` or ``https://``."""
-
-IPv4_PATTERN = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$'
-"""Matches ``"192.168.1.1"`` style IPv4 addresses."""
-
-MAC_PATTERN = r'^([0-9A-Fa-f]{2}[:.-]){5}[0-9A-Fa-f]{2}$'
-"""Matches MAC addresses with ``:``, ``-``, or ``.`` separators."""
-
-BOOL_ENUM_VALUES = (
-    '', ' ', '+', '-', '0', '1',
-    'true', 'false', 'yes', 'no', 't', 'f',
-)
-"""String values considered boolean / binary indicators."""
-
-LAT_LON_NAMES = ('lat', 'latitude', 'lon', 'lng', 'longitude')
-
-# Name-to-type heuristics (checked after config override, before values)
-_NAME_TYPE_MAP: list[tuple[re.Pattern, DataType]] = [
-    # User column names - EXACT match only (no generic fuzzy patterns)
-    # IPv4 columns
-    (re.compile(r'^:ips$|^:ipd$|^server.ip$|^client.ip$', re.I), DataType.IPv4),
-    # ENUM columns
-    (re.compile(r'^:prs$|^:prd$', re.I), DataType.ENUM),
-    (re.compile(r'^(?:cnrs|isrs)$', re.I), DataType.BOOL_ENUM),
-    (re.compile(r'^(?:scnt|dcnt|1ipp|4dbn|4srs|eiph)$', re.I), DataType.ENUM),
-    (re.compile(r'^0ver$', re.I), DataType.ENUM),
-    # FLOAT columns
-    (re.compile(r'^4dur$|^8ses$|^2tmo$|^0rnt$', re.I), DataType.FLOAT),
-    # INT columns
-    (re.compile(r'^4ksz$|^8ack$|^8ppk$|^8pak$|^8seq$|^8did$|^8byt$|^row$', re.I), DataType.INT),
-    # HEX columns
-    (re.compile(r'^0cph$|^0crv$', re.I), DataType.HEX),
-    # TIMESTAMP columns
-    (re.compile(r'^8dbd$|^time$|^timestamp$', re.I), DataType.TIMESTAMP),
-    # Dot-notation suffixes for lat/lon (e.g. :ips.latd, :ipd.lond)
-    (re.compile(r'\.latd$', re.I), DataType.LAT_LON),
-    (re.compile(r'\.lond$', re.I), DataType.LAT_LON),
-    # Dot-notation STRING suffixes (e.g. :ips.ispn, :ipd.city)
-    (re.compile(r'\.(ispn|orgn|city|anon)$', re.I), DataType.STRING),
-    # Standalone string columns - exact match
-    (re.compile(r'^(?:cnam|snam|tabl|name|0rnd|source.node|ecdhe.named.curve|crcc|orga|orgu)$', re.I), DataType.STRING),
-    # Special columns with @ prefix
-    (re.compile(r'^@', re.I), DataType.STRING),
-]
-
-# Mapping from config.yaml type strings to DataType
-_CONFIG_TYPE_MAP: dict[str, DataType] = {
-    'int': DataType.INT,
-    'float': DataType.FLOAT,
-    'enum': DataType.ENUM,
-    'hex': DataType.HEX,
-    'url': DataType.URL,
-    'ipv4': DataType.IPv4,
-    'lat_lon': DataType.LAT_LON,
-    'timestamp': DataType.TIMESTAMP,
-    'bool_enum': DataType.BOOL_ENUM,
-    'string': DataType.STRING,
-}
-
-# Mapping from tls_ref_data ``data_type`` column values to DataType
-TLSDB_TYPE_MAP: dict[str, DataType] = {
-    'IPv4': DataType.IPv4,
-    '浮点': DataType.FLOAT,
-    '字符串': DataType.STRING,
-    '整数': DataType.INT,
-    '2小写字母缩写': DataType.ENUM,
-    'XX:YY.Z': DataType.STRING,
-    '2字节': DataType.HEX,
-    '2大写字母': DataType.STRING,
-    '枚举': DataType.ENUM,
-    '布尔': DataType.BOOL_ENUM,
-    '字符串或整数': DataType.STRING,
-}
-
-
-def _tlsdb_type_to_datatype(tlsdb_type_str: str) -> DataType | None:
-    """Convert a TlsDB ``data_type`` string to a :class:`DataType`.
-
-    Returns ``None`` if the type string is unknown, allowing the
-    caller to fall through to value-based detection.
-    """
-    return TLSDB_TYPE_MAP.get(tlsdb_type_str)
-
-
-# ---------------------------------------------------------------------------
-#  Column classification (single column)
-# ---------------------------------------------------------------------------
-
-
-def _name_to_type(name: str) -> DataType | None:
-    """Match column name against known heuristics.
-
-    Returns a :class:`DataType` if the name matches strongly, otherwise
-    ``None`` (fall through to value-based detection).
-    """
-    for pattern, dtype in _NAME_TYPE_MAP:
-        if pattern.search(name):
-            return dtype
-    return None
-
-
-def _is_port_value(v: str) -> bool:
-    """Check if *v* is an integer string in the registered port range (1-65535)."""
-    try:
-        n = int(v)
-        return 1 <= n <= 65535
-    except ValueError:
-        return False
-
-
-def _can_float(v: str) -> bool:
-    """Return ``True`` when *v* can be parsed as a Python float."""
-    try:
-        float(v)
-        return True
-    except ValueError:
-        return False
-
-
-def _values_to_type(series: pl.Series, col_name: str | None = None) -> DataType:
-    """Infer column type by sampling non-null values.
-
-    Detection order: IPv4 → MAC guard → HEX → URL → PORT → BOOL_ENUM
-                     → LAT_LON → ENUM → FLOAT → STRING
-    """
-    dtype = series.dtype
-
-    # Boolean dtype ──────────────────────────────────────────────────────
-    if dtype == pl.Boolean:
-        return DataType.BOOL_ENUM
-
-    # Numeric dtypes ─────────────────────────────────────────────────────
-    if dtype in (pl.Float32, pl.Float64):
-        # Check LAT_LON possibility before returning FLOAT
-        non_null = series.drop_nulls()
-        if len(non_null) > 0:
-            sample = non_null.head(100)
-            sample_strs = [str(v).strip().lower() for v in sample]
-            non_blank = [v for v in sample_strs if v not in ('', ' ', '+', '-')]
-            if non_blank:
-                try:
-                    floats = [float(v) for v in non_blank]
-                    max_abs = max(abs(f) for f in floats)
-                    if (
-                        all(-180 <= f <= 180 for f in floats)
-                        and any('.' in v for v in non_blank)
-                        and max_abs > 10
-                    ):
-                        return DataType.LAT_LON
-                except ValueError:
-                    pass
-        return DataType.FLOAT
-
-    if dtype in (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
-                 pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64):
-        return DataType.INT
-
-    # String / categorical types ─────────────────────────────────────────
-    non_null = series.drop_nulls()
-    if len(non_null) == 0:
-        return DataType.STRING
-
-    sample = non_null.head(100)
-    # Convert all to string representations for pattern matching
-    sample_strs = [str(v).strip().lower() for v in sample]
-
-    # ── 1. IPv4 detection ──────────────────────────────────────────────
-    if all(re.match(IPv4_PATTERN, v) for v in sample_strs):
-        if all(_valid_ip_octets(v) for v in sample_strs):
-            return DataType.IPv4
-
-    # ── 2. MAC guard (skip HEX if values look like MAC addresses) ──────
-    is_mac = all(re.match(MAC_PATTERN, v) for v in sample_strs)
-
-    # ── 3. HEX detection ───────────────────────────────────────────────
-    if not is_mac and all(re.match(HEX_PATTERN, v) for v in sample_strs):
-        return DataType.HEX
-
-    # ── 4. URL detection ───────────────────────────────────────────────
-    if all(re.match(URL_PATTERN, v) for v in sample_strs):
-        return DataType.URL
-
-    # ── 5. PORT detection → ENUM ───────────────────────────────────────
-    if all(_is_port_value(v) for v in sample_strs):
-        return DataType.ENUM
-
-    # ── 6. BOOL_ENUM detection ─────────────────────────────────────────
-    if all(v in BOOL_ENUM_VALUES for v in sample_strs):
-        return DataType.BOOL_ENUM
-
-    # ── 7. LAT_LON detection (value-based) ──────────────────────────────
-    non_blank = [v for v in sample_strs if v not in ('', ' ', '+', '-')]
-    if non_blank:
-        try:
-            floats = [float(v) for v in non_blank]
-            max_abs = max(abs(f) for f in floats)
-            if (
-                all(-180 <= f <= 180 for f in floats)
-                and any('.' in v for v in non_blank)
-                and max_abs > 10
-            ):
-                return DataType.LAT_LON
-        except ValueError:
-            pass
-
-    # ── 8. ENUM detection (few unique values) ──────────────────────────
-    unique_count = non_null.n_unique()
-    if unique_count < 20:
-        # Safety: if >80 % of the sample values are unique strings,
-        # the column is high-cardinality — classify as STRING, not ENUM.
-        sample_unique_count = len(set(sample_strs))
-        if sample_unique_count > 0.8 * len(sample_strs):
-            return DataType.STRING
-        return DataType.ENUM
-
-    # ── 9. TLS version hex detection (before STRING fallback) ──────────
-    if col_name is not None and col_name.lower() in ('0ver', '_tlsver', 'tls_version', 'version'):
-        sample_hex = sample_strs[0].replace(' ', '').strip()
-        if sample_hex.upper() in TLS_HEX_MAP:
-            return DataType.ENUM
-
-    # ── 10. FLOAT detection (>80 % can float) ──────────────────────────
-    # Exclude strings starting with '+' from FLOAT detection — these are
-    # text markers (phone codes, IDs, error codes), not real numbers.
-    if any(v.startswith('+') for v in sample_strs):
-        return DataType.STRING
-
-    float_count = sum(1 for v in sample_strs if _can_float(v))
-    if float_count / len(sample_strs) > 0.8:
-        # Guard: values with leading "-" that happen to parse as float
-        # should not force FLOAT when non-float values also exist.
-        if any(v.startswith('-') for v in sample_strs):
-            if float_count < len(sample_strs):
-                return DataType.STRING
-        return DataType.FLOAT
-
-    return DataType.STRING
-
-
-def _valid_ip_octets(ip_str: str) -> bool:
-    """Check that each octet in an IPv4 string is 0-255."""
-    parts = ip_str.split('.')
-    if len(parts) != 4:
-        return False
-    for p in parts:
-        try:
-            n = int(p)
-            if n < 0 or n > 255:
-                return False
-        except ValueError:
-            return False
-    return True
-
-
-def classify_column(
-    name: str,
-    series: pl.Series,
-    config_type: str | None = None,
-) -> DataType:
-    """Classify a single column based on its *name* and *series* values.
-
-    Priority
-    --------
-    1. **config_type** — user override from ``config.yaml``.
-       If provided (e.g. ``'ipv4'``), returns the corresponding
-       :class:`DataType` immediately.
-    2. **tls_ref_data** — authoritative type from ``TlsDB.csv`` /
-       ``tls_ref_data`` table.  Looked up by column name; if found,
-       the declared type is used directly (no value-based guessing).
-    3. **Value-based** pattern matching — samples up to 100 non-null
-       values and tests against IPv4 → HEX → URL → PORT → BOOL_ENUM
-       → LAT_LON → ENUM → FLOAT in order.
-    4. **Name-based** heuristics — only consulted when value-based
-       detection returned ``STRING`` (i.e. no value pattern matched).
-
-    Parameters
-    ----------
-    name:
-        Column name (used for name-based heuristics and tls_ref_data
-        lookup).
-    series:
-        Polars :class:`Series` containing the column's data.
-    config_type:
-        Optional user override (case-insensitive, matched to
-        :data:`_CONFIG_TYPE_MAP`).
-
-    Returns
-    -------
-    DataType
-    """
-    # 1. Config override (highest priority)
-    if config_type is not None:
-        mapped = _CONFIG_TYPE_MAP.get(config_type.lower().strip())
-        if mapped is not None:
-            return mapped
-
-    # 2. tls_ref_data lookup (authoritative type from TlsDB)
-    tls_ref = get_tls_ref_dict()
-    if name in tls_ref:
-        dtype = _tlsdb_type_to_datatype(tls_ref[name]['data_type'])
-        if dtype is not None:
-            return dtype
-
-    # 3. Value-based sampling
-    result = _values_to_type(series, name)
-
-    # 4. Name-based heuristics (only when values returned STRING)
-    if result == DataType.STRING:
-        name_match = _name_to_type(name)
-        if name_match is not None:
-            return name_match
-
-    return result
-
-
-# ---------------------------------------------------------------------------
-#  Schema classification (all columns)
-# ---------------------------------------------------------------------------
-
-
-def classify_schema(
-    lf: pl.LazyFrame,
-    config_overrides: dict[str, str] | None = None,
-) -> dict[str, DataType]:
-    """Classify all columns in a :class:`LazyFrame`.
-
-    Parameters
-    ----------
-    lf:
-        A Polars :class:`LazyFrame` whose columns to classify.
-    config_overrides:
-        Optional mapping of ``{col_name: type_string}`` from a
-        ``config.yaml`` ``columns:`` section.  These take highest
-        priority (see :func:`classify_column`).
-
-    Returns
-    -------
-    dict[str, DataType]
-        Mapping from column name to inferred :class:`DataType`.
-    """
-    config_overrides = config_overrides or {}
-    df_sample = lf.collect(streaming=True)
-    result: dict[str, DataType] = {}
-    for col_name in df_sample.columns:
-        config_type = config_overrides.get(col_name)
-        result[col_name] = classify_column(
-            name=col_name,
-            series=df_sample[col_name],
-            config_type=config_type,
-        )
-    return result
+"""Re-export from new types module for backward compatibility."""
+from analysis.types import ColumnType, TYPE_REGISTRY, TYPE_ALIASES, get_type, classify_column, classify_schema
+TLS_HEX_MAP = {"0303": "TLSv1.2", "0304": "TLSv1.3", "0302": "TLSv1.1", "0301": "TLSv1.0"}

analysis/types.py

@@ -0,0 +1,300 @@
+"""天璇类型系统 — 仅8种固定类型，面向对象接口。
+
+每个类型继承 :class:`ColumnType`，实现:
+- ``distance()`` — 为该类型列计算距离矩阵（追加到 LazyFrame）
+- ``display()`` — 将值格式化为可读字符串
+- ``polars_dtype`` — 映射到 Polars 数据类型
+- ``name`` — 中文名称
+
+:data:`TYPE_REGISTRY` 是唯一的类型注册表，按名称索引。
+"""
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+import numpy as np
+import polars as pl
+
+
+# ═══════════════════════════════════════════════════════════════════════
+#  Base type — all 8 types inherit from this
+# ═══════════════════════════════════════════════════════════════════════
+
+class ColumnType(ABC):
+    """Base class for all column types.
+
+    Subclasses MUST define:
+    - ``name`` (class attribute, Chinese)
+    - ``polars_dtype`` (class attribute)
+    - ``distance()``
+    - ``display()``
+    """
+
+    name: str = ""
+    polars_dtype: pl.DataType = pl.Utf8
+
+    @abstractmethod
+    def distance(self, lf: pl.LazyFrame, columns: list[str]) -> pl.LazyFrame:
+        """Compute per-row distances for *columns* of this type.
+
+        Returns a new LazyFrame with distance columns appended
+        (prefixed ``_dist_``).
+        """
+        ...
+
+    @abstractmethod
+    def display(self, value: Any) -> str:
+        """Format *value* for human-readable display."""
+        ...
+
+
+# ═══════════════════════════════════════════════════════════════════════
+#  8 concrete types
+# ═══════════════════════════════════════════════════════════════════════
+
+class IPv4Type(ColumnType):
+    name = "IPv4"
+    polars_dtype = pl.Utf8
+
+    def distance(self, lf: pl.LazyFrame, columns: list[str]) -> pl.LazyFrame:
+        if len(columns) < 2:
+            return lf
+        from analysis.distance import ip_distance
+        return ip_distance(lf, columns)
+
+    def display(self, value: Any) -> str:
+        return str(value) if value else "-"
+
+
+class StringType(ColumnType):
+    name = "字符串"
+    polars_dtype = pl.Utf8
+
+    def distance(self, lf: pl.LazyFrame, columns: list[str]) -> pl.LazyFrame:
+        from analysis.distance import string_distance
+        return string_distance(lf, columns)
+
+    def display(self, value: Any) -> str:
+        return str(value)[:80] if value else "-"
+
+
+class EnumType(ColumnType):
+    name = "枚举"
+    polars_dtype = pl.Utf8
+
+    def distance(self, lf: pl.LazyFrame, columns: list[str]) -> pl.LazyFrame:
+        # 0-1 match distance: compare to mode
+        for col in columns:
+            mode_val = lf.select(pl.col(col).mode().first()).collect(streaming=True).item()
+            lf = lf.with_columns(
+                (pl.col(col) != mode_val).cast(pl.Float64).alias(f'_enum_dist_{col}')
+            )
+        return lf
+
+    def display(self, value: Any) -> str:
+        return str(value) if value else "-"
+
+
+class FloatType(ColumnType):
+    name = "浮点"
+    polars_dtype = pl.Float64
+
+    def distance(self, lf: pl.LazyFrame, columns: list[str]) -> pl.LazyFrame:
+        # Normalised euclidean distance — handled in preprocessing pipeline
+        for col in columns:
+            lf = lf.with_columns(
+                pl.col(col).cast(pl.Float64).alias(f'_raw_{col}')
+            )
+        return lf
+
+    def display(self, value: Any) -> str:
+        try:
+            return f"{float(value):.2f}"
+        except (ValueError, TypeError):
+            return str(value)
+
+
+class IntType(ColumnType):
+    name = "整数"
+    polars_dtype = pl.Int64
+
+    def distance(self, lf: pl.LazyFrame, columns: list[str]) -> pl.LazyFrame:
+        # Normalised manhattan distance — handled in preprocessing pipeline
+        for col in columns:
+            lf = lf.with_columns(
+                pl.col(col).cast(pl.Int64).alias(f'_raw_{col}')
+            )
+        return lf
+
+    def display(self, value: Any) -> str:
+        try:
+            return str(int(value))
+        except (ValueError, TypeError):
+            return str(value)
+
+
+class BoolType(ColumnType):
+    name = "布尔"
+    polars_dtype = pl.Boolean
+
+    def distance(self, lf: pl.LazyFrame, columns: list[str]) -> pl.LazyFrame:
+        # 0-1 match distance
+        for col in columns:
+            lf = lf.with_columns(
+                pl.col(col).cast(pl.Float64).alias(f'_bool_dist_{col}')
+            )
+        return lf
+
+    def display(self, value: Any) -> str:
+        if value is None:
+            return "-"
+        return "是" if str(value).lower() in ('true', '1', '+', 't') else "否"
+
+
+class BytesType(ColumnType):
+    name = "字节"
+    polars_dtype = pl.Utf8
+
+    def distance(self, lf: pl.LazyFrame, columns: list[str]) -> pl.LazyFrame:
+        from analysis.distance import bytes_distance
+        return bytes_distance(lf, columns)
+
+    def display(self, value: Any) -> str:
+        s = str(value).strip() if value else ""
+        if len(s) > 20:
+            return s[:20] + "..."
+        return s or "-"
+
+
+class TimestampType(ColumnType):
+    name = "时间戳"
+    polars_dtype = pl.Float64
+
+    def distance(self, lf: pl.LazyFrame, columns: list[str]) -> pl.LazyFrame:
+        from analysis.distance import timestamp_fft_distance
+        return timestamp_fft_distance(lf, columns)
+
+    def display(self, value: Any) -> str:
+        try:
+            t = float(value)
+            if t < 1e8 or t > 2e12:
+                return str(value)
+            from datetime import datetime
+            return datetime.fromtimestamp(t).strftime("%Y-%m-%d %H:%M")
+        except (ValueError, TypeError, OSError):
+            return str(value) if value else "-"
+
+
+# ═══════════════════════════════════════════════════════════════════════
+#  Registry — single source of truth
+# ═══════════════════════════════════════════════════════════════════════
+
+TYPE_REGISTRY: dict[str, ColumnType] = {
+    "枚举": EnumType(),
+    "字节": BytesType(),
+    "浮点": FloatType(),
+    "整数": IntType(),
+    "布尔": BoolType(),
+    "字符串": StringType(),
+    "IPv4": IPv4Type(),
+    "时间戳": TimestampType(),
+}
+
+# Alias map for config.yaml type strings
+TYPE_ALIASES: dict[str, str] = {
+    'enum': '枚举', '枚举': '枚举',
+    'bytes': '字节', '字节': '字节', 'hex': '字节',
+    'float': '浮点', '浮点': '浮点',
+    'int': '整数', '整数': '整数',
+    'bool': '布尔', '布尔': '布尔', 'boolean': '布尔',
+    'string': '字符串', '字符串': '字符串', 'str': '字符串',
+    'ipv4': 'IPv4', 'IPv4': 'IPv4',
+    'timestamp': '时间戳', '时间戳': '时间戳',
+}
+
+
+def get_type(name: str) -> ColumnType:
+    """Resolve a type name (Chinese or English alias) to a ColumnType."""
+    resolved = TYPE_ALIASES.get(name.strip().lower() if name.isascii() else name.strip())
+    if resolved is None:
+        return StringType()  # fallback
+    return TYPE_REGISTRY[resolved]
+
+
+def classify_column(name: str, series: pl.Series, config_type: str | None = None) -> ColumnType:
+    """Classify a single column to a ColumnType.
+
+    Priority: config override → TlsDB lookup → name heuristics → value sampling.
+    """
+    from analysis.tls_ref import get_tls_ref_dict
+    import re
+
+    # 1. Config override
+    if config_type is not None:
+        t = get_type(config_type)
+        if t.name != "字符串" or config_type.lower().strip() == 'string':
+            return t
+
+    # 2. TlsDB lookup
+    tls_ref = get_tls_ref_dict()
+    if name in tls_ref:
+        t = get_type(tls_ref[name]['data_type'])
+        return t
+
+    # 3. Name heuristics
+    _patterns: list[tuple[re.Pattern, str]] = [
+        (re.compile(r'^(?:src_ip|dst_ip|:ips|:ipd|server.ip|client.ip)$', re.I), 'IPv4'),
+        (re.compile(r'^(?::prs|:prd|scnt|dcnt|1ipp|4dbn|4srs|eiph|name|tabl|0ver|cnrs|isrs)$', re.I), '枚举'),
+        (re.compile(r'^(?:0cph|0crv|cipher|ecdhe)', re.I), '字节'),
+        (re.compile(r'^(?:4dur|8ses|2tmo|0rnt)$', re.I), '浮点'),
+        (re.compile(r'^(?:4ksz|8ack|8ppk|8pak|8seq|8did|8byt|row)$', re.I), '整数'),
+        (re.compile(r'^(?:timestamp|time|8dbd|created|updated)', re.I), '时间戳'),
+        (re.compile(r'\.(ispn|orgn|city|anon|doma|latd|lond)$', re.I), '字符串'),
+        (re.compile(r'^(?:cnam|snam|0rnd|source.node|crcc|orga|orgu|@)', re.I), '字符串'),
+    ]
+    for pat, type_name in _patterns:
+        if pat.search(name):
+            return TYPE_REGISTRY[type_name]
+
+    # 4. Value-based sampling
+    non_null = series.drop_nulls()
+    if len(non_null) == 0:
+        return TYPE_REGISTRY["字符串"]
+
+    strs = [str(v).strip().lower() for v in non_null.head(100)]
+    ip4 = re.compile(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$')
+    hex_p = re.compile(r'^[0-9a-f]{2}(?:\s[0-9a-f]{2})*$')
+
+    if all(ip4.match(v) for v in strs):
+        return TYPE_REGISTRY["IPv4"]
+    if all(hex_p.match(v) for v in strs):
+        return TYPE_REGISTRY["字节"]
+    if series.dtype.is_integer():
+        return TYPE_REGISTRY["整数"]
+    if series.dtype == pl.Float64:
+        return TYPE_REGISTRY["浮点"]
+    if series.dtype == pl.Boolean:
+        return TYPE_REGISTRY["布尔"]
+    if non_null.n_unique() < 50:
+        return TYPE_REGISTRY["枚举"]
+
+    return TYPE_REGISTRY["字符串"]
+
+
+def classify_schema(
+    lf: pl.LazyFrame,
+    config_overrides: dict[str, str] | None = None,
+) -> dict[str, ColumnType]:
+    """Classify all columns in a LazyFrame."""
+    config_overrides = config_overrides or {}
+    df_sample = lf.collect(streaming=True)
+    result = {}
+    for col_name in df_sample.columns:
+        config_type = config_overrides.get(col_name)
+        result[col_name] = classify_column(
+            name=col_name,
+            series=df_sample[col_name],
+            config_type=config_type,
+        )
+    return result

analysis/views/auto.py

@@ -12,7 +12,7 @@ from django.views.decorators.csrf import csrf_exempt
 from config import get_config
 from analysis.models import AnalysisRun
 from .pipeline import _run_pipeline_worker
-from .clustering import _run_clustering_pipeline
+from analysis.services.clustering import run_clustering_pipeline
 from .helpers import _PLANS_DIR, _add_to_auto_index
 
 logger = logging.getLogger(__name__)
@@ -191,8 +191,10 @@ def run_llm_analysis_view(request):
                 save_fields = ['progress_pct', 'progress_msg']
                 if tool_name == 'thinking':
                     # Accumulate LLM thinking text (appended to llm_thinking)
-                    if result:
-                        run.llm_thinking = (run.llm_thinking or '') + f'\n[{step}] {result}'
+                    thought = (result or '').strip()
+                    if not thought:
+                        thought = f'步骤 {step} 分析中...'
+                    run.llm_thinking = (run.llm_thinking or '') + f'\n[{step}] {thought}'
                     save_fields = ['progress_pct', 'progress_msg', 'llm_thinking']
                 elif tool_name == 'complete':
                     pass  # No extra tracking needed
@@ -269,10 +271,10 @@ def run_llm_analysis_view(request):
                             break
 
                 if entity_ds_id:
-                    _run_clustering_pipeline(
+                    run_clustering_pipeline(
                         run=run,
                         store=store,
-                        entity_ds_id=entity_ds_id,
+                        ds_id=entity_ds_id,
                         feature_columns=None,
                         algorithm='agglomerative',
                         min_cluster_size=5,

analysis/views/clustering.py

@@ -114,6 +114,7 @@ def cluster_overview(request, display_id):
         'llm_timeline_json': llm_timeline_json,
         # Globe flow data for the sidebar
         'globe_flows_json': json.dumps(_get_globe_flows(run)),
+        'selected_ds_id': f'upload_{run.display_id}',
     })
 
 
@@ -160,7 +161,9 @@ def _get_globe_flows(run, max_rows=500):
 
 
 def cluster_detail(request, display_id, cluster_label):
-    """Detailed view of a single cluster with entity list, SVD features, and NL summary."""
+    """Detailed view of a single cluster with raw data rows, SVD features, and NL summary."""
+    import json
+    import polars as pl
     run = get_object_or_404(AnalysisRun, display_id=display_id)
     try:
         cluster_label = int(cluster_label)
@@ -168,326 +171,58 @@ def cluster_detail(request, display_id, cluster_label):
         raise Http404("Invalid cluster label")
     cluster = get_object_or_404(ClusterResult, run=run, cluster_label=cluster_label)
     features = cluster.features.all().order_by('-distinguishing_score')[:50]
-    entities = cluster.entities.all()[:100]
+
+    # Load raw data rows for this cluster from EP row indices
+    row_data = []
+    try:
+        from analysis.data_loader import load_csv_directory, load_from_db
+        lf = None
+        if run.sqlite_table:
+            lf = load_from_db(run.sqlite_table)
+        if lf is None and run.csv_glob:
+            lf, _, _, _, _ = load_csv_directory(run.csv_glob)
+        if lf is not None:
+            # Get row indices assigned to this cluster from EntityProfile
+            eps = cluster.entities.all().order_by('entity_value')[:100]
+            indices = []
+            for ep in eps:
+                try:
+                    idx = int(ep.entity_value.replace('row_', ''))
+                    indices.append(idx)
+                except (ValueError, AttributeError):
+                    continue
+            if indices:
+                df = lf.collect(streaming=True)
+                df_filtered = df[indices]
+                # Convert to dict list for template (limit columns for display)
+                cols = df_filtered.columns[:20]  # show first 20 columns
+                for row_idx, row in enumerate(df_filtered.select(cols).iter_rows(named=True)):
+                    entry = {'index': indices[row_idx] if row_idx < len(indices) else row_idx}
+                    for k, v in row.items():
+                        entry[k] = v
+                    row_data.append(entry)
+    except Exception as exc:
+        logger.warning(f'Failed to load raw data for cluster detail: {exc}')
+
     nl_summary = ''
     feats_list = [{'feature_name': f.feature_name, 'score': f.distinguishing_score,
                    'mean': f.mean, 'std': f.std} for f in features[:10]]
     if feats_list:
         nl_summary = nl_describe.describe_cluster(feats_list)
 
-    # Build entity data with feature_json values
-    entity_data = []
-    for e in entities:
-        entry = {'id': e.id, 'value': e.entity_value,
-                 'embedding_x': e.embedding_x, 'embedding_y': e.embedding_y}
-        if e.feature_json:
-            entry['features'] = {k: round(float(v), 4) if isinstance(v, (int, float)) else str(v)
-                                 for k, v in e.feature_json.items()}
-        entity_data.append(entry)
-
     return render(request, 'analysis/cluster_detail.html', {
         'run': run,
         'cluster': cluster,
         'features': features,
-        'entities': entities,
-        'entity_data_json': json.dumps(entity_data),
+        'row_data': row_data,
         'nl_summary': nl_summary,
     })
 
 
 @profile
-def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorithm,
+def _run_clustering_pipeline(run, store, ds_id, feature_columns, algorithm,
                               min_cluster_size, run_umap=True, head=None):
-    """Unified clustering pipeline: clustering → feature extraction → UMAP → ORM save.
-
-    Args:
-        run: AnalysisRun ORM object.
-        store: SessionStore instance.
-        entity_ds_id: dataset ID in SessionStore pointing to entity-aggregated data.
-        feature_columns: list of feature column names (None = auto-detect).
-        algorithm: 'hdbscan' or 'kmeans'.
-        min_cluster_size: int for HDBSCAN.
-        run_umap: bool, whether to compute and save UMAP-2D embeddings.
-        head: optional int, downsample to at most this many rows (min 100).
-    """
-    import warnings
-    warnings.filterwarnings('ignore', category=RuntimeWarning, module='sklearn')
-
-    try:
-        from analysis.tool_registry import _handle_run_clustering, _handle_extract_features
-        import polars as pl
-
-        entry = store.get_dataset(entity_ds_id)
-        if entry is None:
-            run.status = 'failed'
-            run.error_message = 'Entity dataset not found in session store'
-            run.save(update_fields=['status', 'error_message'])
-            return
-
-        schema = entry.get('schema', {})
-
-        # ── Downsampling ──
-        if head is not None and head > 0:
-            lf = entry['lazyframe']
-            try:
-                row_count = lf.select(pl.len()).collect(streaming=True).item()
-            except Exception:
-                logger.error("unknown failed: {}".format(traceback.format_exc()))
-                row_count = 0
-
-            if row_count > head:
-                head = max(head, 100)  # ensure minimum rows for clustering
-                run.progress_msg = f'正在采样 {head}/{row_count} 行...'
-                run.save(update_fields=['progress_msg'])
-                lf = lf.head(head)
-                store.store_dataset(
-                    entity_ds_id, lf,
-                    schema=schema,
-                    metadata=entry.get('metadata', {}),
-                )
-                entry = store.get_dataset(entity_ds_id)  # refresh
-                row_count = head
-
-        # Resolve feature columns — use actual LazyFrame dtypes (not stored schema dict
-        # which may be stale after Utf8 coercion in _background_process).
-        numeric_types = (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
-                         pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
-                         pl.Float32, pl.Float64)
-
-        lf = entry['lazyframe']
-        live_schema = lf.collect_schema()
-        live_names = live_schema.names()
-        live_dtypes = list(live_schema.dtypes())
-
-        if feature_columns:
-            feature_cols = [
-                c for c in feature_columns
-                if c in live_names and live_dtypes[live_names.index(c)] in numeric_types
-            ]
-        else:
-            feature_cols = [
-                live_names[i] for i, dt in enumerate(live_dtypes)
-                if dt in numeric_types and not live_names[i].startswith('_')
-            ][:10]
-
-        # ── Step 1: Clustering ────────────────────────────────────────────────
-        # If no numeric columns detected, pass empty list to let
-        # _handle_run_clustering apply its Utf8→Float64 coercion fallback.
-        run.status = 'clustering'
-        run.progress_pct = 70
-        run.progress_msg = '正在进行聚类分析...'
-        run.save(update_fields=['status', 'progress_pct', 'progress_msg'])
-
-        result = asyncio.run(_handle_run_clustering(
-            dataset_id=entity_ds_id,
-            cluster_columns=feature_cols,
-            algorithm=algorithm,
-            params={'min_cluster_size': min_cluster_size},
-            random_state=RANDOM_SEED,
-        ))
-
-        if 'error' in result:
-            err = result['error']
-            if 'numeric' in err.lower():
-                available = [
-                    f"{k}({v})" for k, v in schema.items()
-                    if v.split('(')[0].strip() in numeric_types and not k.startswith('_')
-                ]
-                err += f"。可用数值列: {available}"
-            raise Exception(err)
-
-        cluster_id = result.get('cluster_result_id', '')
-        run.cluster_count = result.get('n_clusters', 0)
-        run.save(update_fields=['cluster_count'])
-
-        # ── Step 2: Feature extraction ──────────────────────────────────────
-        run.status = 'extracting'
-        run.progress_pct = 90
-        run.progress_msg = '正在提取聚类特征...'
-        run.save(update_fields=['status', 'progress_pct', 'progress_msg'])
-
-        feat_result = asyncio.run(_handle_extract_features(
-            dataset_id=entity_ds_id,
-            cluster_result_id=cluster_id,
-            top_k=10, method='zscore', save_to_db=True,
-            run_id=run.id,
-        ))
-
-        if 'error' in feat_result:
-            raise Exception(feat_result['error'])
-
-        # ── Step 2b: Cluster SVD feature extraction ──────────────────────
-        run.progress_msg = '正在进行聚类SVD特征提取...'
-        run.save(update_fields=['progress_msg'])
-        try:
-            from analysis.distance import cluster_svd_extract
-            lf = entry['lazyframe']
-            cluster_entry = store.get_cluster_result(cluster_id)
-            labels_list = cluster_entry.get('labels', []) if cluster_entry else []
-
-            if labels_list and feature_cols:
-                svd_results = cluster_svd_extract(lf, labels_list, feature_cols)
-
-                # Persist SVD features to ClusterFeature (method='cluster_svd')
-                from analysis.models import ClusterFeature as CF
-                from analysis.models import ClusterResult as CR
-                for label, svd_info in svd_results.items():
-                    cr = CR.objects.filter(run=run, cluster_label=label).first()
-                    if cr is None:
-                        continue
-                    # Delete old zscore entries for this cluster, replace with SVD
-                    CF.objects.filter(cluster=cr, distinguishing_method='zscore').delete()
-                    for feat_name in svd_info.get('features', [])[:10]:
-                        idx = svd_info['features'].index(feat_name) if feat_name in svd_info['features'] else -1
-                        strength = (svd_info.get('feature_strength', [])[idx]
-                                    if idx >= 0 and idx < len(svd_info.get('feature_strength', []))
-                                    else None)
-                        CF.objects.get_or_create(
-                            cluster=cr,
-                            feature_name=feat_name,
-                            defaults={
-                                'mean': None, 'std': None, 'median': None,
-                                'p25': None, 'p75': None, 'missing_rate': None,
-                                'distinguishing_score': float(strength) if strength is not None else None,
-                                'distinguishing_method': 'cluster_svd',
-                            },
-                        )
-        except Exception as svd_err:
-            logger.warning(f'Cluster SVD extraction skipped: {svd_err}')
-
-        # ── Step 3: UMAP-3D embedding (2D fallback) ───────────────────────
-        if run_umap:
-            try:
-                lf = entry['lazyframe']
-                # UMAP: sample max 10K for training, batch-transform rest in 1K batches
-                MAX_UMAP_TRAIN = UMAP_TRAIN_SAMPLE
-                BATCH_SIZE = UMAP_BATCH_SIZE
-                try:
-                    n_total = lf.select(pl.len()).collect(streaming=True).item()
-                except Exception:
-                    logger.error("unknown failed: {}".format(traceback.format_exc()))
-                    n_total = 0
-                from sklearn.preprocessing import StandardScaler
-                import umap
-                import numpy as np
-
-                live_schema = lf.collect_schema()
-                num_cols = [name for name, dt in zip(live_schema.names(), live_schema.dtypes())
-                            if dt in (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
-                                      pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
-                                      pl.Float32, pl.Float64) and not name.startswith('_')]
-
-                if len(num_cols) >= 2:
-                    if n_total > MAX_UMAP_TRAIN:
-                        run.progress_msg = f'UMAP 采样 {MAX_UMAP_TRAIN}/{n_total} 实体训练...'
-                        run.save(update_fields=['progress_msg'])
-                        df_sample = lf.sample(n=MAX_UMAP_TRAIN, seed=RANDOM_SEED).collect(streaming=True)
-                        df_umap = lf.collect(streaming=True)
-                    else:
-                        df_umap = lf.collect(streaming=True)
-
-                    umap_components = 3  # try 3D first
-                    coords = None
-                    reducer = None
-
-                    if len(df_umap) > 2:
-                        for attempt_n in (3, 2):
-                            try:
-                                if n_total > MAX_UMAP_TRAIN:
-                                    # Train UMAP on 10K sample, batch-transform full dataset
-                                    mat_sample = df_sample.select(num_cols).to_numpy()
-                                    scaler = StandardScaler().fit(mat_sample)
-                                    mat_sample_scaled = np.nan_to_num(
-                                        scaler.transform(mat_sample), nan=0.0)
-                                    reducer = umap.UMAP(n_components=attempt_n,
-                                                        random_state=RANDOM_SEED,
-                                                        n_neighbors=15, min_dist=0.1,
-                                                        metric='euclidean')
-                                    reducer.fit(mat_sample_scaled)
-                                    # Batch transform in 1K-row batches
-                                    coords_list = []
-                                    for start in range(0, len(df_umap), BATCH_SIZE):
-                                        end = min(start + BATCH_SIZE, len(df_umap))
-                                        mat_batch = scaler.transform(
-                                            df_umap[start:end].select(num_cols).to_numpy())
-                                        mat_batch = np.nan_to_num(mat_batch, nan=0.0)
-                                        coords_list.append(reducer.transform(mat_batch))
-                                    coords = np.vstack(coords_list)
-                                else:
-                                    mat = np.nan_to_num(StandardScaler().fit_transform(
-                                        df_umap.select(num_cols).to_numpy()), nan=0.0)
-                                    reducer = umap.UMAP(n_components=attempt_n,
-                                                        random_state=RANDOM_SEED,
-                                                        n_neighbors=15, min_dist=0.1,
-                                                        metric='euclidean')
-                                    coords = reducer.fit_transform(mat)
-                                umap_components = attempt_n
-                                break
-                            except Exception as e_umap:
-                                if attempt_n == 2:
-                                    raise
-                                logger.warning(f'UMAP 3D failed ({e_umap}), falling back to 2D')
-
-                        if coords is None:
-                            raise Exception('UMAP produced no output')
-
-                        non_num = [c for c in df_umap.columns
-                                   if c not in num_cols and not c.startswith('_')]
-                        ent_col = non_num[0] if non_num else df_umap.columns[0]
-
-                        cluster_entry = store.get_cluster_result(cluster_id)
-                        labels_list = cluster_entry.get('labels', []) if cluster_entry else []
-
-                        from analysis.models import EntityProfile as EP
-                        from analysis.models import ClusterResult as CR
-
-                        profiles = []
-                        cr_cache = {}
-                        for i, row in enumerate(df_umap.iter_rows(named=True)):
-                            ev = str(row.get(ent_col, '')) or 'entity'
-                            ev = f'{ev}_{i}'
-                            lbl = labels_list[i] if i < len(labels_list) else -1
-                            cache_key = f'{run.id}_{lbl}'
-                            if cache_key not in cr_cache:
-                                cr_cache[cache_key] = CR.objects.filter(
-                                    run=run, cluster_label=lbl).first()
-                            has_z = umap_components >= 3 and coords.shape[1] >= 3
-                            profiles.append(EP(
-                                entity_value=ev, run=run, cluster_label=lbl,
-                                cluster=cr_cache[cache_key],
-                                embedding_x=float(coords[i, 0]) if i < len(coords) else None,
-                                embedding_y=float(coords[i, 1]) if i < len(coords) else None,
-                                embedding_z=float(coords[i, 2]) if has_z
-                                    and i < len(coords) else None,
-                            ))
-                        if profiles:
-                            EP.objects.bulk_create(profiles, ignore_conflicts=True, batch_size=1000)
-                            run.entity_count = len(profiles)
-                            run.save(update_fields=['entity_count'])
-            except Exception as umap_err:
-                logger.warning(f'UMAP embedding skipped: {umap_err}')
-
-        # Fallback: if entity_count not set (UMAP skipped), derive from cluster sizes
-        if run.entity_count is None:
-            from django.db.models import Sum
-            total = run.clusters.aggregate(total=Sum('size'))['total'] or 0
-            run.entity_count = total
-            run.save(update_fields=['entity_count'])
-
-        # ── Done ──
-        run.status = 'completed'
-        run.progress_pct = 100
-        run.progress_msg = '分析完成'
-        run.save(update_fields=['status', 'progress_pct', 'progress_msg', 'entity_count'])
-
-    except Exception:
-        tb = traceback.format_exc()
-        logger.error(tb)
-        try:
-            run.status = 'failed'
-            run.error_message = tb
-            run.progress_msg = f'失败: {tb}'
-            run.run_log += f'\n[ERROR] {tb}'
-            run.save(update_fields=['status', 'error_message', 'progress_msg', 'run_log'])
-        except Exception as e:
-            logger.warning('save failed after pipeline error: %s', e)
+    """Clustering pipeline on raw rows. Delegates to service layer."""
+    from analysis.services.clustering import run_clustering_pipeline
+    run_clustering_pipeline(run, store, ds_id, feature_columns, algorithm,
+                            min_cluster_size, run_umap=run_umap, head=head)

analysis/views/dashboard.py

@@ -33,11 +33,79 @@ def run_list(request):
 
 def run_detail(request, display_id):
     """Details for a single analysis run."""
+    import json
     run = get_object_or_404(AnalysisRun, display_id=display_id)
     clusters = run.clusters.all().order_by('-size')
+
+    # ── Global SVD features (compute on-demand from the stored run data) ──
+    svd_features = []
+    svd_explained_var = []
+    try:
+        from analysis.data_loader import load_csv_directory, load_from_db
+        import polars as pl
+        from sklearn.decomposition import TruncatedSVD
+        import numpy as np
+        import logging
+        logger = logging.getLogger(__name__)
+
+        lf = None
+        if run.sqlite_table:
+            lf = load_from_db(run.sqlite_table)
+        if lf is None and run.csv_glob:
+            lf, _, _, _, _ = load_csv_directory(run.csv_glob)
+
+        if lf is not None:
+            schema = lf.collect_schema()
+            numeric_types = (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
+                             pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
+                             pl.Float32, pl.Float64)
+            num_cols = [n for n, dt in zip(schema.names(), schema.dtypes()) if dt in numeric_types and not n.startswith('_')]
+
+            if len(num_cols) >= 3:
+                n_rows_lf = lf.select(pl.len()).collect(streaming=True).item()
+                sample = min(int(n_rows_lf), 50000)
+                df = lf.sample(n=sample, seed=42).select(num_cols).collect(streaming=True) if n_rows_lf > sample else lf.select(num_cols).collect(streaming=True)
+                mat = np.nan_to_num(df.to_numpy(), nan=0.0)
+                from sklearn.preprocessing import StandardScaler
+                mat = StandardScaler().fit_transform(mat)
+
+                n_comp = min(20, min(mat.shape) - 1)
+                if n_comp >= 2:
+                    svd = TruncatedSVD(n_components=n_comp, random_state=42)
+                    svd.fit(mat)
+                    var_ratio = svd.explained_variance_ratio_.tolist()
+                    svd_explained_var = var_ratio
+                    for comp_idx in range(min(10, n_comp)):
+                        loadings = np.abs(svd.components_[comp_idx])
+                        top_idx = np.argsort(loadings)[-5:][::-1]
+                        features = [{'feature': num_cols[i], 'strength': round(float(loadings[i]), 4)} for i in top_idx]
+                        svd_features.append({
+                            'component': comp_idx + 1,
+                            'explained_var': round(var_ratio[comp_idx] * 100, 2),
+                            'cumulative_var': round(sum(var_ratio[:comp_idx + 1]) * 100, 2),
+                            'top_features': features,
+                        })
+    except Exception as exc:
+        logger = logging.getLogger(__name__)
+        logger.warning(f'SVD feature extraction failed for run {run.display_id}: {exc}')
+
+    # ── LLM workflow timeline ──
+    llm_timeline_json = ''
+    if run.run_type == 'auto':
+        llm_thinking = run.llm_thinking or ''
+        tool_calls = run.tool_calls_json or []
+        if tool_calls or llm_thinking:
+            llm_timeline_json = json.dumps({
+                'thinking': llm_thinking,
+                'tool_calls': tool_calls,
+            })
+
     return render(request, 'analysis/run_detail.html', {
         'run': run,
         'clusters': clusters,
+        'svd_features': svd_features,
+        'svd_explained_var': svd_explained_var,
+        'llm_timeline_json': llm_timeline_json,
     })
 
 
@@ -53,7 +121,7 @@ def run_status_api(request, display_id):
         'progress_pct': run.progress_pct,
         'progress_msg': run.progress_msg,
         'run_log': run.run_log[-2000:],  # last 2000 chars
-        'llm_thinking': run.llm_thinking[-5000:],  # last 5K chars
+        'llm_thinking': run.llm_thinking[-50000:] if run.llm_thinking else '',
         'tool_calls': run.tool_calls_json,
     })
 

analysis/views/globe.py

@@ -4,6 +4,7 @@ import logging
 import traceback
 
 from django.shortcuts import render
+from django.views.decorators.clickjacking import xframe_options_exempt
 
 from analysis.constants import GLOBE_MAX_ROWS
 from analysis.models import AnalysisRun
@@ -112,6 +113,10 @@ def _extract_flows_from_df(df, MAX_ROWS):
     return flows
 
 
+from django.views.decorators.clickjacking import xframe_options_exempt
+
+
+@xframe_options_exempt
 def globe_view(request):
     from analysis.geoip import lookup as geo_lookup
     from analysis.data_loader import load_csv_directory, load_from_db
@@ -122,6 +127,9 @@ def globe_view(request):
 
     # Check for ?data=dataset_id parameter (direct data loading from session store)
     data_param = request.GET.get('data')
+    embed_mode = request.GET.get('embed') == '1'
+    template_name = 'tianxuan/globe_embed.html' if embed_mode else 'tianxuan/globe.html'
+
     if data_param:
         store = SessionStore()
         flows = []
@@ -130,11 +138,11 @@ def globe_view(request):
             entry = store.get_dataset(data_param)
             if entry is not None:
                 lf = entry['lazyframe']
-                df = lf.head(MAX_ROWS_PER_RUN).collect()
+                df = lf.head(MAX_ROWS_PER_RUN).collect(streaming=True)
                 flows = _extract_flows_from_df(df, MAX_ROWS_PER_RUN)
         except Exception as exc:
             logging.getLogger(__name__).warning(f'globe ?data= param failed: {exc}')
-        return render(request, 'tianxuan/globe.html', {
+        return render(request, template_name, {
             'geo_flows': json.dumps(flows),
             'all_runs': all_runs,
             'selected_ids': [],
@@ -172,7 +180,7 @@ def globe_view(request):
                 lf = load_from_db(run.sqlite_table)
             if lf is None:
                 lf, schema, rc, fc, mem = load_csv_directory(run.csv_glob)
-            df = lf.head(MAX_ROWS_PER_RUN).collect()
+            df = lf.head(MAX_ROWS_PER_RUN).collect(streaming=True)
             flows.extend(_extract_flows_from_df(df, MAX_ROWS_PER_RUN))
             all_selected_failed = False
             break  # stop after first successful load
@@ -193,7 +201,7 @@ def globe_view(request):
                     if not latest.csv_glob:
                         continue
                     lf, schema, rc, fc, mem = load_csv_directory(latest.csv_glob)
-                df = lf.head(MAX_ROWS_PER_RUN).collect()
+                df = lf.head(MAX_ROWS_PER_RUN).collect(streaming=True)
                 flows.extend(_extract_flows_from_df(df, MAX_ROWS_PER_RUN))
                 selected_runs = [latest]
                 break

analysis/views/manual.py

@@ -10,7 +10,7 @@ from django.views.decorators.csrf import csrf_exempt
 
 from analysis.models import AnalysisRun
 from .pipeline import _run_pipeline_worker
-from .clustering import _run_clustering_pipeline
+from analysis.services.clustering import run_clustering_pipeline
 
 logger = logging.getLogger(__name__)
 
@@ -164,7 +164,7 @@ def manual_run_analysis(request):
             ds_id = upload_ds_id if store.get_dataset(upload_ds_id) else f'upload_{pk}'
 
             # Use the unified clustering pipeline (clustering → extraction → UMAP)
-            _run_clustering_pipeline(
+            run_clustering_pipeline(
                 run=run, store=store, entity_ds_id=ds_id,
                 feature_columns=feature_columns,
                 algorithm=algorithm, min_cluster_size=min_cluster_size,

analysis/views/pipeline.py

@@ -176,50 +176,28 @@ def _background_process(run_id, upload_dir):
                     pl.col(lat_lon_col).map_batches(_coerce_to_float_fn, return_dtype=pl.Float64).alias(lat_lon_col)
                 )
 
-        # ── Restore numeric types for Utf8 columns coerced during CSV load ──
-        # All columns were forced to Utf8 for safe SQLite storage.  Now try to
-        # recover the original numeric types (Int64 / Float64) from the data.
-        numeric_types = (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
-                         pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
-                         pl.Float32, pl.Float64)
-        # Get current schema to know which columns are still Utf8
-        live_schema = lf.collect_schema()
-        live_names = live_schema.names()
-        live_dtypes = list(live_schema.dtypes())
-        utf8_cols = [
-            live_names[i] for i, dt in enumerate(live_dtypes)
-            if dt == pl.Utf8 and not live_names[i].startswith('_')
-        ]
-        if utf8_cols:
-            # Infer types from a small sample to decide Float64 vs Int64
-            sample_df = lf.select(utf8_cols).head(1000).collect(streaming=True)
-            type_map: dict[str, pl.DataType] = {}
-            for col in utf8_cols:
-                try:
-                    # Try Float64 first (most general numeric type)
-                    casted = sample_df[col].cast(pl.Float64, strict=False)
-                    non_null = casted.is_not_null().sum()
-                    if non_null > len(casted) * 0.5:
-                        # Check if all non-null values are integers → use Int64
-                        all_int = True
-                        for v in casted.head(100).to_list():
-                            if v is not None and v != int(v):
-                                all_int = False
-                                break
-                        type_map[col] = pl.Int64 if all_int else pl.Float64
-                except Exception:
-                    logger.error("unknown failed: {}".format(traceback.format_exc()))
-                    pass
-
-            if type_map:
-                # Update schema dict to reflect restored types
-                for col, dtype in type_map.items():
-                    schema[col] = str(dtype)
-                # Apply casts to the LazyFrame
-                casts = [pl.col(col).cast(dtype, strict=False) for col, dtype in type_map.items()]
+        # ── Restore column types after SQLite storage ──
+        # All columns were stored as Utf8 for SQLite.  Recover original types
+        # from the classification system — ONLY the 8 allowed types.
+        from analysis.type_classifier import classify_schema, DataType
+        type_map = classify_schema(lf)
+        # Apply Polars-level type casts based on classification
+        dtypes = {
+            DataType.FLOAT: pl.Float64,
+            DataType.INT: pl.Int64,
+            DataType.BOOL: pl.Boolean,
+            DataType.TIMESTAMP: pl.Float64,
+        }
+        casts = []
+        for col, dt in type_map.items():
+            target = dtypes.get(dt)
+            if target and col in lf.collect_schema().names():
+                casts.append(pl.col(col).cast(target, strict=False))
+                schema[col] = str(target)
+        if casts:
             lf = lf.with_columns(casts)
-                logger.info('[BACKGROUND] Restored numeric types for %d columns: %s',
-                            len(type_map), list(type_map.keys())[:10])
+            restored = [(col, dt.value) for col, dt in type_map.items() if dt in dtypes][:15]
+            logger.info('[BACKGROUND] Type restoration: %s', restored)
 
         # ── SVD dimensionality reduction ──
         svd_components = 0

data/geoip_cache.json

@@ -45,5 +45,639 @@
     "lon": 77.0,
     "city": "",
     "country": "印度"
+  },
+  "anon, hosting": null,
+  "-108.418446": null,
+  "-63.303819": null,
+  "156.494253": {
+    "lat": 34.053,
+    "lon": -118.2642,
+    "city": "洛杉矶",
+    "country": "美国"
+  },
+  "174.324906": {
+    "lat": 43.6425,
+    "lon": -79.3872,
+    "city": "",
+    "country": "加拿大"
+  },
+  "91.251045": {
+    "lat": 53.55,
+    "lon": 10.0,
+    "city": "汉堡市",
+    "country": "德国"
+  },
+  "136.838644": {
+    "lat": 42.3223,
+    "lon": -83.1763,
+    "city": "迪尔伯恩",
+    "country": "美国"
+  },
+  "-135.232683": null,
+  "134.708535": {
+    "lat": 45.4763,
+    "lon": -122.6408,
+    "city": "Portland",
+    "country": "美国"
+  },
+  "172.779207": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "76.375007": {
+    "lat": 35.6507,
+    "lon": -78.4564,
+    "city": "Clayton",
+    "country": "美国"
+  },
+  "-37.054377": null,
+  "58.368656": {
+    "lat": 35.685,
+    "lon": 139.7514,
+    "city": "东京",
+    "country": "日本"
+  },
+  "-128.312338": null,
+  "-86.295682": null,
+  "-83.710197": null,
+  "-117.654327": null,
+  "-94.931072": null,
+  "-31.118326": null,
+  "41.94697": {
+    "lat": -29.0,
+    "lon": 24.0,
+    "city": "",
+    "country": "南非"
+  },
+  "-175.557379": null,
+  "None": null,
+  "90.484378": {
+    "lat": 50.6846,
+    "lon": 1.7610999999999999,
+    "city": "维尔维涅",
+    "country": "法国"
+  },
+  "91.136808": {
+    "lat": 50.8934,
+    "lon": 7.8856,
+    "city": "Freudenberg",
+    "country": "德国"
+  },
+  "98.398423": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "-37.351993": null,
+  "108.8728": {
+    "lat": 33.5363,
+    "lon": -117.0439,
+    "city": "特曼库拉",
+    "country": "美国"
+  },
+  "-40.490238": null,
+  "-108.320482": null,
+  "-42.599081": null,
+  "60.847895": {
+    "lat": 30.2936,
+    "lon": 120.1614,
+    "city": "杭州",
+    "country": "中国"
+  },
+  "-70.344735": null,
+  "-17.065978": null,
+  "-17.594761": null,
+  "3.748318": {
+    "lat": 41.1412,
+    "lon": -73.2637,
+    "city": "Fairfield",
+    "country": "美国"
+  },
+  "117.90784": {
+    "lat": 21.0333,
+    "lon": 105.85,
+    "city": "Hanoi",
+    "country": "越南"
+  },
+  "-178.325477": null,
+  "47.034714": {
+    "lat": 45.3155,
+    "lon": -75.837,
+    "city": "Ottawa",
+    "country": "加拿大"
+  },
+  "12.6356": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "-4.376563": null,
+  "117.415791": {
+    "lat": 21.0333,
+    "lon": 105.85,
+    "city": "Hanoi",
+    "country": "越南"
+  },
+  "-95.943745": null,
+  "115.811466": {
+    "lat": 37.57,
+    "lon": 126.98,
+    "city": "",
+    "country": "大韩民国"
+  },
+  "82.858229": {
+    "lat": 54.5842,
+    "lon": -1.5631,
+    "city": "Darlington",
+    "country": "英国"
+  },
+  "120.177108": {
+    "lat": 39.8897,
+    "lon": 115.275,
+    "city": "Hebei",
+    "country": "中国"
+  },
+  "12.054574": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "-112.238338": null,
+  "-161.091649": null,
+  "24.207799": {
+    "lat": 40.3698,
+    "lon": -80.634,
+    "city": "斯托本维尔",
+    "country": "美国"
+  },
+  "-148.31237": null,
+  "-132.223679": null,
+  "76.252382": {
+    "lat": 35.6513,
+    "lon": -78.8336,
+    "city": "Holly Springs",
+    "country": "美国"
+  },
+  "104.839037": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "-147.319447": null,
+  "-137.111721": null,
+  "-107.76671": null,
+  "-167.802595": null,
+  "168.500812": {
+    "lat": 29.7176,
+    "lon": -95.4188,
+    "city": "休斯敦",
+    "country": "美国"
+  },
+  "-19.640731": null,
+  "-36.115066": null,
+  "-31.172609": null,
+  "4.746775": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "-128.714202": null,
+  "-95.66714": null,
+  "-19.935247": null,
+  "52.225763": {
+    "lat": 39.5645,
+    "lon": -75.597,
+    "city": "威尔明顿",
+    "country": "美国"
+  },
+  "-39.370987": null,
+  "162.006633": {
+    "lat": 43.7807,
+    "lon": -79.2855,
+    "city": "士嘉堡",
+    "country": "加拿大"
+  },
+  "51.696746": {
+    "lat": 51.5,
+    "lon": -0.13,
+    "city": "",
+    "country": "英国"
+  },
+  "6.673336": {
+    "lat": 31.5273,
+    "lon": -110.3607,
+    "city": "谢拉维斯塔",
+    "country": "美国"
+  },
+  "102.5503": null,
+  "-144.964956": null,
+  "-104.457778": null,
+  "83.087052": null,
+  "-125.101539": null,
+  "90.262005": {
+    "lat": 48.9018,
+    "lon": 2.4893,
+    "city": "邦迪",
+    "country": "法国"
+  },
+  "-83.715609": null,
+  "29.378547": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "-57.090496": null,
+  "142.184372": {
+    "lat": 44.7314,
+    "lon": -63.6482,
+    "city": "Dartmouth",
+    "country": "加拿大"
+  },
+  "165.297813": {
+    "lat": -33.9167,
+    "lon": 18.4167,
+    "city": "Cape Town",
+    "country": "南非"
+  },
+  "105.875829": {
+    "lat": -29.0,
+    "lon": 24.0,
+    "city": "",
+    "country": "南非"
+  },
+  "160.97215": {
+    "lat": 43.286,
+    "lon": -77.6843,
+    "city": "Rochester",
+    "country": "美国"
+  },
+  "-76.327692": null,
+  "-179.768984": null,
+  "-135.649249": null,
+  "-129.397466": null,
+  "-159.604348": null,
+  "123.957119": {
+    "lat": 34.6836,
+    "lon": 113.5325,
+    "city": "郑州",
+    "country": "中国"
+  },
+  "106.508142": {
+    "lat": 28.55,
+    "lon": 115.9333,
+    "city": "南昌",
+    "country": "中国"
+  },
+  "60.737607": {
+    "lat": 45.75,
+    "lon": 126.65,
+    "city": "哈尔滨",
+    "country": "中国"
+  },
+  "12.851572": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "133.109702": {
+    "lat": 35.69,
+    "lon": 139.69,
+    "city": "",
+    "country": "日本"
+  },
+  "97.468122": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "-49.099249": null,
+  "31.114424": {
+    "lat": 52.25,
+    "lon": 21.0,
+    "city": "华沙",
+    "country": "波兰"
+  },
+  "49.613214": {
+    "lat": 37.5985,
+    "lon": 126.9783,
+    "city": "首尔特别市",
+    "country": "大韩民国"
+  },
+  "-76.937967": null,
+  "24.350375": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "-57.188778": null,
+  "-129.431676": null,
+  "-136.186649": null,
+  "-61.694727": null,
+  "-90.513611": null,
+  "91.194896": {
+    "lat": 49.5333,
+    "lon": 11.8,
+    "city": "哈恩巴赫",
+    "country": "德国"
+  },
+  "132.327773": {
+    "lat": 32.404,
+    "lon": -86.2539,
+    "city": "蒙哥马利",
+    "country": "美国"
+  },
+  "-112.197533": null,
+  "155.022955": null,
+  "162.854691": {
+    "lat": 51.5,
+    "lon": -0.13,
+    "city": "",
+    "country": "英国"
+  },
+  "-52.303734": null,
+  "-37.91953": null,
+  "144.662392": {
+    "lat": 37.3762,
+    "lon": -122.1826,
+    "city": "Palo Alto",
+    "country": "美国"
+  },
+  "17.315107": {
+    "lat": 37.323,
+    "lon": -122.0322,
+    "city": "Cupertino",
+    "country": "美国"
+  },
+  "97.359727": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "-37.814213": null,
+  "-57.066252": null,
+  "170.670964": {
+    "lat": 47.6689,
+    "lon": -117.4369,
+    "city": "斯波坎",
+    "country": "美国"
+  },
+  "19.237916": {
+    "lat": 42.3223,
+    "lon": -83.1763,
+    "city": "迪尔伯恩",
+    "country": "美国"
+  },
+  "-77.426219": null,
+  "35.273579": {
+    "lat": 42.2776,
+    "lon": -83.7409,
+    "city": "安娜堡",
+    "country": "美国"
+  },
+  "-29.56673": null,
+  "-173.438083": null,
+  "-164.691055": null,
+  "-140.463216": null,
+  "-168.663276": null,
+  "-138.543279": null,
+  "-47.142859": null,
+  "-102.389184": null,
+  "24.754033": {
+    "lat": 32.981,
+    "lon": -80.0326,
+    "city": "Goose Creek",
+    "country": "美国"
+  },
+  "-109.476174": null,
+  "92.836244": {
+    "lat": 51.5,
+    "lon": -0.13,
+    "city": "",
+    "country": "英国"
+  },
+  "2.089979": null,
+  "82.41569": {
+    "lat": 51.6578,
+    "lon": -0.3954,
+    "city": "沃特福德",
+    "country": "英国"
+  },
+  "-117.765392": null,
+  "-112.753899": null,
+  "-147.563834": null,
+  "-27.4301": null,
+  "66.656057": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "-52.329152": null,
+  "16.783619": {
+    "lat": 37.3762,
+    "lon": -122.1826,
+    "city": "Palo Alto",
+    "country": "美国"
+  },
+  "-160.668335": null,
+  "-155.782242": null,
+  "-43.415128": null,
+  "-151.682652": null,
+  "33.713437": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "151.170164": {
+    "lat": 42.8333,
+    "lon": 12.8333,
+    "city": "",
+    "country": "意大利"
+  },
+  "-2.437103": null,
+  "17.096003": null,
+  "4.372196": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "12.775114": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "-26.847855": null,
+  "-2.96053": null,
+  "-136.421502": null,
+  "156.152711": {
+    "lat": 46.2857,
+    "lon": -119.2845,
+    "city": "里奇兰",
+    "country": "美国"
+  },
+  "-121.186972": null,
+  "-126.924167": null,
+  "-101.388131": null,
+  "-101.03754": null,
+  "77.828572": {
+    "lat": 52.5167,
+    "lon": 13.4,
+    "city": "柏林",
+    "country": "德国"
+  },
+  "-34.813058": null,
+  "123.011334": {
+    "lat": 23.7,
+    "lon": 90.375,
+    "city": "",
+    "country": "孟加拉"
+  },
+  "11.265017": {
+    "lat": 39.9612,
+    "lon": -82.9988,
+    "city": "哥伦布",
+    "country": "美国"
+  },
+  "11.202764": {
+    "lat": 39.9612,
+    "lon": -82.9988,
+    "city": "哥伦布",
+    "country": "美国"
+  },
+  "70.262109": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "-70.027646": null,
+  "26.990087": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "147.506751": {
+    "lat": 47.5,
+    "lon": 19.0833,
+    "city": "布达佩斯",
+    "country": "匈牙利"
+  },
+  "-107.81862": null,
+  "141.75262": {
+    "lat": 51.0,
+    "lon": 9.0,
+    "city": "",
+    "country": "德国"
+  },
+  "30.674809": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "120.327798": {
+    "lat": 39.8897,
+    "lon": 115.275,
+    "city": "Hebei",
+    "country": "中国"
+  },
+  "-158.088393": null,
+  "-121.790773": null,
+  "77.908193": {
+    "lat": 50.3167,
+    "lon": 7.8,
+    "city": "Nassau",
+    "country": "德国"
+  },
+  "57.252522": {
+    "lat": 47.0,
+    "lon": 8.0,
+    "city": "",
+    "country": ""
+  },
+  "-15.466886": null,
+  "-136.800708": null,
+  "-139.560022": null,
+  "155.309838": {
+    "lat": 59.3294,
+    "lon": 18.0686,
+    "city": "",
+    "country": "瑞典"
+  },
+  "158.175545": {
+    "lat": 31.5273,
+    "lon": -110.3607,
+    "city": "谢拉维斯塔",
+    "country": "美国"
+  },
+  "-38.283752": null,
+  "146.597747": {
+    "lat": 42.3755,
+    "lon": -83.0772,
+    "city": "底特律",
+    "country": "美国"
+  },
+  "-146.199812": null,
+  "-60.696207": null,
+  "141.547786": {
+    "lat": 35.8869,
+    "lon": 14.4025,
+    "city": "Mdina",
+    "country": "马耳他"
+  },
+  "10.542121": null,
+  "169.586656": {
+    "lat": 45.0059,
+    "lon": -93.1059,
+    "city": "圣保罗",
+    "country": "美国"
+  },
+  "53.588162": {
+    "lat": 51.0,
+    "lon": 9.0,
+    "city": "",
+    "country": "德国"
+  },
+  "-48.221466": null,
+  "-104.847562": null,
+  "-97.617615": null,
+  "145.767865": {
+    "lat": 51.8425,
+    "lon": 5.8528,
+    "city": "奈梅亨",
+    "country": "荷兰"
+  },
+  "-173.124365": null,
+  "-158.431061": null,
+  "-132.023148": null,
+  "-171.873785": null,
+  "-23.369992": null,
+  "53.281077": {
+    "lat": 51.0,
+    "lon": 9.0,
+    "city": "",
+    "country": "德国"
+  },
+  "28.294006": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
   }
 }

prototype_cluster_ui.html

@@ -0,0 +1,356 @@
+<!DOCTYPE html>
+<html lang="zh-hans">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width,initial-scale=1.0">
+<title>原型 — 聚类UI交互验证</title>
+<style>
+* { margin:0; padding:0; box-sizing:border-box; }
+body { font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif; background:#f5f7fa; color:#1a1a2e; }
+.header { background:#1a1a2e; color:#fff; padding:0.8rem 1.5rem; display:flex; justify-content:space-between; align-items:center; }
+.header h1 { font-size:1.1rem; }
+.header span { font-size:0.8rem; color:#aaa; }
+
+.main-layout { display:flex; position:relative; transition:all 0.3s; min-height:calc(100vh - 48px); }
+.left-panel { flex:1; min-width:0; transition:flex 0.3s; padding:1rem; overflow-y:auto; }
+.left-panel.globe-open { flex:0 0 55%; }
+.right-panel { flex:0 0 0; overflow:hidden; transition:flex 0.3s; }
+.right-panel.globe-open { flex:0 0 45%; }
+
+/* toggle handle */
+.toggle-handle { position:absolute; z-index:10; top:50%; width:20px; height:44px; background:#1a1a2e; color:#fff; border:none; border-radius:4px 0 0 4px; cursor:pointer; display:flex; align-items:center; justify-content:center; font-size:12px; transition:left 0.3s ease; }
+.toggle-handle:hover { background:#4361ee; }
+
+/* scatter */
+.scatter-wrap { position:relative; background:#fff; border-radius:8px; padding:0.75rem; margin-bottom:1rem; box-shadow:0 1px 3px rgba(0,0,0,0.1); }
+.scatter-wrap h3 { font-size:0.95rem; margin-bottom:0.5rem; }
+.scatter-wrap canvas { width:100%; height:360px; display:block; cursor:crosshair; border-radius:4px; background:#fafbfc; }
+
+/* pills */
+.pill-row { display:flex; flex-wrap:wrap; gap:4px; margin-bottom:0.75rem; }
+.pill { display:inline-flex; align-items:center; gap:3px; padding:2px 10px; border-radius:12px; font-size:0.75rem; cursor:pointer; border:2px solid transparent; transition:all 0.15s; color:#fff; white-space:nowrap; }
+.pill.all { background:#e8e8e8; color:#333; }
+.pill.active { border-color:#333; font-weight:700; }
+
+/* cluster cards */
+.card-grid { display:grid; grid-template-columns:1fr 1fr; gap:0.75rem; margin-bottom:1rem; }
+.cluster-card { background:#fff; border:1px solid #e0e0e0; border-radius:8px; padding:0.8rem; cursor:pointer; transition:box-shadow 0.15s; }
+.cluster-card:hover { box-shadow:0 2px 8px rgba(0,0,0,0.1); }
+.cluster-card.selected { border-color:#4361ee; box-shadow:0 0 0 2px rgba(67,97,238,0.2); }
+.cluster-card .label { font-weight:700; font-size:1rem; }
+.cluster-card .meta { font-size:0.78rem; color:#666; margin-top:0.2rem; }
+.cluster-card .desc { font-size:0.8rem; color:#333; margin-top:0.4rem; padding:0.4rem; background:#f8f9fa; border-left:3px solid #4361ee; border-radius:0 4px 4px 0; line-height:1.5; }
+
+/* detail panel */
+.detail-panel { position:fixed; bottom:0; left:0; right:0; z-index:100; transform:translateY(100%); transition:transform 0.35s cubic-bezier(.4,0,.2,1); }
+.detail-panel.open { transform:translateY(0); }
+.detail-body { background:#fff; border-radius:16px 16px 0 0; box-shadow:0 -4px 24px rgba(0,0,0,0.15); max-height:40vh; overflow-y:auto; padding:0.5rem 1.5rem 1.5rem; }
+.detail-handle { width:36px; height:4px; background:#ccc; border-radius:2px; margin:8px auto; cursor:pointer; }
+.detail-close { float:right; cursor:pointer; font-size:1.3rem; color:#999; line-height:1; border:none; background:none; }
+.detail-close:hover { color:#333; }
+.detail-body h3 { font-size:1rem; margin-bottom:0.3rem; }
+.detail-body p { font-size:0.85rem; color:#555; margin:0.2rem 0; }
+.detail-body table { width:100%; border-collapse:collapse; font-size:0.8rem; margin-top:0.5rem; }
+.detail-body th { background:#f5f5f5; padding:0.25rem 0.4rem; text-align:left; font-weight:600; border-bottom:2px solid #ddd; }
+.detail-body td { padding:0.2rem 0.4rem; border-bottom:1px solid #eee; }
+.detail-body td code { font-size:0.72rem; }
+
+/* 3D地球容器 */
+#globeContainer { width:100%; height:calc(100vh - 48px); background:#0a0a1a; position:relative; }
+#globeContainer canvas { display:block; }
+</style>
+</head>
+<body>
+
+<div class="header">
+    <h1>聚类概览 — 运行 #9</h1>
+    <span>200 条流 · 3 个簇 · 12 个噪声点</span>
+</div>
+
+<div class="main-layout" id="mainLayout">
+    <!-- Left: cluster content -->
+    <div class="left-panel" id="leftPanel">
+        <!-- Scatter -->
+        <div class="scatter-wrap">
+            <h3>UMAP 嵌入 <span style="font-weight:400;font-size:0.8rem;color:#888;" id="ptCount">加载中...</span></h3>
+            <canvas id="scatterChart"></canvas>
+        </div>
+
+        <!-- Cluster pills -->
+        <div class="pill-row" id="pillRow">
+            <span class="pill all active" data-lbl="all" onclick="clearSel()">全部</span>
+        </div>
+
+        <!-- Cluster cards -->
+        <div class="card-grid" id="cardGrid"></div>
+    </div>
+
+    <!-- Right: 3D Globe -->
+    <div class="right-panel" id="rightPanel">
+        <div id="globeContainer"></div>
+    </div>
+
+    <!-- Toggle handle -->
+    <button class="toggle-handle" id="globeHandle" onclick="toggleGlobe()">◀</button>
+</div>
+
+<!-- Bottom detail panel -->
+<div class="detail-panel" id="detailPanel">
+    <div class="detail-body" id="detailBody">
+        <button class="detail-close" onclick="closeDetail()">✕</button>
+        <div class="detail-handle" onclick="closeDetail()"></div>
+        <div id="detailContent"><p style="color:#999;">选择一个簇或数据点查看详情</p></div>
+    </div>
+</div>
+
+<script src="static/tianxuan/three.min.js"></script>
+<script>
+// ═══════════════════════════════════════════════
+//  MOCK DATA
+// ═══════════════════════════════════════════════
+const MOCK_CLUSTERS = [
+    { label:0, size:78, prop:0.39, silh:0.72, desc:'低延迟 (23ms)，比全局均值高3.1σ；大数据包 (1520 bytes)，高2.3σ；系统端口 (443)' },
+    { label:1, size:65, prop:0.33, silh:0.68, desc:'高TLS1.3比例 (92%)，比均值高1.8σ；短会话 (12s)，低1.5σ；ECDHE曲线 X25519' },
+    { label:2, size:45, prop:0.23, silh:0.54, desc:'高连接频率 (85次/min)，比均值高4.2σ；非标准端口 (8080/8443)，目标多域名' },
+    { label:-1, size:12, prop:0.06, silh:null, desc:'噪声点 — 异常TLS指纹、零散目标、低流量' },
+];
+
+const COLORS = ['#4361ee','#f72585','#7209b7','#4cc9f0','#f8961e','#43aa8b','#f94144','#577590','#e9c46a','#9b5de5'];
+function colorFor(lbl) { return lbl===-1?'rgba(150,150,150,0.7)':COLORS[Math.abs(lbl)%COLORS.length]; }
+
+// Generate mock UMAP scatter data (200 points)
+const SCATTER = [];
+const RNG = (seed) => { let s=seed%2147483647; return ()=> { s=s*16807%2147483647; return (s-1)/2147483646; }; };
+const rng = RNG(42);
+const centers = {0:[2,1.5],1:[-1.8,0.5],2:[0.5,-2], '-1':[3.5,-1.5]};
+const spreads = {0:[0.6,0.5],1:[0.7,0.6],2:[0.5,0.7], '-1':[1.2,0.8]};
+for (const c of MOCK_CLUSTERS) {
+    for (let i=0;i<c.size;i++) {
+        const cx=centers[c.label][0], cy=centers[c.label][1];
+        const sx=spreads[c.label][0], sy=spreads[c.label][1];
+        SCATTER.push({
+            x:cx+(rng()-0.5)*2*sx, y:cy+(rng()-0.5)*2*sy,
+            label:c.label, entity:`模拟主机 ${1000+i}`
+        });
+    }
+}
+
+// ═══════════════════════════════════════════════
+//  SCATTER 2D CANVAS
+// ═══════════════════════════════════════════════
+let selCluster = null, selPoint = null;
+let globeOpen = false;
+
+function drawScatter(highlight) {
+    const c=document.getElementById('scatterChart');
+    if(!c)return;
+    const W=c.parentElement.clientWidth||700, H=360;
+    c.width=W*2; c.height=H*2; c.style.width=W+'px'; c.style.height=H+'px';
+    const ctx=c.getContext('2d'); ctx.scale(2,2);
+    const pad={top:25,bottom:30,left:50,right:20};
+    const pw=W-pad.left-pad.right, ph=H-pad.top-pad.bottom;
+    const xs=SCATTER.map(d=>d.x), ys=SCATTER.map(d=>d.y);
+    const xMin=Math.min(...xs), xMax=Math.max(...xs), yMin=Math.min(...ys), yMax=Math.max(...ys);
+    const xR=xMax-xMin||1, yR=yMax-yMin||1;
+    const toX=v=>pad.left+(v-xMin)/xR*pw, toY=v=>pad.top+ph-(v-yMin)/yR*ph;
+
+    ctx.clearRect(0,0,W,H);
+    // grid
+    ctx.strokeStyle='#eee'; ctx.lineWidth=1;
+    for(let i=0;i<=5;i++){ctx.beginPath();ctx.moveTo(pad.left,pad.top+i*ph/5);ctx.lineTo(W-pad.right,pad.top+i*ph/5);ctx.stroke();}
+    // points
+    SCATTER.forEach(d=>{
+        const cx=toX(d.x),cy=toY(d.y);
+        let radius=3,color=colorFor(d.label);
+        if(highlight!==null&&highlight!==undefined){
+            if(d.label===highlight){color=colorFor(d.label);radius=5;}
+            else{color='rgba(200,200,200,0.15)';radius=2;}
+        }
+        ctx.beginPath();ctx.arc(cx,cy,radius,0,Math.PI*2);ctx.fillStyle=color;ctx.fill();
+    });
+    // labels
+    ctx.fillStyle='#999'; ctx.font='11px sans-serif'; ctx.textAlign='center';
+    ctx.fillText('UMAP-1',W/2,H-3);
+    ctx.save();ctx.translate(14,H/2);ctx.rotate(-Math.PI/2);ctx.fillText('UMAP-2',0,0);ctx.restore();
+
+    c.onclick=function(e){
+        const r=c.getBoundingClientRect();
+        const mx=e.clientX-r.left, my=e.clientY-r.top;
+        const sx=mx/W*pw, sy=my/H*ph;
+        let best=null,bestD=10;
+        for(const d of SCATTER){
+            const dx=toX(d.x)-mx, dy=toY(d.y)-my;
+            const dist=Math.sqrt(dx*dx+dy*dy);
+            if(dist<bestD){bestD=dist;best=d;}
+        }
+        if(best){selCluster=MOCK_CLUSTERS.find(c=>c.label===best.label);selPoint=best;showDetail(selCluster,selPoint);}
+    };
+    document.getElementById('ptCount').textContent=SCATTER.length+' 个点';
+}
+
+// ═══════════════════════════════════════════════
+//  CLUSTER SELECTION
+// ═══════════════════════════════════════════════
+function selectCluster(lbl){
+    selCluster=MOCK_CLUSTERS.find(c=>c.label===lbl);
+    selPoint=null;
+    drawScatter(lbl);
+    document.querySelectorAll('.pill').forEach(p=>p.classList.toggle('active',parseInt(p.dataset.lbl)===lbl));
+    document.querySelectorAll('.cluster-card').forEach(p=>p.classList.toggle('selected',parseInt(p.dataset.label)===lbl));
+    if(globeOpen)globeFilter(lbl);
+    showDetail(selCluster,null);
+}
+function clearSel(){
+    selCluster=null;selPoint=null;
+    drawScatter(null);
+    document.querySelectorAll('.pill,.cluster-card').forEach(p=>p.classList.remove('active','selected'));
+    closeDetail();
+    if(globeOpen)globeFilter(null);
+}
+
+// ═══════════════════════════════════════════════
+//  DETAIL PANEL
+// ═══════════════════════════════════════════════
+function showDetail(cluster,point){
+    const dc=document.getElementById('detailContent');
+    let html='';
+    if(point){
+        html+='<h3>数据点详情</h3><table><thead><tr><th>字段</th><th>值</th></tr></thead><tbody>'+
+            '<tr><td>标识</td><td><code>'+point.entity+'</code></td></tr>'+
+            '<tr><td>所属簇</td><td>#'+(point.label===-1?'噪声':point.label)+'</td></tr>'+
+            '<tr><td>UMAP-x</td><td>'+point.x.toFixed(4)+'</td></tr>'+
+            '<tr><td>UMAP-y</td><td>'+point.y.toFixed(4)+'</td></tr>'+
+            '</tbody></table>';
+    }else if(cluster){
+        html+='<h3>簇 #'+(cluster.label===-1?'噪声':cluster.label)+'</h3>'+
+            '<p>大小: '+cluster.size+' | 占比: '+(cluster.prop*100).toFixed(0)+'%'+
+            (cluster.silh!==null?' | 轮廓系数: '+cluster.silh.toFixed(4):'')+'</p>'+
+            '<p style="background:#f8f9fa;border-left:3px solid #4361ee;padding:0.5rem;margin:0.5rem 0;border-radius:0 4px 4px 0;font-size:0.85rem;line-height:1.6;">'+cluster.desc+'</p>'+
+            '<table><thead><tr><th>特征</th><th>区分度</th><th>均值</th><th>标准差</th></tr></thead><tbody>'+
+            '<tr><td>延迟 (4dur)</td><td style="color:#2e7d32;">+3.2</td><td>23ms</td><td>5.1</td></tr>'+
+            '<tr><td>数据包大小 (8ppk)</td><td style="color:#2e7d32;">+2.3</td><td>1520B</td><td>128</td></tr>'+
+            '<tr><td>TLS 版本 (0ver)</td><td style="color:#c62828;">-1.8</td><td>1.2</td><td>0.3</td></tr>'+
+            '</tbody></table>';
+    }
+    dc.innerHTML=html;
+    document.getElementById('detailPanel').classList.add('open');
+}
+function closeDetail(){document.getElementById('detailPanel').classList.remove('open');}
+
+// ═══════════════════════════════════════════════
+//  GLOBE SIDEBAR
+// ═══════════════════════════════════════════════
+function toggleGlobe(){
+    globeOpen=!globeOpen;
+    document.getElementById('leftPanel').classList.toggle('globe-open',globeOpen);
+    document.getElementById('rightPanel').classList.toggle('globe-open',globeOpen);
+    const h=document.getElementById('globeHandle');
+    h.innerHTML=globeOpen?'▶':'◀';
+    h.style.left=globeOpen?'calc(55% - 20px)':'calc(100% - 20px)';
+    if(globeOpen&&!window._globeInit)initGlobe3D();
+}
+
+function globeFilter(label){
+    if(!window._arcMeshes)return;
+    window._arcMeshes.forEach(m=>{
+        if(label===null){m.material.opacity=0.5;m.material.color.setHex(parseInt(m.userData.color,16));}
+        else{m.material.opacity=0.06;m.material.color.setHex(0x666666);}
+    });
+}
+
+// ═══════════════════════════════════════════════
+//  3D GLOBE (Three.js, simplified)
+// ═══════════════════════════════════════════════
+function initGlobe3D(){
+    if(window._globeInit)return;
+    const cont=document.getElementById('globeContainer');
+    if(!cont)return;
+    const W=cont.clientWidth||450, H=cont.clientHeight||500;
+
+    const scene=new THREE.Scene();scene.background=new THREE.Color(0x0a0a1a);
+    const cam=new THREE.PerspectiveCamera(45,W/H,0.1,100);cam.position.set(0,0,6);
+    const ren=new THREE.WebGLRenderer({antialias:true});ren.setSize(W,H);ren.setPixelRatio(Math.min(window.devicePixelRatio,2));
+    cont.appendChild(ren.domElement);
+
+    // earth
+    const earth=new THREE.Mesh(new THREE.SphereGeometry(2,48,48),new THREE.MeshPhongMaterial({color:0x1a3a5c,emissive:0x0a1a2a}));
+    scene.add(earth);
+    // glow
+    scene.add(new THREE.Mesh(new THREE.SphereGeometry(2.06,48,48),new THREE.MeshBasicMaterial({color:0x224488,transparent:true,opacity:0.12})));
+    // lights
+    scene.add(new THREE.AmbientLight(0x222244));
+    const dl=new THREE.DirectionalLight(0xffffff,1);dl.position.set(5,10,7);scene.add(dl);
+    scene.add(new THREE.DirectionalLight(0x4488ff,0.3));
+    // stars
+    const s=new THREE.BufferGeometry();const p=new Float32Array(600*3);
+    for(let i=0;i<600*3;i++)p[i]=(Math.random()-0.5)*100;
+    s.setAttribute('position',new THREE.BufferAttribute(p,3));
+    scene.add(new THREE.Points(s,new THREE.PointsMaterial({color:0xffffff,size:0.15})));
+
+    // mock flows → arcs
+    const tlsC={'TLSv1.3':0x4cc9f0,'TLSv1.2':0x43aa8b,'TLSv1.1':0xf8961e,'TLSv1.0':0xf94144};
+    const arcGrp=new THREE.Group();scene.add(arcGrp);
+    const arcs=[];
+    function llz(lat,lon,r){const p=(90-lat)*Math.PI/180,t=(lon+180)*Math.PI/180;return new THREE.Vector3(-r*Math.sin(p)*Math.cos(t),r*Math.cos(p),r*Math.sin(p)*Math.sin(t));}
+
+    // generate ~30 mock flows
+    const cities=[{lat:31.2,lon:121.5},{lat:39.9,lon:116.4},{lat:35.7,lon:139.7},{lat:37.6,lon:127.0},{lat:51.5,lon:-0.13},{lat:48.9,lon:2.35},{lat:40.7,lon:-74.0},{lat:37.8,lon:-122.4},{lat:1.35,lon:103.8},{lat:-33.9,lon:151.2}];
+    const tlsOpts=['TLSv1.3','TLSv1.2','TLSv1.3','TLSv1.2','TLSv1.1'];
+    for(let i=0;i<30;i++){
+        const src=cities[i%cities.length],dst=cities[(i+3)%cities.length];
+        const f=llz(src.lat,src.lon,2),t=llz(dst.lat,dst.lon,2);
+        const mid=new THREE.Vector3().addVectors(f,t).multiplyScalar(0.5).normalize().multiplyScalar(3.2);
+        const curve=new THREE.QuadraticBezierCurve3(f,mid,t);
+        const c=tlsOpts[i%tlsOpts.length];
+        const mat=new THREE.MeshBasicMaterial({color:tlsC[c],transparent:true,opacity:0.5});
+        const mesh=new THREE.Mesh(new THREE.TubeGeometry(curve,16,0.015,4,false),mat);
+        mesh.userData={color:tlsC[c].toString(16)};
+        arcGrp.add(mesh);arcs.push(mesh);
+    }
+    window._arcMeshes=arcs;
+
+    // drag
+    let isD=false,pm={x:0,y:0};
+    ren.domElement.onmousedown=e=>{isD=true;pm={x:e.clientX,y:e.clientY};};
+    window.onmouseup=()=>isD=false;
+    window.onmousemove=e=>{if(!isD)return;const dx=e.clientX-pm.x,dy=e.clientY-pm.y;earth.rotation.y+=dx*0.005;earth.rotation.x+=dy*0.005;arcGrp.rotation.y=earth.rotation.y;arcGrp.rotation.x=earth.rotation.x;pm={x:e.clientX,y:e.clientY};};
+
+    !function anim(){requestAnimationFrame(anim);if(!isD){earth.rotation.y+=0.003;arcGrp.rotation.y=earth.rotation.y;}ren.render(scene,cam);}();
+    window._globeInit=true;
+}
+
+// ═══════════════════════════════════════════════
+//  RENDER PILLS & CARDS
+// ═══════════════════════════════════════════════
+(function init(){
+    const pr=document.getElementById('pillRow');
+    MOCK_CLUSTERS.forEach(c=>{
+        if(c.label===-1)return;
+        const p=document.createElement('span');p.className='pill';p.style.background=colorFor(c.label);
+        p.dataset.lbl=c.label;p.textContent='#'+c.label+' ('+c.size+')';p.onclick=()=>selectCluster(c.label);
+        pr.appendChild(p);
+    });
+    // noise pill
+    const np=document.createElement('span');np.className='pill';np.style.background='#999';
+    np.dataset.lbl='-1';np.textContent='噪声 ('+MOCK_CLUSTERS.find(c=>c.label===-1).size+')';np.onclick=()=>selectCluster(-1);
+    pr.appendChild(np);
+
+    // cards
+    const cg=document.getElementById('cardGrid');
+    MOCK_CLUSTERS.forEach(c=>{
+        if(c.label===-1)return;
+        const card=document.createElement('div');card.className='cluster-card';card.dataset.label=c.label;
+        card.innerHTML='<div class="label">簇 #'+c.label+'</div>'+
+            '<div class="meta">'+c.size+' 个实体 · '+(c.prop*100).toFixed(0)+'%'+(c.silh!==null?' · 轮廓系数: '+c.silh.toFixed(4):'')+'</div>'+
+            '<div class="desc">'+c.desc+'</div>';
+        card.onclick=()=>selectCluster(c.label);
+        cg.appendChild(card);
+    });
+
+    drawScatter(null);
+    // set initial handle pos
+    document.getElementById('globeHandle').style.left='calc(100% - 20px)';
+})();
+</script>
+</body>
+</html>

scripts/fix_merge_conflict.py

@@ -0,0 +1,11 @@
+"""Fix the unresolved merge conflict in views/clustering.py."""
+import re
+with open('analysis/views/clustering.py', 'r', encoding='utf-8-sig') as f:
+    lines = f.readlines()
+
+# Find and remove lines containing conflict markers
+new_lines = [l for l in lines if '<<<<<<<' not in l and '>>>>>>>' not in l and '=======' not in l]
+with open('analysis/views/clustering.py', 'w', encoding='utf-8') as f:
+    f.writelines(new_lines)
+print(f'Removed {len(lines) - len(new_lines)} conflict marker lines')
+

scripts/gen_massive_test.py

@@ -0,0 +1,81 @@
+"""Generate 100 TLS flow CSV files (10000 rows each) matching TlsDB.csv column spec.
+
+Usage:
+    runtime\\python\\python.exe scripts\\gen_massive_test.py [--files 100] [--rows 10000] [--output-dir data/test_csvs]
+"""
+import sys
+import argparse
+from pathlib import Path
+
+# Allow importing from project root
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from scripts.gen_test_data import generate_row, COLUMNS, COVERAGE
+import random
+import csv
+from datetime import datetime
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Generate 100 massive TLS flow CSV files (10,000 rows each)'
+    )
+    parser.add_argument('--files', type=int, default=100,
+                        help='Number of CSV files (default: 100)')
+    parser.add_argument('--rows', type=int, default=10000,
+                        help='Rows per file (default: 10000)')
+    parser.add_argument('--output-dir', type=str, default='data/test_csvs',
+                        help='Output directory (default: data/test_csvs)')
+    parser.add_argument('--seed', type=int, default=42,
+                        help='Random seed for reproducibility (default: 42)')
+    args = parser.parse_args()
+
+    random.seed(args.seed)
+    out_dir = Path(args.output_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    base_time = datetime(2026, 6, 1, 0, 0, 0)
+
+    total_rows = 0
+    for file_idx in range(args.files):
+        file_seed = args.seed + file_idx
+        file_rng = random.Random(file_seed)
+
+        # Generate all rows for this file
+        rows = []
+        for row_i in range(args.rows):
+            row = generate_row(total_rows + row_i, base_time)
+            rows.append(row)
+
+        # Auto-drop columns that are ALL blank
+        non_blank_cols = set()
+        for row in rows:
+            for col in COLUMNS:
+                val = row.get(col, '')
+                if val != '' and val != '+':
+                    non_blank_cols.add(col)
+        file_columns = [c for c in COLUMNS if c in non_blank_cols]
+
+        # Randomize column order per file
+        shuffled = file_columns[:]
+        file_rng.shuffle(shuffled)
+
+        # Write: test_0000.csv .. test_0099.csv
+        out_path = out_dir / f'test_{file_idx:04d}.csv'
+        with open(out_path, 'w', newline='', encoding='utf-8') as f:
+            writer = csv.DictWriter(f, fieldnames=shuffled, extrasaction='ignore')
+            writer.writeheader()
+            for row in rows:
+                writer.writerow(row)
+
+        total_rows += args.rows
+
+        if (file_idx + 1) % 10 == 0 or file_idx == 0 or file_idx == args.files - 1:
+            print(f'  [{file_idx + 1}/{args.files}] {total_rows} rows, '
+                  f'{len(shuffled)} cols, seed={file_seed}')
+
+    print(f'\nDone: {args.files} files x {args.rows} rows = {total_rows} total rows')
+    print(f'Output: {out_dir.resolve()}')
+
+
+if __name__ == '__main__':
+    main()

scripts/rewrite_type_classifier.py

@@ -0,0 +1,8 @@
+"""Rewrite type_classifier.py as thin re-export wrapper."""
+content = '''"""Re-export from new types module for backward compatibility."""
+from analysis.types import ColumnType, TYPE_REGISTRY, TYPE_ALIASES, get_type, classify_column, classify_schema
+TLS_HEX_MAP = {"0303": "TLSv1.2", "0304": "TLSv1.3", "0302": "TLSv1.1", "0301": "TLSv1.0"}
+'''
+with open('analysis/type_classifier.py', 'w', encoding='utf-8') as f:
+    f.write(content)
+print('Rewritten OK')

server_stderr.txt

@@ -1,178 +1,143 @@
-Restored 2 datasets from disk
-[2026-07-23 13:06:51,899] INFO django: Restored 2 datasets from disk
-[2026-07-23 13:06:52,038] INFO analysis.data_loader: [LOAD_FROM_DB_LAZY] table=_data_64 rows=9990 cols=57
-[2026-07-23 13:06:52,043] INFO analysis.data_loader: [LOAD_FROM_DB_LAZY] table=_data_66 rows=2000 cols=9
-[23/Jul/2026 13:21:24] "GET / HTTP/1.1" 200 10790
-[23/Jul/2026 13:21:24,471] - Broken pipe from ('127.0.0.1', 64227)
-[2026-07-23 13:27:21,369] INFO analysis.views: [UPLOAD] Received 1998 files
-[23/Jul/2026 13:27:22] "POST /upload/csv/ HTTP/1.1" 200 178755
-[23/Jul/2026 13:27:22] "POST /runs/5/finalize/ HTTP/1.1" 200 34
-[23/Jul/2026 13:27:22] "GET /runs/5/status/ HTTP/1.1" 200 188
-[2026-07-23 13:27:22,672] INFO analysis.views: [BACKGROUND] Forcing lat/lon column ':ips.latd' to Utf8 for safe parsing
-[2026-07-23 13:27:22,672] INFO analysis.views: [BACKGROUND] Forcing lat/lon column ':ips.lond' to Utf8 for safe parsing
-[2026-07-23 13:27:22,672] INFO analysis.views: [BACKGROUND] Forcing lat/lon column ':ipd.latd' to Utf8 for safe parsing
-[2026-07-23 13:27:22,672] INFO analysis.views: [BACKGROUND] Forcing lat/lon column ':ipd.lond' to Utf8 for safe parsing
-[2026-07-23 13:27:22,672] INFO analysis.views: [BACKGROUND] Found 1998 CSV files in C:\Users\25044\AppData\Roaming\TianXuan\data\uploads\20260723_052721
-[2026-07-23 13:27:23,325] INFO analysis.data_loader: [SAVE_TO_DB] table=_data_68 rows=495 mode=append
-[2026-07-23 13:27:23,609] INFO analysis.data_loader: [SAVE_TO_DB] table=_data_68 rows=495 mode=append
-[2026-07-23 13:27:23,895] INFO analysis.data_loader: [SAVE_TO_DB] table=_data_68 rows=495 mode=append
-[2026-07-23 13:27:24,170] INFO analysis.data_loader: [SAVE_TO_DB] table=_data_68 rows=495 mode=append
-[2026-07-23 13:27:24,447] INFO analysis.data_loader: [SAVE_TO_DB] table=_data_68 rows=495 mode=append
-[2026-07-23 13:27:24,733] INFO analysis.data_loader: [SAVE_TO_DB] table=_data_68 rows=495 mode=append
-[2026-07-23 13:27:25,000] INFO analysis.data_loader: [SAVE_TO_DB] table=_data_68 rows=495 mode=append
-[2026-07-23 13:27:25,268] INFO analysis.data_loader: [SAVE_TO_DB] table=_data_68 rows=495 mode=append
-[2026-07-23 13:27:25,553] INFO analysis.data_loader: [SAVE_TO_DB] table=_data_68 rows=495 mode=append
-[23/Jul/2026 13:27:25] "GET /runs/5/status/ HTTP/1.1" 200 242
-[2026-07-23 13:27:25,838] INFO analysis.data_loader: [SAVE_TO_DB] table=_data_68 rows=495 mode=append
-[2026-07-23 13:27:26,116] INFO analysis.data_loader: [SAVE_TO_DB] table=_data_68 rows=495 mode=append
-[2026-07-23 13:27:26,392] INFO analysis.data_loader: [SAVE_TO_DB] table=_data_68 rows=495 mode=append
-[2026-07-23 13:27:26,668] INFO analysis.data_loader: [SAVE_TO_DB] table=_data_68 rows=495 mode=append
-[2026-07-23 13:27:26,946] INFO analysis.data_loader: [SAVE_TO_DB] table=_data_68 rows=495 mode=append
-[2026-07-23 13:27:27,231] INFO analysis.data_loader: [SAVE_TO_DB] table=_data_68 rows=495 mode=append
-[2026-07-23 13:27:27,497] INFO analysis.data_loader: [SAVE_TO_DB] table=_data_68 rows=495 mode=append
-[2026-07-23 13:27:27,782] INFO analysis.data_loader: [SAVE_TO_DB] table=_data_68 rows=495 mode=append
-[2026-07-23 13:27:28,071] INFO analysis.data_loader: [SAVE_TO_DB] table=_data_68 rows=495 mode=append
-[2026-07-23 13:27:28,349] INFO analysis.data_loader: [SAVE_TO_DB] table=_data_68 rows=495 mode=append
-[2026-07-23 13:27:28,621] INFO analysis.data_loader: [SAVE_TO_DB] table=_data_68 rows=495 mode=append
-[23/Jul/2026 13:27:28] "GET /runs/5/status/ HTTP/1.1" 200 243
-[2026-07-23 13:27:28,713] INFO analysis.data_loader: [SAVE_TO_DB] table=_data_68 rows=90 mode=append
-E:\hjq\澶╃拠\analysis\views.py:584: PerformanceWarning: Determining the column names of a LazyFrame requires resolving its schema, which is a potentially expensive operation. Use `LazyFrame.collect_schema().names()` to get the column names without this warning.
-  if lat_lon_col in lf.columns:
-[2026-07-23 13:27:29,071] INFO analysis.views: [BACKGROUND] Restored numeric types for 16 columns: ['1ipp', '2tmo', '4dbn', '4dur', '4srs', '8ack', '8byt', '8dbd', '8did', '8pak']
-[2026-07-23 13:27:29,150] INFO analysis.views: [BACKGROUND] SVD: 15 components from 16 numeric columns (explained variance ratio sum=1.0000)
-[23/Jul/2026 13:27:31] "GET /runs/5/status/ HTTP/1.1" 200 223
-[23/Jul/2026 13:27:56] "POST /analyze/run/ HTTP/1.1" 200 45
-[23/Jul/2026 13:27:56] "GET /runs/5/status/ HTTP/1.1" 200 223
-[2026-07-23 13:27:57,344] INFO analysis.tool_registry: [CLUSTER_INPUT] rows=9990 cols=10 feature_cols=['1ipp', '2tmo', '4dbn', '4dur', '4srs', '8ack', '8byt', '8dbd', '8did', '8pak']
-[2026-07-23 13:27:57,344] INFO analysis.tool_registry: [CLUSTER_SCALE] Starting StandardScaler...
-[2026-07-23 13:27:57,344] INFO analysis.tool_registry: [CLUSTER_SCALE] Done. shape=(9990, 10)
-E:\hjq\澶╃拠\runtime\python\Lib\site-packages\sklearn\cluster\_hdbscan\hdbscan.py:722: FutureWarning: The default value of `copy` will change from False to True in 1.10. Explicitly set a value for `copy` to silence this warning.
-  warn(
-[23/Jul/2026 13:27:59] "GET /runs/5/status/ HTTP/1.1" 200 240
-[23/Jul/2026 13:28:02] "GET /runs/5/status/ HTTP/1.1" 200 240
-[23/Jul/2026 13:28:06] "GET /runs/5/status/ HTTP/1.1" 200 240
-E:\hjq\澶╃拠\runtime\python\Lib\site-packages\umap\umap_.py:1952: UserWarning: n_jobs value 1 overridden to 1 by setting random_state. Use no seed for parallelism.
-  warn(
-[23/Jul/2026 13:28:09] "GET /runs/5/status/ HTTP/1.1" 200 240
-[23/Jul/2026 13:28:12] "GET /runs/5/status/ HTTP/1.1" 200 240
-[23/Jul/2026 13:28:15] "GET /runs/5/status/ HTTP/1.1" 200 240
-[23/Jul/2026 13:28:18] "GET /runs/5/status/ HTTP/1.1" 200 240
-[23/Jul/2026 13:28:21] "GET /runs/5/status/ HTTP/1.1" 200 240
-[23/Jul/2026 13:28:24] "GET /runs/5/status/ HTTP/1.1" 200 240
-[23/Jul/2026 13:28:27] "GET /runs/5/status/ HTTP/1.1" 200 240
-[23/Jul/2026 13:28:30] "GET /runs/5/status/ HTTP/1.1" 200 240
-[23/Jul/2026 13:28:33] "GET /runs/5/status/ HTTP/1.1" 200 240
-[23/Jul/2026 13:28:37] "GET /runs/5/status/ HTTP/1.1" 200 240
-[2026-07-23 13:28:38,183] INFO analysis.geoip: [GEOIP] loaded 803 ranges from E:\hjq\澶╃拠\data\geoip_data.txt (0 skipped)
-E:\hjq\澶╃拠\runtime\python\Lib\site-packages\umap\umap_.py:1952: UserWarning: n_jobs value 1 overridden to 1 by setting random_state. Use no seed for parallelism.
-  warn(
-[23/Jul/2026 13:28:40] "GET /runs/5/status/ HTTP/1.1" 200 254
-Restored 3 datasets from disk
-[2026-07-23 21:16:22,571] INFO django: Restored 3 datasets from disk
-[2026-07-23 21:16:22,802] INFO analysis.data_loader: [LOAD_FROM_DB_LAZY] table=_data_64 rows=9990 cols=57
-[2026-07-23 21:16:22,804] INFO analysis.data_loader: [LOAD_FROM_DB_LAZY] table=_data_66 rows=2000 cols=9
-[2026-07-23 21:16:22,992] INFO analysis.data_loader: [LOAD_FROM_DB_LAZY] table=_data_68 rows=9990 cols=57
-[23/Jul/2026 21:16:44] "GET / HTTP/1.1" 200 12728
-[23/Jul/2026 21:16:44] "GET /static/tianxuan/favicon.svg HTTP/1.1" 304 0
-[23/Jul/2026 21:17:14] "HEAD / HTTP/1.1" 200 0
-[23/Jul/2026 21:17:20] "HEAD / HTTP/1.1" 200 0
-Restored 3 datasets from disk
-[2026-07-23 21:22:58,596] INFO django: Restored 3 datasets from disk
-[2026-07-23 21:22:58,783] INFO analysis.data_loader: [LOAD_FROM_DB_LAZY] table=_data_64 rows=9990 cols=57
-[2026-07-23 21:22:58,801] INFO analysis.data_loader: [LOAD_FROM_DB_LAZY] table=_data_66 rows=2000 cols=9
-[2026-07-23 21:22:59,009] INFO analysis.data_loader: [LOAD_FROM_DB_LAZY] table=_data_68 rows=9990 cols=57
-[23/Jul/2026 21:22:59] "GET / HTTP/1.1" 200 12728
-Restored 3 datasets from disk
-[2026-07-23 21:37:07[23/Jul/2026 21:37:07] "GET / HTTP/1.1" 200 12728
[[23/Jul/2026 21:37:38] "HEAD / HTTP/1.1" 200 0
-[23/Jul/2026 21:37:49] "HEAD / HTTP/1.1" 200 0
-[23/Jul/2026 21:38:08] "HEAD / HTTP/1.1" 200 0
-[23/Jul/2026 21:38:38] "HEAD / HTTP/1.1" 200 0
-[23/Jul/2026 21:39:08] "HEAD / HTTP/1.1" 200 0
-[23/Jul/2026 21:39:38] "HEAD / HTTP/1.1" 200 0
-[23/Jul/2026 21:39:43] "HEAD / HTTP/1.1" 200 0
-[23/Jul/2026 21:39:45] "HEAD / HTTP/1.1" 200 0
-[23/Jul/2026 21:39:49] "HEAD / HTTP/1.1" 200 0
-[23/Jul/2026 21:40:08] "HEAD / HTTP/1.1" 200 0
-[23/Jul/2026 21:40:38] "HEAD / HTTP/1.1" 200 0
-[23/Jul/2026 21:41:35] "HEAD / HTTP/1.1" 200 0
-Restored 3 datasets from disk
-[2026-07-23 21:42:01,918] INFO django: Restored 3 datasets from disk
-[2026-07-23 21:42:02,145] INFO analysis.data_loader: [LOAD_FROM_DB_LAZY] table=_data_64 rows=9990 cols=57
-[2026-07-23 21:42:02,153] INFO analysis.data_loader: [LOAD_FROM_DB_LAZY] table=_data_66 rows=2000 cols=9
-[2026-07-23 21:42:02,295] INFO analysis.data_loader: [LOAD_FROM_DB_LAZY] table=_data_68 rows=9990 cols=57
-esponse = wrapped_callback(request, *callback_args, **callback_kwargs)
-               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "E:\hjq\天璇\simple_analysis\views.py", line 796, in index
-  File "E:\hjq\天璇\runtime\python\Lib\site-packages\django\shortcuts.py", line 24, in render
-    content = loader.render_to_string(template_name, context, request, using=using)
-              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "E:\hjq\天璇\runtime\python\Lib\site-packages\django\template\loader.py", line 61, in render_to_string
-    template = get_template(template_name, using=using)
-               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "E:\hjq\天璇\runtime\python\Lib\site-packages\django\template\loader.py", line 19, in get_template
-    raise TemplateDoesNotExist(template_name, chain=chain)
-django.template.exceptions.TemplateDoesNotExist: simple_analysis/simple_analysis.html
-[2026-07-23 21:41:56,536] ERROR django.request: Internal Server Error: /simple/
-Traceback (most recent call last):
-  File "E:\hjq\天璇\runtime\python\Lib\site-packages\django\core\handlers\exception.py", line 55, in inner
-    response = get_response(request)
-               ^^^^^^^^^^^^^^^^^^^^^
-  File "E:\hjq\天璇\runtime\python\Lib\site-packages\django\core\handlers\base.py", line 197, in _get_response
-    response = wrapped_callback(request, *callback_args, **callback_kwargs)
-               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "E:\hjq\天璇\simple_analysis\views.py", line 796, in index
-  File "E:\hjq\天璇\runtime\python\Lib\site-packages\django\shortcuts.py", line 24, in render
-    content = loader.render_to_string(template_name, context, request, using=using)
-              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "E:\hjq\天璇\runtime\python\Lib\site-packages\django\template\loader.py", line 61, in render_to_string
-    template = get_template(template_name, using=using)
-               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "E:\hjq\天璇\runtime\python\Lib\site-packages\django\template\loader.py", line 19, in get_template
-    raise TemplateDoesNotExist(template_name, chain=chain)
-django.template.exceptions.TemplateDoesNotExist: simple_analysis/simple_analysis.html
-[23/Jul/2026 21:41:56] "GET /simple/ HTTP/1.1" 500 80559
-Not Found: /favicon.ico
-[2026-07-23 21:41:56,814] WARNING django.request: Not Found: /favicon.ico
-[23/Jul/2026 21:41:56] "GET /favicon.ico HTTP/1.1" 404 6872
-Restored 3 datasets from disk
-[2026-07-23 21:45:23,039] INFO django: Restored 3 datasets from disk
-[2026-07-23 21:45:23,182] INFO analysis.data_loader: [LOAD_FROM_DB_LAZY] table=_data_64 rows=9990 cols=57
-[2026-07-23 21:45:23,190] INFO analysis.data_loader: [LOAD_FROM_DB_LAZY] table=_data_66 rows=2000 cols=9
-[2026-07-23 21:45:23,314] INFO analysis.data_loader: [LOAD_FROM_DB_LAZY] table=_data_68 rows=9990 cols=57
-[23/Jul/2026 21:45:28] "GET / HTTP/1.1" 200 12645
-[23/Jul/2026 21:45:31] "GET /upload/ HTTP/1.1" 200 19078
-[23/Jul/2026 21:45:44] "POST /runs/5/delete/ HTTP/1.1" 200 17
-[23/Jul/2026 21:45:44] "GET /upload/ HTTP/1.1" 200 18534
-[23/Jul/2026 21:45:46] "POST /runs/4/delete/ HTTP/1.1" 200 17
-[23/Jul/2026 21:45:46] "GET /upload/ HTTP/1.1" 200 17939
-[23/Jul/2026 21:45:48] "POST /runs/3/delete/ HTTP/1.1" 200 17
-[23/Jul/2026 21:45:48] "GET /upload/ HTTP/1.1" 200 17395
-[23/Jul/2026 21:45:51] "POST /runs/2/delete/ HTTP/1.1" 200 17
-[23/Jul/2026 21:45:51] "GET /upload/ HTTP/1.1" 200 16853
-[23/Jul/2026 21:45:52] "POST /runs/1/delete/ HTTP/1.1" 200 17
-[23/Jul/2026 21:45:52] "GET /upload/ HTTP/1.1" 200 16136
-[23/Jul/2026 21:46:23] "HEAD / HTTP/1.1" 200 0
-[23/Jul/2026 21:46:53] "HEAD / HTTP/1.1" 200 0
-[23/Jul/2026 21:47:23] "HEAD / HTTP/1.1" 200 0
-[23/Jul/2026 21:47:53] "HEAD / HTTP/1.1" 200 0
-[23/Jul/2026 21:48:23] "HEAD / HTTP/1.1" 200 0
-[23/Jul/2026 21:48:53] "HEAD / HTTP/1.1" 200 0
-[23/Jul/2026 21:49:23] "HEAD / HTTP/1.1" 200 0
-[23/Jul/2026 21:49:53] "HEAD / HTTP/1.1" 200 0
-[23/Jul/2026 21:50:23] "HEAD / HTTP/1.1" 200 0
-Restored 3 datasets from disk
-[2026-07-23 21:50:50,464] INFO django: Restored 3 datasets from disk
-[23/Jul/2026 21:50:54] "GET /analyze/manual/ HTTP/1.1" 200 65847
-[23/Jul/2026 21:50:54] "GET /tools/plan/ HTTP/1.1" 200 13
-[23/Jul/2026 21:50:57] "GET /analyze/auto/ HTTP/1.1" 200 39486
-[23/Jul/2026 21:51:09] "GET /runs/ HTTP/1.1" 200 9674
-[2026-07-23 21:51:10,324] INFO analysis.geoip: [GEOIP] loaded 0 ranges (0 skipped)
-[2026-07-23 21:51:10,325] INFO analysis.geoip: [GEOIP] loaded 12 cached entries
-[23/Jul/2026 21:51:10] "GET /globe/ HTTP/1.1" 200 18520
-[23/Jul/2026 21:51:10] "GET /static/tianxuan/three.min.js HTTP/1.1" 304 0
-[23/Jul/2026 21:51:10] "GET /static/tianxuan/world_borders.js HTTP/1.1" 304 0
-[23/Jul/2026 21:51:10] "GET /static/tianxuan/three.min.js HTTP/1.1" 304 0
-[23/Jul/2026 21:51:10] "GET /static/tianxuan/world_borders.js HTTP/1.1" 304 0
-[23/Jul/2026 21:51:10] "GET /static/tianxuan/earth_atmos_2048.jpg HTTP/1.1" 304 0
-[23/Jul/2026 21:51:27] "GET /config/ HTTP/1.1" 200 14372
-[23/Jul/2026 21:51:31] "GET /analyze/auto/ HTTP/1.1" 200 39486
+Restored 1 datasets from disk
+[2026-07-24 12:30:18,755] INFO analysis.data_loader: [LOAD] files=1 total_rows_est=200
+[2026-07-24 12:30:18,742] INFO django: Restored 1 datasets from disk
+[2026-07-24 12:30:18,811] INFO analysis.data_loader: [LOAD_SCHEMA] col=0cph dtype=String
+[2026-07-24 12:30:18,811] INFO analysis.data_loader: [LOAD_SCHEMA] col=0crv dtype=String
+[2026-07-24 12:30:18,811] INFO analysis.data_loader: [LOAD_SCHEMA] col=0rnd dtype=String
+[2026-07-24 12:30:18,811] INFO analysis.data_loader: [LOAD_SCHEMA] col=0rnt dtype=String
+[2026-07-24 12:30:18,811] INFO analysis.data_loader: [LOAD_SCHEMA] col=0ver dtype=String
+[2026-07-24 12:30:18,811] INFO analysis.data_loader: [LOAD_SCHEMA] col=1ipp dtype=Int64
+[2026-07-24 12:30:18,811] INFO analysis.data_loader: [LOAD_SCHEMA] col=2tmo dtype=Float64
+[2026-07-24 12:30:18,811] INFO analysis.data_loader: [LOAD_SCHEMA] col=4dbn dtype=Int64
+[2026-07-24 12:30:18,817] INFO analysis.data_loader: [LOAD_SCHEMA] col=4dur dtype=Float64
+[2026-07-24 12:30:18,817] INFO analysis.data_loader: [LOAD_SCHEMA] col=4ksz dtype=Int64
+[2026-07-24 12:30:18,817] INFO analysis.data_loader: [LOAD_SCHEMA] col=4srs dtype=Int64
+[2026-07-24 12:30:18,817] INFO analysis.data_loader: [LOAD_SCHEMA] col=8ack dtype=Int64
+[2026-07-24 12:30:18,817] INFO analysis.data_loader: [LOAD_SCHEMA] col=8byt dtype=Int64
+[2026-07-24 12:30:18,817] INFO analysis.data_loader: [LOAD_SCHEMA] col=8dbd dtype=Int64
+[2026-07-24 12:30:18,818] INFO analysis.data_loader: [LOAD_SCHEMA] col=8did dtype=Int64
+[2026-07-24 12:30:18,818] INFO analysis.data_loader: [LOAD_SCHEMA] col=8pak dtype=Int64
+[2026-07-24 12:30:18,818] INFO analysis.data_loader: [LOAD_SCHEMA] col=8ppk dtype=Int64
+[2026-07-24 12:30:18,818] INFO analysis.data_loader: [LOAD_SCHEMA] col=8seq dtype=Float64
+[2026-07-24 12:30:18,818] INFO analysis.data_loader: [LOAD_SCHEMA] col=8ses dtype=Float64
+[2026-07-24 12:30:18,818] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ipd dtype=String
+[2026-07-24 12:30:18,818] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ipd.anon dtype=String
+[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ipd.city dtype=String
+[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ipd.doma dtype=String
+[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ipd.ispn dtype=String
+[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ipd.latd dtype=String
+[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ipd.lond dtype=String
+[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ipd.orgn dtype=String
+[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ips dtype=String
+[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ips.anon dtype=String
+[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ips.city dtype=String
+[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ips.doma dtype=String
+[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ips.ispn dtype=String
+[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ips.latd dtype=String
+[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ips.lond dtype=String
+[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ips.orgn dtype=String
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=:prd dtype=Int64
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=:prs dtype=Int64
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=cipher-suite dtype=String
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=client-ip dtype=String
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=cnam dtype=String
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=crcc dtype=String
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=dcnt dtype=String
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=ecdhe-named-curve dtype=String
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=name dtype=String
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=orga dtype=String
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=orgu dtype=Int64
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=scnt dtype=String
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=server-ip dtype=String
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=snam dtype=String
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=source-node dtype=String
+[2026-07-24 12:30:18,832] INFO analysis.data_loader: [LOAD_SCHEMA] col=tabl dtype=String
+[2026-07-24 12:30:18,832] INFO analysis.data_loader: [LOAD_SCHEMA] col=time dtype=String
+[2026-07-24 12:30:18,832] INFO analysis.data_loader: [LOAD_SCHEMA] col=timestamp dtype=Float64
+[2026-07-24 12:30:18,840] INFO analysis.data_loader: [CLEAN] Generic numeric coercion applied: [':ipd.lond', ':ipd.doma', ':ipd.ispn', 'cnam', ':ips.ispn', ':ips.doma', 'orga', 'snam', 'time', ':ips.orgn', ':ipd.orgn', 'crcc', ':ips.latd', ':ipd.latd', ':ips.city', ':ipd.city', ':ips.lond']
+[2026-07-24 12:30:18,840] INFO analysis.data_loader: [CLEAN] HEX columns added hex_pairs_count: ['0cph', '0crv', '0ver', '0rnt', '0rnd']
+[24/Jul/2026 12:30:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:31:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:31:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:32:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:32:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:33:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:33:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:34:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:34:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:35:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:35:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:36:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:36:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:37:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:37:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:38:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:38:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:39:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:39:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:40:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:40:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:41:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:41:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:42:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:42:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:43:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:43:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:44:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:44:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:45:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:45:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:46:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:46:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:47:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:47:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:48:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:48:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:49:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:49:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:50:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:50:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:51:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:51:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:52:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:52:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:53:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:53:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:54:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:54:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:55:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:55:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:56:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:56:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:57:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:57:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:58:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:58:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:59:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:59:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:00:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:00:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:01:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:01:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:02:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:02:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:03:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:03:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:04:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:04:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:05:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:05:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:06:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:06:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:07:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:07:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:08:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:08:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:09:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:09:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:10:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:10:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:11:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:11:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:12:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:12:37] "HEAD / HTTP/1.1" 200 0

server_stdout.txt

@@ -1,65 +1,8 @@
 Performing system checks...
 
 System check identified no issues (0 silenced).
-July 23, 2026 - 13:06:51
+July 24, 2026 - 12:30:18
 Django version 4.2.30, using settings 'tianxuan.settings'
-Starting development server at http://127.0.0.1:18766/
-Quit the server with CTRL-BREAK.
-
-Performing system checks...
-
-System check identified no issues (0 silenced).
-Performing system checks...
-
-System check identified no issues (0 silenced).
-Performing system checks...
-
-System check identified no issues (0 silenced).
-July 23, 2026 - 21:16:22
-Django version 4.2.30, using settings 'tianxuan.settings'
-Starting development server at http://0.0.0.0:8000/
-Quit the server with CTRL-BREAK.
-
-Performing system checks...
-
-System check identified no issues (0 silenced).
-Performing system checks...
-
-System check identified no issues (0 silenced).
-July 23, 2026 - 21:22:58
-Django version 4.2.30, using settings 'tianxuan.settings'
-Starting development server at http://0.0.0.0:8000/
-Quit the server with CTRL-BREAK.
-
-Performing system checks...
-
-System check identified no issues (0 silenced).
-July 23, 2026 - 21:37:07
-Django version 4.2.30, using settings 'tianxuan.settings'
-Starting development server at http://0.0.0.0:8000/
-Quit the server with CTRL-BREAK.
-
-Performing system checks...
-
-System check identified no issues (0 silenced).
-July 23, 2026 - 21:42:01
-Django version 4.2.30, using settings 'tianxuan.settings'
-Starting development server at http://0.0.0.0:8000/
-Quit the server with CTRL-BREAK.
-
-Performing system checks...
-
-System check identified no issues (0 silenced).
-July 23, 2026 - 21:45:23
-Django version 4.2.30, using settings 'tianxuan.settings'
-Starting development server at http://0.0.0.0:8000/
-Quit the server with CTRL-BREAK.
-
-Performing system checks...
-
-System check identified no issues (0 silenced).
-July 23, 2026 - 21:50:50
-Django version 4.2.30, using settings 'tianxuan.settings'
-Starting development server at http://0.0.0.0:8000/
+Starting development server at http://127.0.0.1:8765/
 Quit the server with CTRL-BREAK.
 

templates/analysis/cluster_detail.html

@@ -1,29 +1,27 @@
-{% extends 'base.html' %}
+{% extends 'base.html' %}
 {% load static %}
-{% block title %}Cluster #{{ cluster.cluster_label }} — Run #{{ run.display_id }}{% endblock %}
+{% block title %}簇 #{{ cluster.cluster_label }} — 运行 #{{ run.display_id }}{% endblock %}
 {% block content %}
 <style>
-.entity-card { border:1px solid #e0e0e0; border-radius:6px; padding:0.6rem; margin-bottom:0.4rem; background:#fafafa; }
-.entity-card .val { font-weight:600; font-size:0.85rem; }
-.entity-card .feat { font-size:0.75rem; color:#666; display:inline-block; margin-right:0.5rem; }
+.feat-table { width:100%; border-collapse:collapse; font-size:0.8rem; }
+.feat-table th { background:#f5f5f5; padding:0.3rem 0.5rem; text-align:left; font-weight:600; border-bottom:2px solid #ddd; white-space:nowrap; }
+.feat-table td { padding:0.25rem 0.5rem; border-bottom:1px solid #eee; }
+.feat-table td code { font-size:0.72rem; }
 .feat-positive { color:#2e7d32; }
 .feat-negative { color:#c62828; }
-.feat-table { width:100%; border-collapse:collapse; font-size:0.8rem; }
-.feat-table th { background:#f5f5f5; padding:0.3rem 0.5rem; text-align:left; font-weight:600; border-bottom:2px solid #ddd; }
-.feat-table td { padding:0.25rem 0.5rem; border-bottom:1px solid #eee; }
 </style>
 
 <div class="card">
-    <a href="{% url 'analysis:cluster_overview' run.display_id %}" style="color:#4361ee;text-decoration:none;font-size:0.9rem;">← Cluster Overview</a>
+    <a href="{% url 'analysis:cluster_overview' run.display_id %}" style="color:#4361ee;text-decoration:none;font-size:0.9rem;">← 聚类概览</a>
     <div style="display:flex;justify-content:space-between;align-items:center;flex-wrap:wrap;gap:0.5rem;margin-top:0.5rem;">
         <div>
-            <h2 style="margin:0;">Cluster #{{ cluster.cluster_label }}
-                {% if cluster.cluster_label == -1 %}<span class="badge badge-warning">Noise</span>{% endif %}
+            <h2 style="margin:0;">簇 #{{ cluster.cluster_label }}
+                {% if cluster.cluster_label == -1 %}<span class="badge badge-warning">噪声</span>{% endif %}
             </h2>
             <p style="margin:0.25rem 0 0;color:#666;font-size:0.9rem;">
-                {{ cluster.size }} entities
-                {% if cluster.proportion %} — {{ cluster.proportion|floatformat:1 }}% of total{% endif %}
-                {% if cluster.silhouette_score is not None %} — Silhouette: {{ cluster.silhouette_score|floatformat:4 }}{% endif %}
+                {{ cluster.size }} 个行
+                {% if cluster.proportion %} — {{ cluster.proportion|floatformat:1 }}% 占总{% endif %}
+                {% if cluster.silhouette_score is not None %} — 轮廓系数: {{ cluster.silhouette_score|floatformat:4 }}{% endif %}
             </p>
         </div>
     </div>
@@ -34,17 +32,17 @@
 
 <!-- ── Feature Analysis ── -->
 <div class="card">
-    <h3>📊 Feature Analysis <span style="font-weight:400;font-size:0.8rem;color:#888;">{{ features|length }} features</span></h3>
+    <h3>📊 特征分析 <span style="font-weight:400;font-size:0.8rem;color:#888;">{{ features|length }} 个特征</span></h3>
     {% if features %}
     <table class="feat-table">
         <thead>
             <tr>
                 <th>#</th>
-                <th>Feature</th>
-                <th>Dist. Score</th>
-                <th>Mean</th>
-                <th>Std</th>
-                <th>Median</th>
+                <th>特征名</th>
+                <th>区分度</th>
+                <th>均值</th>
+                <th>标准差</th>
+                <th>中位数</th>
                 <th>P25</th>
                 <th>P75</th>
             </tr>
@@ -55,17 +53,17 @@
                 <td>{{ forloop.counter }}</td>
                 <td><code>{{ f.feature_name }}</code></td>
                 <td><span class="{% if f.distinguishing_score > 0 %}feat-positive{% else %}feat-negative{% endif %}">{{ f.distinguishing_score|floatformat:3 }}</span></td>
-                <td>{{ f.mean|floatformat:3|default:"-" }}</td>
+                <td class="ts-val" data-col="{{ f.feature_name }}">{{ f.mean|floatformat:3|default:"-" }}</td>
                 <td>{{ f.std|floatformat:3|default:"-" }}</td>
-                <td>{{ f.median|floatformat:3|default:"-" }}</td>
-                <td>{{ f.p25|floatformat:3|default:"-" }}</td>
-                <td>{{ f.p75|floatformat:3|default:"-" }}</td>
+                <td class="ts-val" data-col="{{ f.feature_name }}">{{ f.median|floatformat:3|default:"-" }}</td>
+                <td class="ts-val" data-col="{{ f.feature_name }}">{{ f.p25|floatformat:3|default:"-" }}</td>
+                <td class="ts-val" data-col="{{ f.feature_name }}">{{ f.p75|floatformat:3|default:"-" }}</td>
             </tr>
             {% endfor %}
         </tbody>
     </table>
     {% else %}
-    <div class="empty-state"><p>No feature data for this cluster.</p></div>
+    <div class="empty-state"><p>该簇无特征数据</p></div>
     {% endif %}
 </div>
 
@@ -75,15 +73,15 @@
 {% with svd_features=features|slice:":5" %}
 {% if svd_features %}
 <div class="card">
-    <h3>🔬 SVD Key Features <span style="font-weight:400;font-size:0.8rem;color:#888;">Top 5 by distinguishing score</span></h3>
+    <h3>🔬 SVD 关键特征 <span style="font-weight:400;font-size:0.8rem;color:#888;">Top 5 按区分度</span></h3>
     <div style="display:grid;grid-template-columns:1fr 1fr;gap:0.5rem;">
     {% for f in svd_features %}
         <div style="background:#f8f9fa;border-radius:6px;padding:0.5rem;border:1px solid #e0e0e0;">
             <div style="font-weight:600;font-size:0.8rem;">{{ f.feature_name }}</div>
             <div style="font-size:0.75rem;color:#666;margin-top:0.2rem;">
                 Score: <span class="{% if f.distinguishing_score > 0 %}feat-positive{% else %}feat-negative{% endif %}">{{ f.distinguishing_score|floatformat:2 }}</span>
-                | Mean: {{ f.mean|floatformat:2 }}
-                | Std: {{ f.std|floatformat:2 }}
+                | 均值: {{ f.mean|floatformat:2 }}
+                | 标准差: {{ f.std|floatformat:2 }}
             </div>
         </div>
     {% endfor %}
@@ -93,28 +91,51 @@
 {% endwith %}
 {% endif %}
 
-<!-- ── Entity List ── -->
+<!-- ── Raw Data Rows ── -->
 <div class="card">
-    <h3>👤 Entities <span style="font-weight:400;font-size:0.8rem;color:#888;">{{ entities|length }} shown</span></h3>
-    <div id="entityList">
-    {% for e in entities %}
-        <div class="entity-card">
-            <div style="display:flex;justify-content:space-between;align-items:center;">
-                <span class="val">{{ e.entity_value }}</span>
-                <a href="{% url 'analysis:entity_profile' e.id %}" class="btn btn-sm" style="font-size:0.75rem;padding:0.2rem 0.6rem;background:#4361ee;color:#fff;text-decoration:none;border-radius:4px;">Profile</a>
-            </div>
-            {% if e.feature_json %}
-            <div style="margin-top:0.3rem;">
-                {% for k, v in e.feature_json.items|slice:":8" %}
-                <span class="feat"><strong>{{ k }}</strong>: {% if v is None %}-{% else %}{{ v|floatformat:2 }}{% endif %}</span>
+    <h3>📋 数据行 <span style="font-weight:400;font-size:0.8rem;color:#888;">{{ row_data|length }} 条</span></h3>
+    {% if row_data %}
+    <div style="overflow-x:auto;">
+    <table class="feat-table" style="font-size:0.75rem;">
+        <thead>
+            <tr>
+                <th>#</th>
+                {% for k, v in row_data.0.items %}
+                <th>{{ k }}</th>
                 {% endfor %}
-                {% if e.feature_json.items|length > 8 %}<span class="feat" style="color:#999;">+{{ e.feature_json.items|length|add:"-8" }} more</span>{% endif %}
+            </tr>
+        </thead>
+        <tbody>
+            {% for row in row_data %}
+            <tr>
+                <td>{{ forloop.counter }}</td>
+                {% for k, v in row.items %}
+                <td><code>{{ v|default:"-" }}</code></td>
+                {% endfor %}
+            </tr>
+            {% endfor %}
+        </tbody>
+    </table>
     </div>
+    {% else %}
+    <div class="empty-state"><p>该簇无数据行</p></div>
     {% endif %}
 </div>
-    {% empty %}
-        <div class="empty-state"><p>No entities in this cluster.</p></div>
-    {% endfor %}
-    </div>
-</div>
 {% endblock %}
+
+<script>
+(function(){
+var tsCols=["timestamp","time","epoch","ts","8dbd","created_at","updated_at","event_time","datetime"];
+document.querySelectorAll(".ts-val").forEach(function(el){
+var col=el.dataset.col,isTs=false;
+for(var i=0;i<tsCols.length;i++){if(col.toLowerCase().indexOf(tsCols[i])>=0){isTs=true;break;}}
+if(!isTs)return;
+var raw=parseFloat(el.textContent);
+if(isNaN(raw)||raw<1e8||raw>2e12)return;
+var d=new Date(raw*1000);
+if(d.getFullYear()<2000||d.getFullYear()>2100)return;
+el.textContent=d.toISOString().replace("T"," ").substring(0,16);
+el.title="epoch: "+raw;
+});
+})();
+</script>

templates/analysis/cluster_overview.html

@@ -1,17 +1,17 @@
-{% extends 'base.html' %}
+{% extends 'base.html' %}
 {% load static %}
 {% block title %}聚类概览 — 运行 #{{ run.display_id }}{% endblock %}
 {% block content %}
 <style>
 /* ── 布局 ── */
-.main-layout { display:flex; gap:0; transition:all 0.3s; }
+.main-layout { display:flex; gap:0; transition:all 0.3s; position:relative; }
 .left-panel { flex:1; min-width:0; transition:flex 0.3s; }
 .left-panel.globe-open { flex:0 0 55%; }
-.right-panel { flex:0 0 0; overflow:hidden; transition:flex 0.3s; border-left:1px solid #e0e0e0; position:relative; }
+.right-panel { flex:0 0 0; overflow:hidden; transition:flex 0.3s; border-left:1px solid #e0e0e0; }
 .right-panel.globe-open { flex:0 0 45%; }
-.globe-toggle-handle { position:absolute; left:-24px; top:50%; transform:translateY(-50%); width:24px; height:48px; background:#1a1a2e; color:#fff; border:none; border-radius:6px 0 0 6px; cursor:pointer; display:flex; align-items:center; justify-content:center; font-size:14px; z-index:10; transition:background 0.15s; }
+.globe-toggle-handle { position:absolute; z-index:10; top:50%; left:calc(100% - 24px); transform:translateY(-50%); width:24px; height:48px; background:#1a1a2e; color:#fff; border:none; border-radius:6px 0 0 6px; cursor:pointer; display:flex; align-items:center; justify-content:center; font-size:14px; transition:left 0.3s, background 0.15s; }
 .globe-toggle-handle:hover { background:#4361ee; }
-.globe-toggle-handle.open { left:0; border-radius:0; }
+.globe-toggle-handle.open { left:calc(55% - 24px); border-radius:6px 0 0 6px; }
 /* ── Scatter ── */
 .scatter-wrapper { position:relative; cursor:crosshair; }
 .scatter-wrapper canvas { width:100%; height:400px; display:block; }
@@ -61,7 +61,7 @@
 <!-- ── UMAP 2D / 3D ── -->
 <div class="card">
     <div style="display:flex;justify-content:space-between;align-items:center;flex-wrap:wrap;gap:0.5rem;">
-        <h3 style="margin:0;">UMAP 嵌入 <span style="font-weight:400;font-size:0.8rem;color:#888;"><script>document.write(scatterCount)</script> 个点</span></h3>
+        <h3 style="margin:0;">UMAP 嵌入 <span style="font-weight:400;font-size:0.8rem;color:#888;" id="pointCount">计算中...</span></h3>
         <span id="viewToggle" style="display:none;">
             <button id="btn2D" class="btn btn-sm" style="font-weight:bold;padding:2px 10px;">2D</button>
             <button id="btn3D" class="btn btn-sm btn-outline" style="padding:2px 10px;">3D</button>
@@ -120,9 +120,9 @@
 
 <!-- ── Right Panel: Globe ── -->
 <div class="right-panel" id="rightPanel">
-    <button class="globe-toggle-handle" id="globeHandle" onclick="toggleGlobe()">◀</button>
-    <div id="globeContainer"></div>
+    <iframe id="globeIframe" src="/globe/?embed=1&data={{ selected_ds_id|default:'' }}" style="width:100%;height:100%;border:none;"></iframe>
 </div>
+<button class="globe-toggle-handle" id="globeHandle" onclick="toggleGlobe()">◀</button>
 </div><!-- /main-layout -->
 
 <!-- ── Bottom Detail Panel ── -->
@@ -137,6 +137,7 @@
 <script>
 const scatterData = {{ scatter_data_json|safe }};
 const scatterCount = scatterData.length;
+document.getElementById('pointCount').textContent = scatterCount + ' 个点';
 const hasZ = {{ has_z|yesno:"true,false" }};
 const colors = ['#4361ee','#f72585','#7209b7','#4cc9f0','#f8961e','#43aa8b','#f94144','#577590','#e9c46a','#9b5de5',
     '#1f77b4','#ff7f0e','#2ca02c','#d62728','#9467bd','#8c564b','#e377c2','#7f7f7f','#bcbd22','#17becf',
@@ -144,8 +145,7 @@ const colors = ['#4361ee','#f72585','#7209b7','#4cc9f0','#f8961e','#43aa8b','#f9
     '#393b79','#637939','#8c6d31','#843c39','#7b4173','#5254a3','#8ca252','#bd9e39','#ad494a','#a55194'];
 
 let selectedCluster = null, selectedPoint = null;
-let globeOpen = false, globeInitialized = false, globeRenderer = null;
-let globeFlows = {{ globe_flows_json|safe }};
+let globeOpen = false;
 const clusterSizes = {{ cluster_sizes_json|safe }};
 
 // Color pills after DOM ready
@@ -258,7 +258,10 @@ function selectCluster(label) {
         document.getElementById('detailPanel').classList.add('open');
     }
     // Gray out globe if open
-    if (globeOpen && window.setGlobeFilter) window.setGlobeFilter(label);
+    if (globeOpen) {
+        var ifr = document.getElementById('globeIframe');
+        if (ifr && ifr.contentWindow) ifr.contentWindow.postMessage({ type: 'globe_filter', cluster_label: label }, '*');
+    }
 }
 
 function clearClusterFilter() {
@@ -267,15 +270,14 @@ function clearClusterFilter() {
     document.querySelectorAll('.cluster-card').forEach(el => el.classList.remove('selected'));
     document.querySelectorAll('.pill').forEach(el => el.classList.remove('active'));
     document.getElementById('detailPanel').classList.remove('open');
-    if (globeOpen && window.setGlobeFilter) window.setGlobeFilter(null);
+    if (globeOpen) {
+        var ifr = document.getElementById('globeIframe');
+        if (ifr && ifr.contentWindow) ifr.contentWindow.postMessage({ type: 'globe_filter', cluster_label: null }, '*');
+    }
 }
 
 function closeDetail() { document.getElementById('detailPanel').classList.remove('open'); }
 
-// ══════════════════════════════════════════════════════════
-//  GLOBE SIDEBAR
-// ══════════════════════════════════════════════════════════
-
 function toggleGlobe() {
     globeOpen = !globeOpen;
     document.getElementById('leftPanel').classList.toggle('globe-open', globeOpen);
@@ -283,122 +285,6 @@ function toggleGlobe() {
     var handle = document.getElementById('globeHandle');
     handle.innerHTML = globeOpen ? '▶' : '◀';
     handle.classList.toggle('open', globeOpen);
-    if (globeOpen && !globeInitialized) initGlobe();
-    if (globeOpen) setTimeout(function() { if (globeRenderer) globeRenderer.setSize(document.getElementById('globeContainer').clientWidth, document.getElementById('globeContainer').clientHeight); }, 350);
-}
-
-// ══════════════════════════════════════════════════════════
-//  3D THREE.JS GLOBE
-// ══════════════════════════════════════════════════════════
-
-function initGlobe() {
-    if (globeInitialized) return;
-    const container = document.getElementById('globeContainer');
-    if (!container) return;
-    const W = container.clientWidth || 500, H = container.clientHeight || 500;
-
-    const scene = new THREE.Scene();
-    scene.background = new THREE.Color(0x0a0a1a);
-
-    const camera = new THREE.PerspectiveCamera(45, W / H, 0.1, 100);
-    camera.position.set(0, 0, 5.5);
-
-    const renderer = new THREE.WebGLRenderer({ antialias: true });
-    renderer.setSize(W, H);
-    renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
-    container.appendChild(renderer.domElement);
-    globeRenderer = renderer;
-
-    // Earth sphere
-    const earthGeo = new THREE.SphereGeometry(1.8, 64, 64);
-    var canvas = document.createElement('canvas');
-    canvas.width = 1; canvas.height = 1;
-    var ectx = canvas.getContext('2d');
-    ectx.fillStyle = '#1a3a5c'; ectx.fillRect(0, 0, 1, 1);
-    var tex = new THREE.CanvasTexture(canvas);
-    const earthMat = new THREE.MeshPhongMaterial({ map: tex, transparent: true, opacity: 0.9 });
-    const earth = new THREE.Mesh(earthGeo, earthMat);
-    scene.add(earth);
-
-    // Atmosphere glow
-    const glowGeo = new THREE.SphereGeometry(1.85, 64, 64);
-    const glowMat = new THREE.MeshBasicMaterial({ color: 0x224488, transparent: true, opacity: 0.15 });
-    scene.add(new THREE.Mesh(glowGeo, glowMat));
-
-    // Lights
-    scene.add(new THREE.AmbientLight(0x404060));
-    const sun = new THREE.DirectionalLight(0xffffff, 1);
-    sun.position.set(5, 3, 5);
-    scene.add(sun);
-    scene.add(new THREE.DirectionalLight(0x4488ff, 0.3).position.set(-5, -3, -5));
-
-    // Arc group
-    const arcGroup = new THREE.Group();
-    scene.add(arcGroup);
-
-    // Build arcs from flow data
-    const tlsColors = { 'TLSv1.3': 0x4cc9f0, 'TLSv1.2': 0x43aa8b, 'TLSv1.1': 0xf8961e, 'TLSv1.0': 0xf94144 };
-    var arcMeshes = [];
-
-    globeFlows.forEach(function(flow) {
-        const from = latLonToVec3(flow.slat, flow.slon, 1.8);
-        const to = latLonToVec3(flow.dlat, flow.dlon, 1.8);
-        if (!from || !to) return;
-        const mid = new THREE.Vector3().addVectors(from, to).multiplyScalar(0.5).normalize().multiplyScalar(2.6);
-        const curve = new THREE.QuadraticBezierCurve3(from, mid, to);
-        const tubeGeo = new THREE.TubeGeometry(curve, 20, 0.008, 4, false);
-        const color = tlsColors[flow.tls] || 0x888888;
-        const mat = new THREE.MeshBasicMaterial({ color: color, transparent: true, opacity: 0.5 });
-        const mesh = new THREE.Mesh(tubeGeo, mat);
-        mesh.userData = flow;
-        arcGroup.add(mesh);
-        arcMeshes.push(mesh);
-    });
-
-    // Stars
-    const starGeo = new THREE.BufferGeometry();
-    const starPos = new Float32Array(2000 * 3);
-    for (let i = 0; i < 2000 * 3; i++) starPos[i] = (Math.random() - 0.5) * 100;
-    starGeo.setAttribute('position', new THREE.BufferAttribute(starPos, 3));
-    scene.add(new THREE.Points(starGeo, new THREE.PointsMaterial({ color: 0xffffff, size: 0.05 })));
-
-    // Mouse rotation
-    let isDragging = false, prevMouse = { x: 0, y: 0 };
-    renderer.domElement.addEventListener('mousedown', function(e) { isDragging = true; prevMouse = { x: e.clientX, y: e.clientY }; });
-    window.addEventListener('mouseup', function() { isDragging = false; });
-    window.addEventListener('mousemove', function(e) {
-        if (!isDragging) return;
-        const dx = e.clientX - prevMouse.x, dy = e.clientY - prevMouse.y;
-        earth.rotation.y += dx * 0.005;
-        earth.rotation.x += dy * 0.005;
-        arcGroup.rotation.y = earth.rotation.y;
-        arcGroup.rotation.x = earth.rotation.x;
-        prevMouse = { x: e.clientX, y: e.clientY };
-    });
-
-    // Globe filter function (called when cluster selected)
-    window.setGlobeFilter = function(clusterLabel) {
-        arcMeshes.forEach(function(mesh) {
-            if (clusterLabel === null) { mesh.material.opacity = 0.5; mesh.material.color.setHex(tlsColors[mesh.userData.tls] || 0x888888); }
-            else { mesh.material.opacity = 0.08; mesh.material.color.setHex(0x666666); }
-        });
-    };
-
-    function animate() {
-        requestAnimationFrame(animate);
-        // Auto-rotate when not dragging
-        if (!isDragging) { earth.rotation.y += 0.002; arcGroup.rotation.y = earth.rotation.y; }
-        renderer.render(scene, camera);
-    }
-    animate();
-    globeInitialized = true;
-}
-
-function latLonToVec3(lat, lon, r) {
-    if (lat == null || lon == null) return null;
-    const phi = (90 - lat) * Math.PI / 180;
-    const theta = (lon + 180) * Math.PI / 180;
-    return new THREE.Vector3(-r * Math.sin(phi) * Math.cos(theta), r * Math.cos(phi), r * Math.sin(phi) * Math.sin(theta));
 }
 
 // ══════════════════════════════════════════════════════════

templates/analysis/run_detail.html

@@ -1,38 +1,91 @@
 {% extends 'base.html' %}
-{% block title %}Run #{{ run.display_id }} - TLS Analyzer{% endblock %}
+{% block title %}运行 #{{ run.display_id }} - 天璇{% endblock %}
 {% block content %}
+<style>
+.svd-card { background:#f8f9fa; border:1px solid #e0e0e0; border-radius:8px; padding:0.8rem; }
+.svd-card h4 { margin:0 0 0.3rem; font-size:0.9rem; }
+.var-bar { height:8px; border-radius:4px; background:#4361ee; margin:0.2rem 0; transition:width 0.3s; }
+.feat-tag { display:inline-block; background:#e8f0fe; padding:0.1rem 0.4rem; border-radius:3px; font-size:0.75rem; margin:0.1rem; }
+</style>
+
 <div class="card">
-    <h2>Run #{{ run.display_id }} — {{ run.created_at|date:"Y-m-d H:i" }}</h2>
-    <p>Status: <span class="badge {% if run.status == 'completed' %}badge-success{% elif run.status == 'failed' %}badge-danger{% else %}badge-info{% endif %}">{{ run.get_status_display }}</span></p>
-    <p>CSV: <code>{% if run.sqlite_table %}已存储至SQLite: {{ run.sqlite_table }}{% else %}{{ run.csv_glob }}{% endif %}</code></p>
+    <div style="display:flex;justify-content:space-between;align-items:center;flex-wrap:wrap;gap:0.5rem;">
+        <div>
+            <h2 style="margin:0;">运行 #{{ run.display_id }}</h2>
+            <p style="margin:0.25rem 0 0;color:#666;font-size:0.9rem;">
+                {{ run.created_at|date:"Y-m-d H:i" }}
+                — 状态: <span class="badge {% if run.status == 'completed' %}badge-success{% elif run.status == 'failed' %}badge-danger{% else %}badge-info{% endif %}">{{ run.get_status_display }}</span>
+            </p>
+        </div>
+        <a href="{% url 'analysis:cluster_overview' run.display_id %}" class="btn btn-primary" style="font-size:0.85rem;">聚类概览 →</a>
+    </div>
+    {% if run.error_message %}
+    <pre style="background:#1a1a2e;color:#e0e0e0;padding:0.5rem;border-radius:4px;font-size:0.78rem;margin-top:0.5rem;max-height:200px;overflow:auto;"><code>{{ run.error_message }}</code></pre>
+    {% endif %}
 </div>
 
 <div class="grid-3">
     <div class="card" style="text-align:center;">
         <div class="stat">{{ run.total_flows|default:"0" }}</div>
-        <div class="stat-label">Total Flows</div>
+        <div class="stat-label">总流数</div>
     </div>
     <div class="card" style="text-align:center;">
         <div class="stat">{{ run.entity_count|default:"0" }}</div>
-        <div class="stat-label">Entities</div>
+        <div class="stat-label">行数</div>
     </div>
     <div class="card" style="text-align:center;">
         <div class="stat">{{ run.cluster_count|default:"0" }}</div>
-        <div class="stat-label">Clusters</div>
+        <div class="stat-label">聚类数</div>
     </div>
 </div>
 
+<!-- ── Global SVD Features ── -->
+{% if svd_features %}
 <div class="card">
-    <h2>Clusters</h2>
-    <a href="{% url 'analysis:cluster_overview' run.display_id %}" class="btn btn-primary" style="margin-bottom:1rem;">View Cluster Overview</a>
+    <div style="display:flex;justify-content:space-between;align-items:center;">
+        <h3>🔬 全域 SVD 特征分析</h3>
+        <span style="font-size:0.8rem;color:#888;">基于 {{ run.total_flows }} 行数据</span>
+    </div>
+    <p style="font-size:0.82rem;color:#666;margin-top:0.3rem;">奇异值分解 (TruncatedSVD) 降维后的各主成分解释方差比例与贡献特征。</p>
+    <div style="display:grid;grid-template-columns:1fr 1fr;gap:0.75rem;margin-top:0.75rem;">
+    {% for svd in svd_features %}
+        <div class="svd-card">
+            <h4>PC{{ svd.component }}</h4>
+            <div style="display:flex;justify-content:space-between;font-size:0.78rem;color:#555;">
+                <span>解释方差: {{ svd.explained_var }}%</span>
+                <span>累积: {{ svd.cumulative_var }}%</span>
+            </div>
+            <div class="var-bar" style="width:{{ svd.explained_var }}%;"></div>
+            <div style="margin-top:0.4rem;">
+            {% for f in svd.top_features %}
+                <span class="feat-tag">{{ f.feature }}: {{ f.strength }}</span>
+            {% endfor %}
+            </div>
+        </div>
+    {% endfor %}
+    </div>
+</div>
+{% endif %}
+
+<!-- ── LLM 自动分析流程 ── -->
+{% if run.run_type == 'auto' and llm_timeline_json %}
+<div class="card">
+    <h3>🧠 LLM 自动分析流程</h3>
+    <div id="llmTimeline"></div>
+</div>
+{% endif %}
+
+<!-- ── 聚类列表 ── -->
+<div class="card">
+    <h3>📊 聚类结果</h3>
     {% if clusters %}
     <table>
         <thead>
             <tr>
-                <th>Cluster</th>
-                <th>Size</th>
-                <th>Proportion</th>
-                <th>Silhouette</th>
+                <th>簇</th>
+                <th>大小</th>
+                <th>比例</th>
+                <th>轮廓系数</th>
                 <th></th>
             </tr>
         </thead>
@@ -43,13 +96,83 @@
                 <td>{{ c.size }}</td>
                 <td>{{ c.proportion|floatformat:2 }}</td>
                 <td>{{ c.silhouette_score|floatformat:4|default:"-" }}</td>
-                <td><a href="{% url 'analysis:cluster_detail' run.display_id c.cluster_label %}" class="btn btn-primary">Detail</a></td>
+                <td><a href="{% url 'analysis:cluster_detail' run.display_id c.cluster_label %}" class="btn btn-primary">详情</a></td>
             </tr>
             {% endfor %}
         </tbody>
     </table>
     {% else %}
-    <div class="empty-state"><p>No clusters yet.</p></div>
+    <div class="empty-state"><p>无聚类结果</p></div>
     {% endif %}
 </div>
+
+{% if run.run_type == 'auto' and llm_timeline_json %}
+<script>
+const llmData = {{ llm_timeline_json|safe }};
+(function() {
+    const container = document.getElementById('llmTimeline');
+    if (!container) return;
+
+    function escapeHtml(s) { if (s==null) return ''; return String(s).replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;').replace(/"/g,'&quot;'); }
+    function prettyJson(o) { try { return JSON.stringify(o,null,2); } catch(e) { return String(o); } }
+
+    var thinkingByStep = {}, thinkOrder = [];
+    if (llmData.thinking) {
+        var parts = llmData.thinking.split(/\n(?=\[\d+\])/);
+        for (var pi = 0; pi < parts.length; pi++) {
+            var m = parts[pi].match(/^\[(\d+)\]\s*(.*)/s);
+            if (m) {
+                var step = parseInt(m[1]);
+                if (!(step in thinkingByStep)) thinkOrder.push(step);
+                thinkingByStep[step] = (thinkingByStep[step]||'')+(thinkingByStep[step]?'\n':'')+m[2].trim();
+            }
+        }
+    }
+    var toolCalls = llmData.tool_calls || [];
+    var toolMap = {};
+    for (var ti = 0; ti < toolCalls.length; ti++) {
+        var tc = toolCalls[ti];
+        if (!toolMap[tc.step]) toolMap[tc.step] = [];
+        toolMap[tc.step].push(tc);
+    }
+    var allSteps = new Set(thinkOrder.concat(Object.keys(toolMap).map(Number)));
+    var sortedSteps = Array.from(allSteps).sort(function(a,b){return a-b;});
+
+    for (var si = 0; si < sortedSteps.length; si++) {
+        var stp = sortedSteps[si];
+        var hdr = document.createElement('div');
+        hdr.style.cssText = 'font-weight:700;font-size:0.85rem;color:#4361ee;padding:0.6rem 0 0.3rem;border-top:1px solid #eee;';
+        hdr.textContent = '步骤 ' + stp;
+        container.appendChild(hdr);
+        if (thinkingByStep[stp] !== undefined) {
+            var tdiv = document.createElement('div');
+            tdiv.style.cssText = 'background:#eef2ff;padding:0.5rem;border-radius:6px;font-size:0.78rem;line-height:1.6;color:#333;max-height:300px;overflow:auto;margin-bottom:0.3rem;border:1px solid #d0d8f0;';
+            tdiv.innerHTML = '<span style="font-weight:600;color:#4361ee;font-size:0.75rem;display:block;margin-bottom:0.25rem;">💭 思考</span>' + escapeHtml(thinkingByStep[stp]);
+            container.appendChild(tdiv);
+        }
+        if (toolMap[stp]) {
+            for (var tj = 0; tj < toolMap[stp].length; tj++) {
+                var tc2 = toolMap[stp][tj];
+                var card = document.createElement('div');
+                card.style.cssText = 'border:1px solid #e0e0e0;border-radius:6px;margin-bottom:0.3rem;overflow:hidden;background:#fff;';
+                var inputStr = prettyJson(tc2.input);
+                var outputStr = prettyJson(tc2.output);
+                card.innerHTML =
+                    '<div onclick="this.classList.toggle(\'open\');var b=this.nextElementSibling;if(b)b.classList.toggle(\'open\');" style="padding:0.4rem 0.8rem;cursor:pointer;display:flex;justify-content:space-between;align-items:center;background:#f8f9fa;user-select:none;">' +
+                    '<span><span style="background:#43aa8b;color:#fff;border-radius:3px;padding:0 5px;font-size:0.7rem;font-weight:600;margin-right:6px;">🔧</span>' +
+                    '<span style="font-weight:600;font-size:0.8rem;">' + escapeHtml(tc2.name||'?') + '</span></span>' +
+                    '<span style="font-size:0.75rem;color:#888;">▶</span></div>' +
+                    '<div style="display:none;padding:0.5rem 0.8rem;background:#fafbfc;">' +
+                    '<div style="font-weight:600;font-size:0.72rem;color:#555;margin-bottom:0.2rem;">输入:</div>' +
+                    '<pre style="background:#1a1a2e;color:#e0e0e0;padding:0.5rem;border-radius:4px;font-size:0.7rem;max-height:200px;overflow:auto;margin:0 0 0.3rem;"><code>' + escapeHtml(inputStr) + '</code></pre>' +
+                    '<div style="font-weight:600;font-size:0.72rem;color:#555;margin-bottom:0.2rem;">输出:</div>' +
+                    '<pre style="background:#1a1a2e;color:#e0e0e0;padding:0.5rem;border-radius:4px;font-size:0.7rem;max-height:200px;overflow:auto;margin:0;"><code>' + escapeHtml(outputStr) + '</code></pre>' +
+                    '</div>';
+                container.appendChild(card);
+            }
+        }
+    }
+})();
+</script>
+{% endif %}
 {% endblock %}

templates/tianxuan/auto.html

@@ -428,72 +428,6 @@ async function applyFilter() {
 }
 
 
-// ── Dataset table rendering ──
-function buildDatasetTable(filter) {
-    const container = document.getElementById('datasetTable');
-    let rows = DATASETS_WITH_STATUS.filter(ds => {
-        if (filter === 'all') return true;
-        if (filter === 'ready') return ds.badge_class === 'ready';
-        if (filter === 'analyzing') return ds.badge_class === 'analyzing';
-        if (filter === 'completed') return ds.badge_class === 'completed';
-        if (filter === 'failed') return ds.badge_class === 'failed';
-        return true;
-    });
-
-    if (!rows.length) {
-        container.innerHTML = '<p style="text-align:center;color:#999;padding:1rem;">无匹配的数据集</p>';
-        return;
-    }
-
-    let html = '<table class="ds-table"><thead><tr><th></th><th>ID</th><th>状态</th><th>行数</th><th>文件数</th><th>来源</th></tr></thead><tbody>';
-    rows.forEach(ds => {
-        const label = ds.status_label || '待分析';
-        const badgeClass = ds.badge_class || 'ready';
-        const dispId = ds.display_id || ds.dataset_id;
-        const rowCount = ds.row_count != null ? ds.row_count.toLocaleString() : '?';
-        const fileCount = ds.file_count != null ? ds.file_count : '?';
-        const source = ds.sqlite_table ? ('SQLite: ' + ds.sqlite_table) : (ds.csv_glob || '?');
-        const checked = (selectedRunId === String(ds.dataset_id)) ? 'checked' : '';
-        html += '<tr data-ds-id="' + ds.dataset_id + '" onclick="selectRow(\'' + ds.dataset_id + '\')" class="' + (selectedRunId === String(ds.dataset_id) ? 'selected' : '') + '">' +
-            '<td><input type="radio" name="auto_run_id" value="' + ds.dataset_id + '" ' + checked + ' onclick="event.stopPropagation();selectRow(\'' + ds.dataset_id + '\')"></td>' +
-            '<td style="font-weight:600;">#' + dispId + '</td>' +
-            '<td><span class="status-badge ' + badgeClass + '">' + label + '</span></td>' +
-            '<td>' + rowCount + '</td>' +
-            '<td>' + fileCount + '</td>' +
-            '<td style="max-width:180px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">' + escapeHtml(source) + '</td>' +
-        '</tr>';
-    });
-    html += '</tbody></table>';
-    container.innerHTML = html;
-}
-
-function selectRow(dsId) {
-    selectedRunId = String(dsId);
-    // Update radio buttons and row highlight
-    document.querySelectorAll('#datasetTable input[name="auto_run_id"]').forEach(r => {
-        r.checked = (r.value === String(dsId));
-    });
-    document.querySelectorAll('#datasetTable tr').forEach(tr => tr.classList.remove('selected'));
-    const row = document.querySelector('#datasetTable tr[data-ds-id="' + dsId + '"]');
-    if (row) row.classList.add('selected');
-}
-
-function setDatasetFilter(filter) {
-    activeFilter = filter;
-    document.querySelectorAll('#statusFilter .filter-btn').forEach(btn => {
-        btn.classList.toggle('active', btn.dataset.filter === filter);
-    });
-    buildDatasetTable(filter);
-}
-
-// Init filter tabs
-document.querySelectorAll('#statusFilter .filter-btn').forEach(btn => {
-    btn.addEventListener('click', function() {
-        setDatasetFilter(this.dataset.filter);
-    });
-});
-buildDatasetTable('all');
-
 // ── Timeline rendering ──
 let statusPoll = null;
 
@@ -502,6 +436,44 @@ function escapeHtml(str) {
     return String(str).replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;').replace(/"/g,'&quot;');
 }
 
+function renderMarkdown(text) {
+    if (!text) return '';
+    // Escape HTML first
+    var html = escapeHtml(text);
+    // Code blocks (```...```) — must be before inline code
+    html = html.replace(/```(\w*)\n?([\s\S]*?)```/g, '<pre style="background:#1a1a2e;color:#e0e0e0;padding:0.5rem;border-radius:4px;font-size:0.78rem;overflow:auto;margin:0.4rem 0;"><code>$2</code></pre>');
+    // Inline code
+    html = html.replace(/`([^`]+)`/g, '<code style="background:#1a1a2e;color:#e0e0e0;padding:0.1rem 0.3rem;border-radius:3px;font-size:0.78rem;">$1</code>');
+    // Bold (**text** or __text__)
+    html = html.replace(/\*\*([^*]+)\*\*/g, '<strong>$1</strong>');
+    html = html.replace(/__([^_]+)__/g, '<strong>$1</strong>');
+    // Italic (*text* or _text_)
+    html = html.replace(/\*([^*]+)\*/g, '<em>$1</em>');
+    html = html.replace(/(?<!_)_{1}(?!_)([^_]+)_{1}(?!_)/g, '<em>$1</em>');
+    // Links [text](url)
+    html = html.replace(/\[([^\]]+)\]\(([^)]+)\)/g, '<a href="$2" target="_blank" style="color:#4361ee;">$1</a>');
+    // Headers (# ## ###)
+    html = html.replace(/^### (.+)$/gm, '<h4 style="margin:0.5rem 0 0.3rem;font-size:0.95rem;">$1</h4>');
+    html = html.replace(/^## (.+)$/gm, '<h3 style="margin:0.5rem 0 0.3rem;font-size:1rem;">$1</h3>');
+    html = html.replace(/^# (.+)$/gm, '<h2 style="margin:0.5rem 0 0.3rem;font-size:1.1rem;">$1</h2>');
+    // Blockquotes
+    html = html.replace(/^&gt;\s?(.+)$/gm, '<blockquote style="border-left:3px solid #4361ee;padding:0.3rem 0.6rem;margin:0.3rem 0;background:#f0f4ff;border-radius:0 4px 4px 0;">$1</blockquote>');
+    // Unordered lists
+    html = html.replace(/^[\s]*[-*+]\s+(.+)$/gm, '<li style="margin:0.15rem 0;">$1</li>');
+    html = html.replace(/(<li[\s\S]*?<\/li>)\n(?!<li)/g, function(m) { return '<ul style="padding-left:1.5rem;margin:0.3rem 0;">' + m + '</ul>'; });
+    // Ordered lists
+    html = html.replace(/^\d+\.\s+(.+)$/gm, '<li style="margin:0.15rem 0;">$1</li>');
+    // Horizontal rules
+    html = html.replace(/^---$/gm, '<hr style="border:none;border-top:1px solid #ddd;margin:0.5rem 0;">');
+    // Paragraphs (double newlines → paragraphs)
+    html = html.replace(/\n\n/g, '</p><p style="margin:0.4rem 0;">');
+    // Line breaks (single newlines → br)
+    html = html.replace(/\n/g, '<br>');
+    // Wrap in paragraph if not already wrapped
+    if (!html.startsWith('<')) html = '<p style="margin:0.4rem 0;">' + html + '</p>';
+    return html;
+}
+
 function prettyJson(obj) {
     try { return JSON.stringify(obj, null, 2); }
     catch(e) { return String(obj); }
@@ -596,18 +568,18 @@ function renderTimeline(entries) {
         if (entry.type === 'thinking') {
             // 思考条目 — 浅蓝背景文本块
             const div = document.createElement('div');
-            div.style.cssText = 'background:#eef2ff;padding:0.6rem;border-radius:6px;font-size:0.78rem;line-height:1.6;color:#333;max-height:200px;overflow:auto;margin-bottom:0.3rem;white-space:pre-wrap;border:1px solid #d0d8f0;';
+            div.style.cssText = 'background:#eef2ff;padding:0.6rem;border-radius:6px;font-size:0.78rem;line-height:1.6;color:#333;max-height:300px;overflow:auto;margin-bottom:0.3rem;border:1px solid #d0d8f0;';
             div.innerHTML =
                 '<span style="font-weight:600;color:#4361ee;font-size:0.75rem;display:block;margin-bottom:0.3rem;">💭 思考</span>' +
-                escapeHtml(entry.text);
+                renderMarkdown(entry.text);
             container.appendChild(div);
         } else if (entry.type === 'answer') {
-            // 回答条目 — 绿色背景
+            // 回答条目 — 绿色背景，支持 Markdown
             const div = document.createElement('div');
-            div.style.cssText = 'background:#e8f5e9;padding:0.6rem;border-radius:6px;font-size:0.78rem;line-height:1.6;color:#333;max-height:300px;overflow:auto;margin-bottom:0.3rem;white-space:pre-wrap;border:1px solid #c8e6c9;';
+            div.style.cssText = 'background:#e8f5e9;padding:0.6rem 0.8rem;border-radius:6px;font-size:0.82rem;line-height:1.7;color:#333;max-height:400px;overflow:auto;margin-bottom:0.3rem;border:1px solid #c8e6c9;';
             div.innerHTML =
                 '<span style="font-weight:600;color:#2e7d32;font-size:0.75rem;display:block;margin-bottom:0.3rem;">✅ 回答</span>' +
-                escapeHtml(entry.text);
+                renderMarkdown(entry.text);
             container.appendChild(div);
         } else if (entry.type === 'tool') {
             const card = document.createElement('div');

templates/tianxuan/globe_embed.html

@@ -0,0 +1,141 @@
+{% load static %}
+<!DOCTYPE html>
+<html lang="zh-hans">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width,initial-scale=1.0">
+<title>3D Globe</title>
+<style>
+* { margin:0; padding:0; box-sizing:border-box; }
+body { background:#0a0a1a; overflow:hidden; width:100vw; height:100vh; }
+#globeContainer { width:100%; height:100%; }
+</style>
+</head>
+<body>
+<div id="globeContainer"></div>
+<script src="{% static 'tianxuan/three.min.js' %}"></script>
+<script>
+const geoFlows = {{ geo_flows|safe }};
+const W = window.innerWidth, H = window.innerHeight;
+
+var scene = new THREE.Scene();
+var camera = new THREE.PerspectiveCamera(45, W/H, 0.1, 1000);
+camera.position.set(0, 2, 12);
+
+var renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true });
+renderer.setSize(W, H);
+renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
+document.getElementById('globeContainer').appendChild(renderer.domElement);
+
+// Earth
+var earthGeo = new THREE.SphereGeometry(5, 64, 64);
+var earthMat = new THREE.MeshPhongMaterial({ color: 0x1a3a5c, emissive: 0x0a1a2a, specular: new THREE.Color(0x333333), shininess: 5 });
+var earth = new THREE.Mesh(earthGeo, earthMat);
+scene.add(earth);
+
+// Glow
+var glow = new THREE.Mesh(new THREE.SphereGeometry(5.1, 64, 64), new THREE.MeshBasicMaterial({ color: 0x224488, transparent: true, opacity: 0.1 }));
+scene.add(glow);
+
+// Lights
+scene.add(new THREE.AmbientLight(0x222244));
+var dl = new THREE.DirectionalLight(0xffffff, 1);
+dl.position.set(5, 10, 7);
+scene.add(dl);
+scene.add(new THREE.DirectionalLight(0x4488ff, 0.3));
+
+// Stars
+var starGeo = new THREE.BufferGeometry();
+var starPos = new Float32Array(1000 * 3);
+for (var i = 0; i < 1000 * 3; i++) starPos[i] = (Math.random() - 0.5) * 200;
+starGeo.setAttribute('position', new THREE.BufferAttribute(starPos, 3));
+scene.add(new THREE.Points(starGeo, new THREE.PointsMaterial({ color: 0xffffff, size: 0.15 })));
+
+// Lat/lon grid
+var gridMat = new THREE.LineBasicMaterial({ color: 0x6699cc, transparent: true, opacity: 0.1 });
+for (var lat = -80; lat <= 80; lat += 20) {
+    var pts = [];
+    for (var lon = 0; lon <= 360; lon += 5) {
+        var phi = (90 - lat) * Math.PI / 180;
+        var theta = (lon + 180) * Math.PI / 180;
+        pts.push(new THREE.Vector3(-5.02 * Math.sin(phi) * Math.cos(theta), 5.02 * Math.cos(phi), 5.02 * Math.sin(phi) * Math.sin(theta)));
+    }
+    scene.add(new THREE.Line(new THREE.BufferGeometry().setFromPoints(pts), gridMat));
+}
+for (var lon = 0; lon < 360; lon += 20) {
+    var pts = [];
+    for (var lat = -90; lat <= 90; lat += 5) {
+        var phi = (90 - lat) * Math.PI / 180;
+        var theta = (lon + 180) * Math.PI / 180;
+        pts.push(new THREE.Vector3(-5.02 * Math.sin(phi) * Math.cos(theta), 5.02 * Math.cos(phi), 5.02 * Math.sin(phi) * Math.sin(theta)));
+    }
+    scene.add(new THREE.Line(new THREE.BufferGeometry().setFromPoints(pts), gridMat));
+}
+
+// Flow arcs
+var tlsColors = { 'TLSv1.3': 0x4cc9f0, 'TLSv1.2': 0x43aa8b, 'TLSv1.1': 0xf8961e, 'TLSv1.0': 0xf94144 };
+var flowGroup = new THREE.Group();
+scene.add(flowGroup);
+
+function latLonToVec3(lat, lon, r) {
+    if (lat == null || lon == null) return null;
+    var phi = (90 - lat) * Math.PI / 180;
+    var theta = (lon + 180) * Math.PI / 180;
+    return new THREE.Vector3(-r * Math.sin(phi) * Math.cos(theta), r * Math.cos(phi), r * Math.sin(phi) * Math.sin(theta));
+}
+
+var arcMeshes = [];
+var flowData = Array.isArray(geoFlows) ? geoFlows : (typeof geoFlows === 'string' ? JSON.parse(geoFlows) : []);
+flowData.forEach(function(flow) {
+    var from = latLonToVec3(flow.slat, flow.slon, 5);
+    var to = latLonToVec3(flow.dlat, flow.dlon, 5);
+    if (!from || !to) return;
+    var mid = new THREE.Vector3().addVectors(from, to).multiplyScalar(0.5).normalize().multiplyScalar(7);
+    var curve = new THREE.QuadraticBezierCurve3(from, mid, to);
+    var color = tlsColors[flow.tls] || 0x888888;
+    var mat = new THREE.MeshBasicMaterial({ color: color, transparent: true, opacity: 0.6 });
+    var mesh = new THREE.Mesh(new THREE.TubeGeometry(curve, 20, 0.02, 4, false), mat);
+    mesh.userData = flow;
+    flowGroup.add(mesh);
+    arcMeshes.push(mesh);
+});
+
+// Mouse rotation
+var isDragging = false, prevMouse = { x: 0, y: 0 };
+renderer.domElement.addEventListener('mousedown', function(e) { isDragging = true; prevMouse = { x: e.clientX, y: e.clientY }; });
+window.addEventListener('mouseup', function() { isDragging = false; });
+window.addEventListener('mousemove', function(e) {
+    if (!isDragging) return;
+    var dx = e.clientX - prevMouse.x, dy = e.clientY - prevMouse.y;
+    earth.rotation.y += dx * 0.005;
+    earth.rotation.z += dy * 0.005;
+    flowGroup.rotation.y = earth.rotation.y;
+    flowGroup.rotation.z = earth.rotation.z;
+    prevMouse = { x: e.clientX, y: e.clientY };
+});
+
+// Listen for cluster filter from parent
+window.addEventListener('message', function(e) {
+    if (e.data && e.data.type === 'globe_filter') {
+        var label = e.data.cluster_label;
+        arcMeshes.forEach(function(mesh) {
+            if (label === null) {
+                mesh.material.opacity = 0.6;
+                mesh.material.color.setHex(tlsColors[mesh.userData.tls] || 0x888888);
+            } else {
+                mesh.material.opacity = 0.06;
+                mesh.material.color.setHex(0x666666);
+            }
+        });
+    }
+});
+
+// Auto-rotate
+(function animate() {
+    requestAnimationFrame(animate);
+    if (!isDragging) { earth.rotation.y += 0.003; flowGroup.rotation.y = earth.rotation.y; }
+    renderer.render(scene, camera);
+})();
+</script>
+</body>
+</html>

tianxuan/llm_orchestrator.py

@@ -314,8 +314,10 @@ def run_llm_pipeline(dataset_id: str, entity_col: str = "",
     # Track tool call history for smarter guidance
     called_tools: set[str] = set()
     consecutive_errors = 0
+    step = 0
+    max_steps = 999
 
-    for step in range(25):
+    while step < max_steps:
         resp = call_llm(msgs, config)
         if resp is None:
             if callback:
@@ -328,6 +330,9 @@ def run_llm_pipeline(dataset_id: str, entity_col: str = "",
 
         msg = resp["choices"][0]["message"]
         content = msg.get("content", "")
+        # DeepSeek reasoning fallback (reasoning_content for chain-of-thought)
+        if not content:
+            content = msg.get("reasoning_content", "") or ""
 
         # Pass thinking text to callback
         if callback:
@@ -365,18 +370,9 @@ def run_llm_pipeline(dataset_id: str, entity_col: str = "",
             logger.info(f'[TOOL] step={step} name={name} args={json.dumps(args)[:200]}')
 
             try:
-                async def _call_with_timeout():
-                    return await asyncio.wait_for(
-                        handle_call(name, args),
-                        timeout=120,
-                    )
-                result = asyncio.run(_call_with_timeout())
-            except asyncio.TimeoutError:
-                result = {'error': f'Tool {name} timed out after 120s'}
-                logger.warning(f'[TOOL] {name} timed out')
+                result = asyncio.run(handle_call(name, args))
             except Exception as e:
                 result = {'error': f'MCP tool error: {e}'}
-                logger.error(f'[TOOL] {name} error: {e}')
 
             if callback:
                 callback(step, name, result, meta={"args": args})
@@ -414,6 +410,8 @@ def run_llm_pipeline(dataset_id: str, entity_col: str = "",
                     "content": f"Tool '{name}' failed: {result['error'][:200]}. Call '{suggested}' to diagnose, then retry with corrected parameters.",
                 })
 
+        step += 1
+
     if callback:
-        callback(25, "max_steps", None)
-    return {"error": "Reached max 25 tool calls", "steps": 25}
+        callback(max_steps, "max_steps", None)
+    return {"error": f"Reached max {max_steps} tool calls", "steps": max_steps}