refactor(types): ColumnType.distance() + ColumnProcessor dispatch, distance.py→TYPE_REGISTRY delegation

- globe.py: add streaming=True to all lf.head().collect() calls (3 places)
- entities.py: chunked aggregation (100K batches) for datasets > 1M rows
- constants.py: reduce MAX_SAMPLE_ROWS 50000->30000 for 8GB RAM
- tools/__init__.py: add __all__ for '*' re-export (fix tool_registry import)
- clustering.py: fix misplaced logger/import traceback syntax errors
- analysis.py: fix indentation of pass statement
- anomalies.py: fix misplaced import traceback syntax error

.gitignore

@@ -40,3 +40,15 @@ RUNTIME_CHANGED.txt
 # Test data (auto-generated CSV files)
 data/*.csv
 data/multi_upload/
+
+# User config (contains API keys)
+config/config.yaml
+
+data/test_csvs/
+
+data/geoip_chunks/
+*.mmdb.gz
+*.mmdb
+
+# Playwright MCP sessions
+.playwright-mcp/

.server_pid

@@ -0,0 +1 @@
+11572

AGENTS.md

@@ -1,4 +1,4 @@
-# 天璇 (TianXuan) -Agent Knowledge Base
+# 天璇 (TianXuan) — Agent Knowledge Base
 
 ## Project Identity
 
@@ -6,462 +6,365 @@
 |-------|-------|
 | 项目名称 | 天璇 (TianXuan) |
 | 原始名称 | tls-analyzer |
-| 核心功能 | TLS 流数据分析、实体画像、聚类3D地球可视化、PCA散点图|
+| 核心功能 | TLS 流数据分析、实体画像、聚类 3D 地球可视化、UMAP/SVD 降维散点图 |
 | Python 版本 | 3.12 (embedded portable at `runtime/python/python.exe`) |
 | Polars 版本 | 1.42.1 (精确锁定) |
 | 数据框架 | Polars (LazyFrame + streaming) |
-| 机器学习 | scikit-learn (HDBSCAN, KMeans, UMAP, TruncatedSVD, StandardScaler) |
+| 机器学习 | scikit-learn (HDBSCAN, KMeans, MiniBatchKMeans, IsolationForest), UMAP, TruncatedSVD, StandardScaler |
-| Web框架 | Django 4.2 + SQLite WAL |
+| Web 框架 | Django 4.2 + SQLite WAL |
-| 前端3D | Three.js (603KB, 离线, 地球贴图1.4MB) |
+| 前端 3D | Three.js (603KB, 离线, 地球贴图 1.4MB) |
-| 前端图表 | 纯Canvas散点图(无Chart.js) |
+| 前端图表 | 纯 Canvas 2D 散点图 / Canvas 地理分布 |
-| LLM协议 | MCP (stdio transport) |
+| LLM 协议 | MCP (stdio transport) + OpenAI 兼容 API |
-| 启动方式 | `run.bat`（双击即用）/ `runtime\python\python.exe manage.py runserver`（开发） |
+| 启动方式 | `run.bat` (双击即用, 非阻塞, 打印 PID 后退出) / `runtime\python\python.exe manage.py runserver` (开发) |
-| 目标设备 | Windows 10 1902, 4GB RAM, 无独立显卡（纯核显） |
+| 目标设备 | Windows 10 1909+, 8GB RAM, 无独立显卡 (核显), 支持 2500 CSV × 10000 行 × 50~58 列 |
 
 ## Project Structure
 
 ```
-tianxuan/
+天璇/
-├── tianxuan/ # Django 项目配置
+├── tianxuan/                     Django 项目配置
-│ ├── settings.py # ALLOWED_HOSTS 自动检测 LOGGING 文件+stderr
+│   ├── settings.py               ALLOWED_HOSTS 自动检测, LOGGING 文件+stderr
-│ ├── wsgi.py # SQLite 启动自修复
+│   ├── wsgi.py                   SQLite 启动自修复
-│  ├── urls.py
+│   ├── urls.py                   路由汇总
-│ └── llm_orchestrator.py # LLM 编排(32B/284B 双兼容
+│   └── llm_orchestrator.py       LLM 编排 (策略式提示词, 自动错误恢复)
-│ ├── analysis/ # 核心分析模块 (Django app)
+├── analysis/                     核心分析模块 (Django app)
-│ ├── data_loader.py # CSV 加载, BOM检测 schema_strict, 递归glob, 中文路径
+│   ├── views/                    视图包 (12 模块, 原 views.py 拆分)
-│ ├── data_profiler.py # 列统计+ 相关性矩阵(无pandas回退:numpy)
+│   │   ├── __init__.py           重导出所有视图函数
-│ ├── entity_detector.py # 实体列自动检测(tuple关键词+ unique_ratio + null惩罚)
+│   │   ├── helpers.py            辅助函数
-│ ├── entity_aggregator.py # 实体聚合 (tuple关键词 lat/lon检测 多列group_by, IP子网)
+│   │   ├── dashboard.py          首页 + 运行记录
-│ ├── session_store.py # 线程安全单例内存存储
+│   │   ├── pipeline.py           管道执行器
-│ ├── tool_registry.py # 12个MCP工具 + DB持久化(sync_to_async)
+│   │   ├── clustering.py         聚类概览/详情/地球数据
-│ ├── mcp_server.py # MCP stdio server
+│   │   ├── entity.py             实体画像页
-│ ├── views.py # 所有Django视图 (上传/手动/LLM/配置/LLM测试/日志/地球)
+│   │   ├── upload.py             CSV 上传
-│ ├── urls.py # 16条路由 ├── models.py # ORM: AnalysisRun, ClusterResult, EntityProfile, ClusterFeature
+│   │   ├── manual.py             手动分析工作流构建器
-│ ├── data_validator.py # 列校验(缺失异常值IP有效性
+│   │   ├── auto.py               LLM 自动分析
-│ ├── type_classifier.py # 值优先类型检测(MAC/端口/IPv4/URL/HEX/ENUM/LAT_LON)
+│   │   ├── globe.py              3D 地球数据接口
-│ ├── geoip.py # GeoIP 经纬度查询(data/geoip_data.txt)
+│   │   ├── config.py             配置编辑 + LLM 测试
-│ ├── admin.py
+│   │   ├── log_viewer.py         日志查看器
-│ └── management/commands/
+│   │   └── tools.py              工具实验室
-│ ├── start_mcp.py # MCP服务器启动命令
+│   ├── tools/                    工具包 (17 模块, 30 个 MCP 工具实现)
-│  └── run_pipeline.py # 一键CLI管道命令
+│   │   ├── __init__.py           重导出所有 handler
-│ ├── config/
+│   │   ├── _registry.py          工具元数据 (Tool descriptor + get_tools_meta)
-│ ├── config.yaml # 自动生成默认配置 (含entity.subnet_masks, columns覆盖)
+│   │   ├── _dispatch.py          工具调用分发 (handle_call)
-│ ├── loader.py # Pydantic 配置加载 + 文件修改检测
+│   │   ├── _helpers.py           共享辅助 (过滤表达式构建/响应截断/数据集决议/数值列检测)
-│  └── __init__.py
+│   │   ├── load_data.py          工具 1: load_data
-│ ├── templates/
+│   │   ├── profile.py            工具 2: profile_data
-│ ├── base.html # 导航: 首页/上传/手动/LLM/记录/地球/配置
+│   │   ├── filter.py             工具 3: filter_data
-│ └── analysis/ 
+│   │   ├── preprocess.py         工具 4: preprocess_data
-│ tianxuan/ # 全部11个页面模板
+│   │   ├── clustering.py         工具 5+8: run_clustering, filter_and_cluster
-│      ├── dashboard.html # 首页 - 最近运行
+│   │   ├── evaluate.py           工具 6: evaluate_clustering
-│      ├── upload.html # CSV上传 - 拖拽+多文件删除+进度
+│   │   ├── features.py           工具 7: extract_features (+ UMAP 嵌入)
-│ ├── manual.html # 3步手动向- 选数据→设参数→运行
+│   │   ├── entities.py           工具 9+10: build_entity_profiles, compute_scores
-│ ├── auto.html # LLM自动分析 - 实时日志取消
+│   │   ├── anomalies.py          工具 11+12: detect_anomalies, visualize_anomalies
-│ ├── config.html # 配置编辑 - LLM连通性测试
+│   │   ├── export.py             工具 13: export_results
-│      ├── run_list.html # 运行记录列表
+│   │   ├── analysis.py           工具 14-19: analyze_patterns/temporal/fft/tls_health/geo_distribution/entity_detail
-│ ├── run_detail.html # 运行详情摘要
+│   │   ├── diagnostics.py        工具 20-26: validate_data/explore_distributions/find_outliers/diagnose_clustering/compare_datasets/export_debug_sample/repair_schema
-│ ├── cluster_overview.html # 聚类概览 - 纯Canvas散点PCA+地理
+│   │   ├── data_mgmt.py          工具 27-29: list_datasets/drop_dataset/clone_dataset
-│ ├── cluster_detail.html # 簇详情- 特征实体列表
+│   │   └── distance_matrix.py    工具 30: compute_distance_matrix
-│ ├── entity_profile.html # 实体画像 - 特征偏离+Canvas单点
+│   ├── tool_registry.py          旧编排层 (薄包装, 重导出 tools/)
-│      ├── globe.html # 3D地球可视- Three.js 流量弧线+经纬国境
+│   ├── mcp_server.py             MCP stdio server
-│  └── log_viewer.html # 日志查看
+│   ├── data_loader.py            CSV 加载底层 (BOM/编码/schema 容错/递归 glob/ZIP 解压)
-│ ├── static/tianxuan/ # 离线静态资源
+│   ├── data_profiler.py          列统计 + 相关性矩阵
-│  ├── three.min.js # Three.js 3D引擎 (603KB)
+│   ├── data_validator.py         列校验 (缺失/异常值/IP 有效性)
-│ ├── earth_atmos_2048.jpg # 地球贴图 (1.4MB)
+│   ├── type_classifier.py        值优先类型检测 (MAC/端口/IPv4/URL/HEX/ENUM/LAT_LON/BOOL_ENUM/TIMESTAMP)
-│ └── world_borders.js # 10国精简国境线轮廓 ├── runtime/python/ # 便携Python 3.12运行时(~593MB)
+│   ├── geoip.py                  GeoIP 经纬度查询
-│ ├── scripts/
+│   ├── ip_clustering.py          IP 子网聚类与转换
-│ ├── gen_test_data.py # 简单测试数据生成(--globe模式使用真实GeoIP范围)
+│   ├── session_store.py          线程安全单例内存存储
-│ ├── gen_complex_test.py # 复杂数据生成 (5000行4列含55%缺失)
+│   ├── distance.py               距离计算 (IP 子网/地理/Levenshtein/Hamming/FFT 相位)
-│ ├── column_survey.py # CSV列结构调查(统计各CSV的列类型/分布)
+│   ├── models.py                 ORM: AnalysisRun, ClusterResult, EntityProfile, ClusterFeature
-│ ├── diagnose_compare.py # 跨机日志对比诊断
+│   ├── urls.py                   路由注册
-│ └── debug_db.py # DB持久化调试 ├── tests/ # 92个测试(unit) / 121个测试(all)
+│   ├── nl_describe.py            自然语言数据描述
-│ ├── test_data_loader.py # 26: BOM/schema/中文路径/递归glob
+│   ├── profile_util.py           画像工具函数
-│ ├── test_entity.py # 17: 关键词检测聚合
+│   ├── tls_ref.py                TLS 版本/加密套件参考数据
-│ ├── test_e2e.py # 6: 全端到端
+│   ├── db_utils.py               SQLite WAL 工具 + 锁重试
-│ └── test_clustering_edge.py # HDBSCAN零样本降级 ├── docs/
+│   ├── constants.py              全局常量
-│ ├── 工作流.md # 完整分析流水线文档
+│   ├── appdata.py                应用数据路径 (%APPDATA%/TianXuan)
-│  └── 故障诊断手册.md # 跨机问题诊断指南
+│   └── management/commands/
-│ ├── run.bat # 用户启动 (含PYTHONUTF8=1)
+│       ├── start_mcp.py          MCP 服务器启动命令
-├── shell.bat # Django shell
+│       ├── run_pipeline.py       一键 CLI 管道命令
-├── TlsDB.csv # 参考表头名清单（非实际数据├── README.md
+│       └── import_tlsdb.py       TLS 数据库导入
+├── config/                       配置管理
+│   ├── config.yaml               自动生成默认配置 (entity, server, data, clustering, llm)
+│   ├── loader.py                 Pydantic 配置加载 + 文件修改检测 + 缓存
+│   └── __init__.py
+├── templates/                    页面模板 (13 个 HTML)
+│   ├── base.html                 导航: 首页/上传/手动/LLM/记录/地球/配置
+│   ├── analysis/                 分析页面 (6 个)
+│   │   ├── dashboard.html        首页 — 最近运行
+│   │   ├── run_list.html         运行记录列表
+│   │   ├── run_detail.html       运行详情摘要
+│   │   ├── cluster_overview.html 聚类概览 — Canvas 散点图 (PC1/PC2) + 地理分布 + Silhouette
+│   │   ├── cluster_detail.html   簇详情 — 特征实体列表
+│   │   └── entity_profile.html   实体画像 — 特征偏离 + Canvas 单点
+│   └── tianxuan/                 功能页面 (6 个)
+│       ├── upload.html           CSV 上传 — 拖拽+多文件删除+进度
+│       ├── manual.html           手动分析 — 工作流构建器 (添加/排序/保存/执行)
+│       ├── auto.html             LLM 自动分析 — 实时日志, thinking panel, 工具调用折叠
+│       ├── globe.html            3D 地球 — Three.js 流量弧线 + 经纬线 + 国境线
+│       ├── config.html           配置编辑 — LLM 连通性测试
+│       └── log_viewer.html       日志查看
+├── static/tianxuan/              离线静态资源
+│   ├── three.min.js              Three.js 3D 引擎 (603KB)
+│   ├── earth_atmos_2048.jpg      地球贴图 (1.4MB)
+│   └── world_borders.js          177 国完整国境线 (269KB)
+├── runtime/python/               便携 Python 3.12 运行时 (~593MB)
+├── scripts/                      工具脚本
+│   ├── gen_test_data.py          简单测试数据生成
+│   ├── gen_complex_test.py       复杂数据生成 (5000 行 含 55% 缺失)
+│   ├── column_survey.py          CSV 列结构调查
+│   ├── diagnose_compare.py       跨机日志对比诊断
+│   └── debug_db.py               DB 持久化调试
+├── tests/                        测试
+│   ├── test_e2e_full.py          全端到端测试
+│   ├── test_e2e_batch.py         批量端到端测试
+│   ├── start_srv.py              测试服务器启动辅助
+│   └── _test_hdbscan.py          HDBSCAN 零样本降级测试
+├── data/                         数据目录
+│   └── geoip_data.txt            GeoIP 离线数据库
+├── logs/                         日志输出
+│   └── tianxuan.log              主日志文件
+├── docs/
+│   ├── 工作流.md                  完整分析流水线文档
+│   └── 故障诊断手册.md            跨机问题诊断指南
+├── run.bat                       用户启动入口 (含 PYTHONUTF8=1)
+├── shell.bat                     Django shell
+├── update.bat                    增量更新
+├── rollback.bat                  更新回滚
+├── build.bat                     构建打包
+├── TlsDB.csv                     参考表头名清单
+├── README.md
 ├── AGENTS.md
 └── manage.py
 ```
 
-## 12 MCP Tools
+## 30 MCP Tools
 
-| # | 工具| 功能 | 必需参数 |
+系统现有 **30 个 MCP 工具**, 分四层, 实现于 `analysis/tools/` 包的 14 个 handler 模块:
-|---|--------|------|----------|
-| 1 | `load_data` | 加载 CSV 文件（glob / 递归 / schema容错| `csv_glob` |
-| 2 | `profile_data` | 数据集概要统+ 相关性矩阵| `dataset_id` |
-| 3 | `filter_data` | 按条件过滤（12操作+ AND/OR| `dataset_id`, `filters` |
-| 4 | `preprocess_data` | 预处理（标准化编码/填充)| `dataset_id`, `columns` |
-| 5 | `run_clustering` | 执行聚类（HDBSCAN/KMeans + 质量评估| `dataset_id`, `cluster_columns` |
-| 6 | `evaluate_clustering` | 评估聚类质量（Silhouette/DB/CH| `cluster_result_id`, `dataset_id` |
-| 7 | `extract_features` | 提取各聚类区分特征（Z-Score/ANOVA| `dataset_id`, `cluster_result_id` |
-| 8 | `export_results` | 导出结果到磁盘（CSV/Parquet/JSON| `result_id`, `output_path` |
-| 9 | `list_datasets` | 列出所有活跃数据集 | |
-| 10 | `drop_dataset` | 删除数据集释放内存| `dataset_id` |
-| 11 | `clone_dataset` | 浅拷贝数据集（不复制数据| `dataset_id` |
-| 12 | `build_entity_profiles` | 自动检测实体列 + 聚合画像 | `dataset_id` |
 
-所有工具通过 `analysis/tool_registry.py` 注册，`analysis/mcp_server.py` 暴露标准 MCP stdio 接口
+### ⚡ 核心工具 (14 个) — 执行分析, 可修改数据
 
-## Config.Entity 
+| # | 工具 | 实现模块 | 功能 | 必需参数 |
+|---|------|---------|------|----------|
+| 1 | `load_data` | tools/load_data.py | 加载 CSV 文件 (glob/递归/ZIP 解压/schema 容错) | `csv_glob` |
+| 2 | `profile_data` | tools/profile.py | 数据集概要统计 + 相关性矩阵 | `dataset_id` |
+| 3 | `filter_data` | tools/filter.py | 按条件过滤 (12 操作符 + AND/OR 逻辑) | `dataset_id`, `filters` |
+| 4 | `preprocess_data` | tools/preprocess.py | 预处理 (StandardScaler/OneHot/填充/列删除) | `dataset_id`, `columns` |
+| 5 | `run_clustering` | tools/clustering.py | 执行聚类 (HDBSCAN/KMeans + 自动特征过滤 + 质量评估) | `dataset_id`, `cluster_columns` |
+| 6 | `evaluate_clustering` | tools/evaluate.py | 评估聚类质量 (Silhouette/DB/CH/噪声比/簇大小) | `cluster_result_id`, `dataset_id` |
+| 7 | `extract_features` | tools/features.py | 提取各聚类区分特征 (Z-Score/ANOVA) + UMAP-2D 嵌入 | `dataset_id`, `cluster_result_id` |
+| 8 | `filter_and_cluster` | tools/clustering.py | 一步完成: 过滤 + 自动选数值列 + 聚类 | `dataset_id`, `filters` |
+| 9 | `build_entity_profiles` | tools/entities.py | 自动检测实体列 + 聚合特征 (多列复合键支持) | `dataset_id` |
+| 10 | `compute_scores` | tools/entities.py | 自适应代理/异常/威胁评分 (SVD 权重 + IQR 阈值) | `dataset_id` |
+| 11 | `detect_anomalies` | tools/anomalies.py | Isolation Forest 异常检测 (分块处理大数据集) | `dataset_id` |
+| 12 | `visualize_anomalies` | tools/anomalies.py | UMAP-2D 嵌入 + 聚类可视化 (按风险等级/聚类着色) | `dataset_id` |
+| 13 | `export_results` | tools/export.py | 导出结果到磁盘 (CSV/Parquet/JSON) | `result_id`, `output_path` |
+| - | `export_entities_json` | tools/export.py | 导出实体画像为 JSON (附加工具) | `dataset_id`, `output_path` |
 
-`config/loader.py` 中的 Pydantic 模型
+### 🔍 分析工具 (6 个) — 只读深潜, 安全随时调用
+
+| # | 工具 | 实现模块 | 功能 |
+|---|------|---------|------|
+| 14 | `analyze_patterns` | tools/analysis.py | 流量模式: top 源/目标 IP、端口/TLS 版本/协议分布 |
+| 15 | `analyze_temporal` | tools/analysis.py | 时间维度流量分析: 小时/日流量计数、繁忙/空闲时段 |
+| 16 | `analyze_fft` | tools/analysis.py | FFT 频谱分析: 周期模式、频率特征、周期性评分 |
+| 17 | `analyze_tls_health` | tools/analysis.py | TLS 安全态势: 旧版比例、弱加密、证书问题、SNI 异常 |
+| 18 | `analyze_geo_distribution` | tools/analysis.py | 地理分布: top 来源国家、不可能旅行、ISP 多样性 |
+| 19 | `analyze_entity_detail` | tools/analysis.py | 单实体深度调查: 流/版本/目标/时间模式/异常评分 |
+
+### 🩺 诊断工具 (7 个) — 排查问题
+
+| # | 工具 | 实现模块 | 功能 |
+|---|------|---------|------|
+| 20 | `validate_data` | tools/diagnostics.py | 数据质量: schema 一致性、缺失率、列类型冲突 |
+| 21 | `explore_distributions` | tools/diagnostics.py | 列分布统计: 唯一值、空值率、min/max/mean/std、直方图 |
+| 22 | `find_outliers` | tools/diagnostics.py | IQR 方法检测数值列统计异常值 |
+| 23 | `diagnose_clustering` | tools/diagnostics.py | 聚类效果差时诊断: 方差/相关分析 + 参数建议 |
+| 24 | `compare_datasets` | tools/diagnostics.py | 对比两数据集: schema 差异、行数变化、值分布偏移 |
+| 25 | `export_debug_sample` | tools/diagnostics.py | 导出原始数据样本 JSON 供外部调试 |
+| 26 | `repair_schema` | tools/diagnostics.py | 修复 schema 不匹配: 列名对齐、大小写合并、填充缺失 |
+
+### 📦 数据管理 (3 个)
+
+| # | 工具 | 实现模块 | 功能 |
+|---|------|---------|------|
+| 27 | `list_datasets` | tools/data_mgmt.py | 列出所有活跃数据集和结果 |
+| 28 | `drop_dataset` | tools/data_mgmt.py | 删除数据集释放内存 (支持 dataset/cluster/feature 三种类型) |
+| 29 | `clone_dataset` | tools/data_mgmt.py | 浅拷贝数据集 (LazyFrame 查询计划, 不复制数据) |
+
+### 🧠 LLM 驱动 (1 个)
+
+| # | 工具 | 实现模块 | 功能 |
+|---|------|---------|------|
+| 30 | `compute_distance_matrix` | tools/distance_matrix.py | LLM 编写 Python 距离函数, 逐行执行并返回评分摘要 |
+
+**工具注册流程**: `tools/_registry.py` (Tool 元数据) → `tools/_dispatch.py` (分发到对应 handler) → `analysis/mcp_server.py` (暴露 MCP stdio 接口)
+
+### 手动分析工作流
+
+手动分析页面 (manual.html) 为**工作流构建器**:
+- 添加/移除/重排步骤
+- 每步选择工具 + 填写参数
+- **保存/加载/删除**方案 (存储于 `.omo/plans/<name>.json`)
+- 单步执行 / **全部执行** (自动串行)
+- 执行结果显示在每一步下面
+
+### 标准分析流程
+
+```
+load_data → filter_data → profile_data → build_entity_profiles → compute_scores → run_clustering → extract_features
+```
+
+LLM 自动模式由 `tianxuan/llm_orchestrator.py` 驱动, 策略式提示词: 先 profile 了解数据 → 根据数据决策 → 失败自动诊断 → 重复调用检测 + 自动错误恢复。
+
+## Config.Entity
+
+`config/loader.py` 中的 Pydantic 模型:
 
 ```python
 class Entity(BaseModel):
- subnet_masks: list[int] = [] # 子网掩码列表，如 [24, 28]
+    subnet_masks: list[int] = []                     # 子网掩码列表, 如 [24, 28]
- ip_columns: list[str] = ['src_ip', 'dst_ip'] # 需聚合的IP列```
+    ip_columns: list[str] = ['src_ip', 'dst_ip']     # 需聚合的 IP 列
+```
 
 config.yaml 默认配置:
 ```yaml
 entity:
- subnet_masks: [24, 28]
+  subnet_masks: [24, 28]
- ip_columns: ["src_ip", "dst_ip"]
+  ip_columns: ["src_ip", "dst_ip"]
 ```
 
+IP 子网聚合逻辑位于 `analysis/ip_clustering.py`, `tools/entities.py::_handle_build_entity_profiles` 在 group_by 前对 IP 列自动应用子网掩码。
+
 ---
 
 ## Architecture
 
+- **views 包拆分**: 原单体 `views.py` (2000+ 行) 拆分为 12 个模块: dashboard, pipeline, clustering, entity, upload, manual, auto, globe, config, log_viewer, tools, helpers
+- **tools 包拆分**: 原 `tool_registry.py` 拆分为 17 个模块 (14 个 handler + 3 个基础设施: _registry/_dispatch/_helpers)
 - **tuple 替代 frozenset**: 避免跨机 PYTHONHASHSEED 差异
-- **PYTHONUTF8=1**: 跨机编码一致性- **类型安全聚合**: 聚合前检查dtype，非数值列cast
+- **PYTHONUTF8=1**: 跨机编码一致性 (run.bat 内置)
-- **sync_to_async ORM**: 异步上下文中Django ORM
+- **类型安全聚合**: 聚合前检查 dtype, 非数值列 cast
-- **纯Canvas 图表**: 零外部JS 依赖
+- **sync_to_async ORM**: 异步上下文中 Django ORM 安全调用
-- **上传目录隔离**: %APPDATA%/TianXuan/data/uploads/，与项目路径无关
+- **纯 Canvas 图表**: 零外部 JS 依赖 (散点图/地理分布/Silhouette 柱状图)
-- **通用数值清理_coerce_to_float**: 处理 + / - /空白/N/A 等伪值- **值优先类型检测*: config→值→名称→STRING
+- **上传目录隔离**: `%APPDATA%/TianXuan/data/uploads/`, 与项目路径无关
-- **地球深度测试**: depthTest: true, depthWrite: false，r=5.5
+- **通用数值清理 _coerce_to_float**: 处理 `+` / `-` / 空白 / N/A 等伪值
-- **完整国境线*: Natural Earth 110m, 177国 286多边形 269KB
+- **值优先类型检测**: config → 值 → 名称 → STRING (type_classifier.py)
+- **地球深度测试**: depthTest: true, depthWrite: false, r=5.5
+- **完整国境线**: Natural Earth 110m, 177 国 286 多边形 269KB
+- **UMAP 降维**: extract_features 中为实体数据计算 UMAP-2D 嵌入, 存储到 EntityProfile 表 (embedding_x/embedding_y), 训练 10K 样本 + 批量变换 1K 批次
+- **TruncatedSVD**: 聚类时特征 > 50 维自动降维; compute_scores 中学习自适应评分权重
+- **LLM 驱动距离计算**: compute_distance_matrix 工具允许 LLM 编写 Python 函数在受限命名空间 (math, numpy) 中逐行执行
+- **距离计算多策略**: tools/distance_matrix.py 中的 compute_distance_matrix 和 analysis/distance.py 中的 IP 子网/地理/Levenshtein/Hamming/FFT 相位距离
 
 ---
 
 ## Testing Protocol
 
-### 运行全部测试
+### 运行测试
 
 ```bash
-# 单元测试（不需要Django)runtime\python\python.exe -m pytest tests/test_data_loader.py tests/test_type_classifier.py -q
+# 端到端测试 (自动启动后端、模拟前端行为、监控 stderr)
+runtime\python\python.exe tests\test_e2e_full.py
+runtime\python\python.exe tests\test_e2e_batch.py
 
-# 集成测试（自动启动后端、模拟前端行为、监控stderr报错)runtime\python\python.exe tests/test_integration.py
+# HDBSCAN 零样本降级测试
+runtime\python\python.exe tests\_test_hdbscan.py
 ```
 
-```
+### 跨机一致性测试
-92 passed (unit tests); ALL INTEGRATION TESTS PASSED
-```
-
-> ⚠️ **重要**: `tests/test_integration.py` 模拟前端HTML行为（上传CSV 轮询状态访问页面 删除）> **当前端HTML/API发生变化时，必须同步更新此脚*，否则集成测试将不再反映真实用户流程
->
-### 跨机一致性测```bash
 
+```bash
 # 两台机器各自执行
-
 runtime\python\python.exe manage.py run_pipeline data\complex_test.csv
 
 # 对比日志
-
 runtime\python\python.exe scripts\diagnose_compare.py log_a.txt log_b.txt
-
 ```
 
-### DB持久化测```bash
+### DB 持久化测试
+
+```bash
 runtime\python\python.exe scripts\debug_db.py
 # 期望: n_features=30 db_saved=True
-# 验证: run_analysis ClusterFeature 表有数据
+# 验证: AnalysisRun, ClusterFeature 表有数据
 ```
 
-### LLM编排测试
+### LLM 编排测试
 
 ```bash
 runtime\python\python.exe scripts\test_llm_full.py
-# 或 runtime\python\python.exe -c "from tianxuan.llm_orchestrator import run_llm_pipeline, LLMConfig; ..."
 ```
 
 ### 生成测试数据
 
 ```bash
-# 简单数据runtime\python\python.exe scripts\gen_test_data.py --rows 1000
+# 简单数据
+runtime\python\python.exe scripts\gen_test_data.py --rows 1000
 
-# Globe模式 (IP在真实GeoIP范围)
+# Globe 模式 (IP 在真实 GeoIP 范围)
 runtime\python\python.exe scripts\gen_test_data.py --globe
 
-# 复杂数据 (5000行4列含55%缺失)
+# 复杂数据 (5000 行 4 列含 55% 缺失)
 runtime\python\python.exe scripts\gen_complex_test.py --rows 5000
 ```
 
-### 列结构调查```bash
+### 列结构调查
 
+```bash
 runtime\python\python.exe scripts\column_survey.py --csv "data/*.csv"
-
 ```
 
 ---
 
 ## Polars Version Compatibility
 
-| API | 旧版(<1.0) | 新版(.0) |
+| API | 旧版 (<1.0) | 新版 (>=1.0) |
 |-----|-----------|-----------|
 | mode | `pl.mode()` | `pl.col(col).mode()` |
 | encoding | `encoding='utf-8'` | `encoding='utf8'` |
-| value_counts col | `'index'` | 原列名|
+| value_counts col | `'index'` | 原列名 |
 | streaming | `collect(streaming=True)` | `collect(engine='streaming')` |
 | truth value | `if series:` | `if series.item():` |
 
 ## 32B LLM Optimization
 
-- Step guidance: 每步告诉模型下一步做什么- Truncation: 工具结果截断1200 chars
+- **Step guidance**: 每步告诉模型下一步做什么, 减少 32B 模型推理发散
-- Timeout: 90s (32B 推理更慢)
+- **Truncation**: 工具结果截断 1200 chars (避免上下文窗口溢出)
-- Max steps: 15
+- **Timeout**: 90s (32B 推理更慢)
-- Tool description: 一行简洁描述
+- **Max steps**: 15
----
+- **Tool description**: 一行简洁描述 (中文)
-
+- **策略式提示词**: 先 profile → 根据数据决策 → 失败自动诊断 → 重复调用检测 + 自动错误恢复
-## v1.1.3 修复 (2026-07-21)
+- **工具结果自动截断**: `_truncate_response()` 确保单次返回不超过 64 KB, 长列表/矩阵自动压缩
-
-13 issues fixed across build system, frontend, pipeline, and data handling.
-
-| # | 问题 | 状态| 详情 |
-|---|------|------|------|
-| 1 | 增量包构建修复| | update_info.json含SHA256、文件在files/子目录下、测试数据排除、__delete__支持 |
-| 2 | 构建顺序+三段版本号| | 先commit后bump再tag、bump_version支持vX.Y.Z、update.bat加PYTHONUTF8 |
-| 3 | 前端导航跳转 | | 上传后→手动分析页、LLM完成后→聚类概览|
-| 4 | 聚类前手动筛选| | manual.html下filter_data工具UI |
-| 5 | 地球→态势+拖拽修复 | | 导航改名、拖拽方向修正、惯性仅松开时施加|
-| 6 | DB模型同步 | | 所有模型与SQLite表完全匹配|
-| 7 | LLM后强制聚类| | LLM完成后自动运行clustering+PCA pipeline |
-| 8 | 列名不一致处理| | 保留所有列(union)、不丢弃额外列、gen_test_data支持--missing-cols |
-| 9 | FFT频谱分析 | | 新MCP工具analyze_fft(numpy.fft)、周期性特征提取|
-| 10 | 异常处理修复 | | 24处裸except/except:pass全部修复为带日志处理 |
-| 11 | LLM上下文增加| | 发送列类型+中文含义+空值率+众数给LLM |
-| 12 | 工作流自动保存| | 最0条自动方案保存到手动分析页、支持固定防淘汰 |
-| 13 | 列类型遵循TlsDB | | tls_ref_data作为权威类型源、未知列严格回退 |
-
-## 修复计划 (2026-07-16)
-
-| # | 问题 | 状态| 详情 |
-|---|------|------|------|
-| 1 | 重启后旧数据可分析| | SessionStore 元数据持久化为JSON 文件(`%APPDATA%/TianXuan/.session_store.json`)，wsgi.py 启动时自动恢复|
-| 2 | Web图标所有页| | `<link rel="icon">` 在 base.html 中，所有页面继承|
-| 3 | 上传按钮在顶部| | `position: sticky` |
-| 4 | Traceback完整 | | 移除 views.py 4处 + run_pipeline.py 1处`[:200]` 截断 |
-| 5 | 大数据不卡顿 | | 降采样`head=` 参数支持，手动POST 可传 head，LLM 自动决定，clamp 500|
-| 6 | 进度条| | progress_pct → 10/30/50/70/90/100 各阶段更新|
-| 7 | TLS 0ver hex识别 | | TLS_HEX_MAP 移至 type_classifier.py 值优先检测链ver列自动ENUM("TLSv1.2") |
-| 8 | Globe响应式缩放| | Canvas resize 事件绑定，vh 单位动态计算|
-| 9 | Globe ?data= | | 异常安全：try/except 包裹，空数据仍渲染页面|
-| 10 | 文件上传无限| | DATA_UPLOAD_MAX_NUMBER_FIELDS=1000000, FILE_UPLOAD_MAX_MEMORY_SIZE=100MB |
-| 11 | 上传页删除功能| | removeFile() + 按钮每文件|
-| 12 | LLM日志 | | progress_callback 写入 run_run_log |
-| 13 | 聚类图表 | | cluster_overview.html 含散点图/柱状态Silhouette |特征纯Canvas) |
-| 14 | 低内存不崩溃 | | MiniBatchKMeans + PCA降维 + 降采样head 参数 |
-| 15 | +号字符串识别为数| | type_classifier.py FLOAT检测前排除 `+` 前缀 |
-| 16 | 空闲高IO | | 日志级别 INFO，无需修改 |
-| 17 | 聚类效果 | ⚠️ | 预置了特征过滤降维，但具体效果需业务验证 |
-| 18 | 列名精确匹配 | | `re.match` 精确匹配 + `_find_column` tuple 关键词|
-
-## v7 Final Verification (2026-07-20)
-
-All **19 issues** fixed and verified. 92/92 unit tests passing. Full regression run completed.
-
-| # | 问题 | 状态| 详情 |
-|---|------|------|------|
-| 1 | `handle_call` imported in llm_orchestrator.py | | `from analysis.tool_registry import handle_call` at line 7 |
-| 2 | Retry button + run_type filter | | Retry form in `run_list.html` (line 51), run_type dropdown filter, URL route `retry_run` added |
-| 3 | auto.html logs show tool+result | | `liveLog` polling every 3s from `/logs/?pos=` endpoint |
-| 4 | Data saved to SQLite tables | | `sqlite_table` field on `AnalysisRun` model, `drop_sqlite_table` in data_loader |
-| 5 | Error toast in base.html | | Complete toast system with error/success types, auto-dismiss, slide animations |
-| 6 | display_id in all URLs | | All 6 URL patterns (`run_detail`, `run_status`, `cluster_overview`, `cluster_detail`, `retry_run`, `delete_upload`) use `display_id` |
-| 7 | Delete buttons on dashboard+run_list | | Both pages have styled delete buttons with confirmation dialog |
-| 8 | Progress bar 100% on completion | | `progress_pct` field updated through pipeline stages (10/30/50/70/90/100) |
-| 9 | LLM thinking panel in auto.html | | `thinkingPanel` div with `llm_thinking` text area and step indicator |
-| 10 | Tool call accordion in auto.html | | `renderToolCalls()` with toggle-able accordion sections per tool step |
-| 11 | LazyFrame.sample() fixed | | Uses `lf.sample(n=...)` with explicit row count parameter (not deprecated `.sample()` API) |
-| 12 | `filter_and_cluster` tool exists | | Registered at line 407 in `tool_registry.py` as tool #13 |
-| 13 | Tool descriptions in Chinese | | All tool descriptions and parameter docs use Chinese |
-| 14 | Globe: lines, timestamp, inertia, polar flip | | Flow arcs (Line geometry), timestamp-based dot animation, quaternion momentum decay, camera-relative rotation (no polar flip) |
-| 15 | Config button in nav | | `{% url 'analysis:config' %}` link 配置 in base.html nav bar |
-| 16 | config host:port affects broadcast | | `settings.py` reads `_cfg.server.host` - auto-adds to `ALLOWED_HOSTS` |
-| 17 | SQLite timeout+retry mechanism | | `retry_on_lock` decorator in `db_utils.py`: 3 retries, exponential backoff, logs `[DB_LOCKED]` warning |
-| 18 | Delete crash fixed (null csv_glob) | | `delete_upload` view checks `if run.csv_glob:` before accessing path (line 366) |
-| 19 | No file count limit, streaming upload | | `DATA_UPLOAD_MAX_NUMBER_FIELDS=10000000`, `FILE_UPLOAD_MAX_MEMORY_SIZE=100MB` |
-
-### Remaining issues
-- `test_e2e.py`, `test_entity.py`, `test_pipeline.py` still reference removed `entity_detector` module - need updating for new clustering pipeline
-- `test_clustering_edge.py` does not exist on disk (only referenced in AGENTS.md)
-- `retry_run` URL route was missing from `urls.py` - fixed during final verification
-
-### Test Results (2026-07-20)
-| Test suite | Tests | Result |
-|------------|-------|--------|
-| `test_data_loader.py` | 28 | All passed |
-| `test_type_classifier.py` | 64 | All passed |
-| **Total** | **92** | ** All passed** |
-
-## v8 Fix (2026-07-20)
-
-| # | 问题 | 状态|
-|---|------|------|
-| 1 | 服务器启动config模块找不到| 修复 |
-| 2 | 首次启动无数据库| 修复 |
-| 3 | CSV中+'无法解析为f64 | 修复 |
-| 4 | 上传分块处理+释放临时文件 | 修复 |
-| 5 | 集成测试脚本 | 添加 |
-| 6 | TlsDB.csv文档说明 | 更新 |
-| 7 | load_from_db类型冲突 | 修复 |
-| 8 | CSV未合并处理| 修复 |
-
-### Final Verification (2026-07-20)
-| Check | Result |
-|-------|--------|
-| `manage.py check` (system) | 0 errors |
-| Unit tests (data_loader + type_classifier) | 92 passed |
-| Integration test (full upload→load→analyze) | ALL INTEGRATION TESTS PASSED |
-| Git commit `v8` | `555ec29` |
-
-## MCP 工具系统 (2026-07-19)
-
-系统现有 **27 |MCP 工具**，分为三层：
-
-### 工具架构
-
-| | 数量 | 标签 | 用|
-|---|------|------|------|
-| Core | 7 | | 执行分析，可修改数据 |
-| Analysis | 5 | 🔍 | 只读深潜，安全随时调用|
-| Diagnostic | 7 | 🩺 | 排查问题 |
-| 其他 | 8 | | 数据加载/过滤/导出|
-
-### 手动分析工作流
-手动分析页面已改为 **工作流构建器**:
-- 添加/移除/重排步骤
-- 每步选择工具 + 填写参数
-- **保存/加载/删除**方案（存储`.omo/plans/<name>.json`)- 单步执行 |**▶▶ 全部执行**（自动串行）
-- 执行结果显示在每一步下面
-### LLM 自动编排
-
-LLM orchestrator 重写，支持：
-- 策略式系统提示词（先 profile 根据数据决策 失败自动诊断- 重复调用检测+ 自动错误恢复
-- 27 个工具全部可用- 中文总结分析结果
-
-### 全链路测试
-| 测试 | 用例 | 结果 |
-|------|------|------|
-| 单元测试 | 120 | |
-| 管道测试 | 8 (3 datasets × 2 algos + edge) | |
-| CLI pipeline | e2e 5 stages | |
-| MCP 工具集成 | 10 (profile/patterns/validate/tls/geo/build/scores/detect/detail/visualize) | |
-
-## v3 修复 (2026-07-17)
-
-| # | 问题 | 状态| 详情 |
-|---|------|------|------|
-| 1 | 经纬度列名匹配`:ips.latd`) | | views.py直接字符串匹配+ entity_aggregator.py _LAT_KEYWORDS增加.latd后缀 |
-| 2 | cnrs/isrs布尔类型 | | entity_aggregator.py聚合前将"+"→True/空白→False |
-| 3 | 聚类图表（legend/柱状态负值） | | 纯Canvas：legend用HTML div+overflow、柱状图动态带宽、负值标签重新定义|
-| 4 | 地球缩放改变数据流大小| | Three.js arc sphere尺寸与camera距离反比，zoom时视觉大小不变|
-| 5 | 恶意/代理流量分析 | | entity_aggregator.py规则引擎：non_std_port_rate + modern_tls_rate + sni_missing_ratio + multi_country + proxy_score |
-| 6 | EntityProfile批量IO | | views.py + tool_registry.py 改为 bulk_create(ignore_conflicts=True, batch_size=1000) |
-| 7 | 重试分析 | 🗑| 已移除(v7 新架构不再需要entity_column 概念) |
-| 8 | D5降采样导致特征提取行数不匹配 | | _handle_run_clustering存储_ds_idx抽样索引，_handle_extract_features按索引过滤数据|
-
-## v4 修复 (2026-07-17)
-
-| # | 问题 | 状态| 详情 |
-|---|------|------|------|
-| 1 | 进度条位置| | DOM重排: submitBar → progressArea → fileList progressArea → fileList |
-| 2 | PCA sklearn RuntimeWarning | | `warnings.filterwarnings('ignore', category=RuntimeWarning, module='sklearn')` |
-| 3 | cluster_overview N+1查询 | | `prefetch_related('features')` + `.only()`字段限制 |
-| 4 | SQLite PRAGMA未优| | WAL + synchronous=NORMAL + cache_size=20MB + temp_store=MEMORY + mmap_size=3GB |
-| 5 | 零方差过滤崩| | 全零方差时回退`np.arange(data.shape[1])`保留原始特征 |
-
-### 大规模测试结(2026-07-17)
-
-| 测试 | 规模 | 结果 |
-|------|------|------|
-| 单元测试 | 120 tests | 全部通过 |
-| 管道测试 | 3 datasets × 2 algorithms | 8 tests OK |
-| 中等规模 | **100文件 × 10000行= 1M行 825MB** | 全部完成: 加载→实体检测→聚合→聚2285个实体→特征提取|
-
-## Cleanup (2026-07-16)
-
-对 v6 迭代引入的垃圾文件和代码破坏进行全面清理
-| 操作 | 文件 |
-|------|------|
-| 🗑删除 | 15个垃圾脚(wnl_to_user_csv, eval_clustering, debug_db2/3, debug_persistence, test_orch/orch2, test_pca, test_profile, test_tool_loop/format, test_e2e_full, test_32b, test_chinese_path, verify_runtime) |
-| 🗑删除 | data/wnl_converted.csv, data/globe_test.csv |
-| 🗑删除 | analysis/value_normalizer.py + 关联测试 + data_loader/entity_aggregator中的_norm引用 |
-| 🗑删除 | .omo/plans/ 旧计划文(tianxiu-v4/v5/v6, globe-v6) |
-| 🔧 重构 | views.py: 统一 _run_analysis_worker → _run_llm_analysis → _run_pipeline_worker |
-| 🔧 修复 | settings.py: FILE_UPLOAD_MAX_MEMORY_SIZE 10MB→100MB |
-| 🔧 修复 | upload.html: 添加每文件删除按钮 + removeFile() |
-| 🔧 修复 | tool_registry.py: async函数中的同步ORM调用加sync_to_async包装 |
-| 🔧 修复 | run_pipeline.py: 改为调用 _run_pipeline_worker |
-| 🔧 替换 | gen_test_data.py: 3画像版→简单随机生成版 |
-| 验证 | 120/120 测试通过, 4次端到端管道测试全部成功 |
-
-### v6 (2026-07-16) 已清理（不再维护
-> ⚠️ v6 引入value_normalizer 模块已被删除，_norm 列全部移除> entity_aggregator 回退到使用原始列名（0ver/0cph/cipher-suite等）
-| 模块 | 变动 | 当前状态|
-|------|------|---------|
-| value_normalizer.py | ~~新增: hex→enum归一化~~ | 🗑已删|
-| data_loader.py | ~~集成normalize_lf~~ | 🔧 恢复原始 |
-| entity_aggregator.py | ~~_norm列依赖~~ | 🔧 回退原始列名 |
-| gen_test_data.py | ~~3类流量画像~~ | 🔧 简单随机生|
-| 特征维度 | 20→3| 特征维度保留3维），去掉_norm列后的纯原始列特征|
-
-## Completed (2026-07-16)
-
-18个问题全部修复，3次端到端管道测试全部通过（简单CSV、用户自定义列名CSV、大规模CSV）
-| 工作流| 问题 | 状态|
-|--------|------|------|
-| A1 | 重启后旧数据可分析| 目录不存在时优雅降级，标记为failed |
-| A2 | Web图标所有页面显示| 创建favicon.svg + base.html添加link |
-| A3 | 上传按钮在顶部| 移至文件列表上方，sticky定位 |
-| A10 | 文件上传限制 | 去除了Django字段限制，增大至256MB/50GB |
-| A11 | 上传页删除功能| 所有状态显示删除按钮，处理中弹出确认|
-| B7 | TLS 0ver hex格式 | 添加TLS_HEX_MAP，识别3 03/03 04 |
-| B15 | "+"字符串识别为数字 | FLOAT检测添加/前缀检查|
-| B18 | 列名映射 | 使用精确^exact_name$匹配，无模糊猜测 |
-| C4 | Traceback截断 | tool_registry.py移除[:200] |
-| D5 | 大数据量卡顿 | 添加head参数10K降采样、MAX_ROWS=500 |
-| D6 | 进度条| 添加progress_pct/progress_msg字段 |
-| D14 | 低内存崩溃| MiniBatchKMeans、PCA降维、内存检测|
-| D16 | 空闲高IO | 日志级别已为INFO，无需修改 |
-| E8 | Globe响应式缩放| 动态H计算、vh单位、resize处理 |
-| E9 | Globe ?data=参数 | 支持从SessionStore直接加载数据|
-| F12 | LLM日志 | 添加callback机制、run_log字段 |
-| G13 | 聚类图表 | 添加簇大小柱状图、Silhouette对比|
-| H17 | 聚类质量 | 方差过滤、相关过滤、HDBSCAN自动调参 |
-
-## v1.1.6 (2026-07-22)
-
-Cleanup + minor fixes release.
-
-| # | 问题 | 状态 | 描述 |
-|---|------|------|------|
-| 1 | schema sort fix | 修复 | 排序逻辑修改，确保最后一行被包含 |
-| 2 | batch upload | 修复 | 批量上传流程加固 |
-| 3 | temp dir to D: | 修复 | 临时目录移到 D: 盘避免 C: 盘空间占用 |
-| 4 | update.bat wildcard fix | 修复 | 通配符匹配优化使更新更可靠 |
-| 5 | cleanup test artifacts | 修复 | 删除 test CSV data, temp scripts, gen_fast.py |
-| 6 | VERSION sync | 修复 | VERSION 文件已更新为 v1.1.6 |
-
-### Pending
-- tests/ 需要更新 entity_detector 相关
-- 大文件上传存在内存泄漏需要进一步排查
 
 ---
 
 ## Operating Procedures
 
-1. **不要阻塞式启动后端** — Start Django backend in background (use `start /B` or threading), never block the terminal. Use `start /B runtime\python\python.exe manage.py runserver` to avoid hanging the shell.
+1. **非阻塞启动后端** — `run.bat` 使用 `start /B` 启动 Django, 打印 PID 后退出, 不阻塞终端。日志写入 `logs/tianxuan.log`。
 
-2. **全量测试** — After all changes, run full E2E test: generate 2000 CSVs, upload through frontend, verify complete pipeline. This validates the entire system end-to-end before considering work complete.
+2. **全量测试** — 所有改动完成后, 运行完整 E2E 测试: 生成测试 CSV, 上传前端, 验证全流水线。
 
-3. **更新文档和Git** — After tests pass: update AGENTS.md, commit all changes, push, clean up temp files and running processes. Do not leave dirty state (orphan processes, temp files, uncommitted changes).
+3. **更新文档和 Git** — 测试通过后: 更新 AGENTS.md, 提交所有更改, 清理临时文件和残留进程。不留脏状态 (孤儿进程、临时文件、未提交更改)。
 
-4. **同步依赖** — Use `git pull` to sync latest code, then `runtime\python\python.exe -m pip install -r requirements.txt` to install any new packages. Do this before starting any new work session.
+4. **同步依赖** — 使用 `git pull` 同步最新代码, 然后 `runtime\python\python.exe -m pip install -r requirements.txt` 安装新依赖。每次新工作会话前执行。
 
-5. **编码前缀** — Prefix ALL terminal commands with `chcp 65001` to prevent Opencode encoding errors. PowerShell/CMD encoding issues are the #1 source of inexplicable command failures.
+5. **编码前缀** — 所有终端命令加 `chcp 65001` 避免 Opencode 编码错误。PowerShell/CMD 编码问题是命令失败的首要原因。
 
-6. **Python UTF-8** — When running Python, always add `set PYTHONUTF8=1` before the command. This ensures consistent Unicode handling across all machines and prevents cross-machine encoding drift.
+6. **Python UTF-8** — 运行 Python 时始终加 `set PYTHONUTF8=1`。确保跨机 Unicode 处理一致, 防止编码漂移。
+
+## 近期变更 (beta-clean 分支)
+
+- views.py (2000+ 行单体) → `analysis/views/` 包 (13 模块)
+- tool_registry.py (3000+ 行) → `analysis/tools/` 包 (18 模块)
+- 默认聚类算法改为 AgglomerativeClustering
+- 移除实体概念，直接对原始行聚类 (entity_value = row_N)
+- cluster_detail 显示原始数据行而非实体画像
+- run_detail 新增全域 SVD 特征分析 + LLM 工作流时间线
+- 3D 地球侧栏复用 `/globe/?embed=1`（iframe + postMessage 联动）
+- UI 全面中文化
+- 非阻塞启动 (run.bat 打印 PID 后退出)
+
+## 待办
+
+- [ ] **前端 UI 验证**: 打开 `prototype_cluster_ui.html` 查看交互设计是否符合要求
+- [ ] **更新前端原型**后，将验证通过的交互逻辑合并到正式模板

README.md

@@ -1,541 +1,190 @@
-# 天璇 (TianXuan) — TLS 流数据分析与实体画像系统
+# 天璇 (TianXuan) — TLS 流数据分析平台
 
-基于 Polars + scikit-learn + Three.js 的 TLS 流数据分析工具链。支持 CSV 批量导入、自动实体检测、多维聚合、聚类分析、特征提取、**3D 地球流量可视化**。提供 Django Web 操作界面和 MCP 协议接口，支持 32B 大语言模型编排分析流程。**全离线运行**，所有 JS 库/地图贴图/GeoIP 数据库均已内嵌。
+基于 Polars + scikit-learn + Three.js 的 TLS 流量分析工具。CSV 上传后支持手动分步分析（工作流构建器）、LLM 全自动分析（MCP 工具链）、聚类 3D 地球可视化。全离线运行，内嵌 Python 3.12 运行时。
 
-## TlsDB.csv — 参考表头名清单
+## 功能概述
 
-`TlsDB.csv` 是 TLS 流量数据的**参考表头名清单**，包含常见的列名、含义和数据类型对照表。
+| 模块 | 说明 |
-**此文件为参考对照表，非实际数据文件。**
-实际 CSV 文件的列名可能与表头名不同，分析工具会自动匹配。
-
----
-
-## 目录
-
-1. [系统概述](#1-系统概述)
-2. [快速开始](#2-快速开始)
-3. [用户指南](#3-用户指南)
-4. [页面功能说明](#4-页面功能说明)
-5. [CLI 管道](#5-cli-管道)
-6. [MCP 工具文档](#6-mcp-工具文档)
-7. [测试](#7-测试)
-8. [项目结构](#8-项目结构)
-9. [常见问题](#9-常见问题)
-
----
-
-## 1. 系统概述
-
-### 1.1 解决的问题
-
-企业网络中 TLS 流量通常以 CSV 形式导出（每行一条流记录），数据量大、列名不统一、缺乏标签。本系统提供一条从原始 CSV 到实体画像的自动化分析流水线：
-
-```
-CSV 文件 → 上传/加载 → 自动检测实体列 → 按实体聚合特征
-→ 实体级聚类 → 提取每簇区分特征 → Web 可视化 / MCP 接口
-```
-
-### 1.2 核心能力
-
-| 模块 | 功能说明 |
-|------|---------|
-| **数据加载** | 多文件合并、BOM 自动检测、跨文件 schema 容错、递归 `**/*.csv`、中文路径支持 |
-| **实体检测** | 基于列名关键词 + 唯一值比率 + 数据类型的三维打分，自动推荐实体列（如 `src_ip`、`sni`、`user`） |
-| **实体聚合** | 按实体分组计算流统计、目标多样性、TLS 特征、协议特征、时间模式、地理位置（经纬度） |
-| **IP 子网聚合** | 支持 /24 和 /28 子网掩码，同一子网下所有 IP 合并为一个实体节点 |
-| **多列复合键** | 支持 `(src_ip, dst_ip)` 等多列组合作为实体标识 |
-| **聚类分析** | HDBSCAN（自动确定簇数）/ KMeans，零样本自动降级，Silhouette / Davies-Bouldin 质量评估 |
-| **特征提取** | Z-Score / ANOVA 方法计算每个聚类的区分性特征 |
-| **3D 地球** | 基于 Three.js 的 TLS 流量弧线可视化，多数据源叠加，经纬线网格，国境线轮廓 |
-| **地理可视化** | PCA 散点图 + Canvas 原生渲染（无外部 JS 依赖） |
-| **LLM 集成** | OpenAI 兼容 API、LLM 自动编排分析流程（profile → 实体 → 聚类 → 特征） |
-| **Web 界面** | CSV 上传拖拽、手动分步分析向导、LLM 自动分析、配置管理、运行记录查看 |
-| **MCP 协议** | 12 个 JSON-RPC 工具，供外部 LLM 编排调用 |
-| **便携运行时** | 项目内置 Python 3.12 运行时（593MB），零系统依赖，`run.bat` 双击即用 |
-
-### 1.3 技术栈
-
-| 组件 | 技术选型 |
-|------|---------|
-| **后端框架** | Django 4.2 (SQLite WAL) |
-| **数据处理** | Polars 1.42.1 (LazyFrame 延迟计算 + streaming) |
-| **机器学习** | scikit-learn (StandardScaler, HDBSCAN, KMeans, PCA) |
-| **MCP 协议** | Python MCP SDK (stdio 传输) |
-| **前端 3D** | Three.js (603KB, 离线) |
-| **前端图表** | 纯 Canvas 2D API 原生绘制 |
-| **配置** | Pydantic + YAML |
-| **LLM 接口** | OpenAI chat/completions 兼容 API |
-| **运行时** | 嵌入式 Python 3.12 (Win7 兼容版 adang1345) |
-
----
-
-## 2. 快速开始
-
-### 2.1 用户模式（无需预装 Python）
-
-```bat
-1. 解压项目文件夹到任意目录
-2. 双击 run.bat
-3. 等待命令行显示 "Starting TianXuan..."
-4. 浏览器自动打开 http://127.0.0.1:8000/
-5. 点击「上传数据」→ 选择 CSV 文件 → 上传 → 等待预处理完成
-6. 进入「手动分析」→ 选择数据集 → 配置参数 → 运行分析
-7. 查看结果：聚类概览、簇详情、实体画像、地球分布图
-```
-
-### 2.2 开发者模式
-
-```bash
-# 初始化数据库
-runtime\python\python.exe manage.py migrate
-
-# 生成测试数据
-runtime\python\python.exe scripts\gen_test_data.py --rows 1000
-
-# 启动开发服务器
-runtime\python\python.exe manage.py runserver
-
-# 浏览器打开 http://127.0.0.1:8000/
-```
-
-### 2.3 脚本一览
-
-| 脚本 | 功能 |
 |------|------|
-| `run.bat` | 启动 Django Web 界面（:8000），自动打开浏览器 |
+| **CSV 上传** | 多文件拖拽、BOM 自动检测、schema 容错、ZIP 自动解压 |
-| `shell.bat` | 打开 Django Shell（Python 交互式环境） |
+| **手动分析** | 工作流构建器：添加/排序/保存工具链，串行执行 |
-
+| **LLM 自动分析** | OpenAI 兼容 API，自动编排 filter→profile→entity→cluster→feature |
-### 2.4 一键分析管道
+| **聚类可视化** | HDBSCAN/KMeans，Silhouette/DB 评估，Canvas 散点图 + 地理分布 |
-
+| **3D 地球** | Three.js 流量弧线，TLS 版本着色，经纬线+国境线 |
-```bash
+| **降维** | UMAP + TruncatedSVD 用于高维数据降维与可视化 |
-# 加载 CSV → 自动检测实体 → 聚合 → 聚类 → 特征提取 → 写入数据库
+| **实体画像** | 聚合特征 + Z-Score 簇偏离量，自定义实体列/IP 子网掩码 |
-runtime\python\python.exe manage.py run_pipeline "data/input.csv"
+| **MCP 协议** | 30 个 JSON-RPC 工具供外部 LLM 编排调用 |
-
+| **增量更新** | update.bat + rollback.bat，用户数据/代码分离 |
-# 指定聚类算法
+
-runtime\python\python.exe manage.py run_pipeline "data/*.csv" --algo kmeans
+## 技术栈
-
+
-# 手动指定实体列（跳过自动检测）
+| 组件 | 选型 |
-runtime\python\python.exe manage.py run_pipeline "data/*.csv" --entity-col src_ip
+|------|------|
-
+| 后端 | Django 4.2 + SQLite WAL |
-# 导出结果
+| 数据处理 | Polars 1.42 (LazyFrame + streaming) |
-runtime\python\python.exe manage.py run_pipeline "data/*.csv" --output ./results/
+| 机器学习 | scikit-learn (HDBSCAN, KMeans, MiniBatchKMeans, IsolationForest), UMAP, TruncatedSVD, StandardScaler |
-```
+| 前端 3D | Three.js (603KB, 离线, 地球贴图 1.4MB) |
-
+| 前端图表 | 纯 Canvas 2D (散点图/地理分布/Silhouette) |
-输出示例：
+| 配置管理 | Pydantic + YAML |
-```
+| LLM 协议 | MCP SDK (stdio) + OpenAI 兼容 API |
-[1/5] 加载 CSV: data/test_flows.csv → 500 rows, 1 files, 0.2 MB
+| 运行时 | 内嵌 Python 3.12 (~593MB) |
-[2/5] 检测实体列 → 自动检测: src_ip (共 6 候选)
+
-[3/5] 实体聚合 → 50 个实体, 18 维特征
+## 安装
-[4/5] 聚类 (hdbscan) → 2 个簇, 噪声比 0.24, Silhouette: 0.2557
+
-[5/5] 特征提取 → 30 个特征已保存到数据库
+项目自包含，无需预装 Python。解压后即可使用。
-*** 分析完成! Run ID: #1
-```
-
----
-
-## 3. 用户指南
-
-### 3.1 上传数据
-
-1. 点击导航栏「上传数据」
-2. 拖拽 CSV 文件到虚线区域，或点击选择文件
-3. 支持多文件上传，自动拼接相同 schema 的文件
-4. 支持 `schema_strict=false` 模式（默认）：不同列名的文件会自动合并，缺失列填 null
-5. 支持 `.zip` 压缩包自动解压
-6. 上传后自动在后台进行预处理（加载 → 检测实体列 → 聚合），完成后状态变为 `ready`
-7. 进度条实时显示处理状态
-
-**数据格式要求**：
-- 每行一条 TLS 流记录
-- 建议包含列：`src_ip`、`dst_ip`、`proto`、`bytes_sent`、`bytes_rev`、`duration`、`packets` 等
-- 可选地理列：`latitude` / `lat` / `y` 和 `longitude` / `lon` / `lng` / `x`（自动识别）
-- 可选时间列：`timestamp` / `ts` / `time`（自动识别）
-
-### 3.2 手动分析
-
-3 步向导完成分析：
-
-**Step 1 - 选择数据集**：从已预处理完成的数据集中选择一个（显示行数、已检测的实体列）
-
-**Step 2 - 配置参数**：
-- 聚类算法：HDBSCAN（自动确定簇数，推荐）/ KMeans（需预设 k 值）
-- 最小簇大小（min_cluster_size）：默认 5，数据不足时自动降级
-
-**Step 3 - 确认运行**：回顾配置，点击「开始分析」。后台自动完成聚类 + 特征提取，完成后跳转到结果页。
-
-### 3.3 LLM 自动分析
-
-需要先在「配置」页面设置 LLM 的 base_url 和 API Key：
-
-1. 进入「LLM 分析」页面
-2. 选择一个已预处理的数据集
-3. 点击「开始自动分析」
-4. 后端 LLM 编排器会自动调用 profile → build_entity_profiles → run_clustering → extract_features
-5. 进度条实时显示当前步骤
-
-### 3.4 查看结果
-
-#### 运行详情页
-- 总流数、实体数、簇数 摘要卡片
-- 簇列表：每个簇的大小、占比、Silhouette 分数
-
-#### 聚类概览页
-- **PCA 散点图**：实体在二维主成分空间中的分布，颜色区分聚类
-- **地理分布图**：实体在地理坐标上的分布（需 CSV 包含经纬度列），支持缺失值
-- 每个簇的详情卡片：Top 5 区分特征
-
-#### 簇详情页
-- 特征表：每列的均值、标准差、中位数、区分度分数（Top 50）
-- 实体列表：该簇包含的实体（Top 50），可点击查看详情
-
-#### 实体画像页
-- 实体标识、所属簇
-- 聚合特征值和相对簇均值的偏离（Z-score）
-- 偏差显著的特征以颜色高亮（|Z| > 2）
-
-#### 3D 地球页
-- Three.js 3D 地球，TLS 流量弧线连接源目 IP
-- 弧线颜色标识 TLS 版本（TLSv1.3 蓝 / v1.2 粉 / 其他青）
-- 鼠标拖拽旋转、滚轮缩放
-- 经纬线网格 + 国境线轮廓
-- 多数据源叠加复选框
-- 脉冲光点动画（背面剔除）
-
----
-
-## 4. 页面功能说明
-
-| 页面 | 路由 | 功能 |
-|------|------|------|
-| **首页** | `/` | 最近运行记录概览 |
-| **上传数据** | `/upload/` | 拖拽上传 CSV，自动预处理 |
-| **手动分析** | `/analyze/manual/` | 3 步向导：选数据→设参数→运行 |
-| **LLM 分析** | `/analyze/auto/` | LLM 自动编排分析流程 |
-| **运行记录** | `/runs/` | 所有分析运行列表 |
-| **运行详情** | `/runs/<id>/` | 单次运行摘要 + 簇列表 |
-| **聚类概览** | `/clusters/<run_id>/` | PCA 散点图 + 每簇区分特征 |
-| **簇详情** | `/clusters/<run_id>/<label>/` | 特征表 + 实体列表 |
-| **实体画像** | `/entities/<id>/` | 单实体特征 + 簇偏离 |
-| **3D 地球** | `/globe/` | Three.js 流量弧线可视化 |
-| **配置** | `/config/` | 系统配置编辑 + LLM 测试 |
-| **日志** | `/logs/` | 运行日志查看 |
-
----
-
-## 5. CLI 管道
-
-```bash
-# 完整管道（HDBSCAN 默认）
-runtime\python\python.exe manage.py run_pipeline data/complex_test.csv
-
-# 指定 KMeans
-runtime\python\python.exe manage.py run_pipeline data/*.csv --algo kmeans
-
-# 手动指定实体列
-runtime\python\python.exe manage.py run_pipeline data/*.csv --entity-col src_ip
-
-# 设置子网聚合掩码
-runtime\python\python.exe manage.py run_pipeline data/*.csv --subnet-masks 24 28
-```
-
----
-
-## 6. MCP 工具文档
-
-### 6.1 工具总览
-
-系统通过 MCP (Model Context Protocol) 暴露 12 个工具，支持 LLM 以 JSON-RPC 方式编排调用。
-
-| # | 工具名 | 功能 | 必需参数 |
-|---|--------|------|----------|
-| 1 | `load_data` | 加载 CSV 文件（支持 glob、递归、schema 容错） | `csv_glob` |
-| 2 | `profile_data` | 数据集概要统计 + 相关性矩阵 | `dataset_id` |
-| 3 | `filter_data` | 按条件过滤（12 操作符 + AND/OR） | `dataset_id`, `filters` |
-| 4 | `preprocess_data` | 预处理（标准化/编码/填充） | `dataset_id`, `columns` |
-| 5 | `run_clustering` | 执行聚类（HDBSCAN/KMeans，带质量评估） | `dataset_id`, `cluster_columns` |
-| 6 | `evaluate_clustering` | 评估聚类质量（Silhouette/DB/CH） | `cluster_result_id`, `dataset_id` |
-| 7 | `extract_features` | 提取各聚类区分特征（Z-Score/ANOVA） | `dataset_id`, `cluster_result_id` |
-| 8 | `export_results` | 导出结果到磁盘（CSV/Parquet/JSON） | `result_id`, `output_path` |
-| 9 | `list_datasets` | 列出所有活跃数据集 | 无 |
-| 10 | `drop_dataset` | 删除数据集释放内存 | `dataset_id` |
-| 11 | `clone_dataset` | 浅拷贝数据集（不复制数据） | `dataset_id` |
-| 12 | `build_entity_profiles` | 自动检测实体列 + 聚合画像 | `dataset_id` |
-
-### 6.2 响应约定
-
-- 成功时返回 JSON 对象，含 `truncated: false`
-- 超过阈值时自动截断，设置 `truncated: true` + `omitted_columns`
-- 错误时返回 `{ "error": "描述", "truncated": false }`
-- 所有响应通过 `TextContent` 包装返回
-
-### 6.3 JSON-RPC 调用示例
-
-```
-→ {"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"load_data","arguments":{"csv_glob":"data/*.csv"}}}
-← {"jsonrpc":"2.0","id":1,"result":{"content":[{"type":"text","text":"{\"dataset_id\":\"ds_...\",\"row_count\":500}"}]}}
-
-→ {"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"build_entity_profiles","arguments":{"dataset_id":"ds_...","auto_detect":true}}}
-← {"jsonrpc":"2.0","id":2,"result":{"content":[{"type":"text","text":"{\"entity_dataset_id\":\"entity_...\",\"entity_count\":50,\"entity_column_used\":\"src_ip\"}"}]}}
-```
-
----
-
-## 7. 测试
-
-```bash
-# 运行全部测试
-runtime\python\python.exe -m pytest tests -q
-
-# 运行特定测试文件
-runtime\python\python.exe -m pytest tests/test_entity.py -v
-
-# 运行端到端测试
-runtime\python\python.exe -m pytest tests/test_e2e.py -v
-```
-
-### 生成测试数据
-```bash
-# 简单数据
-runtime\python\python.exe scripts\gen_test_data.py --rows 1000
-
-# Globe 模式（IP 在真实 GeoIP 范围内，弧线可见）
-runtime\python\python.exe scripts\gen_test_data.py --globe
-
-# 复杂数据（5000 行 24 列含 55% 缺失）
-runtime\python\python.exe scripts\gen_complex_test.py --rows 5000
-```
-
-### 列结构调查
-```bash
-# 统计各 CSV 文件的列名、类型、分布
-runtime\python\python.exe scripts\column_survey.py --csv "data/*.csv"
-```
-
----
-
-## 8. 项目结构
-
-```
-tianxuan/
-├── tianxuan/                 # Django 项目配置
-│   ├── settings.py           # Django 配置（数据库、中间件）
-│   ├── urls.py               # 根路由
-│   └── llm_orchestrator.py   # LLM 编排后端（tool_calls 循环）
-│
-├── analysis/                 # 核心分析模块（Django app）
-│   ├── data_loader.py        # CSV 加载、BOM 检测、schema 容错、递归 glob
-│   ├── data_profiler.py      # 数据集概要统计与相关性矩阵
-│   ├── entity_detector.py    # 实体列自动检测（关键词 + unique_ratio 打分）
-│   ├── entity_aggregator.py  # 实体聚合（流→实体画像），含 IP 子网聚合
-│   ├── type_classifier.py    # 值优先类型检测（MAC/端口/IPv4/URL/HEX/ENUM/LAT_LON）
-│   ├── data_validator.py     # 列校验（缺失率/异常值/IP有效性）
-│   ├── geoip.py              # GeoIP 经纬度查询
-│   ├── session_store.py      # 线程安全内存会话存储（Singleton + RLock）
-│   ├── tool_registry.py      # 12 个 MCP 工具定义 + 异步处理函数
-│   ├── mcp_server.py         # MCP stdio 服务器
-│   ├── views.py              # 所有 Django 视图
-│   ├── urls.py               # 16 条 URL 路由
-│   ├── models.py             # ORM 模型
-│   └── management/commands/
-│       ├── start_mcp.py      # manage.py start_mcp
-│       └── run_pipeline.py   # manage.py run_pipeline（一键 CLI 管道）
-│
-├── config/
-│   ├── config.yaml           # 配置文件（自动生成，带注释）
-│   └── loader.py             # Pydantic 配置加载器
-│
-├── templates/tianxuan/       # 11 个页面模板
-├── static/tianxuan/          # Three.js + 地球贴图 + 国境线
-├── runtime/python/           # 便携 Python 3.12 运行时（593MB）
-├── tests/                    # 53 个测试 (pytest)
-├── scripts/                  # 工具脚本
-├── docs/                     # 文档
-├── run.bat                   # 用户启动脚本（双击即用）
-└── manage.py                 # Django 管理入口
-```
-
----
-
-## 9. 常见问题
-
-### Q1: 双击 run.bat 后浏览器没打开？
-A: 手动打开浏览器访问 `http://127.0.0.1:8000/`。如果也无法访问，检查命令行是否有报错。
-
-### Q2: 压缩时提示 `db.sqlite3-wal 被占用`？
-A: 有残留的 Python 进程持有 SQLite 连接。运行以下命令后重试：
-```powershell
-Get-Process python -ErrorAction SilentlyContinue | Stop-Process -Force
-Remove-Item db.sqlite3-wal, db.sqlite3-shm -Force -ErrorAction SilentlyContinue
-```
-
-### Q3: 地图不显示？
-A: CSV 需包含 lat/lon 列（列名含 `lat`/`latitude`/`lon`/`longitude`/`lng` 自动识别），缺失值不显示。页面会提示跳过的数量。
-
-### Q4: 上传 CSV 后显示 "处理失败"？
-A: 在运行详情页查看错误消息。常见原因：
-- CSV 编码不是 UTF-8
-- 所有列都是非数值类型（无法聚类，返回 0 簇）
-- 数据量太小（< 3 条实体记录，HDBSCAN 自动跳过）
-
-### Q5: 如何配置 LLM？
-A: 导航到「配置」页面，填写 base_url 和 API Key，点击「测试连通性」验证。支持 OpenAI 兼容的任何 API。
-
-### Q6: 支持哪些 CSV 编码？
-A: 自动检测 UTF-8 BOM、UTF-16LE BOM、UTF-16BE BOM。无 BOM 时默认 UTF-8。中文路径完全支持。
-
-### Q7: 不同 CSV 文件的列名不一致怎么办？
-A: 默认 `schema_strict=false`，系统自动取所有文件的列名并集，缺失列填 null。如需严格模式，在配置中将 `schema_strict` 改为 `true`。
-
-### Q8: 项目太大，如何缩小？
-```powershell
-Remove-Item -Recurse *.pyc, __pycache__ -Force
-Remove-Item db.sqlite3-wal, db.sqlite3-shm -Force -ErrorAction SilentlyContinue
-```
-
----
-
-## 10. 增量更新（部署到离线机）
-
-天璇支持**增量更新**，只需将变更代码打包成小 zip 包，复制到离线机双击 `update.bat` 即可完成更新，无需每次传输整个项目（1.5GB）。
-
-### 10.1 首次部署
-
-在离线机上第一次部署时，需要完成数据库迁移：
 
 ```bat
-:: 确保数据库在 %APPDATA%/TianXuan/ 下
+:: 首次部署 — 初始化数据库
-runtime\python\python.exe scripts\migrate_db_to_appdata.py
-
-:: 初始化数据库表
 runtime\python\python.exe manage.py migrate
-
-:: 导入 TLS 引用数据
-runtime\python\python.exe manage.py import_tlsdb
 ```
 
-> 如果之前已在使用天璇，`migrate_db_to_appdata.py` 会自动将已有 `db.sqlite3` 复制到新位置。
+## 使用方法
 
-### 10.2 开发机：构建更新包
+```bat
+:: 双击启动 Web 界面 (http://127.0.0.1:8000/)
+run.bat          :: 非阻塞启动，打印 PID 后退出，日志写入 logs/tianxuan.log
 
-在**开发机**上完成代码修改并提交后：
+:: 命令行一键管道
+runtime\python\python.exe manage.py run_pipeline "data/*.csv"
 
-```bash
+:: 开发模式启动
-# 确保工作树干净（无未提交变更）
+set PYTHONUTF8=1
-git status
+runtime\python\python.exe manage.py runserver
-
-# 构建增量更新包
-python scripts/build_update.py
 ```
 
-脚本会自动：
+**Web 工作流:** 上传 CSV → 手动分析（工作流构建器）或 LLM 自动分析 → 检视聚类概览/簇详情/实体画像/3D 地球
-1. 检测当前 git tag 与最近一次 tag 之间的文件变更
-2. 计算每个文件的 SHA256 校验和
-3. 打包为 `tianxuan_update_<旧版本>-<新版本>.zip`
-4. 更新 `VERSION` 文件
-5. 如果检测到依赖变更，会输出提示信息
 
-**输出示例**：
+**CLI 管道:** 加载 CSV → 类型检测 → 实体聚合 → 聚类 → 特征提取 → 写入数据库
-```
-==================================================
-  天璇 增量更新包构建工具
-==================================================
 
-当前 VERSION: v1.0.0
+## 项目结构
-最新 git tag: v1.0
-
-变更文件: 5 个修改/新增, 1 个删除
-
-==================================================
-✅ 增量更新包构建完成！
-  包: tianxuan_update_v1.0-v1.1.zip
-  大小: 0.05 MB
-  从: v1.0
-  到: v1.1
-  文件数: 5
-  删除: 1
-==================================================
-```
-
-### 10.3 离线机：应用更新
-
-在**离线机**上：
-
-1. 将 `tianxuan_update_*.zip` 复制到天璇项目目录（与 `run.bat` 同目录）
-2. **双击 `update.bat`**（或拖拽 zip 到 `update.bat` 上）
-3. 等待更新完成
-4. 关闭窗口，重新双击 `run.bat` 启动新版本
-
-更新脚本会自动：
-1. ✅ 验证更新包完整性（SHA256 校验）
-2. ✅ 备份当前配置（`config/config.yaml`）
-3. ✅ 删除废弃文件
-4. ✅ 覆盖更新文件
-5. ✅ 智能合并配置（保留你的设置，仅增加新配置项）
-6. ✅ 清理 Python 缓存
-7. ✅ 运行数据库迁移
-8. ✅ 更新版本号
-
-### 10.4 回滚
-
-如果更新后出现问题：
 
 ```
-双击 rollback.bat
+天璇/
-→ 输入 y 确认回滚
+├── analysis/                    核心分析模块 (Django app)
-→ 等待恢复完成
+│   ├── views/                   视图包 (12 模块)
-→ 重新双击 run.bat 启动旧版本
+│   │   ├── __init__.py          重导出所有视图函数
+│   │   ├── helpers.py           辅助函数 (_extract_lat, _extract_lon)
+│   │   ├── dashboard.py         首页 + 运行记录
+│   │   ├── pipeline.py          管道执行器 (_run_pipeline_worker)
+│   │   ├── clustering.py        聚类概览/详情/3D 地球数据
+│   │   ├── entity.py            实体画像页
+│   │   ├── upload.py            CSV 上传 (拖拽/多文件/进度)
+│   │   ├── manual.py            手动分析工作流构建器
+│   │   ├── auto.py              LLM 自动分析 (SSE 流式)
+│   │   ├── globe.py             3D 地球数据接口
+│   │   ├── config.py            配置编辑 + LLM 连通性测试
+│   │   ├── log_viewer.py        日志查看器
+│   │   └── tools.py             工具实验室
+│   ├── tools/                   工具包 (17 模块, 30 个 MCP 工具实现)
+│   │   ├── __init__.py          重导出所有 handler
+│   │   ├── _registry.py         工具元数据定义 (Tool schema)
+│   │   ├── _dispatch.py         工具调用分发器
+│   │   ├── _helpers.py          共享辅助 (过滤/截断/数据集决议)
+│   │   ├── load_data.py         CSV 加载 (glob/递归/ZIP/schema 容错)
+│   │   ├── profile.py           数据概要统计 + 相关性矩阵
+│   │   ├── filter.py            数据过滤 (12 操作符 + AND/OR)
+│   │   ├── preprocess.py        预处理 (StandardScaler/OneHot/填充)
+│   │   ├── clustering.py        聚类执行 (HDBSCAN/KMeans + 自动降维)
+│   │   ├── evaluate.py          聚类质量评估 (Silhouette/DB/CH)
+│   │   ├── features.py          特征提取 (Z-Score/ANOVA/UMAP)
+│   │   ├── entities.py          实体画像 + 自适应评分 (SVD 权重)
+│   │   ├── anomalies.py         异常检测 (IsolationForest) + UMAP 可视化
+│   │   ├── export.py            结果导出 (CSV/Parquet/JSON)
+│   │   ├── analysis.py          只读分析 (6 工具: 模式/时序/FFT/TLS/地理/实体)
+│   │   ├── diagnostics.py       诊断 (7 工具: 校验/分布/异常值/聚类诊断/对比/导出/修复)
+│   │   ├── data_mgmt.py         数据管理 (列表/删除/克隆)
+│   │   └── distance_matrix.py   LLM 驱动距离计算
+│   ├── tool_registry.py         旧编排层 (薄包装，重导出 tools/)
+│   ├── mcp_server.py            MCP stdio 服务器
+│   ├── data_loader.py           CSV 加载底层实现
+│   ├── data_profiler.py         列统计 + 相关性矩阵
+│   ├── data_validator.py        列校验 (缺失/异常值/IP 有效性)
+│   ├── type_classifier.py       值优先类型检测 (MAC/端口/IPv4/URL/HEX/经纬度)
+│   ├── geoip.py                 GeoIP 经纬度查询
+│   ├── ip_clustering.py         IP 子网聚类与转换
+│   ├── session_store.py         线程安全内存会话存储
+│   ├── distance.py              距离计算 (IP 子网/地理/Levenshtein/Hamming/FFT)
+│   ├── models.py                ORM: AnalysisRun, ClusterResult, EntityProfile, ClusterFeature
+│   ├── urls.py                  路由注册
+│   ├── nl_describe.py           自然语言数据描述
+│   ├── profile_util.py          画像工具函数
+│   ├── tls_ref.py               TLS 参考数据
+│   ├── db_utils.py              SQLite 工具
+│   ├── constants.py             全局常量
+│   ├── appdata.py               应用数据路径管理
+│   └── management/commands/     CLI 命令
+│       ├── start_mcp.py         MCP 服务器启动
+│       ├── run_pipeline.py      一键 CLI 管道
+│       └── import_tlsdb.py      TLS 数据库导入
+├── tianxuan/                    Django 项目配置
+│   ├── settings.py              ALLOWED_HOSTS 自动检测, LOGGING 文件+stderr
+│   ├── wsgi.py                  SQLite 启动自修复
+│   ├── urls.py                  路由汇总
+│   └── llm_orchestrator.py     LLM 编排 (策略式提示词, 自动错误恢复)
+├── config/                      配置管理
+│   ├── config.yaml              自动生成默认配置 (entity, server, data, clustering, llm)
+│   ├── loader.py                Pydantic 配置加载 + 文件修改检测
+│   └── __init__.py
+├── templates/                   页面模板 (13 个 HTML)
+│   ├── base.html                导航: 首页/上传/手动/LLM/记录/地球/配置
+│   ├── analysis/                分析页面 (6 个)
+│   │   ├── dashboard.html       首页
+│   │   ├── run_list.html        运行记录列表
+│   │   ├── run_detail.html      运行详情
+│   │   ├── cluster_overview.html 聚类概览
+│   │   ├── cluster_detail.html  簇详情
+│   │   └── entity_profile.html  实体画像
+│   └── tianxuan/                功能页面 (6 个)
+│       ├── upload.html          CSV 上传
+│       ├── manual.html          手动分析
+│       ├── auto.html            LLM 自动分析
+│       ├── globe.html           3D 地球
+│       ├── config.html          配置编辑
+│       └── log_viewer.html      日志查看
+├── static/tianxuan/             离线静态资源
+│   ├── three.min.js             Three.js 3D 引擎 (603KB)
+│   ├── earth_atmos_2048.jpg     地球贴图 (1.4MB)
+│   └── world_borders.js         177 国完整国境线 (269KB)
+├── runtime/python/              便携 Python 3.12 运行时 (~593MB)
+├── scripts/                     工具脚本
+│   ├── gen_test_data.py         测试数据生成
+│   ├── gen_complex_test.py      复杂数据生成
+│   ├── column_survey.py         CSV 列结构调查
+│   ├── diagnose_compare.py      跨机日志对比诊断
+│   └── debug_db.py              DB 持久化调试
+├── tests/                       端到端测试
+│   ├── test_e2e_full.py
+│   ├── test_e2e_batch.py
+│   └── _test_hdbscan.py
+├── docs/                        文档
+│   ├── 工作流.md                 完整分析流水线
+│   └── 故障诊断手册.md            问题排查指南
+├── run.bat                      用户启动入口 (含 PYTHONUTF8=1)
+├── shell.bat                    Django shell
+├── update.bat                   增量更新
+├── rollback.bat                 更新回滚
+├── build.bat                    构建打包
+└── manage.py                    Django 管理入口
 ```
 
-回滚会从 `%APPDATA%/TianXuan/backups/` 恢复最近一次备份。
+## 架构要点
 
-### 10.5 依赖变更时的特殊处理
+- **views 包**: 原单体 `views.py` (2000+ 行) 拆分为 12 个模块，按功能域分离 (dashboard, pipeline, clustering, entity, upload, manual, auto, globe, config, log_viewer, tools, helpers)
+- **tools 包**: 原 `tool_registry.py` 拆分为 17 个模块 (14 个 handler 模块 + 3 个基础设施模块)，每个工具领域独立文件
+- **30 个 MCP 工具** 分四层: 核心 (14 个)、只读分析 (6 个)、诊断 (7 个)、数据管理 (3 个)
+- **纯 Canvas 2D 图表**: 零外部 JS 依赖，散点图/地理分布/Silhouette 均纯 Canvas 绘制
+- **tuple 替代 frozenset**: 避免跨机 PYTHONHASHSEED 差异
+- **值优先类型检测**: config → 值采样 → 名称推断 → STRING 回退 (type_classifier.py)
+- **TruncatedSVD 降维**: 特征 >50 维时自动降维至 50；compute_scores 中学习自适应评分权重
+- **UMAP 2D 嵌入**: 训练 10K 样本 + 批量变换 1K 批次，存入 EntityProfile 表
+- **上传目录隔离**: `%APPDATA%/TianXuan/data/uploads/`，与项目路径无关
 
-当 `pyproject.toml` 或 `requirements.txt` 变更时（即添加/更新了 Python 依赖），构建脚本会输出警告：
+## 目标设备
 
-```
+| 条件 | 规格 |
-⚠️  依赖已变更！请手动将 runtime/ 目录同步到离线机
+|------|------|
-```
+| 操作系统 | Windows 10 1909+ |
-
+| 内存 | 8 GB |
-此时需要额外步骤：
+| GPU | 无独立显卡（核显可运行 Three.js） |
-
+| 磁盘 | SSD 推荐（SQLite WAL 优化） |
-1. 在开发机上将 `runtime/` 目录**完整压缩**（约 645MB）
+| 数据规模 | 支持 2500 个 CSV × 10000 行 × 50~58 列 |
-2. 复制到离线机，覆盖 `runtime/` 目录
-3. 或者使用差异更新：仅覆盖变更的包目录（`runtime/python/Lib/site-packages/`）
-
-> 依赖变更是低频操作（通常只在添加新功能时需要），所以这不会影响日常的代码增量更新。
-
-### 10.6 文件说明
-
-| 文件 | 用途 | 需要 git 跟踪？ |
-|------|------|----------------|
-| `VERSION` | 记录当前版本号 | ✅ 是 |
-| `scripts/build_update.py` | 开发机：构建增量包 | ✅ 是 |
-| `scripts/apply_update.py` | 离线机：应用更新 | ✅ 是 |
-| `scripts/merge_config.py` | 合并用户配置与源码配置 | ✅ 是 |
-| `scripts/migrate_db_to_appdata.py` | 首次迁移数据库到 APPDATA | ✅ 是 |
-| `scripts/version_utils.py` | 版本工具函数 | ✅ 是 |
-| `update.bat` | 离线机双击入口 | ✅ 是 |
-| `rollback.bat` | 回滚入口 | ✅ 是 |
-| `analysis/appdata.py` | APPDATA 路径共享模块 | ✅ 是 |
-| `.update_cache/runtime_snapshot.json` | 运行时依赖快照（自动生成）| ❌ 否 |
-| `RUNTIME_CHANGED.txt` | 依赖变更提示（自动生成）| ❌ 否 |
-
-### 10.7 目录结构变化
-
-更新系统将用户数据与项目代码分离：
-
-```
-项目目录（可安全覆盖）              用户数据目录（永不覆盖）
-天璇/                              %APPDATA%/TianXuan/
-├── analysis/          ← 代码       ├── db.sqlite3        ← 数据库
-├── config/config.yaml ← 配置       ├── data/uploads/     ← 上传的 CSV
-├── runtime/           ← 运行时      ├── .session_store.json
-├── templates/         ← 模板        └── backups/          ← 更新备份
-├── static/            ← 静态资源
-├── scripts/           ← 工具脚本
-├── update.bat         ← 更新入口
-└── VERSION            ← 版本号
-```
-
-
-```

VERSION

@@ -1 +1 @@
-v2.0.0beta
+v2.0.1beta

_check_bool.py

@@ -1,9 +0,0 @@
-import csv, os
-os.chdir(r'E:\hjq\天璇')
-with open('TlsDB.csv', 'r', encoding='utf-8') as f:
-    reader = csv.DictReader(f)
-    for row in reader:
-        dtype = row.get('data_type', '').strip().upper()
-        code = row.get('field_code', '').strip().lower()
-        if 'BOOL' in dtype or code in ('cnrs', 'isrs'):
-            print(f"{row['field_code']:20s} {dtype:15s} {row.get('meaning', '')[:50]}")

_pipeline_test.py

@@ -1,73 +0,0 @@
-import urllib.request, json, time, sys
-
-BASE = 'http://127.0.0.1:18766'
-RUN_ID = 2  # the upload we just did
-
-# Step 1: Trigger manual analysis (full pipeline: clustering + UMAP + SVD)
-print('[1] Starting manual analysis pipeline...')
-body = json.dumps({
-    'run_id': int(RUN_ID),
-    'algorithm': 'hdbscan',
-    'min_cluster_size': 5,
-    'cluster_mode': 'raw',
-}).encode('utf-8')
-
-req = urllib.request.Request(f'{BASE}/analyze/run/', data=body,
-    headers={'Content-Type': 'application/json'})
-try:
-    r = urllib.request.urlopen(req, timeout=10)
-    resp = json.loads(r.read())
-    r.close()
-    print(f'[1] Started: {resp}')
-except Exception as e:
-    print(f'[1] FAILED: {e}')
-    sys.exit(1)
-
-# Step 2: Poll until completed or failed
-print('[2] Polling analysis progress...')
-for i in range(240):
-    try:
-        r = urllib.request.urlopen(f'{BASE}/runs/{RUN_ID}/status/', timeout=5)
-        s = json.loads(r.read())
-        r.close()
-    except:
-        time.sleep(2)
-        continue
-
-    status = s['status']
-    pct = s.get('progress_pct', 0)
-    msg = s.get('progress_msg', '')
-    if i % 10 == 0:
-        clusters = s.get('cluster_count') or '?'
-        entities = s.get('entity_count') or '?'
-        print(f'  [{i}] {status} ({pct}%) clusters={clusters} entities={entities} {msg[:60]}')
-
-    if status == 'completed':
-        print(f'\n[2] DONE!')
-        print(f'    status: {status}')
-        print(f'    cluster_count: {s.get("cluster_count")}')
-        print(f'    entity_count: {s.get("entity_count")}')
-        print(f'    progress: {pct}%')
-        break
-    if status == 'failed':
-        err = s.get('error_message', '')[:800]
-        print(f'\n[2] FAILED: {err}')
-        break
-    time.sleep(3)
-else:
-    print('[2] TIMEOUT after 12 minutes')
-
-# Step 3: Verify cluster overview page
-print('[3] Checking cluster overview...')
-try:
-    r = urllib.request.urlopen(f'{BASE}/clusters/{RUN_ID}/', timeout=10)
-    body = r.read().decode('utf-8', errors='replace')
-    r.close()
-    has_scatter = 'scatter_data' in body
-    has_geo = 'geo_data' in body
-    has_umap = 'UMAP' in body
-    print(f'    scatter_data: {has_scatter}')
-    print(f'    geo_data: {has_geo}')
-    print(f'    UMAP text: {has_umap}')
-except Exception as e:
-    print(f'    FAILED: {e}')

analysis/column_processor.py

@@ -0,0 +1,162 @@
+"""Unified column processor — dispatches per-type distance computation.
+
+Replaces the if-elif chain in :func:`distance.run_preprocessing_pipeline`
+with :data:`~analysis.types.TYPE_REGISTRY` dispatch.
+
+Usage::
+
+    from analysis.types import classify_schema
+    from analysis.column_processor import ColumnProcessor
+
+    column_types = classify_schema(lf)
+    proc = ColumnProcessor(lf, column_types)
+    lf = proc.compute_distances()   # Applies each type's .distance()
+    lf = proc.normalize()           # StandardScaler on numeric dist cols
+    lf, noise = proc.svd_denoise()  # SVD with noise profile
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import polars as pl
+
+from sklearn.preprocessing import StandardScaler
+from sklearn.decomposition import TruncatedSVD
+
+
+# ── Prefixes produced by each type's .distance() ──
+_DIST_PREFIXES: tuple[str, ...] = (
+    '_ip_dist_', '_str_dist_', '_bool_dist_', '_bytes_dist_',
+    '_ts_dist_', '_enum_dist_', '_raw_',
+)
+
+# ── Types that produce numeric distance/raw output (all 8 do) ──
+_ALL_TYPES: set[str] = {
+    "枚举", "字节", "浮点", "整数", "布尔", "字符串", "IPv4", "时间戳",
+}
+
+
+class ColumnProcessor:
+    """Processes all columns in a dataset by their registered types.
+
+    Groups columns by type, dispatches each group to the corresponding
+    :class:`~analysis.types.ColumnType` instance in
+    :data:`~analysis.types.TYPE_REGISTRY`, then chains normalisation
+    and SVD denoising.
+
+    Parameters
+    ----------
+    lf:
+        Input LazyFrame with raw data columns.
+    column_types:
+        ``{col_name: ColumnType}`` mapping from
+        :func:`~analysis.types.classify_schema`.
+    """
+
+    def __init__(
+        self,
+        lf: pl.LazyFrame,
+        column_types: dict,
+    ) -> None:
+        self._lf = lf
+        self._column_types = column_types
+        self._feature_cols: list[str] = []
+        self._noise_profile: dict = {}
+
+    # ── public API ──────────────────────────────────────────────────────
+
+    @property
+    def feature_cols(self) -> list[str]:
+        """Numeric feature columns available after :meth:`compute_distances`."""
+        return list(self._feature_cols)
+
+    @property
+    def noise_profile(self) -> dict:
+        """SVD noise profile populated by :meth:`svd_denoise`."""
+        return dict(self._noise_profile)
+
+    @property
+    def lf(self) -> pl.LazyFrame:
+        """Current LazyFrame state."""
+        return self._lf
+
+    def compute_distances(self) -> pl.LazyFrame:
+        """Group columns by type and dispatch to TYPE_REGISTRY.
+
+        Each :class:`~analysis.types.ColumnType` subclass handles all
+        columns of that type in one call.  IP columns with fewer than
+        2 entries are skipped (they need pairs).
+
+        Returns the LazyFrame with distance/raw columns appended.
+        """
+        from analysis.types import TYPE_REGISTRY
+
+        # Group columns by type name
+        groups: dict[str, list[str]] = {}
+        for col_name, ct in self._column_types.items():
+            groups.setdefault(ct.name, []).append(col_name)
+
+        result = self._lf
+
+        # Dispatch each group to its type's distance method
+        for type_name, cols in groups.items():
+            type_obj = TYPE_REGISTRY.get(type_name)
+            if type_obj is None:
+                continue
+            # IP distance needs at least 2 columns (pairs)
+            if type_name == "IPv4" and len(cols) < 2:
+                continue
+            result = type_obj.distance(result, cols)
+
+        # ── Collect feature columns produced by distance computation ──
+        schema = result.collect_schema()
+        self._feature_cols = [
+            c for c in schema.names()
+            if any(c.startswith(p) for p in _DIST_PREFIXES)
+        ]
+
+        self._lf = result
+        return result
+
+    def normalize(self) -> pl.LazyFrame:
+        """Apply StandardScaler normalisation to feature columns.
+
+        Uses :func:`analysis.distance.normalize_features` internally.
+        Requires :meth:`compute_distances` to have been called first.
+        """
+        if not self._feature_cols:
+            return self._lf
+
+        from analysis.distance import normalize_features
+        self._lf = normalize_features(self._lf, self._feature_cols)
+        return self._lf
+
+    def svd_denoise(self, variance_threshold: float = 0.95) -> tuple[pl.LazyFrame, dict]:
+        """SVD noise removal on normalised feature columns.
+
+        Uses :func:`analysis.distance.svd_denoise` internally.
+        Requires :meth:`normalize` to have been called first.
+
+        Parameters
+        ----------
+        variance_threshold:
+            Fraction of total variance to preserve (0.0–1.0).
+
+        Returns
+        -------
+        tuple[pl.LazyFrame, dict]
+            ``(denoised_lf, noise_profile)``.
+        """
+        if not self._feature_cols:
+            self._noise_profile = {
+                'kept_components': 0, 'total_components': 0,
+                'kept_variance': 0.0, 'noise_components': [],
+            }
+            return self._lf, self._noise_profile
+
+        from analysis.distance import svd_denoise as _svd_denoise
+        norm_cols = [f'_norm_{c}' for c in self._feature_cols]
+        self._lf, self._noise_profile = _svd_denoise(
+            self._lf, norm_cols, variance_threshold,
+        )
+        return self._lf, self._noise_profile

analysis/constants.py

@@ -0,0 +1,100 @@
+"""Centralized constants for the TianXuan analysis module.
+
+All magic numbers from analysis/tools/ and analysis/views/ are defined here
+so they can be updated in one place. Import with::
+
+    from analysis.constants import RANDOM_SEED, MAX_SAMPLE_ROWS, ...
+"""
+
+import polars as pl
+
+# ---------------------------------------------------------------------------
+#  Clustering / UMAP — tuned for 25M rows × 50 cols on 8GB RAM
+# ---------------------------------------------------------------------------
+
+MAX_SAMPLE_ROWS = 200000
+"""Maximum rows for clustering sample.  200K ≈ 0.8% of 25M rows, fits 8GB RAM."""
+
+MAX_DISTRIBUTION_SAMPLE = 50000
+"""Default sample for distribution exploration — 50K provides stable statistics."""
+
+UMAP_TRAIN_SAMPLE = 30000
+"""Max rows for UMAP training.  30K is a balance: UMAP O(n²) for neighbors."""
+# Note: UMAP neighbors search is O(n * n_neighbors * log n).  30K × 15 ≈ fine.
+# Going above 50K causes UMAP to allocate several GB for the nearest-neighbor index.
+
+UMAP_BATCH_SIZE = 2000
+"""Batch size for UMAP transform when dataset exceeds UMAP_TRAIN_SAMPLE."""
+
+LOW_MEMORY_THRESHOLD_GB = 2
+"""If available memory < this (in GiB), enable low-memory clustering path."""
+
+LOW_MEMORY_THRESHOLD_ROWS = 500000
+"""Row count above which low-memory downsampling is triggered.
+500K gives 2% coverage of 25M rows while keeping numpy matrix < 200 MB."""
+
+RANDOM_SEED = 42
+"""Default random seed for all sklearn/numpy operations."""
+
+# ---------------------------------------------------------------------------
+#  Globe — 8GB RAM / 25M rows constraints
+# ---------------------------------------------------------------------------
+
+GLOBE_MAX_ROWS = 500
+"""Max rows to load per run for 3D globe visualization.  300→500 for better
+coverage on the globe, still < 1 MB of flow data."""
+
+# ---------------------------------------------------------------------------
+#  Filter
+# ---------------------------------------------------------------------------
+
+FILTER_MAX_ROWS = 5
+"""Maximum visible filter rows in the manual analysis UI (deprecated / reserved)."""
+
+MAX_COLUMNS_PROFILE = 200
+"""Max columns to include in a dataset profile summary."""
+
+# ---------------------------------------------------------------------------
+#  File upload limits
+# ---------------------------------------------------------------------------
+
+FILE_UPLOAD_MAX_SIZE = 500 * 1024 * 1024  # 500 MB
+"""Maximum size for a single uploaded file."""
+
+TOTAL_UPLOAD_MAX_SIZE = 50 * 1024 * 1024 * 1024  # 50 GB
+"""Maximum total upload size across all files."""
+
+# ---------------------------------------------------------------------------
+#  Tool limits (numeric type tuples used across tools/)
+# ---------------------------------------------------------------------------
+
+NUMERIC_DTYPES: tuple[type[pl.DataType], ...] = (
+    pl.Int8, pl.Int16, pl.Int32, pl.Int64,
+    pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
+    pl.Float32, pl.Float64,
+)
+"""Polars numeric data types — used for column selection in clustering,
+entity aggregation, anomaly detection, and feature extraction."""
+
+# ---------------------------------------------------------------------------
+#  Diagnostics / Validation
+# ---------------------------------------------------------------------------
+
+MAX_VALIDATE_SAMPLE = 5000
+"""Max rows sampled during validate_data."""
+
+MAX_DIAGNOSE_SAMPLE = 50000
+"""Max rows sampled for clustering diagnosis (SVD).  50K for stable SVD."""
+
+MAX_CLUSTER_FEATURES = 50
+"""Feature count threshold above which TruncatedSVD is applied."""
+
+CLUSTER_MAX_FEATURES_TOP = 10
+"""Max feature columns selected by auto-detection in clustering tools."""
+
+# ---------------------------------------------------------------------------
+#  Globe / Flow extraction
+# ---------------------------------------------------------------------------
+
+GLOBE_FLOW_MAX_ROWS = 500
+"""Alias for GLOBE_MAX_ROWS — max rows loaded for flow extraction."""

analysis/data_loader.py

@@ -15,7 +15,7 @@ from typing import Optional
 import polars as pl
 import yaml
 
-from analysis.type_classifier import classify_schema, DataType
+from analysis.types import classify_schema, TYPE_REGISTRY
 
 logger = logging.getLogger(__name__)
 
@@ -383,6 +383,79 @@ def load_from_db(
         conn.close()
 
 
+def load_from_db_lazy(
+    table_name: str,
+    schema_overrides: dict[str, pl.DataType | str] | None = None,
+) -> Optional[pl.LazyFrame]:
+    """Load a SQLite table into a LazyFrame using Polars native DB reader.
+
+    Uses ``pl.read_database`` for efficient bulk reading — an order of
+    magnitude faster than the manual batch-processing path in
+    :func:`load_from_db`.  Returns a :class:`pl.LazyFrame` so downstream
+    operations remain lazy (query-plan, not materialised).
+
+    Args:
+        table_name: SQLite table name.
+        schema_overrides: Optional ``{column_name: dtype}`` dict to cast
+            columns after loading.  Values can be ``pl.DataType`` instances
+            (e.g. ``pl.Float64``) or dtype strings (e.g. ``'Float64'``).
+
+    Returns:
+        ``None`` if the table does not exist or is empty.
+    """
+    from django.conf import settings
+
+    db_path = settings.DATABASES['default']['NAME']
+    conn = sqlite3.connect(db_path)
+
+    try:
+        # Check table existence (fast metadata query)
+        cursor = conn.execute(
+            "SELECT name FROM sqlite_master WHERE type='table' AND name=?",
+            (table_name,),
+        )
+        if cursor.fetchone() is None:
+            return None
+
+        # Bulk-read with Polars native engine — leverages the DBAPI2
+        # connection directly for efficient C-level data transfer.
+        query = f'SELECT * FROM "{table_name}"'
+        df = pl.read_database(connection=conn, query=query)
+    finally:
+        conn.close()
+
+    if df.height == 0:
+        return None
+
+    # Apply type overrides to restore original column types
+    # (SQLite stores everything as TEXT/INTEGER/REAL/BLOB;
+    #  we need to cast back to Float64 / Int64 / etc.)
+    if schema_overrides:
+        casts = {}
+        for col, dtype in schema_overrides.items():
+            if col not in df.columns:
+                continue
+            dtype_obj = (
+                dtype if isinstance(dtype, pl.DataType)
+                else _str_to_polars_dtype(dtype)
+            )
+            if df[col].dtype != dtype_obj:
+                try:
+                    casts[col] = pl.col(col).cast(dtype_obj)
+                except Exception:
+                    pass  # uncastable column — keep as-is
+        if casts:
+            df = df.with_columns(list(casts.values()))
+
+    rows = df.height
+    cols = df.width
+    logger.info(
+        '[LOAD_FROM_DB_LAZY] table=%s rows=%d cols=%d',
+        table_name, rows, cols,
+    )
+    return df.lazy()
+
+
 def drop_sqlite_table(table_name: str) -> None:
     """Drop a SQLite data table silently if it exists."""
     from django.conf import settings
@@ -551,7 +624,7 @@ def load_csv_directory(
         merged_schema = {k: v for k, v in merged_schema.items() if k not in existing_drop}
         logger.info('[CLEAN] dropped columns: %s', existing_drop)
 
-    # ── Cleaning: BOOL_ENUM normalisation ────────────────────────────────
+    # ── Cleaning: BOOL normalisation ─────────────────────────────────────
     try:
         type_map = classify_schema(
             merged_lf.head(1000),
@@ -560,21 +633,20 @@ def load_csv_directory(
     except Exception:
         type_map = {}
 
-    bool_enum_cols = [
+    bool_cols = [
         c for c, t in type_map.items()
-        if t == DataType.BOOL_ENUM and c in merged_schema
+        if t.name == "布尔" and c in merged_schema
     ]
-    if bool_enum_cols:
+    if bool_cols:
-        # Only clean string-typed BOOL_ENUM columns; native Boolean is already clean
         schema_dtypes = dict(zip(
             merged_lf.collect_schema().names(),
             merged_lf.collect_schema().dtypes(),
         ))
         bool_exprs = []
         cleaned_names: list[str] = []
-        for c in bool_enum_cols:
+        for c in bool_cols:
             if schema_dtypes.get(c) not in (pl.Utf8, pl.String):
-                continue  # native Boolean / numeric — skip
+                continue
             c_str = pl.col(c).cast(pl.Utf8).str.strip_chars()
             cleaned = (
                 pl.when(c_str == "")
@@ -591,37 +663,12 @@ def load_csv_directory(
             merged_lf = merged_lf.with_columns(bool_exprs)
             for c in cleaned_names:
                 merged_schema[c] = 'Utf8'
-            logger.info('[CLEAN] BOOL_ENUM columns normalised: %s', cleaned_names)
+            logger.info('[CLEAN] BOOL columns normalised: %s', cleaned_names)
 
-    # ── Cleaning: LAT_LON normalisation ────────────────────────────────
+    # ── Generic numeric coercion ────────────────────────────────────────
-    latlon_cols = [
+    # For remaining string columns (not IPv4/BYTES/ENUM/BOOL etc.),
-        c for c, t in type_map.items()
+    # try to coerce to Float64.
-        if t == DataType.LAT_LON and c in merged_schema
+    _SKIP_NUM_COERCE = {"IPv4", "字节", "枚举", "布尔"}
-    ]
-    if latlon_cols:
-        schema_dtypes_ll = dict(zip(
-            merged_lf.collect_schema().names(),
-            merged_lf.collect_schema().dtypes(),
-        ))
-        latlon_exprs = []
-        for c in latlon_cols:
-            dtype = schema_dtypes_ll.get(c)
-            if dtype in (pl.Utf8, pl.String):
-                latlon_exprs.append(
-                    pl.col(c).map_batches(_coerce_to_float, return_dtype=pl.Float64).alias(c)
-                )
-                merged_schema[c] = 'Float64'
-        if latlon_exprs:
-            merged_lf = merged_lf.with_columns(latlon_exprs)
-            logger.info('[CLEAN] LAT_LON columns normalised to Float64: %s', latlon_cols)
-
-    # ── Cleaning: Generic numeric coercion ────────────────────────────
-    # For any remaining string column (not IPv4/URL/HEX/ENUM/etc.),
-    # try to coerce to Float64. Whatever can't parse becomes null.
-    # This prevents "mean on str column" crashes for ANY column.
-    _SKIP_NUM_COERCE = (DataType.IPv4, DataType.URL, DataType.HEX,
-                         DataType.ENUM, DataType.LAT_LON, DataType.BOOL_ENUM)
-    # Also protect columns with these name patterns (MAC addresses, etc.)
     _SKIP_NAME_PATTERNS = ('mac', 'mac_', '_mac', 'addr', 'address')
     schema_dtypes_gn = dict(zip(
         merged_lf.collect_schema().names(),
@@ -629,14 +676,13 @@ def load_csv_directory(
     ))
     generic_numeric_cols = []
     for c, t in type_map.items():
-        if t in _SKIP_NUM_COERCE:
+        if t.name in _SKIP_NUM_COERCE:
             continue
         if c not in merged_schema:
             continue
         dtype = schema_dtypes_gn.get(c)
         if dtype not in (pl.Utf8, pl.String):
-            continue  # already numeric
+            continue
-        # Skip columns with protected name patterns
         cl = c.lower().replace('-', '_').replace(' ', '_')
         if any(p in cl for p in _SKIP_NAME_PATTERNS):
             continue
@@ -654,10 +700,10 @@ def load_csv_directory(
                 merged_schema[c] = 'Float64'
             logger.info('[CLEAN] Generic numeric coercion applied: %s', generic_numeric_cols)
 
-    # ── Cleaning: HEX preprocessing (hex_pairs_count) ────────────────────
+    # ── Cleaning: BYTES (hex) preprocessing ─────────────────────────────
     hex_cols = [
         c for c, t in type_map.items()
-        if t == DataType.HEX and c in merged_schema
+        if t.name == "字节" and c in merged_schema
     ]
     if hex_cols:
         hex_exprs = []
@@ -668,7 +714,7 @@ def load_csv_directory(
         merged_lf = merged_lf.with_columns(hex_exprs)
         for c in hex_cols:
             merged_schema[f"{c}_hex_pairs_count"] = 'UInt32'
-        logger.info('[CLEAN] HEX columns added hex_pairs_count: %s', hex_cols)
+        logger.info('[CLEAN] BYTES columns added hex_pairs_count: %s', hex_cols)
 
     # Row count estimate (fast path: use first file row count × files)
     # For accurate count we'd need to collect, but that defeats lazy.

analysis/geoip.py

@@ -1,14 +1,18 @@
 """
-离线 GeoIP 查询。内嵌 IPv4 区间 → 城市/经纬度映射。
+离线 GeoIP 查询。纯本地运行，无网络请求。
 
-从 ``data/geoip_data.txt`` 加载 IP 段数据，在模块首次导入时构建
+三级存储：
-排序索引，提供 O(log N) 的 IP 查询能力。
+1. 快速缓存 (data/geoip_cache.json) — 已查过的 IP，磁盘持久化
+2. 离线数据库 (data/geoip_data.txt) — 手动整理的优先 IP 段
+3. MMDB 数据库 — 若 python_geoip_geolite2 包已安装，加载其
+   内置的 GeoLite2-City.mmdb 作为最全面回退。
 
-数据约 200KB，覆盖 23 个主要城市 IP 段。
+查询链：缓存 > geoip_data.txt > MMDB > 写入缓存。
-可通过编辑 ``data/geoip_data.txt`` 扩展。
 """
 import bisect
+import json
 import logging
+import os
 from pathlib import Path
 from typing import Optional
 
@@ -18,32 +22,22 @@ logger = logging.getLogger(__name__)
 #  Data structures
 # ---------------------------------------------------------------------------
 
-# [(start_int, end_int, lat, lon, city, country), ...]
-# Built at module load time from data/geoip_data.txt
 _GEO_DATA: list[tuple[int, int, float, float, str, str]] = []
-
-# Sorted list of start integers (for bisect lookup) + parallel data list
 _STARTS: list[int] = []
 _RANGES: list[tuple[int, int, float, float, str, str]] = []
 
+_MMDB_READER = None
+
+_CACHE: dict[str, Optional[dict]] = {}
+_CACHE_DIRTY = False
+_CACHE_PATH = Path(__file__).resolve().parent.parent / 'data' / 'geoip_cache.json'
+
 
 # ---------------------------------------------------------------------------
 #  IP conversion
 # ---------------------------------------------------------------------------
 
 def ip_to_int(ip_str: str) -> Optional[int]:
-    """Convert an IPv4 dotted-string to its 32-bit integer representation.
-
-    Returns ``None`` if *ip_str* is not a valid IPv4 address (including
-    when any octet is outside 0-255).
-
-    Examples
-    --------
-    >>> ip_to_int('8.8.8.8')
-    134744072
-    >>> ip_to_int('1.0.0.0')
-    16777216
-    """
     parts = ip_str.strip().split('.')
     if len(parts) != 4:
         return None
@@ -57,78 +51,144 @@ def ip_to_int(ip_str: str) -> Optional[int]:
 
 
 # ---------------------------------------------------------------------------
-#  Index building
+#  geoip_data.txt loader
 # ---------------------------------------------------------------------------
 
-def _build_index() -> tuple[list[int], list[tuple[int, int, float, float, str, str]]]:
+def _build_index():
-    """Build sorted parallel lists for bisect-based IP lookup.
-
-    Processes ``_GEO_DATA``, sorts by start integer, and returns
-    ``(starts, ranges)`` where *starts* is a sorted list of start integers
-    and *ranges* is the corresponding data list at the same indices.
-    """
     if not _GEO_DATA:
         return [], []
-
     sorted_data = sorted(_GEO_DATA, key=lambda x: x[0])
     starts = [entry[0] for entry in sorted_data]
     return starts, sorted_data
 
 
 def _load_data_file() -> None:
-    """Load IP range data from ``data/geoip_data.txt`` into ``_GEO_DATA``.
+    data_path = Path(__file__).resolve().parent.parent / 'data' / 'geoip_data.txt'
-
-    Called once at module import time. Each non-blank, non-comment line
-    in the file should contain six comma-separated fields::
-
-        start_ip,end_ip,lat,lon,city,country
-    """
-    data_path = Path(__file__).parent.parent / 'data' / 'geoip_data.txt'
-
     if not data_path.exists():
         logger.warning('[GEOIP] data file not found: %s', data_path)
         return
-
+    loaded = []
-    loaded: list[tuple[int, int, float, float, str, str]] = []
     skipped = 0
-
     with open(data_path, 'r', encoding='utf-8') as f:
-        for line_no, raw in enumerate(f, start=1):
+        for raw in f:
             stripped = raw.strip()
             if not stripped or stripped.startswith('#'):
                 continue
-
             parts = [p.strip() for p in stripped.split(',')]
             if len(parts) != 6:
                 skipped += 1
                 continue
-
+            sip, eip, lat_s, lon_s, city, country = parts
-            start_ip_str, end_ip_str, lat_str, lon_str, city, country = parts
+            si = ip_to_int(sip)
-
+            ei = ip_to_int(eip)
-            start_int = ip_to_int(start_ip_str)
+            if si is None or ei is None or si > ei:
-            end_int = ip_to_int(end_ip_str)
-
-            if start_int is None or end_int is None:
                 skipped += 1
                 continue
-
             try:
-                lat = float(lat_str)
+                lat = float(lat_s)
-                lon = float(lon_str)
+                lon = float(lon_s)
             except ValueError:
                 skipped += 1
                 continue
-
+            loaded.append((si, ei, lat, lon, city, country))
-            loaded.append((start_int, end_int, lat, lon, city, country))
-
     _GEO_DATA.clear()
     _GEO_DATA.extend(loaded)
-
     global _STARTS, _RANGES
     _STARTS, _RANGES = _build_index()
+    logger.info('[GEOIP] loaded %d ranges (%d skipped)', len(loaded), skipped)
 
-    logger.info('[GEOIP] loaded %d ranges from %s (%d skipped)',
+
-                len(loaded), data_path, skipped)
+# ---------------------------------------------------------------------------
+#  MMDB reader (GeoLite2-City.mmdb bundled with python_geoip_geolite2)
+# ---------------------------------------------------------------------------
+
+def _get_mmdb_reader():
+    global _MMDB_READER
+    if _MMDB_READER is not None:
+        return _MMDB_READER
+    # Try multiple locations for the MMDB file
+    candidates = []
+    try:
+        import _geoip_geolite2
+        candidates.append(Path(_geoip_geolite2.__file__).parent / 'GeoLite2-City.mmdb')
+    except Exception:
+        pass
+    # Fallback: temp directory without Chinese chars in path
+    candidates.append(Path(os.environ.get('TEMP', 'C:/temp')) / 'opencode' / 'GeoLite2-City.mmdb')
+    candidates.append(Path.home() / 'AppData' / 'Local' / 'Temp' / 'opencode' / 'GeoLite2-City.mmdb')
+
+    for mmdb_path in candidates:
+        if mmdb_path.exists():
+            try:
+                import maxminddb
+                _MMDB_READER = maxminddb.open_database(mmdb_path)
+                logger.info('[GEOIP] loaded MMDB: %s (%.1f MB)',
+                            mmdb_path.name, mmdb_path.stat().st_size / 1e6)
+                break
+            except Exception as exc:
+                logger.debug('[GEOIP] failed to open %s: %s', mmdb_path, exc)
+                continue
+
+    if _MMDB_READER is None:
+        logger.debug('[GEOIP] MMDB not available — falling back to txt database only')
+    return _MMDB_READER
+
+
+def _mmdb_lookup(ip_str: str) -> Optional[dict]:
+    reader = _get_mmdb_reader()
+    if reader is None:
+        return None
+    try:
+        result = reader.get(ip_str)
+        if result is None:
+            return None
+        location = result.get('location', {})
+        city_data = result.get('city', {})
+        country_data = result.get('country', {})
+        lat = location.get('latitude')
+        lon = location.get('longitude')
+        if lat is None or lon is None:
+            return None
+        city = ''
+        if city_data and 'names' in city_data:
+            city = city_data['names'].get('zh-CN') or city_data['names'].get('en', '')
+        country = ''
+        if country_data and 'names' in country_data:
+            country = country_data['names'].get('zh-CN') or country_data['names'].get('en', '')
+        return {'lat': float(lat), 'lon': float(lon), 'city': city, 'country': country}
+    except Exception as exc:
+        logger.debug('[GEOIP] MMDB lookup failed for %s: %s', ip_str, exc)
+        return None
+
+
+# ---------------------------------------------------------------------------
+#  Disk cache
+# ---------------------------------------------------------------------------
+
+def _load_cache() -> None:
+    if not _CACHE_PATH.exists():
+        return
+    try:
+        data = json.loads(_CACHE_PATH.read_text(encoding='utf-8'))
+        _CACHE.update(data)
+        logger.info('[GEOIP] loaded %d cached entries', len(data))
+    except Exception:
+        pass
+
+
+def _save_cache() -> None:
+    global _CACHE_DIRTY
+    if not _CACHE_DIRTY:
+        return
+    try:
+        _CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
+        _CACHE_PATH.write_text(
+            json.dumps(_CACHE, ensure_ascii=False, indent=2),
+            encoding='utf-8',
+        )
+        _CACHE_DIRTY = False
+    except Exception as exc:
+        logger.warning('[GEOIP] cache write failed: %s', exc)
 
 
 # ---------------------------------------------------------------------------
@@ -136,55 +196,44 @@ def _load_data_file() -> None:
 # ---------------------------------------------------------------------------
 
 def lookup(ip_str: str) -> Optional[dict]:
-    """查询 IP 对应的地理位置。
+    """查询 IP 地理位置。纯本地，无网络。
 
-    Uses binary search (``bisect_right``) on the sorted start-IP list to
+    查询链：磁盘缓存 > geoip_data.txt (O(log N)) > MMDB (GeoLite2) > 写入缓存。
-    locate the range *ip_str* falls within, then checks the end bound.
-
-    Parameters
-    ----------
-    ip_str:
-        IPv4 address as a dotted string (e.g. ``'8.8.8.8'``).
-
-    Returns
-    -------
-    dict or None
-        A dictionary with keys ``lat``, ``lon``, ``city``, ``country``
-        if the IP is found in the database; ``None`` otherwise.
-
-    Examples
-    --------
-    >>> result = lookup('8.8.8.8')
-    >>> result['city']
-    'Mountain View'
-    >>> lookup('10.0.0.1') is None
-    True
     """
+    # 1. Cache hit
+    cached = _CACHE.get(ip_str)
+    if cached is not None:
+        return cached
+    if ip_str in _CACHE:
+        return None
+
+    result = None
+
+    # 2. geoip_data.txt
     target = ip_to_int(ip_str)
-    if target is None:
+    if target is not None and _STARTS:
-        return None
+        idx = bisect.bisect_right(_STARTS, target) - 1
+        if idx >= 0:
+            start, end, lat, lon, city, country = _RANGES[idx]
+            if start <= target <= end:
+                result = {'lat': lat, 'lon': lon, 'city': city, 'country': country}
 
-    if not _STARTS:
+    # 3. MMDB fallback (GeoLite2, ~3M entries world-wide)
-        return None
+    if result is None:
+        result = _mmdb_lookup(ip_str)
 
-    idx = bisect.bisect_right(_STARTS, target) - 1
+    # 4. Cache for next time
-    if idx < 0:
+    global _CACHE_DIRTY
-        return None
+    _CACHE[ip_str] = result
+    _CACHE_DIRTY = True
+    _save_cache()
 
-    start, end, lat, lon, city, country = _RANGES[idx]
+    return result
-    if start <= target <= end:
-        return {
-            'lat': lat,
-            'lon': lon,
-            'city': city,
-            'country': country,
-        }
-
-    return None
 
 
 # ---------------------------------------------------------------------------
-#  Module-level initialisation
+#  Module-level init
 # ---------------------------------------------------------------------------
 
 _load_data_file()
+_load_cache()

analysis/management/commands/run_pipeline.py

@@ -69,7 +69,7 @@ class Command(BaseCommand):
             run.save(update_fields=['total_flows'])
             from analysis.views import _run_clustering_pipeline
             _run_clustering_pipeline(
-                run=run, store=store, entity_ds_id=ds_id,
+                run=run, store=store, ds_id=ds_id,
                 feature_columns=feature_cols,
                 algorithm=algo, min_cluster_size=5,
                 run_umap=True,

analysis/migrations/0001_squashed_0008_entityprofile_embedding_z_and_more.py

@@ -1,11 +1,19 @@
-# Generated by Django 4.2.30 on 2026-07-13 14:36
+# Generated by Django 4.2.30 on 2026-07-23 11:15
 
 from django.db import migrations, models
+from django.db.models import F
+import django.db.migrations.operations.special
 import django.db.models.deletion
 
 
+def populate_display_id(apps, schema_editor):
+    AnalysisRun = apps.get_model('tianxuan_analysis', 'AnalysisRun')
+    AnalysisRun.objects.filter(display_id__isnull=True).update(display_id=F('id'))
+
 class Migration(migrations.Migration):
 
+    replaces = [('tianxuan_analysis', '0001_initial'), ('tianxuan_analysis', '0002_analysisrun_progress_msg_analysisrun_progress_pct_and_more'), ('tianxuan_analysis', '0003_analysisrun_sqlite_table'), ('tianxuan_analysis', '0004_tls_ref_data'), ('tianxuan_analysis', '0005_remove_entity_column'), ('tianxuan_analysis', '0006_analysisrun_display_id_analysisrun_llm_thinking_and_more'), ('tianxuan_analysis', '0007_populate_display_id'), ('tianxuan_analysis', '0008_entityprofile_embedding_z_and_more')]
+
     initial = True
 
     dependencies = [
@@ -26,6 +34,10 @@ class Migration(migrations.Migration):
                 ('entity_count', models.IntegerField(blank=True, null=True)),
                 ('cluster_count', models.IntegerField(blank=True, null=True)),
                 ('entity_column', models.CharField(blank=True, default='', help_text='Auto-detected entity column name', max_length=256)),
+                ('progress_msg', models.CharField(blank=True, default='', max_length=256)),
+                ('progress_pct', models.IntegerField(default=0)),
+                ('run_log', models.TextField(blank=True, default='')),
+                ('sqlite_table', models.CharField(blank=True, default='', help_text='SQLite table name for persisted raw data', max_length=256)),
             ],
             options={
                 'ordering': ['-created_at'],
@@ -81,4 +93,51 @@ class Migration(migrations.Migration):
                 'unique_together': {('cluster', 'feature_name')},
             },
         ),
+        migrations.RunSQL(
+            sql="\nCREATE TABLE IF NOT EXISTS tls_ref_data (\n    id          INTEGER PRIMARY KEY AUTOINCREMENT,\n    field_code  TEXT    NOT NULL UNIQUE,\n    meaning     TEXT    NOT NULL DEFAULT '',\n    data_type   TEXT    NOT NULL DEFAULT ''\n);\n",
+            reverse_sql='\nDROP TABLE IF EXISTS tls_ref_data;\n',
+        ),
+        migrations.RemoveField(
+            model_name='analysisrun',
+            name='entity_column',
+        ),
+        migrations.AddField(
+            model_name='analysisrun',
+            name='display_id',
+            field=models.IntegerField(blank=True, help_text='User-facing numeric ID (recycles gaps)', null=True, unique=True),
+        ),
+        migrations.AddField(
+            model_name='analysisrun',
+            name='llm_thinking',
+            field=models.TextField(blank=True, default='', help_text='Accumulated LLM reasoning / thinking text'),
+        ),
+        migrations.AddField(
+            model_name='analysisrun',
+            name='run_type',
+            field=models.CharField(choices=[('upload', '手动上传'), ('manual', '手动分析'), ('auto', 'LLM自动')], default='upload', max_length=16),
+        ),
+        migrations.AddField(
+            model_name='analysisrun',
+            name='tool_calls_json',
+            field=models.JSONField(blank=True, default=list, help_text='Structured tool call list: [{step, name, input, output}, ...]'),
+        ),
+        migrations.RunPython(
+            code=populate_display_id,
+            reverse_code=django.db.migrations.operations.special.RunPython.noop,
+        ),
+        migrations.AddField(
+            model_name='entityprofile',
+            name='embedding_z',
+            field=models.FloatField(blank=True, help_text='UMAP-3D Z coordinate', null=True),
+        ),
+        migrations.AlterField(
+            model_name='entityprofile',
+            name='embedding_x',
+            field=models.FloatField(blank=True, help_text='UMAP-2D X coordinate', null=True),
+        ),
+        migrations.AlterField(
+            model_name='entityprofile',
+            name='embedding_y',
+            field=models.FloatField(blank=True, help_text='UMAP-2D Y coordinate', null=True),
+        ),
     ]

analysis/migrations/0002_analysisrun_progress_msg_analysisrun_progress_pct_and_more.py

@@ -1,28 +0,0 @@
-# Generated by Django 4.2.30 on 2026-07-16 05:11
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('tianxuan_analysis', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='analysisrun',
-            name='progress_msg',
-            field=models.CharField(blank=True, default='', max_length=256),
-        ),
-        migrations.AddField(
-            model_name='analysisrun',
-            name='progress_pct',
-            field=models.IntegerField(default=0),
-        ),
-        migrations.AddField(
-            model_name='analysisrun',
-            name='run_log',
-            field=models.TextField(blank=True, default=''),
-        ),
-    ]

analysis/migrations/0003_analysisrun_sqlite_table.py

@@ -1,18 +0,0 @@
-# Generated by Django 4.2.30 on 2026-07-20 05:11
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('tianxuan_analysis', '0002_analysisrun_progress_msg_analysisrun_progress_pct_and_more'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='analysisrun',
-            name='sqlite_table',
-            field=models.CharField(blank=True, default='', help_text='SQLite table name for persisted raw data', max_length=256),
-        ),
-    ]

analysis/migrations/0004_tls_ref_data.py

@@ -1,38 +0,0 @@
-"""Migration 0004 — create raw SQLite table ``tls_ref_data``.
-
-Replaces the ``TlsDB.csv`` reference file with a proper SQLite table.
-The table stores TLS field-code metadata: human-readable meaning and
-data type for each field code used in TLS flow data.
-"""
-
-from django.db import migrations
-
-# Raw SQL for creating the tls_ref_data table
-CREATE_TLS_REF_DATA = """
-CREATE TABLE IF NOT EXISTS tls_ref_data (
-    id          INTEGER PRIMARY KEY AUTOINCREMENT,
-    field_code  TEXT    NOT NULL UNIQUE,
-    meaning     TEXT    NOT NULL DEFAULT '',
-    data_type   TEXT    NOT NULL DEFAULT ''
-);
-"""
-
-DROP_TLS_REF_DATA = """
-DROP TABLE IF EXISTS tls_ref_data;
-"""
-
-
-class Migration(migrations.Migration):
-    """Create ``tls_ref_data`` table via raw SQL — no Django model needed."""
-
-    dependencies = [
-        ('tianxuan_analysis', '0003_analysisrun_sqlite_table'),
-    ]
-
-    operations = [
-        migrations.RunSQL(
-            sql=CREATE_TLS_REF_DATA,
-            reverse_sql=DROP_TLS_REF_DATA,
-            state_operations=None,
-        ),
-    ]

analysis/migrations/0005_remove_entity_column.py

@@ -1,17 +0,0 @@
-# Generated by Django 4.2.30 on 2026-07-20 05:20
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('tianxuan_analysis', '0004_tls_ref_data'),
-    ]
-
-    operations = [
-        migrations.RemoveField(
-            model_name='analysisrun',
-            name='entity_column',
-        ),
-    ]

analysis/migrations/0006_analysisrun_display_id_analysisrun_llm_thinking_and_more.py

@@ -1,33 +0,0 @@
-# Generated by Django 4.2.30 on 2026-07-20 05:25
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('tianxuan_analysis', '0005_remove_entity_column'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='analysisrun',
-            name='display_id',
-            field=models.IntegerField(blank=True, help_text='User-facing numeric ID (recycles gaps)', null=True, unique=True),
-        ),
-        migrations.AddField(
-            model_name='analysisrun',
-            name='llm_thinking',
-            field=models.TextField(blank=True, default='', help_text='Accumulated LLM reasoning / thinking text'),
-        ),
-        migrations.AddField(
-            model_name='analysisrun',
-            name='run_type',
-            field=models.CharField(choices=[('upload', '手动上传'), ('manual', '手动分析'), ('auto', 'LLM自动')], default='upload', max_length=16),
-        ),
-        migrations.AddField(
-            model_name='analysisrun',
-            name='tool_calls_json',
-            field=models.JSONField(blank=True, default=list, help_text='Structured tool call list: [{step, name, input, output}, ...]'),
-        ),
-    ]

analysis/migrations/0007_populate_display_id.py

@@ -1,20 +0,0 @@
-# Generated by Django 4.2.30 on 2026-07-20 05:26
-
-from django.db import migrations
-from django.db.models import F
-
-
-def populate_display_id(apps, schema_editor):
-    AnalysisRun = apps.get_model('tianxuan_analysis', 'AnalysisRun')
-    AnalysisRun.objects.filter(display_id__isnull=True).update(display_id=F('id'))
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('tianxuan_analysis', '0006_analysisrun_display_id_analysisrun_llm_thinking_and_more'),
-    ]
-
-    operations = [
-        migrations.RunPython(populate_display_id, migrations.RunPython.noop),
-    ]

analysis/models.py

@@ -96,6 +96,7 @@ class EntityProfile(models.Model):
                                      help_text="Aggregated entity features as dict")
     embedding_x = models.FloatField(null=True, blank=True, help_text="UMAP-2D X coordinate")
     embedding_y = models.FloatField(null=True, blank=True, help_text="UMAP-2D Y coordinate")
+    embedding_z = models.FloatField(null=True, blank=True, help_text="UMAP-3D Z coordinate")
 
     class Meta:
         unique_together = ['run', 'entity_value']

analysis/nl_describe.py

@@ -0,0 +1,412 @@
+"""
+Template-based natural language feature descriptions.
+
+Uses value-range → text mappings to generate Chinese natural language
+descriptions of cluster features. Consumed by cluster_overview.html and
+entity_profile.html for human-readable cluster summaries.
+
+Data sources
+------------
+- ``analysis.tls_ref.get_tls_ref_dict()`` : ``field_code → {meaning, data_type}``
+- ``ClusterFeature`` model rows : feature_name, mean, std, distinguishing_score
+
+Usage::
+
+    from analysis.nl_describe import describe_feature, describe_cluster
+
+    text = describe_feature("8ack", 720.5, std_dev=2.1)
+    # "中等数据包 (720 bytes)，比全局均值高2.1个标准差"
+
+    summary = describe_cluster(features, top_n=5)
+    # "此簇特征: 低延迟(23ms), 大数据包(1520 bytes), ..."
+"""
+
+from __future__ import annotations
+
+from typing import Optional, Sequence
+
+# ---------------------------------------------------------------------------
+# Category mapping: field_code → semantic category
+# ---------------------------------------------------------------------------
+
+CATEGORY_EXACT: dict[str, str] = {
+    # ── latency ──
+    "4dur": "latency",
+    "2tmo": "latency",
+    "8dbd": "latency",
+    # ── payload bytes ──
+    "8ack": "packet_bytes",
+    "8byt": "packet_bytes",
+    # ── packet count ──
+    "8pak": "packet_count",
+    "8ppk": "packet_count",
+    # ── port ──
+    ":prs": "port",
+    ":prd": "port",
+    # ── TLS version ──
+    "0ver": "tls_version",
+    # ── key size ──
+    "4ksz": "key_size",
+    # ── cipher code ──
+    "0cph": "cipher_code",
+    # ── curve code ──
+    "0crv": "curve_code",
+    # ── session / offset ──
+    "8seq": "session_offset",
+    "8ses": "session_offset",
+    "8did": "session_offset",
+    # ── IP protocol ──
+    "1ipp": "ip_protocol",
+    # ── TLS random ──
+    "0rnt": "tls_random_time",
+    "0rnd": "tls_random",
+    # ── database refs ──
+    "4dbn": "database_ref",
+    "4srs": "device_ref",
+    # ── recovery flags ──
+    "cnrs": "recovery",
+    "isrs": "recovery",
+}
+
+# Substring-based matching (checked after exact match fails)
+CATEGORY_SUBSTR: list[tuple[str, str]] = [
+    ("latd", "latitude"),
+    ("lond", "longitude"),
+    ("orgn", "organization"),
+    ("ispn", "isp"),
+    ("anon", "anonymity"),
+    ("doma", "domain"),
+    ("city", "city"),
+    ("_lat", "latitude"),
+    ("_lon", "longitude"),
+    ("latitude", "latitude"),
+    ("longitude", "longitude"),
+    ("duration", "latency"),
+    ("delay", "latency"),
+    ("bytes", "packet_bytes"),
+    ("packets", "packet_count"),
+    ("count", "count"),
+    ("ratio", "ratio"),
+    ("tls_ver", "tls_version"),
+    ("version", "tls_version"),
+]
+
+# ---------------------------------------------------------------------------
+# TLS version code → human description
+# ---------------------------------------------------------------------------
+
+TLS_VERSION_MAP: dict[int, str] = {
+    0x0000: "SSL 2.0",
+    0x0300: "SSL 3.0",
+    0x0301: "TLS 1.0",
+    0x0302: "TLS 1.1",
+    0x0303: "TLS 1.2",
+    0x0304: "TLS 1.3",
+    0x7F00: "TLS 1.3 (draft)",
+    0x7F12: "TLS 1.3 (draft 18)",
+    0xFEFF: "DTLS 1.0",
+}
+
+# ---------------------------------------------------------------------------
+# Templates: category → [(predicate, template_string), ...]
+#
+# Template variables available:
+#   {v}      — value (formatted to 1 decimal unless overridden)
+#   {v:.0f}  — value with custom format
+#   {sv}     — signed value (e.g. "+2.1" for positive zscore)
+#   {av}     — absolute value (e.g. "2.1")
+#   {z:.1f}  — z-score signed (preserved in deviation suffix)
+# ---------------------------------------------------------------------------
+
+TEMPLATES: dict[str, list[tuple]] = {
+    "latency": [
+        (lambda v: v > 1000, "极高延迟 ({v:.0f}ms)"),
+        (lambda v: v > 500, "高延迟 ({v:.0f}ms)"),
+        (lambda v: v > 200, "中等延迟 ({v:.0f}ms)"),
+        (lambda v: v > 50, "较低延迟 ({v:.0f}ms)"),
+        (lambda v: True, "低延迟 ({v:.0f}ms)"),
+    ],
+    "packet_bytes": [
+        (lambda v: v > 1500, "大数据包 ({v:.0f} bytes, 超过MTU)"),
+        (lambda v: v > 1000, "较大数据包 ({v:.0f} bytes)"),
+        (lambda v: v > 400, "中等数据包 ({v:.0f} bytes)"),
+        (lambda v: True, "小数据包 ({v:.0f} bytes)"),
+    ],
+    "packet_count": [
+        (lambda v: v > 200, "大量数据包 ({v:.0f} 个)"),
+        (lambda v: v > 50, "中等数据包数 ({v:.0f} 个)"),
+        (lambda v: True, "少量数据包 ({v:.0f} 个)"),
+    ],
+    "port": [
+        (lambda v: v <= 1024, "系统端口 ({v:.0f})"),
+        (lambda v: v <= 49151, "注册端口 ({v:.0f})"),
+        (lambda v: True, "动态/私有端口 ({v:.0f})"),
+    ],
+    "tls_version": [
+        (lambda v: 0x0304 <= v <= 0x0304, "TLS 1.3 (最新)"),
+        (lambda v: 0x0303 <= v < 0x0304, "TLS 1.2 (主流)"),
+        (lambda v: 0x0302 <= v < 0x0303, "TLS 1.1 (已弃用)"),
+        (lambda v: 0x0301 <= v < 0x0302, "TLS 1.0 (过时)"),
+        (lambda v: 0x0300 <= v < 0x0301, "SSL 3.0 (不安全)"),
+        (lambda v: v < 0x0300, "SSL 2.0 (极不安全)"),
+        (lambda v: v > 0x0304, "DTLS/实验性TLS (0x{v:.0X})"),
+        (lambda v: True, "未知TLS版本 (0x{v:.0X})"),
+    ],
+    "key_size": [
+        (lambda v: v >= 384, "高强度密钥 ({v:.0f} bits)"),
+        (lambda v: v >= 256, "强密钥 ({v:.0f} bits)"),
+        (lambda v: v >= 128, "标准密钥 ({v:.0f} bits)"),
+        (lambda v: v >= 64, "弱密钥 ({v:.0f} bits)"),
+        (lambda v: True, "极弱密钥 ({v:.0f} bits)"),
+    ],
+    "ip_protocol": [
+        (lambda v: abs(v - 6) < 0.5, "TCP 协议 ({v:.0f})"),
+        (lambda v: abs(v - 17) < 0.5, "UDP 协议 ({v:.0f})"),
+        (lambda v: abs(v - 1) < 0.5, "ICMP 协议 ({v:.0f})"),
+        (lambda v: abs(v - 2) < 0.5, "IGMP 协议 ({v:.0f})"),
+        (lambda v: True, "IP 协议 #{v:.0f}"),
+    ],
+    "session_offset": [
+        (lambda v: v > 10000, "大型会话 (偏移 {v:.0f})"),
+        (lambda v: v > 1000, "中型会话 (偏移 {v:.0f})"),
+        (lambda v: True, "小型会话 (偏移 {v:.0f})"),
+    ],
+    "latitude": [
+        (lambda v: v > 60, "高纬度地区 ({v:.1f}°)"),
+        (lambda v: v > 23.5, "中纬度地区 — 北温带 ({v:.1f}°)"),
+        (lambda v: v >= -23.5, "低纬度地区 — 热带 ({v:.1f}°)"),
+        (lambda v: v >= -60, "中纬度地区 — 南温带 ({v:.1f}°)"),
+        (lambda v: True, "高纬度地区 — 南极 ({v:.1f}°)"),
+    ],
+    "longitude": [
+        (lambda v: v > 120 or v < -120, "远东地区 ({v:.1f}°)"),
+        (lambda v: v > 60 or v < -60, "中亚/东欧地区 ({v:.1f}°)"),
+        (lambda v: v > 0 or v < 0, "西欧/非洲地区 ({v:.1f}°)"),
+        (lambda v: True, "本初子午线附近 ({v:.1f}°)"),
+    ],
+    "cipher_code": [
+        (lambda v: True, "加密套件代码 0x{v:.0X}"),
+    ],
+    "curve_code": [
+        (lambda v: True, "椭圆曲线代码 0x{v:.0X}"),
+    ],
+    "tls_random_time": [
+        (lambda v: True, "TLS 随机时间戳 ({v:.0f})"),
+    ],
+    "tls_random": [
+        (lambda v: True, "TLS 随机值"),
+    ],
+    "database_ref": [
+        (lambda v: True, "数据库引用 #{v:.0f}"),
+    ],
+    "device_ref": [
+        (lambda v: True, "设备标识 #{v:.0f}"),
+    ],
+    "recovery": [
+        (lambda v: v >= 0.5, "可恢复流"),
+        (lambda v: True, "不可恢复流"),
+    ],
+    "organization": [
+        (lambda v: True, "组织指标 ({v:.1f})"),
+    ],
+    "isp": [
+        (lambda v: True, "ISP 指标 ({v:.1f})"),
+    ],
+    "anonymity": [
+        (lambda v: True, "匿名状态指标 ({v:.1f})"),
+    ],
+    "domain": [
+        (lambda v: True, "域名指标 ({v:.1f})"),
+    ],
+    "city": [
+        (lambda v: True, "城市指标 ({v:.1f})"),
+    ],
+    "count": [
+        (lambda v: v > 100, "计数很高 ({v:.0f})"),
+        (lambda v: v > 50, "计数较高 ({v:.0f})"),
+        (lambda v: True, "计数正常 ({v:.0f})"),
+    ],
+    "ratio": [
+        (lambda v: v > 0.8, "占比极高 ({v:.1%})"),
+        (lambda v: v > 0.5, "占比偏高 ({v:.1%})"),
+        (lambda v: v >= 0.0, "占比较低 ({v:.1%})"),
+        (lambda v: True, "占比 ({v:.1%})"),
+    ],
+}
+
+# Default template for unrecognised categories / fallback
+DEFAULT_TEMPLATES: list[tuple] = [
+    (lambda v: True, "{meaning} = {v:.2f}"),
+]
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def _resolve_category(feature_name: str) -> str | None:
+    """Map a ``feature_name`` (column name) to a template category string.
+
+    Tries exact match against ``CATEGORY_EXACT`` first, then substring
+    match against ``CATEGORY_SUBSTR``.  Returns ``None`` if no match.
+    """
+    # Exact match
+    cat = CATEGORY_EXACT.get(feature_name)
+    if cat is not None:
+        return cat
+
+    # Substring match
+    lowered = feature_name.lower()
+    for sub, cat_label in CATEGORY_SUBSTR:
+        if sub in lowered:
+            return cat_label
+
+    return None
+
+
+def _lookup_meaning(feature_name: str) -> str:
+    """Return the Chinese meaning of *feature_name* from the TLS reference DB.
+
+    Falls back to *feature_name* itself when no entry is found.
+    """
+    try:
+        from analysis.tls_ref import get_tls_ref_dict
+
+        ref = get_tls_ref_dict()
+        entry = ref.get(feature_name)
+        if entry is not None:
+            return entry.get("meaning", feature_name)
+    except Exception:
+        pass
+    return feature_name
+
+
+def describe_feature(
+    feature_name: str,
+    value: float,
+    std_dev: float | None = None,
+    cluster_std: float | None = None,
+) -> str:
+    """Generate a Chinese natural-language description for a single feature.
+
+    Parameters
+    ----------
+    feature_name : str
+        Column name (e.g. ``"8ack"``, ``":ips.latd"``).
+    value : float
+        The feature's mean value in the current context (cluster mean).
+    std_dev : float | None
+        Optional Z-score magnitude (``abs(distinguishing_score)``).  When
+        provided and >= 1.0, appends a deviation suffix.
+    cluster_std : float | None
+        Optional within-cluster standard deviation. Describes how spread
+        the values are within this cluster (high = scattered, low = tight).
+
+    Returns
+    -------
+    str
+        Natural-language description string.
+    """
+    meaning = _lookup_meaning(feature_name)
+    category = _resolve_category(feature_name)
+
+    # Select templates
+    tmpl_list = TEMPLATES.get(category) if category else None
+    if tmpl_list is None:
+        tmpl_list = DEFAULT_TEMPLATES
+
+    # Evaluate predicates
+    text = ""
+    for predicate, template in tmpl_list:
+        try:
+            if predicate(value):
+                text = template.format(v=value, sv=_signed(value), av=abs(value),
+                                       meaning=meaning)
+                break
+        except Exception:
+            continue
+
+    if not text:
+        text = f"{meaning} = {value:.2f}"
+
+    # Append deviation context
+    if std_dev is not None and std_dev >= 1.0:
+        text += f"，比全局均值高{std_dev:.1f}个标准差"
+
+    # Append within-cluster variance context
+    if cluster_std is not None and cluster_std > 0:
+        if cluster_std > 2.0:
+            text += "（簇内离散度高）"
+        elif cluster_std < 0.3:
+            text += "（簇内高度集中）"
+
+    return text
+
+
+def describe_cluster(
+    features: Sequence[dict],
+    top_n: int = 5,
+) -> str:
+    """Generate a 2-3 sentence natural-language summary for a cluster.
+
+    Parameters
+    ----------
+    features : list[dict]
+        Each dict must include:
+
+        - ``feature_name`` : str
+        - ``mean`` : float (cluster mean)
+        - ``distinguishing_score`` : float (Z-score; may be signed)
+
+        Optional keys used when present:
+
+        - ``std`` : float (cluster std dev)
+
+    top_n : int
+        Number of most-distinguishing features to include (default 5).
+
+    Returns
+    -------
+    str
+        A short paragraph describing the cluster's distinguishing traits.
+
+        Example:
+        ``"低延迟 (23ms)，比全局均值高3.1个标准差; 大数据包 (1520 bytes)，比全局均值高2.3个标准差; 系统端口 (443)"``
+    """
+    if not features:
+        return "无显著区分特征"
+
+    # Sort by |distinguishing_score| descending
+    ranked = sorted(
+        features,
+        key=lambda f: abs(f.get("distinguishing_score", 0.0)),
+        reverse=True,
+    )[:top_n]
+
+    sentences: list[str] = []
+    for f in ranked:
+        name = f.get("feature_name", "")
+        mean = f.get("mean")
+        score = abs(f.get("distinguishing_score", 0.0))
+        cstd = f.get("std")
+
+        if mean is None or name == "":
+            continue
+
+        desc = describe_feature(name, float(mean), std_dev=score if score >= 1.0 else None,
+                                cluster_std=float(cstd) if cstd is not None else None)
+        sentences.append(desc)
+
+    if not sentences:
+        return "无显著区分特征"
+
+    return "；".join(sentences) + "。"
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _signed(value: float) -> str:
+    """Return a signed string representation, e.g. ``"+2.1"``."""
+    return f"{value:+.1f}"

analysis/profile_util.py

@@ -0,0 +1,91 @@
+"""Line profiling utility for the TianXuan backend.
+
+Usage in any backend module::
+
+    from analysis.profile_util import profile
+
+    @profile
+    def my_function(...):
+        ...
+
+When ``TIANXUAN_PROFILE=1`` env var is set AND line_profiler is installed,
+the decorator activates real line-level profiling and prints stats at exit.
+Otherwise the decorator is a no-op.
+
+To auto-profile ALL public functions in a module automatically, add at the
+bottom of the file::
+
+    from analysis.profile_util import auto_profile_module
+    auto_profile_module(__name__)
+
+Auto-profiling is also gated by ``TIANXUAN_PROFILE=1``.
+"""
+import os
+import sys
+import logging
+import atexit
+
+logger = logging.getLogger(__name__)
+
+_PROFILER_ACTIVE = os.environ.get('TIANXUAN_PROFILE', '') == '1'
+
+if _PROFILER_ACTIVE:
+    try:
+        from line_profiler import profile as _line_profile
+    except ImportError:
+        logger.warning('TIANXUAN_PROFILE=1 but line_profiler not installed')
+        _PROFILER_ACTIVE = False
+
+
+def profile(func):
+    """Decorate a function for line-level profiling.
+
+    Active only when ``TIANXUAN_PROFILE=1`` env var is set and
+    line_profiler is installed.
+    """
+    if _PROFILER_ACTIVE:
+        return _line_profile(func)
+    return func
+
+
+def auto_profile_module(module_name: str) -> int:
+    """Apply @profile to every public (non-underscore) function in *module_name*.
+
+    Must be called at module level, typically at the bottom of the file::
+
+        from analysis.profile_util import auto_profile_module
+        auto_profile_module(__name__)
+
+    Active only when ``TIANXUAN_PROFILE=1`` env var is set and
+    line_profiler is installed.  Returns number of functions wrapped, or 0
+    when inactive.
+    """
+    if not _PROFILER_ACTIVE:
+        return 0
+
+    try:
+        from line_profiler import LineProfiler
+    except ImportError:
+        return 0
+
+    module = sys.modules.get(module_name)
+    if module is None:
+        logger.warning('auto_profile_module: module %r not found', module_name)
+        return 0
+
+    lp = LineProfiler()
+    count = 0
+    for name in dir(module):
+        if name.startswith('_'):
+            continue
+        obj = getattr(module, name)
+        if callable(obj):
+            try:
+                lp.add_function(obj)
+                count += 1
+            except (TypeError, ValueError):
+                pass  # skip non-Python or special objects
+
+    atexit.register(lambda: lp.print_stats(output_unit=1e-6))
+    logger.info('auto_profile: wrapped %d functions in %s', count, module_name)
+    return count

analysis/session_store.py

@@ -46,6 +46,18 @@ def _generate_id(prefix: str = 'ds') -> str:
     return f"{prefix}_{uuid.uuid4().hex[:12]}"
 
 
+def _estimate_row_count(lf: pl.LazyFrame) -> int:
+    """Estimate row count from a LazyFrame (fast, streaming).
+
+    Uses ``select(pl.len()).collect(streaming=True)`` which scans the
+    underlying source efficiently without loading all columns.
+    """
+    try:
+        return lf.select(pl.len()).collect(streaming=True).item()
+    except Exception:
+        return 0
+
+
 def _get_store_path() -> Path:
     """Return the persistent JSON file path for dataset metadata.
 
@@ -78,6 +90,7 @@ class SessionStore:
                     cls._instance = super().__new__(cls)
                     cls._instance._stores: dict[str, dict[str, Any]] = {}
                     cls._instance._datalock = threading.RLock()
+                    cls._instance._restore_done = threading.Event()
         return cls._instance
 
     # ── persistence ────────────────────────────────────────────────────
@@ -85,8 +98,9 @@ class SessionStore:
     def _save_to_disk(self) -> None:
         """Persist current dataset metadata to JSON file.
 
-        Only metadata (schema, csv_glob, row_count, …) is saved — never the
+        Only metadata (schema, row_count, file_count, sqlite_table, …) is
-        LazyFrame itself, which is an in-memory query plan.
+        saved — never the LazyFrame itself.  The resulting file is typically
+        under 10 KB even with many datasets.
         """
         with self._datalock:
             serializable = {}
@@ -94,9 +108,16 @@ class SessionStore:
                 if entry.get('type') != 'dataset':
                     continue
                 meta = entry.get('metadata', {})
+                # Store only restore-critical fields; omit runtime noise
+                slim_meta = {
+                    k: v for k, v in meta.items()
+                    if k in ('row_count', 'file_count', 'sqlite_table',
+                              'csv_glob', 'columns', 'memory_mb',
+                              'schema_overrides', 'svd_components')
+                }
                 serializable[ds_id] = {
                     'schema': entry.get('schema'),
-                    'metadata': meta,
+                    'metadata': slim_meta,
                     'csv_glob': meta.get('csv_glob'),
                 }
         path = _get_store_path()
@@ -105,17 +126,15 @@ class SessionStore:
             encoding='utf-8',
         )
 
-    def restore_from_disk(self) -> int:
+    def _restore_metadata(self) -> int:
-        """Re-register datasets from persisted metadata.
+        """Read metadata from disk and store as lightweight *stubs*.
 
-        Attempts two strategies in order:
+        A stub contains everything needed to reconstruct the LazyFrame on
-        1. **SQLite table** — when ``metadata.sqlite_table`` exists (fast,
+        demand but performs **no I/O** beyond reading the JSON file.  Real
-           always available, no CSV dependency).
+        data loading is deferred until the first :meth:`get_dataset` call
-        2. **CSV glob** — falls back to ``load_csv_directory`` for datasets
+        (or the background thread).
-           that were stored before the SQLite migration or whose SQLite table
-           was dropped.
 
-        Returns the number of successfully restored datasets.
+        Returns the number of stubs created.
         """
         path = _get_store_path()
         if not path.exists():
@@ -125,47 +144,131 @@ class SessionStore:
         except (json.JSONDecodeError, OSError):
             return 0
 
-        # Lazy imports to avoid circular dependency at module level
+        count = 0
-        from analysis.data_loader import load_csv_directory, load_from_db  # fmt: skip
+        with self._datalock:
+            for ds_id, info in data.items():
+                # Skip if already loaded (idempotent on re-calls)
+                if ds_id in self._stores:
+                    continue
+                meta = dict(info.get('metadata', {}))
+                schema = dict(info.get('schema', {}))
+                csv_glob = info.get('csv_glob')
+                self._stores[ds_id] = {
+                    'type': 'dataset',
+                    '_stub': {
+                        'sqlite_table': meta.get('sqlite_table'),
+                        'csv_glob': csv_glob,
+                        'schema_overrides': meta.get('schema_overrides'),
+                    },
+                    'schema': schema,
+                    'metadata': meta,
+                    'parent_id': None,
+                }
+                count += 1
+        self._restore_done.set()
+        return count
 
-        restored = 0
+    def _load_pending_dataset(self, ds_id: str) -> bool:
-        for ds_id, info in data.items():
+        """Resolve a stub entry into a real LazyFrame.
-            meta = dict(info.get('metadata', {}))
-            schema = dict(info.get('schema', {}))
-            restored_ok = False
 
-            # Strategy 1: SQLite table (fast path)
+        Called on the first :meth:`get_dataset` for a dataset that was
-            sqlite_table = meta.get('sqlite_table')
+        restored via :meth:`_restore_metadata`.
-            if sqlite_table:
-                try:
-                    lf = load_from_db(sqlite_table)
-                    if lf is not None:
-                        self.store_dataset(ds_id, lf, schema=schema, metadata=meta)
-                        restored += 1
-                        restored_ok = True
-                except Exception:
-                    pass  # fall through to CSV restore
 
-            if restored_ok:
+        Attempts two strategies:
-                continue
+        1.  **SQLite table** (fast — uses :func:`load_from_db_lazy`).
+        2.  **CSV glob** (legacy fallback — scans the filesystem).
 
-            # Strategy 2: CSV glob (legacy / fallback)
+        Returns ``True`` if the dataset was successfully loaded.
-            csv_glob = info.get('csv_glob')
+        """
-            if not csv_glob:
+        from analysis.data_loader import load_csv_directory, load_from_db_lazy  # fmt: skip
-                continue
+
+        entry = self._stores.get(ds_id)
+        if not entry or '_stub' not in entry:
+            return True  # nothing to do
+
+        stub = entry.pop('_stub')
+        meta = entry.get('metadata', {})
+        schema = entry.get('schema', {})
+
+        # Strategy 1: SQLite table (fast, native Polars reader)
+        sqlite_table = stub.get('sqlite_table')
+        if sqlite_table:
             try:
-                lf, schema_csv, row_count, file_count, memory_mb = load_csv_directory(csv_glob)
+                lf = load_from_db_lazy(sqlite_table, schema_overrides=stub.get('schema_overrides'))
-                # Preserve original metadata counts from disk — CSV re-scan may
+                if lf is not None:
-                # detect different file/row counts if files changed on disk.
+                    entry['lazyframe'] = lf
-                if 'file_count' not in meta and file_count is not None:
+                    if 'row_count' not in meta:
-                    meta['file_count'] = file_count
+                        meta['row_count'] = _estimate_row_count(lf)
-                if 'row_count' not in meta and row_count is not None:
+                    return True
-                    meta['row_count'] = row_count
-                self.store_dataset(ds_id, lf, schema=schema_csv, metadata=meta)
-                restored += 1
             except Exception:
-                # File(s) no longer exist or schema changed — skip gracefully
+                pass  # fall through to CSV restore
-                continue
+
+        # Strategy 2: CSV glob (legacy fallback)
+        csv_glob = stub.get('csv_glob')
+        if not csv_glob:
+            # No recoverable data source — entry remains without LazyFrame
+            return False
+        try:
+            lf, schema_csv, row_count, file_count, memory_mb = load_csv_directory(csv_glob)
+            if 'file_count' not in meta and file_count is not None:
+                meta['file_count'] = file_count
+            if 'row_count' not in meta and row_count is not None:
+                meta['row_count'] = row_count
+            # Merge CSV-scanned schema (may have more/different columns)
+            merged_schema = {**schema, **schema_csv}
+            entry['lazyframe'] = lf
+            entry['schema'] = merged_schema
+            return True
+        except Exception:
+            return False
+
+    def start_async_restore(self) -> int:
+        """Restore dataset metadata immediately; load data in background.
+
+        At startup this reads the small metadata JSON and stores *stubs*
+        (metadata only, no data I/O).  A daemon thread then resolves each
+        stub into a real LazyFrame without blocking the main thread.
+
+        Returns the number of datasets queued for restoration.
+        """
+        count = self._restore_metadata()
+        if count > 0:
+            t = threading.Thread(
+                target=self._background_restore,
+                daemon=True,
+                name='session-restore',
+            )
+            t.start()
+        return count
+
+    def _background_restore(self) -> None:
+        """Pre-load all stubs in a daemon thread (best-effort)."""
+        with self._datalock:
+            ds_ids = [
+                ds_id for ds_id, entry in self._stores.items()
+                if entry.get('type') == 'dataset' and '_stub' in entry
+            ]
+        for ds_id in ds_ids:
+            try:
+                self._load_pending_dataset(ds_id)
+            except Exception:
+                pass  # best-effort; stub remains for on-demand fallback
+
+    def restore_from_disk(self) -> int:
+        """Legacy synchronous restore (kept for backward compatibility).
+
+        Prefer :meth:`start_async_restore` for non-blocking startup.
+        """
+        self._restore_metadata()
+        with self._datalock:
+            ds_ids = [
+                ds_id for ds_id, entry in self._stores.items()
+                if entry.get('type') == 'dataset' and '_stub' in entry
+            ]
+        restored = 0
+        for ds_id in ds_ids:
+            if self._load_pending_dataset(ds_id):
+                restored += 1
         return restored
 
     # ── dataset operations ──────────────────────────────────────────────
@@ -199,22 +302,41 @@ class SessionStore:
         self._save_to_disk()
 
     def get_dataset(self, dataset_id: str) -> Optional[dict]:
-        """Retrieve a stored dataset entry (or *None*)."""
+        """Retrieve a stored dataset entry (or *None*).
+
+        If the entry is a *stub* (restored from metadata but not yet
+        materialised), this method triggers on-demand loading of the
+        LazyFrame before returning.
+        """
         with self._datalock:
             entry = self._stores.get(dataset_id)
-            if entry and entry.get('type') == 'dataset':
+            if not entry or entry.get('type') != 'dataset':
-                return entry
+                return None
-            return None
+            needs_load = '_stub' in entry
+
+        if needs_load:
+            self._load_pending_dataset(dataset_id)
+
+        with self._datalock:
+            return self._stores.get(dataset_id)
 
     def drop_dataset(self, dataset_id: str) -> bool:
         """Remove a dataset from the store and force garbage collection.
 
+        Cascades to child datasets (e.g. filtered results) via parent_id tracking.
         Returns True if the dataset existed.
         """
         with self._datalock:
             existed = dataset_id in self._stores and self._stores[dataset_id].get('type') == 'dataset'
             if existed:
                 del self._stores[dataset_id]
+            # Cascade: drop all child datasets that reference the dropped one as parent
+            child_ids = [
+                k for k, v in self._stores.items()
+                if v.get('type') == 'dataset' and v.get('parent_id') == dataset_id
+            ]
+            for cid in child_ids:
+                del self._stores[cid]
         if existed:
             gc.collect()
             self._save_to_disk()
@@ -255,6 +377,7 @@ class SessionStore:
                     'svd_components': meta.get('svd_components', 0),
                     'columns': list(entry.get('schema', {}).keys()),
                     'column_count': len(entry.get('schema', {})),
+                    'is_stub': '_stub' in entry,
                 })
         return results
 

analysis/tools/__init__.py

@@ -0,0 +1,103 @@
+"""Tool registry package — split from analysis/tool_registry.py.
+
+Re-exports all tool metadata, dispatcher, and handlers.
+"""
+
+# ── Helpers ────────────────────────────────────────────────────────────────
+from ._helpers import (
+    _json_size,
+    _truncate_response,
+    _build_filter_expr,
+    _resolve_dataset,
+    _count_rows,
+    _get_numeric_columns,
+    NUMERIC_DTYPES,
+    NUMERIC_TYPE_NAMES,
+)
+
+# ── Registry + dispatch ────────────────────────────────────────────────────
+from ._registry import get_tools_meta
+from ._dispatch import handle_call
+
+# ── Handlers (tool implementations) ────────────────────────────────────────
+from .load_data import _handle_load_data
+from .profile import _handle_profile_data
+from .filter import _handle_filter_data
+from .preprocess import _handle_preprocess_data
+from .clustering import _handle_run_clustering, _handle_filter_and_cluster
+from .evaluate import _handle_evaluate_clustering
+from .features import _handle_extract_features, _save_features_to_db
+from .export import _handle_export_results
+from .entities import (
+    _handle_build_entity_profiles,
+    _handle_compute_scores,
+    _add_adaptive_scores,
+    _ENTITY_KEYWORDS,
+    _NUMERIC_TYPES_FOR_AGG,
+    _ANOMALY_FEATURE_COLS,
+)
+from .anomalies import _handle_detect_anomalies, _handle_visualize_anomalies
+from .diagnostics import (
+    _handle_validate_data,
+    _handle_explore_distributions,
+    _handle_find_outliers,
+    _handle_diagnose_clustering,
+    _handle_compare_datasets,
+    _handle_export_debug_sample,
+    _handle_repair_schema,
+)
+from .analysis import (
+    _handle_analyze_patterns,
+    _handle_analyze_temporal,
+    _handle_analyze_fft,
+    _handle_analyze_tls_health,
+    _handle_analyze_geo_distribution,
+    _handle_analyze_entity_detail,
+)
+from .data_mgmt import (
+    _handle_list_datasets,
+    _handle_drop_dataset,
+    _handle_clone_dataset,
+)
+from .distance_matrix import _handle_compute_distance_matrix
+
+__all__ = [
+    # Helpers
+    '_json_size', '_truncate_response', '_build_filter_expr', '_resolve_dataset',
+    '_count_rows', '_get_numeric_columns', 'NUMERIC_DTYPES', 'NUMERIC_TYPE_NAMES',
+    # Registry + dispatch
+    'get_tools_meta', 'handle_call',
+    # Load
+    '_handle_load_data',
+    # Profile
+    '_handle_profile_data',
+    # Filter
+    '_handle_filter_data',
+    # Preprocess
+    '_handle_preprocess_data',
+    # Clustering
+    '_handle_run_clustering', '_handle_filter_and_cluster',
+    # Evaluate
+    '_handle_evaluate_clustering',
+    # Features
+    '_handle_extract_features', '_save_features_to_db',
+    # Export
+    '_handle_export_results',
+    # Entities
+    '_handle_build_entity_profiles', '_handle_compute_scores', '_add_adaptive_scores',
+    '_ENTITY_KEYWORDS', '_NUMERIC_TYPES_FOR_AGG', '_ANOMALY_FEATURE_COLS',
+    # Anomalies
+    '_handle_detect_anomalies', '_handle_visualize_anomalies',
+    # Diagnostics
+    '_handle_validate_data', '_handle_explore_distributions', '_handle_find_outliers',
+    '_handle_diagnose_clustering', '_handle_compare_datasets',
+    '_handle_export_debug_sample', '_handle_repair_schema',
+    # Analysis
+    '_handle_analyze_patterns', '_handle_analyze_temporal', '_handle_analyze_fft',
+    '_handle_analyze_tls_health', '_handle_analyze_geo_distribution',
+    '_handle_analyze_entity_detail',
+    # Data management
+    '_handle_list_datasets', '_handle_drop_dataset', '_handle_clone_dataset',
+    # Distance matrix
+    '_handle_compute_distance_matrix',
+]

analysis/tools/_dispatch.py

@@ -0,0 +1,99 @@
+"""Tool call dispatcher — maps tool names to handler functions."""
+import asyncio
+from ._helpers import _truncate_response
+from .load_data import _handle_load_data
+from .profile import _handle_profile_data
+from .filter import _handle_filter_data
+from .preprocess import _handle_preprocess_data
+from .clustering import _handle_run_clustering, _handle_filter_and_cluster
+from .evaluate import _handle_evaluate_clustering
+from .features import _handle_extract_features
+from .export import _handle_export_results
+from .entities import _handle_build_entity_profiles, _handle_compute_scores
+from .anomalies import _handle_detect_anomalies, _handle_visualize_anomalies
+from .diagnostics import (
+    _handle_validate_data, _handle_explore_distributions, _handle_find_outliers,
+    _handle_diagnose_clustering, _handle_compare_datasets,
+    _handle_export_debug_sample, _handle_repair_schema,
+)
+from .analysis import (
+    _handle_analyze_patterns, _handle_analyze_temporal, _handle_analyze_fft,
+    _handle_analyze_tls_health, _handle_analyze_geo_distribution,
+    _handle_analyze_entity_detail,
+)
+from .data_mgmt import _handle_list_datasets, _handle_drop_dataset, _handle_clone_dataset
+from .distance_matrix import _handle_compute_distance_matrix
+
+
+async def handle_call(name: str, arguments: dict) -> dict:
+    """Dispatch a tool call to the appropriate handler.
+
+    Extracts the mandatory ``_timeout`` parameter (set by the LLM) from
+    *arguments* and enforces it as an ``asyncio.wait_for`` deadline.
+
+    Args:
+        name: Tool name (must match one of the 30 tools).
+        arguments: Tool-specific parameters, must include ``_timeout``.
+
+    Returns:
+        A JSON-serialisable dict.
+
+    Raises:
+        ValueError: If *name* is not a recognised tool.
+    """
+    _handlers = {
+        'load_data': _handle_load_data,
+        'profile_data': _handle_profile_data,
+        'filter_data': _handle_filter_data,
+        'preprocess_data': _handle_preprocess_data,
+        'run_clustering': _handle_run_clustering,
+        'evaluate_clustering': _handle_evaluate_clustering,
+        'extract_features': _handle_extract_features,
+        'export_results': _handle_export_results,
+        'list_datasets': _handle_list_datasets,
+        'drop_dataset': _handle_drop_dataset,
+        'clone_dataset': _handle_clone_dataset,
+        'build_entity_profiles': _handle_build_entity_profiles,
+        'compute_scores': _handle_compute_scores,
+        'filter_and_cluster': _handle_filter_and_cluster,
+        'detect_anomalies': _handle_detect_anomalies,
+        'visualize_anomalies': _handle_visualize_anomalies,
+        'validate_data': _handle_validate_data,
+        'explore_distributions': _handle_explore_distributions,
+        'find_outliers': _handle_find_outliers,
+        'diagnose_clustering': _handle_diagnose_clustering,
+        'compare_datasets': _handle_compare_datasets,
+        'export_debug_sample': _handle_export_debug_sample,
+        'repair_schema': _handle_repair_schema,
+        'analyze_patterns': _handle_analyze_patterns,
+        'analyze_temporal': _handle_analyze_temporal,
+        'analyze_fft': _handle_analyze_fft,
+        'analyze_tls_health': _handle_analyze_tls_health,
+        'analyze_geo_distribution': _handle_analyze_geo_distribution,
+        'analyze_entity_detail': _handle_analyze_entity_detail,
+        'compute_distance_matrix': _handle_compute_distance_matrix,
+    }
+    handler = _handlers.get(name)
+    if handler is None:
+        raise ValueError(f"Unknown tool: '{name}'")
+
+    # Extract mandatory _timeout from the LLM-provided arguments
+    timeout = arguments.pop('_timeout', 0)
+    cleaned_args = arguments  # _timeout already removed by pop
+
+    # Execute the handler with optional timeout
+    if timeout and timeout > 0:
+        try:
+            result = await asyncio.wait_for(
+                handler(**cleaned_args),
+                timeout=timeout,
+            )
+        except asyncio.TimeoutError:
+            result = {'error': f'工具 {name} 执行超时 ({timeout}秒)'}
+    else:
+        result = await handler(**cleaned_args)
+
+    # Always ensure truncated key exists
+    if 'truncated' not in result:
+        result['truncated'] = False
+    return _truncate_response(result)

analysis/tools/_helpers.py

@@ -0,0 +1,138 @@
+"""Shared utilities for tool handlers.
+
+Provides:
+- NUMERIC_DTYPES / NUMERIC_TYPE_NAMES: shared constants for dtype filtering
+- _json_size, _truncate_response: LLM context-window helpers
+- _build_filter_expr: Polars filter expression builder
+- _resolve_dataset: one-liner SessionStore lookup (replaces 30+ repeated blocks)
+- _count_rows: fast streaming row count
+- _get_numeric_columns: extract numeric column names from schema
+"""
+import json
+import logging
+import traceback
+from typing import Any
+
+import polars as pl
+
+from ..session_store import SessionStore
+logger = logging.getLogger(__name__)
+
+
+# ═══════════════════════════════════════════════════════════════════════
+#  Shared constants
+# ═══════════════════════════════════════════════════════════════════════
+
+NUMERIC_DTYPES = (
+    pl.Int8, pl.Int16, pl.Int32, pl.Int64,
+    pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
+    pl.Float32, pl.Float64,
+)
+
+NUMERIC_TYPE_NAMES = {
+    'Int8', 'Int16', 'Int32', 'Int64',
+    'UInt8', 'UInt16', 'UInt32', 'UInt64',
+    'Float32', 'Float64',
+}
+
+
+# ═══════════════════════════════════════════════════════════════════════
+#  LLM response helpers
+# ═══════════════════════════════════════════════════════════════════════
+
+_MAX_RESPONSE_KB = 64
+
+
+def _json_size(obj: Any) -> int:
+    """Estimate JSON byte size of *obj*."""
+    return len(json.dumps(obj, default=str, ensure_ascii=False))
+
+
+def _truncate_response(result: dict, max_kb: int = _MAX_RESPONSE_KB) -> dict:
+    """Truncate large lists/keys in *result* to stay under *max_kb*."""
+    if _json_size(result) <= max_kb * 1024:
+        result.setdefault('truncated', False)
+        return result
+
+    # Drop verbose keys first
+    for costly_key in ('column_stats', 'features', 'correlation_matrix',
+                        'labels', 'cluster_labels'):
+        if costly_key in result and isinstance(result[costly_key], dict):
+            result[costly_key] = {'__truncated__': True,
+                                  'key_count': len(result[costly_key])}
+        elif costly_key in result and isinstance(result[costly_key], list):
+            lst = result[costly_key]
+            if len(lst) > 10:
+                result[costly_key] = lst[:10] + [{'__truncated__': True,
+                                                   'omitted': len(lst) - 10}]
+    result['truncated'] = True
+    return result
+
+
+# ═══════════════════════════════════════════════════════════════════════
+#  Filter expression builder
+# ═══════════════════════════════════════════════════════════════════════
+
+def _build_filter_expr(column: str, op: str, value: Any) -> pl.Expr:
+    """Build a single Polars expression from a filter descriptor."""
+    col = pl.col(column)
+    op_map = {
+        'eq': col == value,
+        'neq': col != value,
+        'gt': col > value,
+        'gte': col >= value,
+        'lt': col < value,
+        'lte': col <= value,
+        'contains': col.str.contains(value) if value is not None else col.cast(str).str.contains(''),
+        'not_contains': ~(col.str.contains(value) if value is not None else col.cast(str).str.contains('')),
+        'in': col.is_in(value) if isinstance(value, (list, tuple)) else col.is_in([value]),
+        'not_in': ~(col.is_in(value) if isinstance(value, (list, tuple)) else col.is_in([value])),
+        'is_null': col.is_null(),
+        'is_not_null': col.is_not_null(),
+    }
+    expr = op_map.get(op)
+    if expr is None:
+        raise ValueError(f"Unsupported filter operator: {op}")
+    return expr
+
+
+# ═══════════════════════════════════════════════════════════════════════
+#  SessionStore helpers
+# ═══════════════════════════════════════════════════════════════════════
+
+def _resolve_dataset(dataset_id: str) -> tuple:
+    """Resolve a dataset from SessionStore.
+
+    Returns:
+        (entry, None) on success, where *entry* is the SessionStore entry dict.
+        (None, error_dict) if the dataset is not found.
+    """
+    store = SessionStore()
+    entry = store.get_dataset(dataset_id)
+    if entry is None:
+        return None, {'error': f'Dataset not found: {dataset_id}', 'truncated': False}
+    return entry, None
+
+
+def _count_rows(lf: pl.LazyFrame) -> int:
+    """Count rows in a LazyFrame via fast streaming count."""
+    try:
+        return lf.select(pl.len()).collect(streaming=True).item()
+    except Exception:
+        logger.error("_count_rows failed: {}".format(traceback.format_exc()))
+        return 0
+
+
+def _get_numeric_columns(lf: pl.LazyFrame, schema: dict, max_cols: int = 10) -> list[str]:
+    """Get numeric column names from a dataset.
+
+    Prefers *schema* (string-keyed); falls back to LazyFrame collect_schema().
+    """
+    if schema:
+        return [c for c in schema
+                if schema[c].split('(')[0].strip() in NUMERIC_TYPE_NAMES
+                and not c.startswith('_')][:max_cols]
+    # Fallback to LazyFrame schema
+    available = lf.collect_schema()
+    return [available.names()[i] for i, dt in enumerate(available.dtypes())
+            if dt in NUMERIC_DTYPES and not available.names()[i].startswith('_')][:max_cols]

analysis/tools/_registry.py

@@ -0,0 +1,777 @@
+"""Tool metadata definitions for the MCP server.
+
+All 30 Tool(...) descriptor objects and the get_tools_meta() function.
+"""
+from mcp.types import Tool
+
+
+def get_tools_meta() -> list[Tool]:
+    """Return the list of Tool metadata objects for MCP server registration.
+
+    Each tool gets a mandatory ``_timeout`` parameter so the LLM must specify
+    how many seconds to allow for each tool invocation (0 = no limit).
+    """
+    raw_tools = [
+        Tool(
+            name="load_data",
+            description=(
+                "加载匹配glob模式的CSV文件。自动检测每个文件的BOM，验证schema一致性，"
+                "纵向合并，并将结果作为惰性数据集存储在会话存储中。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "csv_glob": {
+                        "type": "string",
+                        "description": "Glob pattern for CSV files (e.g. 'data/*.csv')",
+                    },
+                    "config_path": {
+                        "type": "string",
+                        "description": "Optional path to a YAML config file",
+                    },
+                    "encoding": {
+                        "type": "string",
+                        "description": "Fallback encoding when no BOM is found",
+                        "default": "utf-8",
+                    },
+                    "delimiter": {
+                        "type": "string",
+                        "description": "Column delimiter",
+                        "default": ",",
+                    },
+                    "schema_strict": {
+                        "type": "boolean",
+                        "description": "When True, raise on column mismatch; when False, merge leniently with nulls",
+                        "default": False,
+                    },
+                    "recursive": {
+                        "type": "boolean",
+                        "description": "When True, use recursive glob for **/*.csv patterns",
+                        "default": False,
+                    },
+                },
+                "required": ["csv_glob"],
+            },
+        ),
+        Tool(
+            name="profile_data",
+            description=(
+                "计算数据集的每列统计信息和数值相关矩阵。结果自动截断至约16 KB以"
+                "节省LLM上下文窗口空间。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "dataset_id": {
+                        "type": "string",
+                        "description": "Dataset identifier from a previous load_data call",
+                    },
+                    "sample_size": {
+                        "type": "integer",
+                        "description": "Maximum rows to profile (sampled if larger)",
+                        "default": 1000,
+                    },
+                    "columns": {
+                        "type": "array",
+                        "items": {"type": "string"},
+                        "description": "Columns to profile (default: all)",
+                    },
+                },
+                "required": ["dataset_id"],
+            },
+        ),
+        Tool(
+            name="filter_data",
+            description=(
+                "对数据集应用列级过滤器，将结果存储为新的派生数据集。支持eq/neq/gt/gte/"
+                "lt/lte/contains/not_contains/in/not_in/is_null/is_not_null运算符。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "dataset_id": {
+                        "type": "string",
+                        "description": "Source dataset identifier",
+                    },
+                    "filters": {
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "properties": {
+                                "column": {"type": "string"},
+                                "op": {
+                                    "type": "string",
+                                    "enum": [
+                                        "eq", "neq", "gt", "gte", "lt", "lte",
+                                        "contains", "not_contains",
+                                        "in", "not_in",
+                                        "is_null", "is_not_null",
+                                    ],
+                                },
+                                "value": {"description": "Operand value"},
+                            },
+                            "required": ["column", "op"],
+                        },
+                        "description": "List of filter expressions",
+                    },
+                    "logic": {
+                        "type": "string",
+                        "enum": ["and", "or"],
+                        "description": "Combine filters with AND or OR logic",
+                        "default": "and",
+                    },
+                },
+                "required": ["dataset_id", "filters"],
+            },
+        ),
+        Tool(
+            name="preprocess_data",
+            description=(
+                "预处理选定列：标准化（StandardScaler）、独热编码、缺失值填充和列删除。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "dataset_id": {
+                        "type": "string",
+                        "description": "Source dataset identifier",
+                    },
+                    "columns": {
+                        "type": "array",
+                        "items": {"type": "string"},
+                        "description": "Columns to preprocess",
+                    },
+                    "config": {
+                        "type": "object",
+                        "properties": {
+                            "standardize": {
+                                "type": "boolean",
+                                "description": "Apply StandardScaler to numeric columns",
+                                "default": False,
+                            },
+                            "onehot": {
+                                "type": "boolean",
+                                "description": "One-hot encode categorical columns",
+                                "default": False,
+                            },
+                            "fillna": {
+                                "type": "string",
+                                "enum": ["mean", "median", "zero", "drop"],
+                                "description": "Missing value strategy",
+                            },
+                            "drop": {
+                                "type": "array",
+                                "items": {"type": "string"},
+                                "description": "Columns to drop entirely",
+                            },
+                        },
+                    },
+                },
+                "required": ["dataset_id", "columns"],
+            },
+        ),
+        Tool(
+            name="run_clustering",
+            description=(
+                "对选定列执行聚类。使用HDBSCAN（默认）或KMeans。特征在聚类前进行"
+                "标准化缩放。返回聚类标签和质量指标。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "dataset_id": {
+                        "type": "string",
+                        "description": "Dataset identifier",
+                    },
+                    "cluster_columns": {
+                        "type": "array",
+                        "items": {"type": "string"},
+                        "description": "Columns to use as clustering features",
+                    },
+                    "algorithm": {
+                        "type": "string",
+                        "enum": ["agglomerative", "hdbscan", "kmeans"],
+                        "description": "Clustering algorithm (default: agglomerative/Ward)",
+                        "default": "agglomerative",
+                    },
+                    "params": {
+                        "type": "object",
+                        "description": (
+                            "Algorithm parameters. Agglomerative: n_clusters (5), metric ('euclidean'), "
+                            "linkage ('ward'). HDBSCAN: min_cluster_size (5), "
+                            "min_samples (None), metric ('euclidean'). "
+                            "KMeans: n_clusters (3), random_state (42)."
+                        ),
+                    },
+                    "random_state": {
+                        "type": "integer",
+                        "description": "Random seed for reproducibility",
+                        "default": 42,
+                    },
+                },
+                "required": ["dataset_id", "cluster_columns"],
+            },
+        ),
+        Tool(
+            name="evaluate_clustering",
+            description=(
+                "评估数据集的聚类结果。当前支持轮廓系数（silhouette）、Davies-Bouldin指数、"
+                "Calinski-Harabasz指数、噪声比（noise_ratio）和簇大小（cluster_sizes）指标。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "cluster_result_id": {
+                        "type": "string",
+                        "description": "Result ID from run_clustering",
+                    },
+                    "dataset_id": {
+                        "type": "string",
+                        "description": "Original dataset for metric computation",
+                    },
+                    "metrics": {
+                        "type": "array",
+                        "items": {"type": "string"},
+                        "description": (
+                            "Metrics to compute. Options: silhouette, "
+                            "davies_bouldin, calinski_harabasz, noise_ratio, "
+                            "cluster_sizes (default: all)"
+                        ),
+                    },
+                },
+                "required": ["cluster_result_id", "dataset_id"],
+            },
+        ),
+        Tool(
+            name="extract_features",
+            description=(
+                "提取每个聚类的区分特征并自动保存到Django ORM（ClusterFeature模型）。"
+                "使用z-score或ANOVA对每个聚类的特征重要性进行排序。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "dataset_id": {
+                        "type": "string",
+                        "description": "Dataset containing the feature columns",
+                    },
+                    "cluster_result_id": {
+                        "type": "string",
+                        "description": "Result ID from run_clustering",
+                    },
+                    "top_k": {
+                        "type": "integer",
+                        "description": "Top distinguishing features per cluster",
+                        "default": 10,
+                    },
+                    "method": {
+                        "type": "string",
+                        "enum": ["zscore", "anova"],
+                        "description": "Method to rank feature importance",
+                        "default": "zscore",
+                    },
+                    "save_to_db": {
+                        "type": "boolean",
+                        "description": "Save features to Django ORM if True",
+                        "default": True,
+                    },
+                },
+                "required": ["dataset_id", "cluster_result_id"],
+            },
+        ),
+        Tool(
+            name="export_results",
+            description=(
+                "将数据集或聚类结果以CSV或Parquet格式导出到磁盘。数据集物化后写入；"
+                "聚类结果序列化为JSON。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "result_id": {
+                        "type": "string",
+                        "description": "Dataset ID or cluster result ID to export",
+                    },
+                    "output_path": {
+                        "type": "string",
+                        "description": "Output file path or directory",
+                    },
+                    "format": {
+                        "type": "string",
+                        "enum": ["csv", "parquet", "json"],
+                        "description": "Output format",
+                        "default": "csv",
+                    },
+                    "overwrite": {
+                        "type": "boolean",
+                        "description": "Overwrite existing file",
+                        "default": False,
+                    },
+                },
+                "required": ["result_id", "output_path"],
+            },
+        ),
+        Tool(
+            name="list_datasets",
+            description="列出会话存储中的所有活跃数据集和结果。",
+            inputSchema={
+                "type": "object",
+                "properties": {},
+            },
+        ),
+        Tool(
+            name="drop_dataset",
+            description=(
+                "从会话存储中移除数据集（或结果）并通过垃圾回收释放内存。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "dataset_id": {
+                        "type": "string",
+                        "description": "Dataset or result identifier to remove",
+                    },
+                },
+                "required": ["dataset_id"],
+            },
+        ),
+        Tool(
+            name="clone_dataset",
+            description=(
+                "浅拷贝数据集。由于LazyFrame是查询计划，不会复制实际数据。适用于分支分析。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "dataset_id": {
+                        "type": "string",
+                        "description": "Source dataset identifier",
+                    },
+                },
+                "required": ["dataset_id"],
+            },
+        ),
+        Tool(
+            name="filter_and_cluster",
+            description=(
+                "一步完成：先应用列级过滤器，再对过滤结果执行聚类。支持eq/neq/gt/gte/lt/lte/"
+                "contains/not_contains/in/not_in/is_null/is_not_null运算符和HDBSCAN/KMeans聚类。"
+                "自动选择数值列作为聚类特征。将过滤后的数据集和聚类结果都存储在会话中。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "dataset_id": {
+                        "type": "string",
+                        "description": "Source dataset identifier",
+                    },
+                    "filters": {
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "properties": {
+                                "column": {"type": "string"},
+                                "op": {
+                                    "type": "string",
+                                    "enum": [
+                                        "eq", "neq", "gt", "gte", "lt", "lte",
+                                        "contains", "not_contains",
+                                        "in", "not_in",
+                                        "is_null", "is_not_null",
+                                    ],
+                                },
+                                "value": {"description": "Operand value"},
+                            },
+                            "required": ["column", "op"],
+                        },
+                        "description": "List of filter expressions",
+                    },
+                    "logic": {
+                        "type": "string",
+                        "enum": ["and", "or"],
+                        "description": "Combine filters with AND or OR logic",
+                        "default": "and",
+                    },
+                    "algorithm": {
+                        "type": "string",
+                        "enum": ["agglomerative", "hdbscan", "kmeans"],
+                        "description": "Clustering algorithm (default: agglomerative/Ward)",
+                        "default": "agglomerative",
+                    },
+                    "params": {
+                        "type": "object",
+                        "description": (
+                            "Algorithm parameters. Agglomerative: n_clusters (5), metric ('euclidean'), "
+                            "linkage ('ward'). HDBSCAN: min_cluster_size (5), "
+                            "min_samples (None), metric ('euclidean'). "
+                            "KMeans: n_clusters (3), random_state (42)."
+                        ),
+                    },
+                },
+                "required": ["dataset_id", "filters"],
+            },
+        ),
+        Tool(
+            name="build_entity_profiles",
+            description=(
+                "按实体列（源IP、SNI等）分组原始流数据。自动检测实体列（如 src_ip, "
+                "dst_ip, sni, host, mac, domain），按列分组并计算每个实体的聚合特征："
+                "流数量、唯一端口数、TLS版本分布、字节总量、唯一目标数。"
+                "生成的数据集每个实体一行。结果存储在会话中，可供后续 compute_scores 使用。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "dataset_id": {
+                        "type": "string",
+                        "description": "Dataset ID from load_data or profile_data",
+                    },
+                    "entity_column": {
+                        "type": "string",
+                        "description": "Specific entity column to group by (e.g. 'src_ip')",
+                    },
+                    "entity_columns": {
+                        "type": "array",
+                        "items": {"type": "string"},
+                        "description": "Multiple entity columns to group by",
+                    },
+                    "auto_detect": {
+                        "type": "boolean",
+                        "description": "Auto-detect entity columns (IP, SNI, host, MAC, domain)",
+                        "default": True,
+                    },
+                },
+                "required": ["dataset_id"],
+            },
+        ),
+        Tool(
+            name="compute_scores",
+            description=(
+                "在实体数据上计算自适应代理/异常/威胁评分。从数据自动学习"
+                "特征权重，计算加权代理评分（proxy_score）、基于IQR的风险等级"
+                "（normal/watch/suspicious/critical）和威胁评分（threat_score）。"
+                "返回评分摘要与阈值。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "dataset_id": {
+                        "type": "string",
+                        "description": "Entity dataset ID from filter_and_cluster or a scored dataset",
+                    },
+                },
+                "required": ["dataset_id"],
+            },
+        ),
+        Tool(
+            name="detect_anomalies",
+            description=(
+                "对已评分的实体数据运行孤立森林（Isolation Forest）异常检测。识别偏离"
+                "正常模式的实体。分块处理以应对大数据集。返回异常实体ID列表。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "dataset_id": {
+                        "type": "string",
+                        "description": "Scored entity dataset ID (after compute_scores)",
+                    },
+                    "contamination": {
+                        "type": "number",
+                        "description": "Expected proportion of anomalies (0.0-0.5)",
+                        "default": 0.05,
+                    },
+                },
+                "required": ["dataset_id"],
+            },
+        ),
+        Tool(
+            name="visualize_anomalies",
+            description=(
+                "为高风险实体生成UMAP-2D嵌入和聚类可视化数据。按风险等级和聚类着色。"
+                "返回用于前端渲染的散点图JSON数据。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "dataset_id": {
+                        "type": "string",
+                        "description": "Scored entity dataset ID (after compute_scores)",
+                    },
+                },
+                "required": ["dataset_id"],
+            },
+        ),
+        # ── Edge-case / Diagnostic tools (unexpected situations) ────────
+        Tool(
+            name="validate_data",
+            description=(
+                "诊断工具：检查数据质量问题——文件间schema一致性、缺失值比例、列类型冲突、"
+                "编码问题和空列。当流水线步骤因schema错误失败，或怀疑CSV数据损坏/格式异常时使用。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "dataset_id": {"type": "string", "description": "Dataset ID from load_data"},
+                    "fix_issues": {"type": "boolean", "description": "Auto-fix schema inconsistencies", "default": False},
+                },
+                "required": ["dataset_id"],
+            },
+        ),
+        Tool(
+            name="explore_distributions",
+            description=(
+                "诊断工具：计算每列分布统计（唯一值计数、空值比例、最小/最大/均值/标准差、"
+                "前10高频值、直方图分箱）。当聚类产生意外结果、实体检测失败，或需要在选择"
+                "分析策略前了解数据时使用。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "dataset_id": {"type": "string", "description": "Dataset ID from load_data or entity data"},
+                    "columns": {"type": "array", "items": {"type": "string"},
+                                "description": "Specific columns to analyze (default: auto-detect numeric)"},
+                    "max_sample": {"type": "integer", "description": "Max rows to sample", "default": 10000},
+                },
+                "required": ["dataset_id"],
+            },
+        ),
+        Tool(
+            name="find_outliers",
+            description=(
+                "诊断工具：使用IQR方法检测数值列中的统计异常值。返回任何数值列超过中位数3倍IQR的行。"
+                "当怀疑数据质量问题或极端值偏离分析时使用。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "dataset_id": {"type": "string", "description": "Dataset ID from load_data"},
+                    "columns": {"type": "array", "items": {"type": "string"},
+                                "description": "Columns to check (default: all numeric)"},
+                    "threshold": {"type": "number", "description": "IQR multiplier (default 3.0)", "default": 3.0},
+                },
+                "required": ["dataset_id"],
+            },
+        ),
+        Tool(
+            name="diagnose_clustering",
+            description=(
+                "诊断工具：当聚类产生太少/太多聚类、全部为噪声或轮廓系数低时——诊断原因。"
+                "检查：特征方差、相关矩阵、特征降维分析、数据稀疏性，并建议参数调整"
+                "（min_cluster_size、算法选择、特征选择）。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "dataset_id": {"type": "string", "description": "Entity dataset ID"},
+                    "cluster_result_id": {"type": "string", "description": "Cluster result ID from previous run_clustering"},
+                },
+                "required": ["dataset_id", "cluster_result_id"],
+            },
+        ),
+        Tool(
+            name="compare_datasets",
+            description=(
+                "诊断工具：比较两个数据集（如处理前/后，或两次不同运行）。报告schema差异、"
+                "行数变化、列值分布偏移。用于验证处理步骤未损坏数据或比较不同运行的结果。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "dataset_id_a": {"type": "string", "description": "First dataset ID"},
+                    "dataset_id_b": {"type": "string", "description": "Second dataset ID"},
+                },
+                "required": ["dataset_id_a", "dataset_id_b"],
+            },
+        ),
+        Tool(
+            name="export_debug_sample",
+            description=(
+                "诊断工具：导出原始数据样本为JSON用于外部调试。当流水线深处发生错误且需要"
+                "检查实际数据值时使用。返回前N行所有列。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "dataset_id": {"type": "string", "description": "Dataset ID"},
+                    "max_rows": {"type": "integer", "description": "Rows to export", "default": 50},
+                },
+                "required": ["dataset_id"],
+            },
+        ),
+        Tool(
+            name="repair_schema",
+            description=(
+                "诊断工具：通过对齐列名（不区分大小写合并、下划线/连字符规范化）并用空值填充"
+                "缺失列，尝试修复文件间的schema不匹配。当load_data因CSV文件具有不同列集而"
+                "失败时使用。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "dataset_id": {"type": "string", "description": "Failed dataset ID from load_data"},
+                },
+                "required": ["dataset_id"],
+            },
+        ),
+        # ── Read-only analysis tools (LLM deep-dive, no side effects) ────
+        Tool(
+            name="analyze_patterns",
+            description=(
+                "分析工具：检测流量模式——top源/目标IP、端口分布、TLS版本分布、协议混合。"
+                "返回汇总统计。只读，不修改数据。用于了解高层流量结构。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "dataset_id": {"type": "string", "description": "Dataset ID from load_data"},
+                    "top_n": {"type": "integer", "description": "Number of top items per category", "default": 10},
+                },
+                "required": ["dataset_id"],
+            },
+        ),
+        Tool(
+            name="analyze_temporal",
+            description=(
+                "分析工具：分析随时间变化的流量——小时/日流量计数、繁忙时段、空闲时段。"
+                "需要时间戳列。只读。用于了解时间相关模式和周期性。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "dataset_id": {"type": "string", "description": "Dataset ID from load_data"},
+                    "timestamp_column": {"type": "string", "description": "Column name for timestamps (auto-detected if omitted)"},
+                },
+                "required": ["dataset_id"],
+            },
+        ),
+        Tool(
+            name="analyze_fft",
+            description=(
+                "FFT频谱分析——从时间戳列提取周期模式和频率特征。适用于检测时间周期性"
+                "（如每日/每周流量模式）。返回频率、幅度、top周期、频谱质心、周期性评分。"
+                "只读，无副作用。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "dataset_id": {
+                        "type": "string",
+                        "description": "Dataset ID from load_data",
+                    },
+                    "timestamp_column": {
+                        "type": "string",
+                        "description": "Column name containing timestamps or numeric time values",
+                    },
+                    "top_n": {
+                        "type": "integer",
+                        "description": "Number of top frequency peaks to return",
+                        "default": 5,
+                    },
+                },
+                "required": ["dataset_id", "timestamp_column"],
+            },
+        ),
+        Tool(
+            name="analyze_tls_health",
+            description=(
+                "分析工具：TLS安全态势评估——检查过时的TLS版本（1.0/1.1）、弱加密套件、"
+                "证书问题、SNI异常。只读。用于评估流量的TLS安全性。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "dataset_id": {"type": "string", "description": "Dataset ID from load_data"},
+                },
+                "required": ["dataset_id"],
+            },
+        ),
+        Tool(
+            name="analyze_geo_distribution",
+            description=(
+                "分析工具：流量地理分布——top源/目标国家、异常地点对（如不可能的行程）、"
+                "ISP多样性。只读。需要国家或经纬度列。用于检测地理异常。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "dataset_id": {"type": "string", "description": "Dataset ID from load_data"},
+                },
+                "required": ["dataset_id"],
+            },
+        ),
+        Tool(
+            name="analyze_entity_detail",
+            description=(
+                "分析工具：深入分析单个实体——其所有流、使用的TLS版本、目标多样性、时间模式、"
+                "异常评分。只读。在compute_scores后调查特定可疑实体时使用。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "dataset_id": {"type": "string", "description": "Entity dataset ID (after filter_and_cluster or compute_scores)"},
+                    "entity_value": {"type": "string", "description": "Entity value to investigate (e.g. an IP address or SNI)"},
+                },
+                "required": ["dataset_id", "entity_value"],
+            },
+        ),
+        Tool(
+            name="compute_distance_matrix",
+            description=(
+                "LLM驱动工具：对数据集的每一行执行用户编写的Python函数，返回距离/相似度评分摘要。"
+                "LLM提供函数体，签名格式为`def distance_fn(row: dict) -> float:`。执行器将其包装，"
+                "在受限命名空间（仅math、numpy）中逐行运行，返回得分的最小/最大/均值/标准差以及"
+                "评分行样本。用于现有工具无法表达的一次性度量、过滤或评分任务。"
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "dataset_id": {
+                        "type": "string",
+                        "description": "Source dataset identifier",
+                    },
+                    "python_function": {
+                        "type": "string",
+                        "description": (
+                            "Body of a Python function with signature `def distance_fn(row: dict) -> float:`. "
+                            "Example: \"return abs(row.get('col1', 0)) + row.get('col2', 0)\". "
+                            "Available namespace: math, numpy (as np). No dangerous modules."
+                        ),
+                    },
+                    "columns": {
+                        "type": "array",
+                        "items": {"type": "string"},
+                        "description": "Columns to pass to the function (default: auto-detect numeric columns)",
+                    },
+                    "max_sample": {
+                        "type": "integer",
+                        "description": "Maximum rows to evaluate (default 10000 for performance)",
+                        "default": 10000,
+                    },
+                },
+                "required": ["dataset_id", "python_function"],
+            },
+        ),
+    ]  # end of raw_tools
+
+    # Inject mandatory _timeout parameter into every tool
+    result = []
+    for tool in raw_tools:
+        schema = dict(tool.inputSchema)
+        props = dict(schema.get("properties", {}))
+        req = list(schema.get("required", []))
+        # Add _timeout as the first required property
+        props["_timeout"] = {
+            "type": "integer",
+            "description": "工具调用超时秒数（必填，LLM决定每次调用的最长等待时间；0表示不限制）",
+            "default": 120,
+        }
+        req.insert(0, "_timeout")
+        schema["properties"] = props
+        schema["required"] = req
+        result.append(Tool(
+            name=tool.name,
+            description=tool.description,
+            inputSchema=schema,
+        ))
+    return result

analysis/tools/analysis.py

@@ -0,0 +1,335 @@
+"""Read-only analysis handlers — patterns, temporal, FFT, TLS health, geo, entity detail."""
+import traceback
+from analysis.constants import LOW_MEMORY_THRESHOLD_ROWS
+import numpy as np
+import polars as pl
+
+from ..session_store import SessionStore
+from ._helpers import _resolve_dataset, _count_rows, NUMERIC_TYPE_NAMES
+import logging
+logger = logging.getLogger(__name__)
+
+
+async def _handle_analyze_patterns(dataset_id: str, top_n: int = 10) -> dict:
+    """Read-only: traffic pattern analysis."""
+    entry, err = _resolve_dataset(dataset_id)
+    if err:
+        return err
+    lf = entry['lazyframe']
+    schema = entry.get('schema', {})
+    n = _count_rows(lf)
+    df = lf.head(LOW_MEMORY_THRESHOLD_ROWS).collect(streaming=True)
+
+    result = {'total_rows': n, 'sampled': min(n, LOW_MEMORY_THRESHOLD_ROWS)}
+
+    for col_name, label in [(':ips', 'src_ips'), (':ipd', 'dst_ips'),
+                              ('snam', 'snis'), ('cnam', 'cert_names')]:
+        if col_name in df.columns:
+            try:
+                top = df[col_name].value_counts(sort=True).head(top_n)
+                result[label] = {
+                    'top': [{'value': str(r[col_name]), 'count': int(r['count'])}
+                            for r in top.iter_rows(named=True)],
+                    'unique': int(df[col_name].n_unique()),
+                }
+            except Exception:
+                logger.error("_handle_analyze_patterns failed: {}".format(traceback.format_exc()))
+                pass
+
+    for port_col in [':prs', ':prd']:
+        if port_col in df.columns:
+            try:
+                top = df[port_col].value_counts(sort=True).head(top_n)
+                result[f'{port_col}_dist'] = {
+                    'top': [{'port': int(r[port_col]), 'count': int(r['count'])}
+                            for r in top.iter_rows(named=True)],
+                    'unique': int(df[port_col].n_unique()),
+                }
+            except Exception:
+                logger.error("unknown failed: {}".format(traceback.format_exc()))
+                pass
+
+    for tls_col in ['0ver']:
+        if tls_col in df.columns:
+            try:
+                vc = df[tls_col].value_counts(sort=True)
+                result['tls_versions'] = [{'version': str(r[tls_col]), 'count': int(r['count'])}
+                                          for r in vc.iter_rows(named=True)]
+            except Exception:
+                logger.error("unknown failed: {}".format(traceback.format_exc()))
+                pass
+
+    return result
+
+
+async def _handle_analyze_temporal(dataset_id: str, timestamp_column: str = None) -> dict:
+    """Read-only: temporal traffic pattern analysis."""
+    entry, err = _resolve_dataset(dataset_id)
+    if err:
+        return err
+    lf = entry['lazyframe']
+    schema = entry.get('schema', {})
+
+    if not timestamp_column:
+        for cand in ['timestamp', 'time', '8dbd', '8did']:
+            if cand in schema:
+                timestamp_column = cand
+                break
+    if not timestamp_column:
+        return {'error': 'No timestamp column found', 'truncated': False}
+
+    n = _count_rows(lf)
+    return {'total_rows': n, 'timestamp_column': timestamp_column,
+            'note': 'Temporal analysis requires parsed timestamps; raw values shown in explore_distributions'}
+
+
+async def _handle_analyze_fft(
+    dataset_id: str,
+    timestamp_column: str,
+    top_n: int = 5,
+) -> dict:
+    """FFT spectrum analysis on a timestamp column to detect periodic patterns."""
+    entry, err = _resolve_dataset(dataset_id)
+    if err:
+        return err
+
+    lf: pl.LazyFrame = entry['lazyframe']
+    schema = entry.get('schema', {})
+
+    col = None
+    if timestamp_column in schema:
+        col = timestamp_column
+    else:
+        col_lower = timestamp_column.lower()
+        for c in schema:
+            if c.lower() == col_lower:
+                col = c
+                break
+    if col is None:
+        return {'error': f"Timestamp column '{timestamp_column}' not found in dataset. Available: {list(schema.keys())[:20]}",
+                'truncated': False}
+
+    df = lf.select(pl.col(col).alias('__ts__')).collect(streaming=True)
+    n_raw = len(df)
+    if n_raw < 4:
+        return {'error': f'Need at least 4 samples for FFT; got {n_raw}', 'truncated': False}
+
+    ts_series = df['__ts__']
+    dtype = ts_series.dtype
+    str_dtypes = (pl.Utf8, pl.String)
+
+    if dtype in str_dtypes:
+        try:
+            ts_parsed = ts_series.str.strptime(pl.Datetime, format=None, strict=False)
+            ts_numeric = ts_parsed.cast(pl.Int64) / 1_000_000_000.0
+        except Exception:
+            logger.error("unknown failed: {}".format(traceback.format_exc()))
+            try:
+                ts_numeric = ts_series.cast(pl.Float64)
+            except Exception as e:
+                return {'error': f'Cannot convert string column to numeric or datetime: {e}',
+                        'truncated': False}
+    elif dtype in (pl.Datetime, pl.Date):
+        ts_numeric = ts_series.cast(pl.Int64) / 1_000_000_000.0
+    else:
+        try:
+            ts_numeric = ts_series.cast(pl.Float64)
+        except Exception as e:
+            return {'error': f'Cannot cast column to float: {e}', 'truncated': False}
+
+    df_sorted = pl.DataFrame({'__val__': ts_numeric}).drop_nulls().sort('__val__')
+    n = len(df_sorted)
+    if n < 4:
+        return {'error': f'After removing nulls, only {n} samples remain (need ≥4)', 'truncated': False}
+
+    values = df_sorted['__val__'].to_numpy().astype(np.float64)
+
+    values_centered = values - np.mean(values)
+    fft_complex = np.fft.fft(values_centered)
+    fft_magnitudes = np.abs(fft_complex)
+    freqs = np.fft.fftfreq(n)
+
+    pos_mask = freqs > 0
+    freqs_pos = freqs[pos_mask]
+    mags_pos = fft_magnitudes[pos_mask]
+
+    if len(freqs_pos) == 0:
+        return {'error': 'Insufficient positive frequencies for FFT analysis', 'truncated': False}
+
+    peak_indices = np.argsort(mags_pos)[::-1][:min(top_n, len(mags_pos))]
+    top_periods = []
+    for idx in peak_indices:
+        freq = float(freqs_pos[idx])
+        mag = float(mags_pos[idx])
+        period = float(1.0 / freq) if freq > 1e-10 else float('inf')
+        if n > 1:
+            time_span = float(values[-1] - values[0])
+            sample_period = time_span / max(n - 1, 1)
+            time_period = period * sample_period
+        else:
+            time_period = float('inf')
+        top_periods.append({
+            'period': round(period, 2),
+            'magnitude': round(mag, 4),
+            'frequency': round(freq, 6),
+            'time_domain_period_seconds': round(time_period, 2) if time_period != float('inf') else None,
+        })
+
+    mag_sum = float(np.sum(mags_pos))
+    if mag_sum > 0:
+        spectral_centroid = float(np.sum(freqs_pos * mags_pos) / mag_sum)
+    else:
+        spectral_centroid = 0.0
+
+    mean_mag = float(np.mean(mags_pos)) if len(mags_pos) > 0 else 1.0
+    max_mag = float(mags_pos[peak_indices[0]]) if len(peak_indices) > 0 else 0.0
+    periodicity_score = round(max_mag / max(mean_mag, 1e-10), 4)
+
+    decimate_step = max(1, len(freqs_pos) // 200)
+    freqs_decimated = freqs_pos[::decimate_step].tolist()
+    mags_decimated = mags_pos[::decimate_step].tolist()
+
+    return {
+        'frequencies': [round(float(f), 6) for f in freqs_decimated],
+        'magnitudes': [round(float(m), 4) for m in mags_decimated],
+        'top_periods': top_periods,
+        'spectral_centroid': round(spectral_centroid, 6),
+        'periodicity_score': periodicity_score,
+        'n_samples': n,
+        'n_raw': n_raw,
+        'nan_dropped': n_raw - n,
+        'timestamp_column': col,
+    }
+
+
+async def _handle_analyze_tls_health(dataset_id: str) -> dict:
+    """Read-only: TLS security assessment."""
+    entry, err = _resolve_dataset(dataset_id)
+    if err:
+        return err
+    lf = entry['lazyframe']
+    schema = entry.get('schema', {})
+    n = _count_rows(lf)
+    df = lf.head(LOW_MEMORY_THRESHOLD_ROWS).collect(streaming=True)
+
+    result = {'total_rows': n, 'sampled': min(n, LOW_MEMORY_THRESHOLD_ROWS)}
+
+    if '0ver' in df.columns:
+        vc = df['0ver'].value_counts(sort=True)
+        versions = {}
+        for r in vc.iter_rows(named=True):
+            v = str(r['0ver']).replace(' ', '')
+            if v in ('0303', '0304'):
+                versions['tls_1_2_plus'] = versions.get('tls_1_2_plus', 0) + int(r['count'])
+            else:
+                versions['tls_legacy'] = versions.get('tls_legacy', 0) + int(r['count'])
+        total = sum(versions.values())
+        result.update({
+            'tls_1_2_plus_pct': round(versions.get('tls_1_2_plus', 0) / max(total, 1) * 100, 1),
+            'tls_legacy_pct': round(versions.get('tls_legacy', 0) / max(total, 1) * 100, 1),
+            'tls_legacy_versions': [{'version': str(r['0ver']), 'count': int(r['count'])}
+                                     for r in vc.iter_rows(named=True)
+                                     if str(r['0ver']).replace(' ', '') not in ('0303', '0304')],
+        })
+
+    if 'snam' in df.columns:
+        empty_sni = df['snam'].is_null().sum() + df['snam'].cast(pl.Utf8).is_in(['', 'null', 'None']).sum()
+        result['sni_missing_pct'] = round(float(empty_sni) / max(len(df), 1) * 100, 1)
+
+    if 'cipher-suite' in df.columns:
+        weak_ciphers = ['rc4', 'cbc', 'null', 'export']
+        total_ciphers = len(df)
+        weak_count = 0
+        for w in weak_ciphers:
+            try:
+                weak_count += df['cipher-suite'].str.to_lowercase().str.contains(w).sum()
+            except Exception:
+                logger.error("unknown failed: {}".format(traceback.format_exc()))
+                pass
+        result['weak_cipher_pct'] = round(float(weak_count) / max(total_ciphers, 1) * 100, 1) if total_ciphers else 0
+
+    if result.get('tls_legacy_pct', 0) > 20:
+        result['warning'] = 'High proportion of legacy TLS — possible downgrade attacks'
+    if result.get('sni_missing_pct', 0) > 50:
+        result['warning'] = 'Most flows lack SNI — possible IP-direct connections (proxy/VPN)'
+    return result
+
+
+async def _handle_analyze_geo_distribution(dataset_id: str) -> dict:
+    """Read-only: geographic traffic analysis."""
+    entry, err = _resolve_dataset(dataset_id)
+    if err:
+        return err
+    lf = entry['lazyframe']
+    schema = entry.get('schema', {})
+    n = _count_rows(lf)
+    df = lf.head(LOW_MEMORY_THRESHOLD_ROWS).collect(streaming=True)
+
+    result = {'total_rows': n, 'sampled': min(n, LOW_MEMORY_THRESHOLD_ROWS)}
+
+    for geo_col, label in [('scnt', 'src_countries'), ('dcnt', 'dst_countries')]:
+        if geo_col in df.columns:
+            try:
+                vc = df[geo_col].value_counts(sort=True).head(15)
+                result[label] = [{'country': str(r[geo_col]), 'count': int(r['count'])}
+                                 for r in vc.iter_rows(named=True)]
+                result[f'{label}_unique'] = int(df[geo_col].n_unique())
+            except Exception:
+                logger.error("_handle_analyze_geo_distribution failed: {}".format(traceback.format_exc()))
+                pass
+
+    if 'scnt' in df.columns and 'dcnt' in df.columns:
+        try:
+            same = (df['scnt'] == df['dcnt']).sum()
+            result['domestic_pct'] = round(float(same) / max(len(df), 1) * 100, 1)
+            result['international_pct'] = round(100.0 - result['domestic_pct'], 1)
+        except Exception:
+            logger.error("_handle_analyze_geo_distribution failed: {}".format(traceback.format_exc()))
+            pass
+
+    return result
+
+
+async def _handle_analyze_entity_detail(dataset_id: str, entity_value: str) -> dict:
+    """Read-only: deep-dive into a single entity."""
+    entry, err = _resolve_dataset(dataset_id)
+    if err:
+        return err
+    lf = entry['lazyframe']
+    schema = entry.get('schema', {})
+
+    entity_col = None
+    for c in schema:
+        if c.startswith('_') or schema[c].split('(')[0].strip() in NUMERIC_TYPE_NAMES or '.' in c:
+            continue
+        entity_col = c
+        break
+    if not entity_col:
+        entity_col = list(schema.keys())[0]
+
+    filtered = lf.filter(pl.col(entity_col).cast(pl.Utf8).str.contains(entity_value, literal=True))
+    n = _count_rows(filtered)
+    if n == 0:
+        return {'error': f"Entity '{entity_value}' not found in column '{entity_col}'", 'truncated': False}
+    df = filtered.collect(streaming=True)
+
+    result = {
+        'entity_value': entity_value,
+        'entity_column': entity_col,
+        'flow_count': n,
+        'columns': list(df.columns[:15]),
+    }
+
+    for c in df.columns:
+        if df[c].dtype in (pl.Float32, pl.Float64, pl.Int32, pl.Int64):
+            try:
+                result[f'{c}_stats'] = {
+                    'mean': round(float(df[c].mean()), 2),
+                    'min': float(df[c].min()) if df[c].min() is not None else None,
+                    'max': float(df[c].max()) if df[c].max() is not None else None,
+                }
+            except Exception:
+                logger.error("unknown failed: {}".format(traceback.format_exc()))
+                pass
+
+    return result

analysis/tools/anomalies.py

@@ -0,0 +1,94 @@
+"""Anomaly detection + visualization handlers."""
+import traceback
+import numpy as np
+from analysis.constants import MAX_DISTRIBUTION_SAMPLE, RANDOM_SEED
+import polars as pl
+
+from ..session_store import SessionStore
+from ._helpers import _resolve_dataset, NUMERIC_TYPE_NAMES
+import logging
+logger = logging.getLogger(__name__)
+
+
+async def _handle_detect_anomalies(dataset_id: str, contamination: float = 0.05) -> dict:
+    """Run Isolation Forest on scored entity data."""
+    from sklearn.ensemble import IsolationForest
+    entry, err = _resolve_dataset(dataset_id)
+    if err:
+        return err
+
+    lf: pl.LazyFrame = entry['lazyframe']
+    schema = entry.get('schema', {})
+    num_cols = [c for c in schema if schema[c].split('(')[0].strip() in NUMERIC_TYPE_NAMES and not c.startswith('_')]
+    if not num_cols:
+        return {'error': 'No numeric columns for anomaly detection', 'truncated': False}
+
+    df = lf.select(num_cols).collect(streaming=True)
+    mat = df.to_numpy()
+    mat = np.nan_to_num(mat, nan=0.0)
+
+    n = len(mat)
+    chunk_size = min(MAX_SAMPLE_ROWS, n)
+    labels = np.zeros(n, dtype=int)
+    for start in range(0, n, chunk_size):
+        end = min(start + chunk_size, n)
+        iso = IsolationForest(n_estimators=100, max_samples=min(256, chunk_size),
+                              contamination=contamination, random_state=RANDOM_SEED, n_jobs=-1)
+        labels[start:end] = iso.fit_predict(mat[start:end])
+
+    anomaly_count = int((labels == -1).sum())
+    return {
+        'status': 'completed',
+        'total_entities': n,
+        'anomaly_count': anomaly_count,
+        'anomaly_rate': round(anomaly_count / max(n, 1), 4),
+        'dataset_id': dataset_id,
+    }
+
+
+async def _handle_visualize_anomalies(dataset_id: str) -> dict:
+    """Generate UMAP-2D embedding for anomaly visualization."""
+    from sklearn.preprocessing import StandardScaler
+    import umap
+    entry, err = _resolve_dataset(dataset_id)
+    if err:
+        return err
+
+    lf: pl.LazyFrame = entry['lazyframe']
+    schema = entry.get('schema', {})
+    num_cols = [c for c in schema if schema[c].split('(')[0].strip() in NUMERIC_TYPE_NAMES and not c.startswith('_')]
+    if len(num_cols) < 2:
+        return {'error': 'Need at least 2 numeric columns for UMAP', 'truncated': False}
+
+    try:
+        n_total = lf.select(pl.len()).collect(streaming=True).item()
+    except Exception:
+        logger.error("_handle_visualize_anomalies failed: {}".format(traceback.format_exc()))
+        n_total = 0
+    sample_lf = lf if n_total <= MAX_DISTRIBUTION_SAMPLE else lf.collect(streaming=True).sample(n=MAX_DISTRIBUTION_SAMPLE, seed=RANDOM_SEED).lazy()
+    df = sample_lf.select(num_cols).collect(streaming=True)
+    mat = StandardScaler().fit_transform(df.to_numpy())
+    mat = np.nan_to_num(mat, nan=0.0)
+    reducer = umap.UMAP(n_components=2, random_state=RANDOM_SEED,
+                        n_neighbors=15, min_dist=0.1, metric='euclidean')
+    coords = reducer.fit_transform(mat)
+
+    risk_col = next((c for c in schema if 'risk' in c.lower()), None)
+    score_col = next((c for c in schema if 'proxy_score' in c.lower() or 'threat' in c.lower()), None)
+
+    non_num = [c for c in df.columns if c not in num_cols and not c.startswith('_')]
+    ent_col = non_num[0] if non_num else df.columns[0]
+    entities = df.select(ent_col).to_series().to_list() if ent_col in df.columns else []
+
+    scatter = [
+        {'x': float(coords[i, 0]), 'y': float(coords[i, 1]),
+         'entity': str(entities[i]) if i < len(entities) else '',
+         'score': round(float(mat[i].mean()), 4)}
+        for i in range(len(coords))
+    ]
+    return {
+        'status': 'completed',
+        'scatter_points': len(scatter),
+        'scatter_data': scatter[:200],
+        'dataset_id': dataset_id,
+    }

analysis/tools/clustering.py

@@ -0,0 +1,458 @@
+"""Clustering handlers — HDBSCAN, KMeans, AgglomerativeClustering.
+
+Contains:
+- _handle_run_clustering: cluster selected columns
+- _handle_filter_and_cluster: filter then auto-cluster
+- _cluster_core: shared clustering pipeline extracted from both handlers
+"""
+import logging
+import traceback
+from analysis.constants import LOW_MEMORY_THRESHOLD_GB, LOW_MEMORY_THRESHOLD_ROWS, MAX_CLUSTER_FEATURES, MAX_SAMPLE_ROWS, RANDOM_SEED
+from typing import Optional
+
+import numpy as np
+import polars as pl
+
+from ..session_store import SessionStore, _generate_id
+from ._helpers import (
+    NUMERIC_DTYPES,
+    _build_filter_expr,
+    _resolve_dataset,
+    _count_rows,
+)
+
+logger = logging.getLogger(__name__)
+_clust_logger = logging.getLogger(__name__)
+
+
+def _cluster_core(
+    data: np.ndarray,
+    feature_cols: list[str],
+    algorithm: str,
+    params: dict,
+    random_state: int = RANDOM_SEED,
+):
+    """Shared clustering pipeline: scale, filter, reduce, fit, score.
+
+    Args:
+        data: Numpy array (n_samples, n_features).
+        feature_cols: Column names matching data columns.
+        algorithm: 'hdbscan', 'kmeans', or 'agglomerative'.
+        params: Algorithm-specific parameters dict.
+        random_state: Random seed.
+
+    Returns:
+        (labels_array, n_clusters_found, noise_count, quality_metrics, _ds_idx)
+    """
+    p = dict(params)
+
+    # ── Low memory check ───────────────────────────────────────────────
+    low_memory = False
+    try:
+        import psutil
+        mem = psutil.virtual_memory()
+        low_memory = mem.available < LOW_MEMORY_THRESHOLD_GB * 1024 ** 3  # 2 GB
+    except ImportError:
+        pass
+
+    if low_memory and len(data) > LOW_MEMORY_THRESHOLD_ROWS:
+        rng = np.random.default_rng(RANDOM_SEED)
+        idx = rng.choice(len(data), MAX_SAMPLE_ROWS, replace=False)
+        data = data[idx]
+
+    # ── Downsampling to 50K max ────────────────────────────────────────
+    _ds_idx = None
+    if len(data) > MAX_SAMPLE_ROWS:
+        rng = np.random.default_rng(RANDOM_SEED)
+        _ds_idx = rng.choice(len(data), MAX_SAMPLE_ROWS, replace=False)
+        data = data[_ds_idx]
+
+    # ── Variance filtering ─────────────────────────────────────────────
+    if data.shape[1] > 1:
+        variances = np.var(data, axis=0)
+        keep_var = np.where(variances >= 1e-10)[0]
+        if len(keep_var) == 0:
+            keep_var = np.arange(data.shape[1])
+        if len(keep_var) < data.shape[1]:
+            data = data[:, keep_var]
+            feature_cols = [feature_cols[i] for i in keep_var]
+
+    # Log clustering input
+    _clust_logger.info(
+        '[CLUSTER_INPUT] rows=%d cols=%d feature_cols=%s',
+        len(data), data.shape[1] if len(data.shape) > 1 else 1, feature_cols,
+    )
+
+    # Standard scale
+    from sklearn.preprocessing import StandardScaler
+    _clust_logger.info('[CLUSTER_SCALE] Starting StandardScaler...')
+    data_scaled = StandardScaler().fit_transform(data)
+    _clust_logger.info('[CLUSTER_SCALE] Done. shape=%s', data_scaled.shape)
+
+    # ── Correlation filtering ──────────────────────────────────────────
+    if data_scaled.shape[1] > 1:
+        corr = np.corrcoef(data_scaled.T)
+        upper = np.triu(np.abs(corr), 1)
+        to_drop = [j for j in range(upper.shape[1]) if any(upper[:, j] > 0.95)]
+        if to_drop:
+            keep = [j for j in range(data_scaled.shape[1]) if j not in to_drop]
+            data_scaled = data_scaled[:, keep]
+            feature_cols = [feature_cols[j] for j in keep]
+
+    # ── SVD dimensionality reduction (if >50 features) ─────────────────
+    n_features = data_scaled.shape[1]
+    if n_features > MAX_CLUSTER_FEATURES:
+        from sklearn.decomposition import TruncatedSVD
+        svd = TruncatedSVD(n_components=MAX_CLUSTER_FEATURES, random_state=random_state)
+        data_scaled = svd.fit_transform(data_scaled)
+
+    # ── Hyperparameter auto-tuning ─────────────────────────────────────
+    n = len(data_scaled)
+    if algorithm == 'hdbscan':
+        min_cluster = p.get('min_cluster_size', max(3, min(MAX_CLUSTER_FEATURES, n // 20)))
+        p['min_cluster_size'] = min_cluster
+        p['min_samples'] = p.get('min_samples', min_cluster)
+        p['cluster_selection_epsilon'] = p.get('cluster_selection_epsilon', 0.0)
+    else:
+        p['min_cluster_size'] = min(
+            p.get('min_cluster_size', 5),
+            max(n // 2, 2),
+        )
+
+    # ── Model fit ──────────────────────────────────────────────────────
+    from sklearn.cluster import HDBSCAN, MiniBatchKMeans, AgglomerativeClustering
+    if algorithm == 'kmeans':
+        n_clusters_param = p.get('n_clusters', 3)
+        model = MiniBatchKMeans(
+            n_clusters=n_clusters_param, random_state=random_state,
+            batch_size=1024, n_init='auto',
+        )
+    elif algorithm in ('agglomerative', 'ward'):
+        n_clusters_param = p.get('n_clusters', 5)
+        model = AgglomerativeClustering(
+            n_clusters=n_clusters_param,
+            metric=p.get('metric', 'euclidean'),
+            linkage=p.get('linkage', 'ward'),
+            distance_threshold=None,
+        )
+    else:
+        model = HDBSCAN(
+            min_cluster_size=p['min_cluster_size'],
+            min_samples=p.get('min_samples', None),
+            metric=p.get('metric', 'euclidean'),
+            cluster_selection_epsilon=p.get('cluster_selection_epsilon', 0.0),
+        )
+
+    labels_array = model.fit_predict(data_scaled)
+    n_clusters_found = int(len(set(labels_array)) - (1 if -1 in labels_array else 0))
+    noise_count = int((labels_array == -1).sum()) if algorithm == 'hdbscan' else 0
+    noise_ratio = round(noise_count / len(labels_array), 4) if algorithm == 'hdbscan' else 0.0
+
+    # ── Quality metrics ────────────────────────────────────────────────
+    quality = {
+        'n_clusters': n_clusters_found,
+        'n_noise': noise_count,
+        'noise_ratio': noise_ratio,
+        'algorithm': algorithm,
+    }
+
+    if n_clusters_found > 1:
+        try:
+            from sklearn.metrics import silhouette_score
+            score = silhouette_score(data_scaled, labels_array, random_state=random_state)
+            quality['silhouette_score'] = round(float(score), 4)
+        except Exception:
+            logger.error("unknown failed: {}".format(traceback.format_exc()))
+            quality['silhouette_score'] = None
+        try:
+            from sklearn.metrics import davies_bouldin_score
+            dbs = davies_bouldin_score(data_scaled, labels_array)
+            quality['davies_bouldin_score'] = round(float(dbs), 4)
+        except Exception:
+            logger.error("unknown failed: {}".format(traceback.format_exc()))
+            quality['davies_bouldin_score'] = None
+        try:
+            from sklearn.metrics import calinski_harabasz_score
+            chs = calinski_harabasz_score(data_scaled, labels_array)
+            quality['calinski_harabasz_score'] = round(float(chs), 4)
+        except Exception:
+            logger.error("unknown failed: {}".format(traceback.format_exc()))
+            quality['calinski_harabasz_score'] = None
+
+    # Cluster sizes
+    cluster_sizes = {}
+    for label in np.unique(labels_array):
+        cluster_sizes[int(label)] = int((labels_array == label).sum())
+    quality['cluster_sizes'] = cluster_sizes
+
+    return labels_array, n_clusters_found, noise_count, quality, _ds_idx
+
+
+# ═══════════════════════════════════════════════════════════════════════
+#  _handle_run_clustering
+# ═══════════════════════════════════════════════════════════════════════
+
+async def _handle_run_clustering(
+    dataset_id: str,
+    cluster_columns: list[str],
+    algorithm: str = 'agglomerative',
+    params: Optional[dict] = None,
+    random_state: int = RANDOM_SEED,
+) -> dict:
+    """Run HDBSCAN, KMeans, or Agglomerative clustering on selected columns."""
+    entry, err = _resolve_dataset(dataset_id)
+    if err:
+        return err
+
+    lf: pl.LazyFrame = entry['lazyframe']
+    p = params or {}
+
+    # Collect cluster features - numeric only
+    available = lf.collect_schema()
+    available_names = available.names()
+    available_dtypes = available.dtypes()
+    feature_cols = [
+        c for c in cluster_columns
+        if c in available_names
+        and available_dtypes[available_names.index(c)] in NUMERIC_DTYPES
+    ]
+    if not feature_cols:
+        # Fall back: auto-select all numeric columns (exclude internal ones)
+        feature_cols = [
+            available_names[i] for i, dt in enumerate(available_dtypes)
+            if dt in NUMERIC_DTYPES and not available_names[i].startswith('_')
+        ][:10]
+    if not feature_cols:
+        # Fallback: try coercing Utf8 columns to Float64
+        utf8_cols = [
+            available_names[i] for i, dt in enumerate(available_dtypes)
+            if dt == pl.Utf8 and not available_names[i].startswith('_')
+        ][:10]
+        if utf8_cols:
+            for col in utf8_cols:
+                try:
+                    casted = lf.select(
+                        pl.col(col).cast(pl.Float64, strict=False).alias(col)
+                    ).collect(streaming=True)
+                    null_ratio = casted[col].is_null().sum() / max(len(casted), 1)
+                    if null_ratio < 0.9:
+                        lf = lf.with_columns(
+                            pl.col(col).cast(pl.Float64, strict=False)
+                        )
+                        feature_cols.append(col)
+                except Exception:
+                    logger.error("unknown failed: {}".format(traceback.format_exc()))
+                    pass
+        if not feature_cols:
+            all_numeric = [f"{available_names[i]}" for i, dt in enumerate(available_dtypes)
+                           if dt in NUMERIC_DTYPES]
+            return {
+                'error': f'No numeric cluster columns found in dataset. Available: {all_numeric}',
+                'truncated': False
+            }
+
+    # Downsample before collect
+    row_count = _count_rows(lf)
+    if row_count > LOW_MEMORY_THRESHOLD_ROWS:
+        lf = lf.collect(streaming=True).sample(n=LOW_MEMORY_THRESHOLD_ROWS, seed=RANDOM_SEED).lazy()
+
+    df_features = lf.select(feature_cols).collect(streaming=True)
+
+    # Drop all-NaN columns
+    for col in df_features.columns:
+        if df_features[col].is_null().all():
+            df_features = df_features.drop(col)
+            feature_cols = [c for c in feature_cols if c != col]
+
+    if df_features.width == 0:
+        return {'error': 'No valid feature columns after dropping all-NaN columns',
+                'truncated': False}
+
+    data = df_features.to_numpy()
+
+    # Fill remaining NaN with column mean
+    if data.dtype.kind == 'f':
+        col_mean = np.nanmean(data, axis=0)
+        col_mean = np.nan_to_num(col_mean, nan=0.0)
+        data = np.where(np.isnan(data), col_mean, data)
+
+    # Zero-sample guard
+    if len(data) < 3:
+        return {
+            'cluster_result_id': _generate_id('clust'),
+            'n_clusters': 0,
+            'n_noise': 0,
+            'quality_metrics': {'note': 'too few samples'},
+            'truncated': False,
+        }
+
+    labels_array, n_found, noise_count, quality, _ds_idx = _cluster_core(
+        data, feature_cols, algorithm, p, random_state,
+    )
+
+
+    result_id = _generate_id('clust')
+    store = SessionStore()
+    store.store_cluster_result(
+        result_id=result_id,
+        labels=labels_array.tolist(),
+        n_clusters=n_found,
+        params={
+            'algorithm': algorithm,
+            'params': p,
+            'random_state': random_state,
+            'feature_columns': feature_cols,
+            '_ds_idx': _ds_idx.tolist() if _ds_idx is not None else None,
+        },
+        parent_dataset_id=dataset_id,
+    )
+
+    return {
+        'cluster_result_id': result_id,
+        'n_clusters': n_found,
+        'n_noise': noise_count,
+        'labels_sample': labels_array[:50].tolist(),
+        'quality_metrics': quality,
+    }
+
+
+# ═══════════════════════════════════════════════════════════════════════
+#  _handle_filter_and_cluster
+# ═══════════════════════════════════════════════════════════════════════
+
+async def _handle_filter_and_cluster(
+    dataset_id: str,
+    filters: list[dict],
+    logic: str = 'and',
+    algorithm: str = 'hdbscan',
+    params: dict | None = None,
+) -> dict:
+    """Apply column-level filters then run clustering on the filtered result."""
+    entry, err = _resolve_dataset(dataset_id)
+    if err:
+        return err
+
+    lf: pl.LazyFrame = entry['lazyframe']
+    full_schema = entry.get('schema', {})
+
+    # ── Step 1: Apply filters ─────────────────────────────────────────────
+    exprs = [_build_filter_expr(
+        f['column'], f.get('op', 'eq'), f.get('value'),
+    ) for f in filters]
+
+    if exprs:
+        if logic == 'and':
+            combined = exprs[0]
+            for e in exprs[1:]:
+                combined = combined & e
+        else:
+            combined = exprs[0]
+            for e in exprs[1:]:
+                combined = combined | e
+        filtered_lf = lf.filter(combined)
+    else:
+        filtered_lf = lf
+
+    try:
+        row_count = _count_rows(filtered_lf)
+    except Exception:
+        logger.error("unknown failed: {}".format(traceback.format_exc()))
+        row_count = None
+
+
+    # ── Step 2: Auto-detect numeric feature columns ───────────────────────
+    available = filtered_lf.collect_schema()
+    available_names = available.names()
+    available_dtypes = available.dtypes()
+    feature_cols = [
+        available_names[i] for i, dt in enumerate(available_dtypes)
+        if dt in NUMERIC_DTYPES and not available_names[i].startswith('_')
+    ][:10]
+
+    if not feature_cols:
+        return {
+            'error': 'No numeric columns found for clustering after filtering',
+            'filtered_dataset_id': None,
+            'truncated': False,
+        }
+
+    # ── Step 3: Store filtered dataset ────────────────────────────────────
+    filtered_dataset_id = _generate_id('flt')
+    store = SessionStore()
+    store.store_dataset(
+        dataset_id=filtered_dataset_id,
+        lazyframe=filtered_lf,
+        schema=full_schema,
+        metadata={
+            'row_count': row_count,
+            'parent_dataset_id': dataset_id,
+            'filter_count': len(filters),
+            'logic': logic,
+        },
+        parent_id=dataset_id,
+    )
+
+    # ── Step 4: Run clustering ────────────────────────────────────────────
+    p = params or {}
+    total_count = _count_rows(filtered_lf)
+    clamp_lf = filtered_lf
+    if total_count > LOW_MEMORY_THRESHOLD_ROWS:
+        clamp_lf = filtered_lf.collect(streaming=True).sample(n=LOW_MEMORY_THRESHOLD_ROWS, seed=RANDOM_SEED).lazy()
+
+    df_features = clamp_lf.select(feature_cols).collect(streaming=True)
+
+    for col in df_features.columns:
+        if df_features[col].is_null().all():
+            df_features = df_features.drop(col)
+            feature_cols = [c for c in feature_cols if c != col]
+
+    if df_features.width == 0:
+        return {'error': 'No valid feature columns after dropping all-NaN columns',
+                'filtered_dataset_id': filtered_dataset_id, 'truncated': False}
+
+    data = df_features.to_numpy()
+
+    if data.dtype.kind == 'f':
+        col_mean = np.nanmean(data, axis=0)
+        col_mean = np.nan_to_num(col_mean, nan=0.0)
+        data = np.where(np.isnan(data), col_mean, data)
+
+    if len(data) < 3:
+        return {
+            'cluster_result_id': _generate_id('clust'),
+            'n_clusters': 0,
+            'n_noise': 0,
+            'quality_metrics': {'note': 'too few samples'},
+            'filtered_dataset_id': filtered_dataset_id,
+            'truncated': False,
+        }
+
+    labels_array, n_found, noise_count, quality, _ds_idx = _cluster_core(
+        data, feature_cols, algorithm, p, RANDOM_SEED,
+    )
+
+
+    result_id = _generate_id('clust')
+    store.store_cluster_result(
+        result_id=result_id,
+        labels=labels_array.tolist(),
+        n_clusters=n_found,
+        params={
+            'algorithm': algorithm,
+            'params': p,
+            'random_state': RANDOM_SEED,
+            'feature_columns': feature_cols,
+            '_ds_idx': _ds_idx.tolist() if _ds_idx is not None else None,
+        },
+        parent_dataset_id=filtered_dataset_id,
+    )
+
+    return {
+        'cluster_result_id': result_id,
+        'filtered_dataset_id': filtered_dataset_id,
+        'n_clusters': n_found,
+        'n_noise': noise_count,
+        'labels_sample': labels_array[:50].tolist(),
+        'quality_metrics': quality,
+        'feature_columns': feature_cols,
+    }

analysis/tools/data_mgmt.py

@@ -0,0 +1,36 @@
+"""Data management handlers — list, drop, clone datasets."""
+from typing import Optional
+
+from ..session_store import SessionStore
+
+
+async def _handle_list_datasets(
+    _filter_type: Optional[str] = None,
+) -> dict:
+    """List all active datasets and results."""
+    store = SessionStore()
+    datasets = store.list_datasets()
+    return {'datasets': datasets, 'count': len(datasets)}
+
+
+async def _handle_drop_dataset(dataset_id: str) -> dict:
+    """Remove a dataset/result and release memory."""
+    store = SessionStore()
+
+    dropped = (store.drop_dataset(dataset_id)
+               or store.drop_cluster_result(dataset_id)
+               or store.drop_feature_result(dataset_id))
+
+    if dropped:
+        return {'dropped': True, 'dataset_id': dataset_id}
+    return {'error': f'No dataset or result found: {dataset_id}',
+            'truncated': False}
+
+
+async def _handle_clone_dataset(dataset_id: str) -> dict:
+    """Shallow-clone a dataset."""
+    store = SessionStore()
+    new_id = store.clone_dataset(dataset_id)
+    if new_id is None:
+        return {'error': f'Dataset not found: {dataset_id}', 'truncated': False}
+    return {'original_dataset_id': dataset_id, 'cloned_dataset_id': new_id}

analysis/tools/diagnostics.py

@@ -0,0 +1,253 @@
+"""Diagnostic tool handlers — validate, explore, diagnose, compare, repair."""
+import traceback
+from analysis.constants import LOW_MEMORY_THRESHOLD_ROWS, RANDOM_SEED
+import polars as pl
+
+from ..session_store import SessionStore
+from ._helpers import _resolve_dataset, _count_rows, NUMERIC_TYPE_NAMES
+import logging
+logger = logging.getLogger(__name__)
+
+
+async def _handle_validate_data(dataset_id: str, fix_issues: bool = False) -> dict:
+    """Check data quality: schema consistency, missing values, type conflicts."""
+    entry, err = _resolve_dataset(dataset_id)
+    if err:
+        return err
+    lf = entry['lazyframe']
+    schema = entry.get('schema', {})
+    issues = []
+    df_sample = lf.head(1000).collect(streaming=True)
+    for col in schema:
+        null_count = df_sample[col].is_null().sum()
+        if null_count > 500:
+            issues.append(f'{col}: {null_count/10:.0f}% null')
+        try:
+            unique = df_sample[col].n_unique()
+            if unique <= 1 and null_count == 0:
+                issues.append(f'{col}: constant value (no variation)')
+        except Exception:
+            logger.error("_handle_validate_data failed: {}".format(traceback.format_exc()))
+            pass
+    total_rows = _count_rows(lf)
+    return {
+        'status': 'ok',
+        'total_rows': total_rows,
+        'columns': len(schema),
+        'issues_found': len(issues),
+        'issues': issues[:20],
+        'recommendation': 'Try explore_distributions for detailed column analysis' if issues else 'No issues found',
+    }
+
+
+async def _handle_explore_distributions(dataset_id: str, columns: list[str] = None,
+                                        max_sample: int = 10000) -> dict:
+    """Per-column distribution statistics."""
+    entry, err = _resolve_dataset(dataset_id)
+    if err:
+        return err
+    lf = entry['lazyframe']
+    schema = entry.get('schema', {})
+
+    if not columns:
+        columns = [c for c in schema if schema[c].split('(')[0].strip() in NUMERIC_TYPE_NAMES][:10]
+    if len(columns) > 20:
+        columns = columns[:20]
+
+    try:
+        n = _count_rows(lf)
+        if n > max_sample:
+            sample_lf = lf.select(columns).sample(n=max_sample, seed=RANDOM_SEED)
+        else:
+            sample_lf = lf.select(columns)
+        df = sample_lf.collect(streaming=True)
+    except Exception as exc:
+        return {'error': f'explore_distributions failed: {exc}', 'truncated': False}
+
+    stats = {}
+    for col in columns:
+        try:
+            s = df[col]
+            stats[col] = {
+                'dtype': str(s.dtype),
+                'null_pct': round(float(s.is_null().mean() * 100), 2),
+                'unique': int(s.n_unique()),
+                'top_values': [str(v) for v in s.head(5).to_list()],
+            }
+            if s.dtype in (pl.Float32, pl.Float64, pl.Int32, pl.Int64):
+                stats[col]['min'] = float(s.min()) if s.min() is not None else None
+                stats[col]['max'] = float(s.max()) if s.max() is not None else None
+                stats[col]['mean'] = round(float(s.mean()), 4) if s.mean() is not None else None
+                stats[col]['std'] = round(float(s.std()), 4) if s.std() is not None else None
+        except Exception:
+            logger.error("unknown failed: {}".format(traceback.format_exc()))
+            stats[col] = {'error': 'failed to compute'}
+    return {'columns': stats, 'total_rows': n, 'sampled': min(n, max_sample)}
+
+
+async def _handle_find_outliers(dataset_id: str, columns: list[str] = None,
+                                threshold: float = 3.0) -> dict:
+    """Find statistical outliers using IQR method."""
+    entry, err = _resolve_dataset(dataset_id)
+    if err:
+        return err
+    lf = entry['lazyframe']
+    schema = entry.get('schema', {})
+    if not columns:
+        columns = [c for c in schema if schema[c].split('(')[0].strip() in NUMERIC_TYPE_NAMES]
+    n = _count_rows(lf)
+    sample_lf = lf if n <= 100000 else lf.collect(streaming=True).sample(n=LOW_MEMORY_THRESHOLD_ROWS, seed=RANDOM_SEED).lazy()
+    df = sample_lf.select(columns).collect(streaming=True)
+
+    outlier_counts = {}
+    for col in columns:
+        try:
+            s = df[col]
+            q1, q3 = s.quantile(0.25), s.quantile(0.75)
+            iqr = q3 - q1
+            lo = q1 - threshold * iqr
+            hi = q3 + threshold * iqr
+            n_out = ((s < lo) | (s > hi)).sum()
+            if n_out > 0:
+                outlier_counts[col] = int(n_out)
+        except Exception:
+            logger.error("_handle_find_outliers failed: {}".format(traceback.format_exc()))
+            pass
+    return {'outlier_counts': outlier_counts, 'total_sampled': len(df), 'threshold_iqr': threshold}
+
+
+async def _handle_diagnose_clustering(dataset_id: str, cluster_result_id: str) -> dict:
+    """Diagnose clustering quality issues."""
+    from sklearn.decomposition import TruncatedSVD
+    store = SessionStore()
+    entry = store.get_dataset(dataset_id)
+    cluster_entry = store.get_cluster_result(cluster_result_id)
+    if entry is None:
+        return {'error': f'Dataset not found: {dataset_id}', 'truncated': False}
+    if cluster_entry is None:
+        return {'error': f'Cluster result not found: {cluster_result_id}', 'truncated': False}
+    lf = entry['lazyframe']
+    schema = entry.get('schema', {})
+    num_cols = [c for c in schema if schema[c].split('(')[0].strip() in NUMERIC_TYPE_NAMES and not c.startswith('_')]
+
+    n_clusters = cluster_entry.get('n_clusters', 0)
+    n_noise = cluster_entry.get('n_noise', 0)
+    quality = cluster_entry.get('quality_metrics', {})
+    labels = cluster_entry.get('labels', [])
+    params = cluster_entry.get('params', {})
+    algo = params.get('algorithm', 'unknown')
+    feature_cols = params.get('feature_columns', [])
+
+    diagnosis = []
+    if n_clusters <= 1:
+        diagnosis.append('Only 1 cluster — data may lack structure or need feature scaling')
+    if n_noise > 0.5 * len(labels) if labels else False:
+        diagnosis.append(f'High noise ratio ({n_noise}/{len(labels)}) — try HDBSCAN with larger min_cluster_size')
+    sil = quality.get('silhouette_score')
+    if sil and sil < 0.2:
+        diagnosis.append(f'Low silhouette ({sil:.3f}) — clusters overlap, try KMeans or feature selection')
+    if len(feature_cols) < 3:
+        diagnosis.append(f'Only {len(feature_cols)} features — add more columns for meaningful clusters')
+
+    try:
+        df_sample = lf.select(num_cols).head(5000).collect(streaming=True)
+        mat = df_sample.to_numpy()
+        n_comp = min(mat.shape)
+        svd = TruncatedSVD(n_components=n_comp, random_state=RANDOM_SEED).fit(mat)
+        var_explained = svd.explained_variance_ratio_
+        if var_explained[0] > 0.95:
+            diagnosis.append(f'First SVD component explains {var_explained[0]:.0%} variance — data nearly 1D')
+    except Exception:
+        logger.error("unknown failed: {}".format(traceback.format_exc()))
+        pass
+
+    return {
+        'diagnosis': diagnosis,
+        'n_clusters': n_clusters,
+        'n_noise': n_noise,
+        'silhouette': sil,
+        'algorithm': algo,
+        'n_features': len(feature_cols),
+        'suggestions': [
+            f'Try algorithm={"kmeans" if algo == "hdbscan" else "hdbscan"}',
+            'Increase min_cluster_size for fewer, larger clusters',
+            'Use explore_distributions to check feature quality',
+        ],
+    }
+
+
+async def _handle_compare_datasets(dataset_id_a: str, dataset_id_b: str) -> dict:
+    """Compare two datasets."""
+    store = SessionStore()
+    entry_a, entry_b = store.get_dataset(dataset_id_a), store.get_dataset(dataset_id_b)
+    if entry_a is None or entry_b is None:
+        return {'error': 'One or both datasets not found', 'truncated': False}
+
+    def _get_info(entry):
+        schema = entry.get('schema', {})
+        lf = entry['lazyframe']
+        try:
+            n = _count_rows(lf)
+        except Exception:
+            logger.error("_get_info failed: {}".format(traceback.format_exc()))
+            n = -1
+        return {'schema': set(schema.keys()), 'row_count': n, 'columns': len(schema)}
+
+    info_a, info_b = _get_info(entry_a), _get_info(entry_b)
+    common_cols = info_a['schema'] & info_b['schema']
+    only_a = info_a['schema'] - info_b['schema']
+    only_b = info_b['schema'] - info_a['schema']
+    return {
+        'dataset_a': {'id': dataset_id_a, **info_a},
+        'dataset_b': {'id': dataset_id_b, **info_b},
+        'common_columns': len(common_cols),
+        'columns_only_in_a': list(only_a)[:10] if only_a else [],
+        'columns_only_in_b': list(only_b)[:10] if only_b else [],
+        'row_diff': info_a['row_count'] - info_b['row_count'],
+        'identical_schema': len(only_a) == 0 and len(only_b) == 0,
+    }
+
+
+async def _handle_export_debug_sample(dataset_id: str, max_rows: int = 50) -> dict:
+    """Export raw data sample as JSON for debugging."""
+    entry, err = _resolve_dataset(dataset_id)
+    if err:
+        return err
+    df = entry['lazyframe'].head(max_rows).collect(streaming=True)
+    rows = df.to_dict(as_series=False)
+    result = []
+    for i in range(len(df)):
+        row = {}
+        for col in df.columns:
+            val = rows[col][i]
+            if isinstance(val, (float, int, str, bool)):
+                row[col] = val
+            elif val is None:
+                row[col] = None
+            else:
+                row[col] = str(val)
+        result.append(row)
+    return {'rows': result, 'count': len(result), 'columns': df.columns}
+
+
+async def _handle_repair_schema(dataset_id: str) -> dict:
+    """Attempt to repair schema mismatches by normalising column names."""
+    entry, err = _resolve_dataset(dataset_id)
+    if err:
+        return err
+    lf = entry['lazyframe']
+    schema = entry.get('schema', {})
+    col_map = {}
+    for c in schema:
+        norm = c.lower().replace('-', '_').replace('.', '_')
+        col_map[c] = norm
+    if len(set(col_map.values())) < len(col_map):
+        return {'status': 'conflict', 'message': 'Normalised names conflict', 'column_map': col_map}
+    renames = {old: new for old, new in col_map.items() if old != new}
+    if renames:
+        lf = lf.rename(renames)
+        new_schema = {col_map.get(c, c): dtype for c, dtype in schema.items()}
+        store = SessionStore()
+        store.store_dataset(dataset_id, lf, schema=new_schema, metadata=entry.get('metadata', {}))
+        return {'status': 'repaired', 'renamed': len(renames), 'column_map': col_map}
+    return {'status': 'no_change', 'message': 'Column names already normalised'}

analysis/tools/distance_matrix.py

@@ -0,0 +1,146 @@
+"""compute_distance_matrix handler — LLM-driven per-row evaluation."""
+import traceback
+from analysis.constants import RANDOM_SEED
+from typing import Optional
+
+import numpy as np
+import polars as pl
+
+from ..session_store import SessionStore
+from ._helpers import _resolve_dataset, NUMERIC_DTYPES
+import logging
+logger = logging.getLogger(__name__)
+
+
+def _indent_function_body(body: str, indent: str = '    ') -> str:
+    """Indent each non-empty line of a code block for wrapping in a def."""
+    return '\n'.join(
+        indent + line if line.strip() else line
+        for line in body.split('\n')
+    )
+
+
+async def _handle_compute_distance_matrix(
+    dataset_id: str,
+    python_function: str,
+    columns: Optional[list[str]] = None,
+    max_sample: int = 10000,
+) -> dict:
+    """Evaluate a user-provided Python function on each row of a dataset."""
+    entry, err = _resolve_dataset(dataset_id)
+    if err:
+        return err
+
+    lf: pl.LazyFrame = entry['lazyframe']
+    df = lf.collect(streaming=True)
+
+    if columns is None:
+        columns = [c for c in df.columns
+                   if df[c].dtype in NUMERIC_DTYPES]
+
+    available = set(df.columns)
+    columns = [c for c in columns if c in available]
+    if not columns:
+        return {'error': 'No valid columns found in dataset',
+                'truncated': False}
+
+    n_total = len(df)
+
+    if n_total > max_sample:
+        df_sampled = df.sample(n=max_sample, seed=RANDOM_SEED)
+        n_evaluated = max_sample
+    else:
+        df_sampled = df
+        n_evaluated = n_total
+
+    rows = df_sampled.select(columns).to_dicts()
+
+    safe_builtins = {
+        'abs': abs, 'len': len, 'range': range,
+        'min': min, 'max': max, 'sum': sum, 'round': round,
+        'int': int, 'float': float, 'str': str, 'bool': bool,
+        'list': list, 'dict': dict, 'tuple': tuple,
+        'isinstance': isinstance, 'type': type,
+        'True': True, 'False': False, 'None': None,
+        'enumerate': enumerate, 'zip': zip, 'map': map,
+        'filter': filter, 'sorted': sorted, 'reversed': reversed,
+    }
+
+    safe_ns = {
+        '__builtins__': safe_builtins,
+        'math': __import__('math'),
+        'np': __import__('numpy'),
+    }
+
+    body = python_function.strip()
+    if body.startswith('def '):
+        full_code = body
+    else:
+        indented = _indent_function_body(body)
+        full_code = f'def distance_fn(row):\n{indented}'
+
+    try:
+        exec(full_code, safe_ns)
+        distance_fn = safe_ns.get('distance_fn')
+        if distance_fn is None:
+            for _name, _obj in safe_ns.items():
+                if callable(_obj) and _name not in ('math', 'np', '__builtins__'):
+                    distance_fn = _obj
+                    break
+        if distance_fn is None:
+            return {'error': 'Could not locate the compiled function',
+                    'truncated': False}
+    except SyntaxError as e:
+        return {'error': f'Function syntax error: {e}', 'truncated': False}
+    except Exception as e:
+        return {'error': f'Function compilation failed: {e}',
+                'truncated': False}
+
+    scores: list[float | None] = []
+    errors = 0
+    for row in rows:
+        try:
+            score = distance_fn(row)
+            if isinstance(score, (int, float)):
+                scores.append(float(score))
+            else:
+                scores.append(None)
+                errors += 1
+        except Exception:
+            logger.error("unknown failed: {}".format(traceback.format_exc()))
+            scores.append(None)
+            errors += 1
+
+    valid_scores = [s for s in scores if s is not None]
+    if not valid_scores:
+        return {'error': 'No valid scores computed — all rows raised errors',
+                'truncated': False}
+
+    arr = np.array(valid_scores)
+    summary = {
+        'min': float(np.min(arr)),
+        'max': float(np.max(arr)),
+        'mean': float(np.mean(arr)),
+        'std': float(np.std(arr)),
+        'count': len(arr),
+        'errors': errors,
+        'total_rows_in_sample': len(rows),
+        'matrix_shape': [len(rows), len(columns)],
+    }
+
+    scored_sample = []
+    for i, row in enumerate(rows[:30]):
+        if i < len(scores):
+            scored_sample.append({
+                'index': i,
+                'values': {c: row.get(c) for c in columns[:3]},
+                'score': scores[i],
+            })
+
+    return {
+        'summary': summary,
+        'scored_sample': scored_sample,
+        'columns_used': columns,
+        'total_rows_in_dataset': n_total,
+        'rows_evaluated': n_evaluated,
+    }

analysis/tools/entities.py

@@ -0,0 +1,294 @@
+"""Entity profile building + adaptive scoring handlers."""
+import traceback
+from analysis.constants import MAX_DISTRIBUTION_SAMPLE, RANDOM_SEED
+import logging
+from typing import Optional
+
+import numpy as np
+import polars as pl
+
+from ..session_store import SessionStore, _generate_id
+from ..profile_util import profile
+from ._helpers import _resolve_dataset, _count_rows, NUMERIC_DTYPES
+logger = logging.getLogger(__name__)
+
+
+# ═══════════════════════════════════════════════════════════════════════
+#  Constants
+# ═══════════════════════════════════════════════════════════════════════
+
+_ENTITY_KEYWORDS = [
+    'src_ip', 'dst_ip', 'source_ip', 'destination_ip', 'src_addr', 'dst_addr',
+    'sip', 'dip', 'ip_src', 'ip_dst', 'saddr', 'daddr',
+    'sni', 'host', 'domain', 'uri', 'url', 'fqdn', 'server_name',
+    'mac', 'mac_addr', 'src_mac', 'dst_mac',
+    'user_agent', 'ua', 'asn', 'organization', 'org',
+]
+
+_NUMERIC_TYPES_FOR_AGG = (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
+                           pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
+                           pl.Float32, pl.Float64)
+
+_ANOMALY_FEATURE_COLS = [
+    'non_std_port_rate', 'modern_tls_rate', 'sni_missing_ratio',
+    'recoverable_ratio', 'flow_count', 'total_bytes_sent',
+    'total_packets', 'unique_dst_ips', 'unique_dst_ports',
+    'unique_protocols', 'has_sni', 'avg_sni_length',
+    'countries_visited',
+]
+
+
+# ═══════════════════════════════════════════════════════════════════════
+#  _handle_build_entity_profiles
+# ═══════════════════════════════════════════════════════════════════════
+
+async def _handle_build_entity_profiles(
+    dataset_id: str,
+    entity_column: Optional[str] = None,
+    entity_columns: Optional[list[str]] = None,
+    auto_detect: bool = True,
+) -> dict:
+    """Group raw flow data by entity column(s) and compute per-entity aggregate features."""
+    entry, err = _resolve_dataset(dataset_id)
+    if err:
+        return err
+
+    lf: pl.LazyFrame = entry['lazyframe']
+    schema = entry.get('schema', {})
+    col_names = list(schema.keys())
+
+    # ── Resolve entity columns ──────────────────────────────────────────
+    if entity_column and entity_column in col_names:
+        group_cols = [entity_column]
+    elif entity_columns:
+        group_cols = [c for c in entity_columns if c in col_names]
+        if not group_cols:
+            return {'error': f'None of entity_columns {entity_columns} exist in dataset. Available: {col_names[:20]}',
+                    'truncated': False}
+    elif auto_detect:
+        matched = []
+        for kw in _ENTITY_KEYWORDS:
+            found = [c for c in col_names if kw.lower() in c.lower()]
+            matched.extend(found)
+        seen = set()
+        group_cols = [c for c in matched if not (c in seen or seen.add(c))]
+        if not group_cols:
+            for c in col_names:
+                if c.startswith('_'):
+                    continue
+                group_cols = [c]
+                break
+        group_cols = group_cols[:3]
+    else:
+        return {'error': 'No entity column specified and auto_detect is disabled',
+                'truncated': False}
+
+    if not group_cols:
+        return {'error': 'Could not determine entity column(s) for grouping',
+                'truncated': False}
+
+    # ── Build aggregation expressions ──────────────────────────────────
+    agg_exprs: list[pl.Expr] = [
+        pl.len().alias('flow_count'),
+        pl.col(group_cols[0]).n_unique().alias(f'unique_{group_cols[0]}'),
+    ]
+
+    numeric_cols = [c for c in col_names if c not in group_cols
+                    and schema[c].startswith(('Int', 'UInt', 'Float'))]
+    for c in numeric_cols[:20]:
+        agg_exprs.append(pl.sum(c).alias(f'total_{c}'))
+        agg_exprs.append(pl.mean(c).alias(f'avg_{c}'))
+
+    str_cols = [c for c in col_names if c not in group_cols
+                and schema[c].startswith(('Utf8', 'String', 'Cat'))]
+    for c in str_cols[:10]:
+        agg_exprs.append(pl.col(c).n_unique().alias(f'unique_{c}'))
+
+    for kw in ['dst_ip', 'dip', 'dst_addr', 'daddr', 'ip_dst', 'dest_ip',
+               'dst_port', 'dport', 'destination_port', 'dest_port',
+               'port', 'protocol', 'proto', 'sni', 'server_name',
+               'cipher_suite', 'cipher', 'tls_version', 'version', '0ver',
+               'country', 'scnt', 'dcnt', 'asn', 'organization']:
+        found = [c for c in col_names if c not in group_cols and kw.lower() in c.lower()]
+        for c in found:
+            alias_name = f'unique_{c}'
+            if alias_name not in [str(a.meta.output_name()) if hasattr(a, 'meta') else '' for a in agg_exprs]:
+                agg_exprs.append(pl.col(c).n_unique().alias(alias_name))
+
+    # ── Execute grouping ───────────────────────────────────────────────
+    row_count = _count_rows(lf)
+    ENTITY_CHUNK_THRESHOLD = 1_000_000   # 1M rows
+    ENTITY_CHUNK_SIZE = 100_000          # 100K rows per chunk
+
+    if row_count >= ENTITY_CHUNK_THRESHOLD:
+        # Chunked path: process in 100K batches → group → merge across chunks
+        all_chunk_dfs = []
+        for offset in range(0, row_count, ENTITY_CHUNK_SIZE):
+            chunk_df = lf.slice(offset, ENTITY_CHUNK_SIZE).collect(streaming=True)
+            chunk_grouped = chunk_df.group_by(group_cols).agg(agg_exprs)
+            all_chunk_dfs.append(chunk_grouped)
+        merged_df = pl.concat(all_chunk_dfs)
+
+        # Re-aggregate across chunks (same entity may appear in multiple chunks)
+        re_agg_exprs: list[pl.Expr] = []
+        for col_name in merged_df.columns:
+            if col_name in group_cols:
+                continue
+            if col_name == 'flow_count':
+                re_agg_exprs.append(pl.sum('flow_count').alias('flow_count'))
+            elif col_name.startswith('total_'):
+                re_agg_exprs.append(pl.sum(col_name).alias(col_name))
+            elif col_name.startswith('avg_'):
+                # Weighted average: sum(total) / sum(flow_count)
+                base = col_name[4:]
+                total_col = f'total_{base}'
+                if total_col in merged_df.columns:
+                    re_agg_exprs.append(
+                        (pl.sum(total_col) / pl.sum('flow_count')).alias(col_name))
+            elif col_name.startswith('unique_'):
+                # Approximate: max of per-chunk distinct counts
+                re_agg_exprs.append(pl.max(col_name).alias(col_name))
+
+        entity_lf = merged_df.lazy().group_by(group_cols).agg(re_agg_exprs)
+        logger.info('[ENTITY] chunked aggregation: %d rows → %d chunks → %d merged rows',
+                    row_count, len(all_chunk_dfs), len(merged_df))
+    else:
+        entity_lf = lf.group_by(group_cols).agg(agg_exprs)
+
+    n_entities = _count_rows(entity_lf)
+
+    # ── Store result ───────────────────────────────────────────────────
+    new_schema = {name: str(dtype) for name, dtype in
+                  zip(entity_lf.collect_schema().names(), entity_lf.collect_schema().dtypes())}
+    new_id = f'entity_{_generate_id("ds")}'
+    store = SessionStore()
+    store.store_dataset(
+        dataset_id=new_id,
+        lazyframe=entity_lf,
+        schema=new_schema,
+        metadata={
+            'row_count': n_entities,
+            'entity_columns': group_cols,
+            'parent_dataset_id': dataset_id,
+            'n_agg_features': len(agg_exprs),
+        },
+        parent_id=dataset_id,
+    )
+
+    return {
+        'dataset_id': new_id,
+        'entity_columns': group_cols,
+        'n_entities': n_entities,
+        'n_features': len(agg_exprs),
+        'aggregated_columns': list(new_schema.keys())[:20],
+    }
+
+
+# ═══════════════════════════════════════════════════════════════════════
+#  Adaptive scoring helpers
+# ═══════════════════════════════════════════════════════════════════════
+
+@profile
+def _add_adaptive_scores(lf: pl.LazyFrame, feature_names: list[str]) -> pl.LazyFrame:
+    """Add anomaly scores using data-driven weights (SVD + IQR + percentile)."""
+    _agg_cols = lf.collect_schema().names()
+    avail = [c for c in _ANOMALY_FEATURE_COLS if c in _agg_cols]
+    if not avail:
+        return lf
+
+    try:
+        n_rows = _count_rows(lf)
+        sample_lf = lf if n_rows <= MAX_DISTRIBUTION_SAMPLE else lf.sample(n=MAX_DISTRIBUTION_SAMPLE, seed=RANDOM_SEED)
+        df_sample = sample_lf.select(avail).collect(streaming=True)
+        mat = df_sample.to_numpy()
+        mat = np.nan_to_num(mat, nan=0.0)
+
+        from sklearn.decomposition import TruncatedSVD
+        n_comp = min(1, min(mat.shape) - 1)
+        if n_comp < 1 or mat.shape[1] < 2:
+            return lf
+        svd = TruncatedSVD(n_components=1, random_state=RANDOM_SEED).fit(mat)
+        raw_weights = np.abs(svd.components_[0])
+        weights = dict(zip(avail, raw_weights / raw_weights.sum()))
+
+        expr_parts = []
+        for col, w in weights.items():
+            if col == 'modern_tls_rate':
+                expr_parts.append((1.0 - pl.col(col).fill_null(0.5)) * w)
+            else:
+                expr_parts.append(pl.col(col).fill_null(0.0) * w)
+        if expr_parts:
+            proxy_expr = expr_parts[0]
+            for p in expr_parts[1:]:
+                proxy_expr = proxy_expr + p
+        else:
+            return lf
+
+        lf = lf.with_columns(proxy_expr.clip(0, 1).alias('_raw_score'))
+
+        sample_scores = df_sample.select(
+            pl.sum_horizontal([
+                (1.0 - pl.col('modern_tls_rate').fill_null(0.5)) * weights.get('modern_tls_rate', 0)
+                if c == 'modern_tls_rate'
+                else pl.col(c).fill_null(0.0) * weights.get(c, 0)
+                for c in avail
+            ]).clip(0, 1)
+        ).to_series().to_numpy()
+
+        q25, q50, q75 = np.percentile(sample_scores, [25, 50, 75])
+        iqr = q75 - q25
+        upper_fence = float(q75 + 1.5 * iqr) if iqr > 0 else 0.9
+        extreme_fence = float(q75 + 3.0 * iqr) if iqr > 0 else 0.99
+
+        lf = lf.with_columns([
+            pl.col('_raw_score').alias('proxy_score'),
+            pl.when(pl.col('_raw_score') >= extreme_fence)
+            .then(pl.lit('critical'))
+            .when(pl.col('_raw_score') >= upper_fence)
+            .then(pl.lit('suspicious'))
+            .when(pl.col('_raw_score') >= q50)
+            .then(pl.lit('watch'))
+            .otherwise(pl.lit('normal'))
+            .alias('risk_level'),
+            (pl.col('_raw_score') * 0.6 + 0.4 * (pl.col('_raw_score') > q75).cast(pl.Float64))
+            .alias('threat_score'),
+        ]).drop('_raw_score')
+
+        valid_counts = {c: int((mat[:, i] != 0).sum()) for i, c in enumerate(avail)}
+        logger = logging.getLogger(__name__)
+        logger.info(
+            '[SCORE] adaptive weights from SVD: %s | thresholds: p50=%.3f upper=%.3f extreme=%.3f | sample=%d/%d',
+            {c: f'{w:.3f}' for c, w in sorted(weights.items(), key=lambda x: -x[1])},
+            float(q50), upper_fence, extreme_fence,
+            min(n_rows, MAX_DISTRIBUTION_SAMPLE), n_rows,
+        )
+
+    except Exception as exc:
+        logging.getLogger(__name__).warning('[SCORE] adaptive scoring skipped: %s', exc)
+
+    return lf
+
+
+# ═══════════════════════════════════════════════════════════════════════
+#  _handle_compute_scores
+# ═══════════════════════════════════════════════════════════════════════
+
+async def _handle_compute_scores(dataset_id: str) -> dict:
+    """Compute adaptive proxy/anomaly/threat scores on entity data."""
+    entry, err = _resolve_dataset(dataset_id)
+    if err:
+        return err
+
+    lf = entry['lazyframe']
+    lf = _add_adaptive_scores(lf, [])
+    try:
+        updated_schema = lf.collect_schema()
+        new_schema = {name: str(dtype) for name, dtype in
+                      zip(updated_schema.names(), updated_schema.dtypes())}
+    except Exception:
+        logger.error("_handle_compute_scores failed: {}".format(traceback.format_exc()))
+        new_schema = entry.get('schema', {})
+    store = SessionStore()
+    store.store_dataset(dataset_id, lf, schema=new_schema,
+                        metadata=entry.get('metadata', {}))
+    return {'status': 'scored', 'dataset_id': dataset_id}

analysis/tools/evaluate.py

@@ -0,0 +1,76 @@
+"""evaluate_clustering handler — compute clustering quality metrics."""
+from typing import Optional
+from analysis.constants import RANDOM_SEED
+
+import numpy as np
+import polars as pl
+
+from ..session_store import SessionStore
+
+
+async def _handle_evaluate_clustering(
+    cluster_result_id: str,
+    dataset_id: str,
+    metrics: Optional[list[str]] = None,
+) -> dict:
+    """Evaluate clustering result against the dataset."""
+    store = SessionStore()
+    cluster_entry = store.get_cluster_result(cluster_result_id)
+    if cluster_entry is None:
+        return {'error': f'Cluster result not found: {cluster_result_id}',
+                'truncated': False}
+
+    dataset_entry = store.get_dataset(dataset_id)
+    if dataset_entry is None:
+        return {'error': f'Dataset not found: {dataset_id}', 'truncated': False}
+
+    labels = cluster_entry['labels']
+    default_metrics = ['silhouette', 'davies_bouldin', 'calinski_harabasz',
+                       'noise_ratio', 'cluster_sizes']
+    requested = metrics or default_metrics
+
+    scores: dict = {'n_clusters': cluster_entry['n_clusters']}
+
+    lf: pl.LazyFrame = dataset_entry['lazyframe']
+    params = cluster_entry.get('params', {})
+    feature_cols = params.get('feature_columns', [])
+    if not feature_cols and 'feature_columns' in params:
+        pass
+    elif feature_cols:
+        df_features = lf.select(feature_cols).collect(streaming=True)
+        data = df_features.to_numpy()
+
+        # Align labels with data (handle missing rows)
+        valid_mask = ~np.isnan(data).any(axis=1) if data.dtype.kind == 'f' else np.ones(len(data), dtype=bool)
+        data_clean = data[valid_mask]
+        labels_clean = np.array(labels)[:len(data_clean)]
+
+        unique_labels = set(labels_clean)
+        n_clusters_actual = len(unique_labels - {-1})
+
+        for m in requested:
+            if m == 'silhouette' and n_clusters_actual >= 2:
+                from sklearn.metrics import silhouette_score
+                scores['silhouette'] = round(float(
+                    silhouette_score(data_clean, labels_clean, random_state=RANDOM_SEED)), 4)
+            elif m == 'davies_bouldin' and n_clusters_actual >= 2:
+                from sklearn.metrics import davies_bouldin_score
+                scores['davies_bouldin'] = round(float(
+                    davies_bouldin_score(data_clean, labels_clean)), 4)
+            elif m == 'calinski_harabasz' and n_clusters_actual >= 2:
+                from sklearn.metrics import calinski_harabasz_score
+                scores['calinski_harabasz'] = round(float(
+                    calinski_harabasz_score(data_clean, labels_clean)), 4)
+            elif m == 'noise_ratio':
+                noise = int((labels_clean == -1).sum())
+                scores['noise_ratio'] = round(noise / len(labels_clean), 4)
+            elif m == 'cluster_sizes':
+                sizes = {}
+                for label in unique_labels:
+                    sizes[int(label)] = int((labels_clean == label).sum())
+                scores['cluster_sizes'] = sizes
+
+    scores['n_clusters'] = cluster_entry['n_clusters']
+
+
+    return {'scores': scores}

analysis/tools/export.py

@@ -0,0 +1,91 @@
+"""export_results handler — write datasets/results to disk."""
+import json
+from pathlib import Path
+
+import polars as pl
+
+from ..session_store import SessionStore
+
+
+async def _handle_export_results(
+    result_id: str,
+    output_path: str,
+    format: str = 'csv',
+    overwrite: bool = False,
+) -> dict:
+    """Export a dataset (or result) to disk."""
+    store = SessionStore()
+
+    out = Path(output_path)
+    file_paths: list[str] = []
+
+    # Check if it's a dataset
+    dataset_entry = store.get_dataset(result_id)
+    if dataset_entry is not None:
+        lf: pl.LazyFrame = dataset_entry['lazyframe']
+        df = lf.collect(streaming=True)
+        if out.suffix == '':
+            out = out / f"export_{result_id}.{format}"
+
+        if out.exists() and not overwrite:
+            return {'error': f'Output exists and overwrite=False: {out}',
+                    'truncated': False}
+
+        out.parent.mkdir(parents=True, exist_ok=True)
+
+        if format == 'csv':
+            df.write_csv(out)
+        elif format == 'parquet':
+            df.write_parquet(out)
+        elif format == 'json':
+            df.write_json(out)
+        else:
+            return {'error': f'Unsupported format: {format}',
+                    'truncated': False}
+
+        file_paths.append(str(out.resolve()))
+
+        return {'file_paths': file_paths, 'format': format,
+                'row_count': len(df)}
+
+    # Check if it's a cluster result
+    cluster_entry = store.get_cluster_result(result_id)
+    if cluster_entry is not None:
+        if out.suffix == '':
+            out = out / f"cluster_{result_id}.json"
+        out.parent.mkdir(parents=True, exist_ok=True)
+
+        if out.exists() and not overwrite:
+            return {'error': f'Output exists and overwrite=False: {out}',
+                    'truncated': False}
+
+        export_data = {
+            'result_id': result_id,
+            'n_clusters': cluster_entry['n_clusters'],
+            'params': cluster_entry['params'],
+            'labels_sample': cluster_entry['labels'][:1000],
+        }
+        with open(out, 'w', encoding='utf-8') as f:
+            json.dump(export_data, f, default=str)
+        file_paths.append(str(out.resolve()))
+
+        return {'file_paths': file_paths, 'format': 'json'}
+
+    # Check feature result
+    feat_entry = store.get_feature_result(result_id)
+    if feat_entry is not None:
+        if out.suffix == '':
+            out = out / f"features_{result_id}.json"
+        out.parent.mkdir(parents=True, exist_ok=True)
+
+        if out.exists() and not overwrite:
+            return {'error': f'Output exists and overwrite=False: {out}',
+                    'truncated': False}
+
+        with open(out, 'w', encoding='utf-8') as f:
+            json.dump(feat_entry['features'], f, default=str)
+        file_paths.append(str(out.resolve()))
+        return {'file_paths': file_paths, 'format': 'json'}
+
+    return {'error': f'No dataset or result found for ID: {result_id}',
+            'truncated': False}

analysis/tools/features.py

@@ -0,0 +1,314 @@
+"""extract_features handler — per-cluster feature ranking + ORM persistence."""
+import sys
+import traceback
+
+import numpy as np
+import polars as pl
+
+from ..session_store import SessionStore, _generate_id
+from ..profile_util import profile
+from analysis.constants import RANDOM_SEED, UMAP_BATCH_SIZE, UMAP_TRAIN_SAMPLE
+import logging
+logger = logging.getLogger(__name__)
+
+
+async def _handle_extract_features(
+    dataset_id: str,
+    cluster_result_id: str,
+    top_k: int = 10,
+    method: str = 'zscore',
+    save_to_db: bool = True,
+    run_id: int | None = None,
+) -> dict:
+    """Compute per-cluster distinguishing features and optionally persist."""
+    store = SessionStore()
+    dataset_entry = store.get_dataset(dataset_id)
+    if dataset_entry is None:
+        return {'error': f'Dataset not found: {dataset_id}', 'truncated': False}
+
+    cluster_entry = store.get_cluster_result(cluster_result_id)
+    if cluster_entry is None:
+        return {'error': f'Cluster result not found: {cluster_result_id}',
+                'truncated': False}
+
+    lf: pl.LazyFrame = dataset_entry['lazyframe']
+    labels = cluster_entry['labels']
+
+    # D5: If clustering downsampled the data, filter matching rows
+    _ds_idx = cluster_entry.get('params', {}).get('_ds_idx')
+    if _ds_idx is not None:
+        df = (lf.with_row_index('__ds_idx')
+              .filter(pl.col('__ds_idx').is_in(_ds_idx))
+              .drop('__ds_idx')
+              .collect(streaming=True))
+    else:
+        df = lf.collect(streaming=True)
+
+    # Truncate labels if df was filtered but labels weren't
+    if len(labels) != len(df):
+        labels = labels[:len(df)]
+
+    # Only use numeric columns for feature extraction
+    numeric_cols = [c for c in df.columns
+                    if df[c].dtype in (pl.Float32, pl.Float64,
+                                        pl.Int32, pl.Int64,
+                                        pl.UInt32, pl.UInt64)]
+    if len(labels) != len(df):
+        labels = labels[:len(df)]
+
+    # Global statistics per column
+    global_stats = {}
+    for col in numeric_cols:
+        s = df[col].drop_nulls()
+        if len(s) > 0:
+            global_stats[col] = {
+                'mean': float(s.mean()),
+                'std': float(s.std()) if s.std() is not None else 1.0,
+            }
+
+    unique_labels = sorted(set(labels))
+    all_features = []
+
+    for label in unique_labels:
+        mask = np.array(labels) == label
+        if mask.sum() < 2:
+            continue
+
+        cluster_df = df.filter(pl.Series('__mask__', mask))
+        cluster_features = []
+
+        for col in numeric_cols:
+            c_series = cluster_df[col].drop_nulls()
+            gs = global_stats.get(col)
+            if gs is None or gs['std'] == 0 or len(c_series) == 0:
+                continue
+
+            c_mean = float(c_series.mean())
+            c_std = float(c_series.std()) if c_series.std() is not None else 0.0
+
+            if method == 'zscore':
+                score = (c_mean - gs['mean']) / max(gs['std'], 1e-10)
+            else:
+                score = abs(c_mean - gs['mean']) / max(c_std + gs['std'], 1e-10)
+
+            cluster_features.append({
+                'feature_name': col,
+                'cluster_label': int(label),
+                'mean': round(c_mean, 4),
+                'std': round(c_std, 4),
+                'global_mean': round(gs['mean'], 4),
+                'global_std': round(gs['std'], 4),
+                'distinguishing_score': round(float(score), 4),
+                'distinguishing_method': method,
+            })
+
+        cluster_features.sort(key=lambda x: abs(x['distinguishing_score']), reverse=True)
+        all_features.extend(cluster_features[:top_k])
+
+    # ── Compute UMAP-2D embedding ──────────────────────────────────
+    if len(numeric_cols) >= 2 and len(df) > 2:
+        try:
+            from sklearn.preprocessing import StandardScaler
+            import umap
+            MAX_UMAP_TRAIN = UMAP_TRAIN_SAMPLE
+            BATCH_SIZE = UMAP_BATCH_SIZE
+            data_matrix = df.select(numeric_cols).to_numpy()
+            if len(df) > MAX_UMAP_TRAIN:
+                idx_sample = np.random.RandomState(RANDOM_SEED).choice(
+                    len(df), size=MAX_UMAP_TRAIN, replace=False)
+                mat_sample = data_matrix[idx_sample]
+                scaler = StandardScaler().fit(mat_sample)
+                mat_sample_scaled = np.nan_to_num(scaler.transform(mat_sample),
+                                                  nan=0.0, posinf=0.0, neginf=0.0)
+                reducer = umap.UMAP(n_components=2, random_state=RANDOM_SEED,
+                                    n_neighbors=15, min_dist=0.1,
+                                    metric='euclidean')
+                reducer.fit(mat_sample_scaled)
+                coords_list = []
+                for start in range(0, len(df), BATCH_SIZE):
+                    end = min(start + BATCH_SIZE, len(df))
+                    mat_batch = scaler.transform(data_matrix[start:end])
+                    mat_batch = np.nan_to_num(mat_batch, nan=0.0, posinf=0.0, neginf=0.0)
+                    coords_list.append(reducer.transform(mat_batch))
+                coords = np.vstack(coords_list)
+            else:
+                data_scaled = StandardScaler().fit_transform(data_matrix)
+                data_scaled = np.nan_to_num(data_scaled, nan=0.0, posinf=0.0, neginf=0.0)
+                reducer = umap.UMAP(n_components=2, random_state=RANDOM_SEED,
+                                    n_neighbors=15, min_dist=0.1,
+                                    metric='euclidean')
+                coords = reducer.fit_transform(data_scaled)
+            non_numeric = [c for c in df.columns if c not in numeric_cols and not c.startswith('_')]
+            entity_col_umap = non_numeric[0] if non_numeric else df.columns[0]
+            labels_list = cluster_entry.get('labels', [])
+            from analysis.models import EntityProfile as EP
+            from analysis.models import ClusterResult as CR
+            from analysis.models import AnalysisRun as AR
+            from asgiref.sync import sync_to_async
+            csv_glob = dataset_entry.get('metadata', {}).get('csv_glob', 'manual')
+            run_obj = await sync_to_async(
+                AR.objects.filter(csv_glob=csv_glob).order_by('-id').first,
+                thread_sensitive=True
+            )()
+            if run_obj:
+                run_id_for_ep = run_obj.id
+                profiles = []
+                cr_cache = {}
+                for i, row in enumerate(df.iter_rows(named=True)):
+                    if i >= len(coords): break
+                    ev = str(row.get(entity_col_umap, ''))
+                    if not ev: continue
+                    lbl = labels_list[i] if i < len(labels_list) else -1
+                    cache_key = f'{run_id_for_ep}_{lbl}'
+                    if cache_key not in cr_cache:
+                        cr_cache[cache_key] = CR.objects.filter(run_id=run_id_for_ep, cluster_label=lbl).first()
+                    profiles.append(EP(
+                        entity_value=ev, run_id=run_id_for_ep, cluster_label=lbl,
+                        cluster=cr_cache[cache_key],
+                        embedding_x=float(coords[i, 0]),
+                        embedding_y=float(coords[i, 1]),
+                    ))
+                if profiles:
+                    EP.objects.bulk_create(profiles, ignore_conflicts=True, batch_size=1000)
+        except Exception:
+            print(f'[UMAP_ERR] {traceback.format_exc()}', file=sys.stderr)
+
+    # Persist to Django ORM
+    db_saved = False
+    if save_to_db:
+        try:
+            from asgiref.sync import sync_to_async
+            from analysis.models import AnalysisRun as AnalysisRunModel
+            ce = cluster_entry
+            async def _do_save(ce_inner):
+                if run_id is not None:
+                    run = await sync_to_async(AnalysisRunModel.objects.get)(id=run_id)
+                else:
+                    csv_glob_fallback = dataset_entry.get('metadata', {}).get('csv_glob', 'manual')
+                    run = await sync_to_async(
+                        AnalysisRunModel.objects.filter(csv_glob=csv_glob_fallback).order_by('-id').first,
+                        thread_sensitive=True
+                    )()
+                    if run is None:
+                        run = await sync_to_async(AnalysisRunModel.objects.create)(
+                            csv_glob=csv_glob_fallback, status='running', total_flows=0,
+                        )
+                run.status = 'completed'
+                run.cluster_count = ce_inner['n_clusters']
+                await sync_to_async(run.save)()
+                return await sync_to_async(_save_features_to_db)(cluster_result_id, all_features, run_id=run.id)
+            db_saved = await _do_save(ce)
+        except Exception as e:
+            print(f'[DB_SAVE_ERROR] {e}\n{traceback.format_exc()}', file=sys.stderr)
+            db_saved = False
+
+
+    result_id = _generate_id('feat')
+    store.store_feature_result(
+        result_id=result_id,
+        features=all_features,
+        parent_cluster_result_id=cluster_result_id,
+    )
+
+    return {
+        'feature_result_id': result_id,
+        'features': all_features,
+        'n_features': len(all_features),
+        'db_saved': db_saved,
+    }
+
+
+@profile
+def _save_features_to_db(
+    cluster_result_id: str,
+    features: list[dict],
+    run_id: int | None = None,
+) -> bool:
+    """Save extracted features to the Django ORM ClusterFeature model."""
+    if run_id is not None:
+        from analysis.models import AnalysisRun as AnalysisRun2
+        run = AnalysisRun2.objects.filter(id=run_id).first()
+        if run is None:
+            return False
+    else:
+        store = SessionStore()
+        cluster_entry = store.get_cluster_result(cluster_result_id)
+        if cluster_entry is None:
+            return False
+
+        parent_ds_id = cluster_entry.get('parent_dataset_id', '')
+        from analysis.models import AnalysisRun
+        ds_entry = store.get_dataset(parent_ds_id)
+        if ds_entry is None:
+            return False
+
+        csv_glob = ds_entry.get('metadata', {}).get('csv_glob', '')
+        unique_global = sorted(set(f['cluster_label'] for f in features))
+        n_clusters_global = len([l for l in unique_global if l >= 0])
+        try:
+            run = AnalysisRun.objects.filter(csv_glob=csv_glob).order_by('-created_at').first()
+            if run is None:
+                run = AnalysisRun.objects.create(
+                    csv_glob=csv_glob,
+                    status='completed',
+                    cluster_count=n_clusters_global,
+                )
+        except Exception:
+            logger.error("unknown failed: {}".format(traceback.format_exc()))
+            return False
+
+    from analysis.models import ClusterResult as ClusterResultModel
+
+    store = SessionStore()
+    cluster_entry = store.get_cluster_result(cluster_result_id)
+    label_counts = {}
+    if cluster_entry:
+        full_labels = cluster_entry.get('labels', [])
+        for lbl in full_labels:
+            label_counts[lbl] = label_counts.get(lbl, 0) + 1
+
+    unique_labels = sorted(set(
+        f['cluster_label'] for f in features
+    ))
+    n_clusters = len([l for l in unique_labels if l >= 0])
+    for label in unique_labels:
+        if label == -1:
+            continue
+        label_features = [f for f in features if f['cluster_label'] == label]
+        if not label_features:
+            continue
+
+        cluster_db, _ = ClusterResultModel.objects.get_or_create(
+            run=run,
+            cluster_label=label,
+            defaults={'size': label_counts.get(label, 1)},
+        )
+        if label_counts.get(label) is not None:
+            ClusterResultModel.objects.filter(
+                run=run, cluster_label=label
+            ).update(size=label_counts[label])
+
+        from analysis.models import ClusterFeature as ClusterFeatureModel
+        for feat in label_features:
+            try:
+                ClusterFeatureModel.objects.get_or_create(
+                    cluster=cluster_db,
+                    feature_name=feat['feature_name'],
+                    defaults={
+                        'mean': feat.get('mean'),
+                        'std': feat.get('std'),
+                        'median': None,
+                        'p25': None,
+                        'p75': None,
+                        'missing_rate': None,
+                        'distinguishing_score': feat.get('distinguishing_score'),
+                        'distinguishing_method': feat.get('distinguishing_method', 'zscore'),
+                    }
+                )
+            except Exception:
+                logger.error("unknown failed: {}".format(traceback.format_exc()))
+                continue
+
+    run.cluster_count = n_clusters
+    run.save(update_fields=['cluster_count'])
+    return True

analysis/tools/filter.py

@@ -0,0 +1,68 @@
+"""filter_data handler — apply column-level filters to a dataset."""
+import traceback
+import polars as pl
+
+from ..session_store import SessionStore, _generate_id
+from ._helpers import _build_filter_expr, _resolve_dataset
+import logging
+logger = logging.getLogger(__name__)
+
+
+async def _handle_filter_data(
+    dataset_id: str,
+    filters: list[dict],
+    logic: str = 'and',
+) -> dict:
+    """Apply filters and store filtered dataset."""
+    entry, err = _resolve_dataset(dataset_id)
+    if err:
+        return err
+
+    lf: pl.LazyFrame = entry['lazyframe']
+    full_schema = entry.get('schema', {})
+
+    exprs = [_build_filter_expr(
+        f['column'], f.get('op', 'eq'), f.get('value'),
+    ) for f in filters]
+
+    if not exprs:
+        return {'error': 'No filter expressions could be built', 'truncated': False}
+
+    if logic == 'and':
+        combined = exprs[0]
+        for e in exprs[1:]:
+            combined = combined & e
+    else:
+        combined = exprs[0]
+        for e in exprs[1:]:
+            combined = combined | e
+
+    filtered_lf = lf.filter(combined)
+
+    # Estimate row count (fast streaming count)
+    try:
+        row_count = filtered_lf.select(pl.len()).collect(streaming=True).item()
+    except Exception:
+        logger.error("unknown failed: {}".format(traceback.format_exc()))
+        row_count = None
+
+    new_id = _generate_id('flt')
+    store = SessionStore()
+    store.store_dataset(
+        dataset_id=new_id,
+        lazyframe=filtered_lf,
+        schema=full_schema,
+        metadata={
+            'row_count': row_count,
+            'parent_dataset_id': dataset_id,
+            'filter_count': len(filters),
+            'logic': logic,
+        },
+        parent_id=dataset_id,
+    )
+
+    return {
+        'filtered_dataset_id': new_id,
+        'row_count': row_count,
+        'column_deltas': {'kept': list(full_schema.keys()), 'dropped': []},
+    }

analysis/tools/load_data.py

@@ -0,0 +1,58 @@
+"""load_data handler — load CSV files and store as dataset."""
+import os
+from pathlib import Path
+from typing import Optional
+
+import polars as pl
+
+from ..data_loader import load_csv_directory
+from ..session_store import SessionStore, _generate_id
+from analysis.constants import MAX_DISTRIBUTION_SAMPLE
+
+
+async def _handle_load_data(
+    csv_glob: str,
+    config_path: Optional[str] = None,
+    encoding: str = 'utf-8',
+    delimiter: str = ',',
+    schema_strict: bool = False,
+    recursive: bool = False,
+) -> dict:
+    """Load, validate, merge CSVs, and store in session."""
+    try:
+        lf, schema, row_count, file_count, memory_mb = load_csv_directory(
+            glob_pattern=csv_glob,
+            encoding=encoding,
+            delimiter=delimiter,
+            config_path=config_path,
+            sample_rows=MAX_DISTRIBUTION_SAMPLE,
+            schema_strict=schema_strict,
+            recursive=recursive,
+        )
+    except FileNotFoundError as e:
+        return {'error': f'File not found: {e}', 'truncated': False}
+    except ValueError as e:
+        return {'error': f'Schema validation failed: {e}', 'truncated': False}
+
+    store = SessionStore()
+    dataset_id = _generate_id('ds')
+    store.store_dataset(
+        dataset_id=dataset_id,
+        lazyframe=lf,
+        schema=schema,
+        metadata={
+            'row_count': row_count,
+            'file_count': file_count,
+            'memory_mb': round(memory_mb, 2),
+            'csv_glob': csv_glob,
+            'encoding': encoding,
+        },
+    )
+
+    return {
+        'dataset_id': dataset_id,
+        'schema': schema,
+        'row_count': row_count,
+        'file_count': file_count,
+        'memory_mb': round(memory_mb, 2),
+    }

analysis/tools/preprocess.py

@@ -0,0 +1,107 @@
+"""preprocess_data handler — scale, encode, impute, drop columns."""
+from typing import Optional
+
+import polars as pl
+
+from ..session_store import SessionStore, _generate_id
+from ._helpers import _resolve_dataset
+
+
+async def _handle_preprocess_data(
+    dataset_id: str,
+    columns: list[str],
+    config: Optional[dict] = None,
+) -> dict:
+    """Preprocess selected columns (scale, encode, impute, drop)."""
+    entry, err = _resolve_dataset(dataset_id)
+    if err:
+        return err
+
+    lf: pl.LazyFrame = entry['lazyframe']
+    cfg = config or {}
+    column_mapping: dict[str, str] = {}
+
+    df = lf.collect(streaming=True)
+
+    # Resolve columns that actually exist
+    available = set(df.columns)
+    target_cols = [c for c in columns if c in available]
+    if not target_cols:
+        return {'error': 'None of the specified columns exist in the dataset',
+                'truncated': False}
+
+    # Drop columns
+    drop_cols = cfg.get('drop', [])
+    if drop_cols:
+        df = df.drop([c for c in drop_cols if c in df.columns])
+        for c in drop_cols:
+            if c in available:
+                column_mapping[c] = f'<dropped:{c}>'
+
+    # Fill missing values
+    fill_strategy = cfg.get('fillna')
+    if fill_strategy and fill_strategy != 'drop':
+        for col in target_cols:
+            if col not in df.columns:
+                continue
+            if df[col].null_count() == 0:
+                continue
+            if fill_strategy == 'mean':
+                mean_val = df[col].drop_nulls().mean()
+                if mean_val is not None:
+                    df = df.with_columns(pl.col(col).fill_null(mean_val))
+            elif fill_strategy == 'median':
+                med_val = df[col].drop_nulls().median()
+                if med_val is not None:
+                    df = df.with_columns(pl.col(col).fill_null(med_val))
+            elif fill_strategy == 'zero':
+                df = df.with_columns(pl.col(col).fill_null(0))
+    elif fill_strategy == 'drop':
+        df = df.drop_nulls(subset=target_cols)
+
+    # Standard scaling (z-score)
+    if cfg.get('standardize'):
+        from sklearn.preprocessing import StandardScaler
+        num_cols = [c for c in target_cols if c in df.columns
+                    and df[c].dtype in (pl.Float32, pl.Float64, pl.Int32, pl.Int64)]
+        if num_cols:
+            scaler = StandardScaler()
+            scaled = scaler.fit_transform(df.select(num_cols).to_numpy())
+            for i, col in enumerate(num_cols):
+                col_scaled = f'{col}_scaled'
+                df = df.with_columns(pl.Series(col_scaled, scaled[:, i]))
+                column_mapping[col] = col_scaled
+
+    # One-hot encoding
+    if cfg.get('onehot'):
+        str_cols = [c for c in target_cols if c in df.columns
+                    and df[c].dtype in (pl.Utf8, pl.String, pl.Categorical)]
+        if str_cols:
+            df = df.to_dummies(columns=str_cols, drop_first=False)
+            for c in str_cols:
+                column_mapping[c] = f'<onehot:{c}>'
+
+    new_id = _generate_id('pp')
+    new_lf = df.lazy()
+    new_schema = {name: str(dtype) for name, dtype in
+                  zip(new_lf.collect_schema().names(), new_lf.collect_schema().dtypes())}
+
+    store = SessionStore()
+    store.store_dataset(
+        dataset_id=new_id,
+        lazyframe=new_lf,
+        schema=new_schema,
+        metadata={
+            'row_count': len(df),
+            'parent_dataset_id': dataset_id,
+            'preprocessing_config': cfg,
+        },
+        parent_id=dataset_id,
+    )
+
+    return {
+        'preprocessed_dataset_id': new_id,
+        'column_mapping': column_mapping,
+        'row_count': len(df),
+        'column_count': len(new_schema),
+    }

analysis/tools/profile.py

@@ -0,0 +1,22 @@
+"""profile_data handler — compute per-column statistics and correlations."""
+from typing import Optional
+
+import polars as pl
+
+from ..data_profiler import profile_dataset as _profile_dataset
+from ._helpers import _resolve_dataset
+
+
+async def _handle_profile_data(
+    dataset_id: str,
+    sample_size: int = 1000,
+    columns: Optional[list[str]] = None,
+) -> dict:
+    """Profile dataset columns and correlations."""
+    entry, err = _resolve_dataset(dataset_id)
+    if err:
+        return err
+
+    lf: pl.LazyFrame = entry['lazyframe']
+    profile = _profile_dataset(lf, sample_size=sample_size, columns=columns)
+    return profile

analysis/type_classifier.py

@@ -1,415 +1,3 @@
-"""Data type classifier for column-level type inference.
+"""Re-export from new types module for backward compatibility."""
-
+from analysis.types import ColumnType, TYPE_REGISTRY, TYPE_ALIASES, get_type, classify_column, classify_schema
-Provides :class:`DataType` enumeration and functions to classify Polars
+TLS_HEX_MAP = {"0303": "TLSv1.2", "0304": "TLSv1.3", "0302": "TLSv1.1", "0301": "TLSv1.0"}
-columns based on name heuristics and value sampling.
-
-Typical usage::
-
-    from analysis.type_classifier import classify_column, classify_schema, DataType
-
-    # Single column
-    dtype = classify_column('src_ip', series, config_type='ipv4')
-
-    # Full schema
-    type_map = classify_schema(lazyframe, config_overrides={'src_ip': 'ipv4'})
-"""
-from __future__ import annotations
-
-import re
-from enum import Enum
-
-import polars as pl
-
-from analysis.tls_ref import get_tls_ref_dict
-
-# ---------------------------------------------------------------------------
-#  TLS version hex-to-label mapping
-#  (Duplicated in views.py for globe visualization — keep both in sync.)
-# ---------------------------------------------------------------------------
-
-TLS_HEX_MAP: dict[str, str] = {
-    '0303': 'TLSv1.2',
-    '0304': 'TLSv1.3',
-    '0302': 'TLSv1.1',
-    '0301': 'TLSv1.0',
-}
-
-# ---------------------------------------------------------------------------
-#  DataType enumeration
-# ---------------------------------------------------------------------------
-
-
-class DataType(Enum):
-    """Enumerated column types recognised by the classifier engine."""
-
-    INT = 1
-    FLOAT = 2
-    ENUM = 3
-    HEX = 4
-    URL = 5
-    IPv4 = 6
-    LAT_LON = 7
-    TIMESTAMP = 8
-    BOOL_ENUM = 9
-    STRING = 10
-
-
-# ---------------------------------------------------------------------------
-#  Detection patterns (raw strings for regex)
-# ---------------------------------------------------------------------------
-
-HEX_PATTERN = r'^[0-9a-f]{2}(?:\s[0-9a-f]{2})*$'
-"""Matches ``"0e a6 3f"``, ``"ab cd ef"`` etc. (space-separated hex pairs)."""
-
-URL_PATTERN = r'^https?://'
-"""Matches URLs starting with ``http://`` or ``https://``."""
-
-IPv4_PATTERN = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$'
-"""Matches ``"192.168.1.1"`` style IPv4 addresses."""
-
-MAC_PATTERN = r'^([0-9A-Fa-f]{2}[:.-]){5}[0-9A-Fa-f]{2}$'
-"""Matches MAC addresses with ``:``, ``-``, or ``.`` separators."""
-
-BOOL_ENUM_VALUES = (
-    '', ' ', '+', '-', '0', '1',
-    'true', 'false', 'yes', 'no', 't', 'f',
-)
-"""String values considered boolean / binary indicators."""
-
-LAT_LON_NAMES = ('lat', 'latitude', 'lon', 'lng', 'longitude')
-
-# Name-to-type heuristics (checked after config override, before values)
-_NAME_TYPE_MAP: list[tuple[re.Pattern, DataType]] = [
-    # User column names - EXACT match only (no generic fuzzy patterns)
-    # IPv4 columns
-    (re.compile(r'^:ips$|^:ipd$|^server.ip$|^client.ip$', re.I), DataType.IPv4),
-    # ENUM columns
-    (re.compile(r'^:prs$|^:prd$', re.I), DataType.ENUM),
-    (re.compile(r'^(?:cnrs|isrs)$', re.I), DataType.BOOL_ENUM),
-    (re.compile(r'^(?:scnt|dcnt|1ipp|4dbn|4srs|eiph)$', re.I), DataType.ENUM),
-    (re.compile(r'^0ver$', re.I), DataType.ENUM),
-    # FLOAT columns
-    (re.compile(r'^4dur$|^8ses$|^2tmo$|^0rnt$', re.I), DataType.FLOAT),
-    # INT columns
-    (re.compile(r'^4ksz$|^8ack$|^8ppk$|^8pak$|^8seq$|^8did$|^8byt$|^row$', re.I), DataType.INT),
-    # HEX columns
-    (re.compile(r'^0cph$|^0crv$', re.I), DataType.HEX),
-    # TIMESTAMP columns
-    (re.compile(r'^8dbd$|^time$|^timestamp$', re.I), DataType.TIMESTAMP),
-    # Dot-notation suffixes for lat/lon (e.g. :ips.latd, :ipd.lond)
-    (re.compile(r'\.latd$', re.I), DataType.LAT_LON),
-    (re.compile(r'\.lond$', re.I), DataType.LAT_LON),
-    # Dot-notation STRING suffixes (e.g. :ips.ispn, :ipd.city)
-    (re.compile(r'\.(ispn|orgn|city|anon)$', re.I), DataType.STRING),
-    # Standalone string columns - exact match
-    (re.compile(r'^(?:cnam|snam|tabl|name|0rnd|source.node|ecdhe.named.curve|crcc|orga|orgu)$', re.I), DataType.STRING),
-    # Special columns with @ prefix
-    (re.compile(r'^@', re.I), DataType.STRING),
-]
-
-# Mapping from config.yaml type strings to DataType
-_CONFIG_TYPE_MAP: dict[str, DataType] = {
-    'int': DataType.INT,
-    'float': DataType.FLOAT,
-    'enum': DataType.ENUM,
-    'hex': DataType.HEX,
-    'url': DataType.URL,
-    'ipv4': DataType.IPv4,
-    'lat_lon': DataType.LAT_LON,
-    'timestamp': DataType.TIMESTAMP,
-    'bool_enum': DataType.BOOL_ENUM,
-    'string': DataType.STRING,
-}
-
-# Mapping from tls_ref_data ``data_type`` column values to DataType
-TLSDB_TYPE_MAP: dict[str, DataType] = {
-    'IPv4': DataType.IPv4,
-    '浮点': DataType.FLOAT,
-    '字符串': DataType.STRING,
-    '整数': DataType.INT,
-    '2小写字母缩写': DataType.ENUM,
-    'XX:YY.Z': DataType.STRING,
-    '2字节': DataType.HEX,
-    '2大写字母': DataType.STRING,
-    '枚举': DataType.ENUM,
-    '布尔': DataType.BOOL_ENUM,
-    '字符串或整数': DataType.STRING,
-}
-
-
-def _tlsdb_type_to_datatype(tlsdb_type_str: str) -> DataType | None:
-    """Convert a TlsDB ``data_type`` string to a :class:`DataType`.
-
-    Returns ``None`` if the type string is unknown, allowing the
-    caller to fall through to value-based detection.
-    """
-    return TLSDB_TYPE_MAP.get(tlsdb_type_str)
-
-
-# ---------------------------------------------------------------------------
-#  Column classification (single column)
-# ---------------------------------------------------------------------------
-
-
-def _name_to_type(name: str) -> DataType | None:
-    """Match column name against known heuristics.
-
-    Returns a :class:`DataType` if the name matches strongly, otherwise
-    ``None`` (fall through to value-based detection).
-    """
-    for pattern, dtype in _NAME_TYPE_MAP:
-        if pattern.search(name):
-            return dtype
-    return None
-
-
-def _is_port_value(v: str) -> bool:
-    """Check if *v* is an integer string in the registered port range (1-65535)."""
-    try:
-        n = int(v)
-        return 1 <= n <= 65535
-    except ValueError:
-        return False
-
-
-def _can_float(v: str) -> bool:
-    """Return ``True`` when *v* can be parsed as a Python float."""
-    try:
-        float(v)
-        return True
-    except ValueError:
-        return False
-
-
-def _values_to_type(series: pl.Series, col_name: str | None = None) -> DataType:
-    """Infer column type by sampling non-null values.
-
-    Detection order: IPv4 → MAC guard → HEX → URL → PORT → BOOL_ENUM
-                     → LAT_LON → ENUM → FLOAT → STRING
-    """
-    dtype = series.dtype
-
-    # Boolean dtype ──────────────────────────────────────────────────────
-    if dtype == pl.Boolean:
-        return DataType.BOOL_ENUM
-
-    # Numeric dtypes ─────────────────────────────────────────────────────
-    if dtype in (pl.Float32, pl.Float64):
-        # Check LAT_LON possibility before returning FLOAT
-        non_null = series.drop_nulls()
-        if len(non_null) > 0:
-            sample = non_null.head(100)
-            sample_strs = [str(v).strip().lower() for v in sample]
-            non_blank = [v for v in sample_strs if v not in ('', ' ', '+', '-')]
-            if non_blank:
-                try:
-                    floats = [float(v) for v in non_blank]
-                    max_abs = max(abs(f) for f in floats)
-                    if (
-                        all(-180 <= f <= 180 for f in floats)
-                        and any('.' in v for v in non_blank)
-                        and max_abs > 10
-                    ):
-                        return DataType.LAT_LON
-                except ValueError:
-                    pass
-        return DataType.FLOAT
-
-    if dtype in (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
-                 pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64):
-        return DataType.INT
-
-    # String / categorical types ─────────────────────────────────────────
-    non_null = series.drop_nulls()
-    if len(non_null) == 0:
-        return DataType.STRING
-
-    sample = non_null.head(100)
-    # Convert all to string representations for pattern matching
-    sample_strs = [str(v).strip().lower() for v in sample]
-
-    # ── 1. IPv4 detection ──────────────────────────────────────────────
-    if all(re.match(IPv4_PATTERN, v) for v in sample_strs):
-        if all(_valid_ip_octets(v) for v in sample_strs):
-            return DataType.IPv4
-
-    # ── 2. MAC guard (skip HEX if values look like MAC addresses) ──────
-    is_mac = all(re.match(MAC_PATTERN, v) for v in sample_strs)
-
-    # ── 3. HEX detection ───────────────────────────────────────────────
-    if not is_mac and all(re.match(HEX_PATTERN, v) for v in sample_strs):
-        return DataType.HEX
-
-    # ── 4. URL detection ───────────────────────────────────────────────
-    if all(re.match(URL_PATTERN, v) for v in sample_strs):
-        return DataType.URL
-
-    # ── 5. PORT detection → ENUM ───────────────────────────────────────
-    if all(_is_port_value(v) for v in sample_strs):
-        return DataType.ENUM
-
-    # ── 6. BOOL_ENUM detection ─────────────────────────────────────────
-    if all(v in BOOL_ENUM_VALUES for v in sample_strs):
-        return DataType.BOOL_ENUM
-
-    # ── 7. LAT_LON detection (value-based) ──────────────────────────────
-    non_blank = [v for v in sample_strs if v not in ('', ' ', '+', '-')]
-    if non_blank:
-        try:
-            floats = [float(v) for v in non_blank]
-            max_abs = max(abs(f) for f in floats)
-            if (
-                all(-180 <= f <= 180 for f in floats)
-                and any('.' in v for v in non_blank)
-                and max_abs > 10
-            ):
-                return DataType.LAT_LON
-        except ValueError:
-            pass
-
-    # ── 8. ENUM detection (few unique values) ──────────────────────────
-    unique_count = non_null.n_unique()
-    if unique_count < 20:
-        # Safety: if >80 % of the sample values are unique strings,
-        # the column is high-cardinality — classify as STRING, not ENUM.
-        sample_unique_count = len(set(sample_strs))
-        if sample_unique_count > 0.8 * len(sample_strs):
-            return DataType.STRING
-        return DataType.ENUM
-
-    # ── 9. TLS version hex detection (before STRING fallback) ──────────
-    if col_name is not None and col_name.lower() in ('0ver', '_tlsver', 'tls_version', 'version'):
-        sample_hex = sample_strs[0].replace(' ', '').strip()
-        if sample_hex.upper() in TLS_HEX_MAP:
-            return DataType.ENUM
-
-    # ── 10. FLOAT detection (>80 % can float) ──────────────────────────
-    # Exclude strings starting with '+' from FLOAT detection — these are
-    # text markers (phone codes, IDs, error codes), not real numbers.
-    if any(v.startswith('+') for v in sample_strs):
-        return DataType.STRING
-
-    float_count = sum(1 for v in sample_strs if _can_float(v))
-    if float_count / len(sample_strs) > 0.8:
-        # Guard: values with leading "-" that happen to parse as float
-        # should not force FLOAT when non-float values also exist.
-        if any(v.startswith('-') for v in sample_strs):
-            if float_count < len(sample_strs):
-                return DataType.STRING
-        return DataType.FLOAT
-
-    return DataType.STRING
-
-
-def _valid_ip_octets(ip_str: str) -> bool:
-    """Check that each octet in an IPv4 string is 0-255."""
-    parts = ip_str.split('.')
-    if len(parts) != 4:
-        return False
-    for p in parts:
-        try:
-            n = int(p)
-            if n < 0 or n > 255:
-                return False
-        except ValueError:
-            return False
-    return True
-
-
-def classify_column(
-    name: str,
-    series: pl.Series,
-    config_type: str | None = None,
-) -> DataType:
-    """Classify a single column based on its *name* and *series* values.
-
-    Priority
-    --------
-    1. **config_type** — user override from ``config.yaml``.
-       If provided (e.g. ``'ipv4'``), returns the corresponding
-       :class:`DataType` immediately.
-    2. **tls_ref_data** — authoritative type from ``TlsDB.csv`` /
-       ``tls_ref_data`` table.  Looked up by column name; if found,
-       the declared type is used directly (no value-based guessing).
-    3. **Value-based** pattern matching — samples up to 100 non-null
-       values and tests against IPv4 → HEX → URL → PORT → BOOL_ENUM
-       → LAT_LON → ENUM → FLOAT in order.
-    4. **Name-based** heuristics — only consulted when value-based
-       detection returned ``STRING`` (i.e. no value pattern matched).
-
-    Parameters
-    ----------
-    name:
-        Column name (used for name-based heuristics and tls_ref_data
-        lookup).
-    series:
-        Polars :class:`Series` containing the column's data.
-    config_type:
-        Optional user override (case-insensitive, matched to
-        :data:`_CONFIG_TYPE_MAP`).
-
-    Returns
-    -------
-    DataType
-    """
-    # 1. Config override (highest priority)
-    if config_type is not None:
-        mapped = _CONFIG_TYPE_MAP.get(config_type.lower().strip())
-        if mapped is not None:
-            return mapped
-
-    # 2. tls_ref_data lookup (authoritative type from TlsDB)
-    tls_ref = get_tls_ref_dict()
-    if name in tls_ref:
-        dtype = _tlsdb_type_to_datatype(tls_ref[name]['data_type'])
-        if dtype is not None:
-            return dtype
-
-    # 3. Value-based sampling
-    result = _values_to_type(series, name)
-
-    # 4. Name-based heuristics (only when values returned STRING)
-    if result == DataType.STRING:
-        name_match = _name_to_type(name)
-        if name_match is not None:
-            return name_match
-
-    return result
-
-
-# ---------------------------------------------------------------------------
-#  Schema classification (all columns)
-# ---------------------------------------------------------------------------
-
-
-def classify_schema(
-    lf: pl.LazyFrame,
-    config_overrides: dict[str, str] | None = None,
-) -> dict[str, DataType]:
-    """Classify all columns in a :class:`LazyFrame`.
-
-    Parameters
-    ----------
-    lf:
-        A Polars :class:`LazyFrame` whose columns to classify.
-    config_overrides:
-        Optional mapping of ``{col_name: type_string}`` from a
-        ``config.yaml`` ``columns:`` section.  These take highest
-        priority (see :func:`classify_column`).
-
-    Returns
-    -------
-    dict[str, DataType]
-        Mapping from column name to inferred :class:`DataType`.
-    """
-    config_overrides = config_overrides or {}
-    df_sample = lf.collect(streaming=True)
-    result: dict[str, DataType] = {}
-    for col_name in df_sample.columns:
-        config_type = config_overrides.get(col_name)
-        result[col_name] = classify_column(
-            name=col_name,
-            series=df_sample[col_name],
-            config_type=config_type,
-        )
-    return result

analysis/types.py

@@ -0,0 +1,300 @@
+"""天璇类型系统 — 仅8种固定类型，面向对象接口。
+
+每个类型继承 :class:`ColumnType`，实现:
+- ``distance()`` — 为该类型列计算距离矩阵（追加到 LazyFrame）
+- ``display()`` — 将值格式化为可读字符串
+- ``polars_dtype`` — 映射到 Polars 数据类型
+- ``name`` — 中文名称
+
+:data:`TYPE_REGISTRY` 是唯一的类型注册表，按名称索引。
+"""
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+import numpy as np
+import polars as pl
+
+
+# ═══════════════════════════════════════════════════════════════════════
+#  Base type — all 8 types inherit from this
+# ═══════════════════════════════════════════════════════════════════════
+
+class ColumnType(ABC):
+    """Base class for all column types.
+
+    Subclasses MUST define:
+    - ``name`` (class attribute, Chinese)
+    - ``polars_dtype`` (class attribute)
+    - ``distance()``
+    - ``display()``
+    """
+
+    name: str = ""
+    polars_dtype: pl.DataType = pl.Utf8
+
+    @abstractmethod
+    def distance(self, lf: pl.LazyFrame, columns: list[str]) -> pl.LazyFrame:
+        """Compute per-row distances for *columns* of this type.
+
+        Returns a new LazyFrame with distance columns appended
+        (prefixed ``_dist_``).
+        """
+        ...
+
+    @abstractmethod
+    def display(self, value: Any) -> str:
+        """Format *value* for human-readable display."""
+        ...
+
+
+# ═══════════════════════════════════════════════════════════════════════
+#  8 concrete types
+# ═══════════════════════════════════════════════════════════════════════
+
+class IPv4Type(ColumnType):
+    name = "IPv4"
+    polars_dtype = pl.Utf8
+
+    def distance(self, lf: pl.LazyFrame, columns: list[str]) -> pl.LazyFrame:
+        if len(columns) < 2:
+            return lf
+        from analysis.distance import ip_distance
+        return ip_distance(lf, columns)
+
+    def display(self, value: Any) -> str:
+        return str(value) if value else "-"
+
+
+class StringType(ColumnType):
+    name = "字符串"
+    polars_dtype = pl.Utf8
+
+    def distance(self, lf: pl.LazyFrame, columns: list[str]) -> pl.LazyFrame:
+        from analysis.distance import string_distance
+        return string_distance(lf, columns)
+
+    def display(self, value: Any) -> str:
+        return str(value)[:80] if value else "-"
+
+
+class EnumType(ColumnType):
+    name = "枚举"
+    polars_dtype = pl.Utf8
+
+    def distance(self, lf: pl.LazyFrame, columns: list[str]) -> pl.LazyFrame:
+        # 0-1 match distance: compare to mode
+        for col in columns:
+            mode_val = lf.select(pl.col(col).mode().first()).collect(streaming=True).item()
+            lf = lf.with_columns(
+                (pl.col(col) != mode_val).cast(pl.Float64).alias(f'_enum_dist_{col}')
+            )
+        return lf
+
+    def display(self, value: Any) -> str:
+        return str(value) if value else "-"
+
+
+class FloatType(ColumnType):
+    name = "浮点"
+    polars_dtype = pl.Float64
+
+    def distance(self, lf: pl.LazyFrame, columns: list[str]) -> pl.LazyFrame:
+        # Normalised euclidean distance — handled in preprocessing pipeline
+        for col in columns:
+            lf = lf.with_columns(
+                pl.col(col).cast(pl.Float64).alias(f'_raw_{col}')
+            )
+        return lf
+
+    def display(self, value: Any) -> str:
+        try:
+            return f"{float(value):.2f}"
+        except (ValueError, TypeError):
+            return str(value)
+
+
+class IntType(ColumnType):
+    name = "整数"
+    polars_dtype = pl.Int64
+
+    def distance(self, lf: pl.LazyFrame, columns: list[str]) -> pl.LazyFrame:
+        # Normalised manhattan distance — handled in preprocessing pipeline
+        for col in columns:
+            lf = lf.with_columns(
+                pl.col(col).cast(pl.Int64).alias(f'_raw_{col}')
+            )
+        return lf
+
+    def display(self, value: Any) -> str:
+        try:
+            return str(int(value))
+        except (ValueError, TypeError):
+            return str(value)
+
+
+class BoolType(ColumnType):
+    name = "布尔"
+    polars_dtype = pl.Boolean
+
+    def distance(self, lf: pl.LazyFrame, columns: list[str]) -> pl.LazyFrame:
+        # 0-1 match distance
+        for col in columns:
+            lf = lf.with_columns(
+                pl.col(col).cast(pl.Float64).alias(f'_bool_dist_{col}')
+            )
+        return lf
+
+    def display(self, value: Any) -> str:
+        if value is None:
+            return "-"
+        return "是" if str(value).lower() in ('true', '1', '+', 't') else "否"
+
+
+class BytesType(ColumnType):
+    name = "字节"
+    polars_dtype = pl.Utf8
+
+    def distance(self, lf: pl.LazyFrame, columns: list[str]) -> pl.LazyFrame:
+        from analysis.distance import bytes_distance
+        return bytes_distance(lf, columns)
+
+    def display(self, value: Any) -> str:
+        s = str(value).strip() if value else ""
+        if len(s) > 20:
+            return s[:20] + "..."
+        return s or "-"
+
+
+class TimestampType(ColumnType):
+    name = "时间戳"
+    polars_dtype = pl.Float64
+
+    def distance(self, lf: pl.LazyFrame, columns: list[str]) -> pl.LazyFrame:
+        from analysis.distance import timestamp_fft_distance
+        return timestamp_fft_distance(lf, columns)
+
+    def display(self, value: Any) -> str:
+        try:
+            t = float(value)
+            if t < 1e8 or t > 2e12:
+                return str(value)
+            from datetime import datetime
+            return datetime.fromtimestamp(t).strftime("%Y-%m-%d %H:%M")
+        except (ValueError, TypeError, OSError):
+            return str(value) if value else "-"
+
+
+# ═══════════════════════════════════════════════════════════════════════
+#  Registry — single source of truth
+# ═══════════════════════════════════════════════════════════════════════
+
+TYPE_REGISTRY: dict[str, ColumnType] = {
+    "枚举": EnumType(),
+    "字节": BytesType(),
+    "浮点": FloatType(),
+    "整数": IntType(),
+    "布尔": BoolType(),
+    "字符串": StringType(),
+    "IPv4": IPv4Type(),
+    "时间戳": TimestampType(),
+}
+
+# Alias map for config.yaml type strings
+TYPE_ALIASES: dict[str, str] = {
+    'enum': '枚举', '枚举': '枚举',
+    'bytes': '字节', '字节': '字节', 'hex': '字节',
+    'float': '浮点', '浮点': '浮点',
+    'int': '整数', '整数': '整数',
+    'bool': '布尔', '布尔': '布尔', 'boolean': '布尔',
+    'string': '字符串', '字符串': '字符串', 'str': '字符串',
+    'ipv4': 'IPv4', 'IPv4': 'IPv4',
+    'timestamp': '时间戳', '时间戳': '时间戳',
+}
+
+
+def get_type(name: str) -> ColumnType:
+    """Resolve a type name (Chinese or English alias) to a ColumnType."""
+    resolved = TYPE_ALIASES.get(name.strip().lower() if name.isascii() else name.strip())
+    if resolved is None:
+        return StringType()  # fallback
+    return TYPE_REGISTRY[resolved]
+
+
+def classify_column(name: str, series: pl.Series, config_type: str | None = None) -> ColumnType:
+    """Classify a single column to a ColumnType.
+
+    Priority: config override → TlsDB lookup → name heuristics → value sampling.
+    """
+    from analysis.tls_ref import get_tls_ref_dict
+    import re
+
+    # 1. Config override
+    if config_type is not None:
+        t = get_type(config_type)
+        if t.name != "字符串" or config_type.lower().strip() == 'string':
+            return t
+
+    # 2. TlsDB lookup
+    tls_ref = get_tls_ref_dict()
+    if name in tls_ref:
+        t = get_type(tls_ref[name]['data_type'])
+        return t
+
+    # 3. Name heuristics
+    _patterns: list[tuple[re.Pattern, str]] = [
+        (re.compile(r'^(?:src_ip|dst_ip|:ips|:ipd|server.ip|client.ip)$', re.I), 'IPv4'),
+        (re.compile(r'^(?::prs|:prd|scnt|dcnt|1ipp|4dbn|4srs|eiph|name|tabl|0ver|cnrs|isrs)$', re.I), '枚举'),
+        (re.compile(r'^(?:0cph|0crv|cipher|ecdhe)', re.I), '字节'),
+        (re.compile(r'^(?:4dur|8ses|2tmo|0rnt)$', re.I), '浮点'),
+        (re.compile(r'^(?:4ksz|8ack|8ppk|8pak|8seq|8did|8byt|row)$', re.I), '整数'),
+        (re.compile(r'^(?:timestamp|time|8dbd|created|updated)', re.I), '时间戳'),
+        (re.compile(r'\.(ispn|orgn|city|anon|doma|latd|lond)$', re.I), '字符串'),
+        (re.compile(r'^(?:cnam|snam|0rnd|source.node|crcc|orga|orgu|@)', re.I), '字符串'),
+    ]
+    for pat, type_name in _patterns:
+        if pat.search(name):
+            return TYPE_REGISTRY[type_name]
+
+    # 4. Value-based sampling
+    non_null = series.drop_nulls()
+    if len(non_null) == 0:
+        return TYPE_REGISTRY["字符串"]
+
+    strs = [str(v).strip().lower() for v in non_null.head(100)]
+    ip4 = re.compile(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$')
+    hex_p = re.compile(r'^[0-9a-f]{2}(?:\s[0-9a-f]{2})*$')
+
+    if all(ip4.match(v) for v in strs):
+        return TYPE_REGISTRY["IPv4"]
+    if all(hex_p.match(v) for v in strs):
+        return TYPE_REGISTRY["字节"]
+    if series.dtype.is_integer():
+        return TYPE_REGISTRY["整数"]
+    if series.dtype == pl.Float64:
+        return TYPE_REGISTRY["浮点"]
+    if series.dtype == pl.Boolean:
+        return TYPE_REGISTRY["布尔"]
+    if non_null.n_unique() < 50:
+        return TYPE_REGISTRY["枚举"]
+
+    return TYPE_REGISTRY["字符串"]
+
+
+def classify_schema(
+    lf: pl.LazyFrame,
+    config_overrides: dict[str, str] | None = None,
+) -> dict[str, ColumnType]:
+    """Classify all columns in a LazyFrame."""
+    config_overrides = config_overrides or {}
+    df_sample = lf.collect(streaming=True)
+    result = {}
+    for col_name in df_sample.columns:
+        config_type = config_overrides.get(col_name)
+        result[col_name] = classify_column(
+            name=col_name,
+            series=df_sample[col_name],
+            config_type=config_type,
+        )
+    return result

analysis/urls.py

@@ -15,7 +15,7 @@ urlpatterns = [
     path('runs/<int:display_id>/', views.run_detail, name='run_detail'),
     path('runs/<int:display_id>/status/', views.run_status_api, name='run_status'),
     path('clusters/<int:display_id>/', views.cluster_overview, name='cluster_overview'),
-    path('clusters/<int:display_id>/<int:cluster_label>/', views.cluster_detail, name='cluster_detail'),
+    path('clusters/<int:display_id>/<path:cluster_label>/', views.cluster_detail, name='cluster_detail'),
     path('entities/<int:entity_id>/', views.entity_profile, name='entity_profile'),
     path('config/', views.config_view, name='config'),
     path('api/llm/test/', views.llm_test, name='llm_test'),
@@ -25,6 +25,7 @@ urlpatterns = [
     path('logs/', views.log_viewer, name='log_viewer'),
     path('globe/', views.globe_view, name='globe'),
     path('tools/run/', views.tool_lab_run, name='tool_lab_run'),
+    path('tools/reload-run/', views.reload_run_data, name='reload_run_data'),
     path('tools/plan/', views.tool_plan, name='tool_plan'),
     path('tools/filter/', views.apply_filter, name='apply_filter'),
 ]

analysis/views/__init__.py

@@ -0,0 +1,13 @@
+"""Analysis views package — re-exports all public view functions."""
+from .helpers import _extract_lat, _extract_lon
+from .dashboard import dashboard, run_list, run_detail, run_status_api, start_analysis
+from .pipeline import _run_pipeline_worker, _background_process
+from .clustering import cluster_overview, cluster_detail, _run_clustering_pipeline, _get_globe_flows
+from .entity import entity_profile
+from .upload import upload_page, upload_csv, upload_csv_batch, finalize_upload, delete_upload
+from .manual import manual_page, manual_run_analysis
+from .auto import auto_page, run_llm_analysis_view
+from .globe import globe_view, _extract_flows_from_df
+from .config import config_view, llm_test
+from .log_viewer import log_viewer
+from .tools import tool_lab, tool_lab_run, tool_plan, apply_filter, reload_run_data, retry_run

analysis/views/auto.py

@@ -0,0 +1,348 @@
+"""LLM auto analysis views."""
+import json
+import asyncio
+import threading
+import traceback
+import logging
+
+from django.shortcuts import render, get_object_or_404
+from django.http import JsonResponse
+from django.views.decorators.csrf import csrf_exempt
+
+from config import get_config
+from analysis.models import AnalysisRun
+from .pipeline import _run_pipeline_worker
+from .clustering import _run_clustering_pipeline
+from .helpers import _PLANS_DIR, _add_to_auto_index
+
+logger = logging.getLogger(__name__)
+
+
+def auto_page(request):
+    """LLM-driven auto analysis page."""
+    from analysis.session_store import SessionStore
+    import json as _json
+
+    cfg = get_config()
+
+    # Get datasets from SessionStore + ALL runs from DB
+    store = SessionStore()
+    session_datasets = store.list_datasets() if hasattr(store, 'list_datasets') else []
+    session_ds_map = {ds['dataset_id']: ds for ds in session_datasets}
+
+    # Build dataset entries from ALL AnalysisRun records
+    all_runs = AnalysisRun.objects.all().order_by('-created_at')[:100]
+    datasets_with_status = []
+
+    for run in all_runs:
+        if run.run_type == 'upload':
+            ds_id = f'upload_{run.display_id}'
+        else:
+            ds_id = f'run_{run.display_id}'
+
+        session_entry = session_ds_map.get(ds_id)
+
+        if session_entry:
+            datasets_with_status.append({
+                **session_entry,
+                'dataset_id': ds_id,
+                'run_status': run.status,
+                'total_flows': run.total_flows,
+                'display_id': run.display_id,
+                'needs_reload': False,
+                'csv_glob': run.csv_glob or '',
+                'sqlite_table': run.sqlite_table or '',
+            })
+        elif run.csv_glob or run.sqlite_table:
+            datasets_with_status.append({
+                'dataset_id': ds_id,
+                'run_status': run.status,
+                'display_id': run.display_id,
+                'total_flows': run.total_flows,
+                'row_count': run.total_flows,
+                'file_count': None,
+                'columns': [],
+                'column_count': 0,
+                'svd_components': 0,
+                'needs_reload': True,
+                'csv_glob': run.csv_glob or '',
+                'sqlite_table': run.sqlite_table or '',
+            })
+
+    for ds_info in datasets_with_status:
+        s = ds_info['run_status']
+        if s in ('completed',):
+            ds_info['status_label'] = '已分析'
+            ds_info['badge_class'] = 'completed'
+        elif s in ('failed',):
+            ds_info['status_label'] = '错误'
+            ds_info['badge_class'] = 'failed'
+        elif s in ('pending', 'loading', 'profiling', 'aggregating', 'clustering', 'extracting'):
+            ds_info['status_label'] = '分析中'
+            ds_info['badge_class'] = 'analyzing'
+        else:
+            ds_info['status_label'] = '待分析'
+            ds_info['badge_class'] = 'ready'
+
+    return render(request, 'tianxuan/auto.html', {
+        'llm_configured': bool(cfg.llm.base_url and cfg.llm.api_key),
+        'datasets_with_status_json': _json.dumps(datasets_with_status, ensure_ascii=False),
+    })
+
+
+@csrf_exempt
+def run_llm_analysis_view(request):
+    """AJAX POST: run LLM-driven analysis pipeline."""
+    if request.method != 'POST':
+        return JsonResponse({'error': 'POST required'}, status=405)
+    body = json.loads(request.body)
+    display_id = body.get('run_id')
+    run = get_object_or_404(AnalysisRun, display_id=display_id)
+    pk = run.id
+    run.run_type = 'auto'
+    run.save(update_fields=['run_type'])
+    head = body.get('head')
+
+    from analysis.session_store import SessionStore
+    store = SessionStore()
+
+    # Determine dataset key format (consistent with manual_page)
+    upload_ds_id = f'upload_{display_id}'
+    run_ds_id = f'run_{display_id}'
+    entry = store.get_dataset(upload_ds_id)
+    if entry is None:
+        entry = store.get_dataset(run_ds_id)
+
+    if entry is None:
+        # Try primary key fallback
+        upload_pk_id = f'upload_{pk}'
+        entity_pk_id = f'entity_{pk}'
+        entry = store.get_dataset(upload_pk_id)
+        if entry is None:
+            entry = store.get_dataset(entity_pk_id)
+
+    if entry is None:
+        # Auto-reload from sqlite_table or csv_glob
+        from analysis.data_loader import load_csv_directory, load_from_db
+        lf = None
+        schema = {}
+        if run.sqlite_table:
+            try:
+                lf = load_from_db(run.sqlite_table)
+            except Exception:
+                logger.error("unknown failed: {}".format(traceback.format_exc()))
+                pass
+        if lf is None and run.csv_glob:
+            try:
+                lf, schema, _, _, _ = load_csv_directory(run.csv_glob)
+            except Exception:
+                logger.error("unknown failed: {}".format(traceback.format_exc()))
+                pass
+        if lf is None:
+            return JsonResponse({'error': '数据集不存在，请先上传并等待预处理完成'})
+        entry_key = upload_ds_id if run.run_type == 'upload' else run_ds_id
+        store.store_dataset(entry_key, lf, schema=schema, metadata={
+            'row_count': run.total_flows,
+            'csv_glob': run.csv_glob or '',
+            'sqlite_table': run.sqlite_table or '',
+        })
+        # Re-fetch entry
+        entry = store.get_dataset(entry_key)
+
+    # Determine the active dataset_id
+    for candidate in [upload_ds_id, run_ds_id, f'upload_{pk}', f'entity_{pk}']:
+        if store.get_dataset(candidate):
+            dataset_id = candidate
+            break
+    else:
+        return JsonResponse({'error': '数据集不存在，请先上传并等待预处理完成'})
+
+    # ── Optional: apply filter before LLM pipeline ──
+    filters = body.get('filters', [])
+    if filters and len(filters) > 0:
+        logic = body.get('logic', 'and')
+        try:
+            from analysis.tool_registry import _handle_filter_data
+            filter_result = asyncio.run(_handle_filter_data(
+                dataset_id=dataset_id,
+                filters=filters,
+                logic=logic,
+            ))
+            if 'error' not in filter_result:
+                filtered_id = filter_result.get('filtered_dataset_id', '')
+                if filtered_id and store.get_dataset(filtered_id):
+                    dataset_id = filtered_id
+        except Exception:
+            logger.warning(f'[LLM auto] Filter failed (non-fatal): {traceback.format_exc()}')
+
+    def _analysis_fn(run, ctx):
+        from tianxuan.llm_orchestrator import run_llm_pipeline, LLMConfig
+        dataset_id = ctx['dataset_id']
+        pk = ctx['pk']
+
+        def progress_callback(step, tool_name, result=None, meta=None):
+            nonlocal run
+            try:
+                run = AnalysisRun.objects.get(id=pk)
+                pct = min(10 + step * 6, 95)
+                run.progress_pct = pct
+                run.progress_msg = f'步骤 {step}: {tool_name}'
+
+                save_fields = ['progress_pct', 'progress_msg']
+                if tool_name == 'thinking':
+                    # Accumulate LLM thinking text (appended to llm_thinking)
+                    thought = (result or '').strip()
+                    if not thought:
+                        thought = f'步骤 {step} 分析中...'
+                    run.llm_thinking = (run.llm_thinking or '') + f'\n[{step}] {thought}'
+                    save_fields = ['progress_pct', 'progress_msg', 'llm_thinking']
+                elif tool_name == 'complete':
+                    pass  # No extra tracking needed
+                else:
+                    # Single tool call — always has result
+                    if isinstance(result, dict):
+                        summary = json.dumps(result, default=str, ensure_ascii=False)[:500]
+                    else:
+                        summary = str(result)[:500]
+                    run.run_log += f'[{step}] {tool_name}: {summary}...\n'
+
+                    # One entry per tool call
+                    if meta and isinstance(meta, dict) and 'args' in meta:
+                        tool_calls = list(run.tool_calls_json or [])
+                        tool_calls.append({
+                            'step': step,
+                            'name': tool_name,
+                            'input': meta['args'],
+                            'output': result,
+                        })
+                        run.tool_calls_json = tool_calls
+                    save_fields = ['progress_pct', 'progress_msg', 'run_log', 'tool_calls_json']
+
+                run.save(update_fields=save_fields)
+            except Exception as e:
+                logger.warning('progress_callback error: %s', e)
+
+        try:
+            from config import get_config
+            cfg = get_config()
+            llm_cfg = LLMConfig(cfg.llm.base_url, cfg.llm.api_key, cfg.llm.model)
+            run.run_log = '启动 LLM 分析...\n'
+            run.status = 'loading'
+            run.progress_pct = 5
+            run.progress_msg = 'LLM 初始化中...'
+            run.save(update_fields=['status', 'progress_pct', 'progress_msg', 'run_log'])
+
+            result = run_llm_pipeline(dataset_id, config=llm_cfg, callback=progress_callback)
+
+            if 'error' in result:
+                run.status = 'failed'
+                run.progress_msg = f'失败: {result["error"]}'
+                run.run_log += f'\n[ERROR] {result["error"]}'
+                run.save(update_fields=['status', 'progress_pct', 'progress_msg', 'run_log'])
+            else:
+                run.progress_pct = 55
+                run.progress_msg = f'LLM分析完成: {result.get("result", "")[:200]}'
+                run.run_log += f'\n[完成] {result.get("result", "")}\n[INFO] 开始自动聚类流程...'
+                run.save(update_fields=['progress_pct', 'progress_msg', 'run_log'])
+
+                # ── Auto-run clustering pipeline after LLM analysis ──
+                entity_ds_id = None
+                tool_calls = run.tool_calls_json or []
+                for tc in tool_calls:
+                    tool_name = tc.get('name', '')
+                    tool_input = tc.get('input', {})
+                    if tool_name in ('build_entity_profiles', 'run_clustering'):
+                        entity_ds_id = tool_input.get('dataset_id', '')
+                        if entity_ds_id and store.get_dataset(entity_ds_id):
+                            break
+                        entity_ds_id = None
+
+                # Fallback: try common dataset ID patterns
+                if not entity_ds_id:
+                    candidates = [
+                        ctx.get('dataset_id', ''),
+                        f'entity_{pk}',
+                        f'entity_{run.display_id}',
+                        f'upload_{run.display_id}',
+                    ]
+                    for cid in candidates:
+                        if cid and store.get_dataset(cid):
+                            entity_ds_id = cid
+                            break
+
+                if entity_ds_id:
+                    _run_clustering_pipeline(
+                        run=run,
+                        store=store,
+                        ds_id=entity_ds_id,
+                        feature_columns=None,
+                        algorithm='agglomerative',
+                        min_cluster_size=5,
+                        run_umap=True,
+                        head=ctx.get('head'),
+                    )
+                else:
+                    logger.warning(
+                        f'[LLM auto] No entity dataset found for clustering. '
+                        f'Tool calls: {[tc.get("name") for tc in tool_calls]}. '
+                        f'Marking run as completed without clustering.'
+                    )
+                    run.status = 'completed'
+                    run.progress_pct = 100
+                    run.progress_msg = 'LLM分析完成（无实体数据，跳过聚类）'
+                    run.save(update_fields=['status', 'progress_pct', 'progress_msg'])
+
+                # ── Auto-save workflow as loadable plan ──
+                try:
+                    tool_calls_data = run.tool_calls_json or []
+                    steps = []
+                    for tc in tool_calls_data:
+                        tc_name = tc.get('name', '')
+                        if tc_name == 'thinking':
+                            continue
+                        steps.append({
+                            'tool_name': tc_name,
+                            'params': tc.get('input', {}),
+                        })
+                    if steps:
+                        plan_data = {
+                            'name': f'_auto_{run.display_id}',
+                            'steps': steps,
+                            'auto': True,
+                            'display_id': run.display_id,
+                            'created_at': str(run.created_at) if run.created_at else '',
+                        }
+                        plan_path = _PLANS_DIR / f'_auto_{run.display_id}.json'
+                        plan_path.write_text(
+                            json.dumps(plan_data, ensure_ascii=False, indent=2),
+                            encoding='utf-8',
+                        )
+                        _add_to_auto_index(f'_auto_{run.display_id}.json')
+                        logger.info(
+                            '[LLM auto] Saved auto plan %s (%d steps)',
+                            plan_path.name, len(steps),
+                        )
+                except Exception as save_err:
+                    logger.warning('[LLM auto] Failed to save auto plan: %s', save_err)
+
+        except Exception:
+            tb = traceback.format_exc()
+            logger.error(tb)
+            try:
+                run = AnalysisRun.objects.get(id=pk)
+                run.status = 'failed'
+                run.progress_msg = f'失败: {tb}'
+                run.run_log += f'\n[ERROR] {tb}'
+                run.error_message = tb
+                run.save(update_fields=['status', 'progress_msg', 'run_log', 'error_message'])
+            except Exception as e:
+                logger.warning('save failed after pipeline error: %s', e)
+
+    t = threading.Thread(
+        target=_run_pipeline_worker,
+        args=(pk, _analysis_fn),
+        kwargs=dict(dataset_id=dataset_id, head=head, pk=pk),
+        daemon=True,
+    )
+    t.start()
+    return JsonResponse({'status': 'started', 'run_id': run.display_id})

analysis/views/clustering.py

@@ -0,0 +1,527 @@
+"""Cluster views: overview, detail, globe flows, and the clustering pipeline."""
+import asyncio
+import json
+import traceback
+import logging
+
+from django.shortcuts import render, get_object_or_404
+from django.http import Http404
+
+from analysis.models import AnalysisRun, ClusterResult, EntityProfile
+from analysis import nl_describe
+from analysis.constants import RANDOM_SEED, UMAP_BATCH_SIZE, UMAP_TRAIN_SAMPLE
+from analysis.profile_util import profile
+
+from .helpers import _extract_lat, _extract_lon
+
+logger = logging.getLogger(__name__)
+
+
+def cluster_overview(request, display_id):
+    """Overview of all clusters for a run, with UMAP scatter data and geo scatter data."""
+    run = get_object_or_404(AnalysisRun, display_id=display_id)
+    clusters = run.clusters.exclude(cluster_label=-1).order_by('-size').prefetch_related('features')
+    noise = run.clusters.filter(cluster_label=-1).first()
+
+    # Build UMAP scatter data for Canvas / Three.js 3D
+    entities = run.entities.exclude(embedding_x=None).exclude(embedding_y=None).only(
+        'embedding_x', 'embedding_y', 'embedding_z', 'cluster_label', 'entity_value'
+    )
+    has_z = any(e.embedding_z is not None for e in entities)
+    scatter_data = [
+        {'x': e.embedding_x, 'y': e.embedding_y, 'z': e.embedding_z if has_z else 0,
+         'label': e.cluster_label if e.cluster_label is not None else -1, 'entity': e.entity_value}
+        for e in entities
+    ]
+
+    # Noise cluster: features + entity detail
+    noise_features = []
+    noise_summary = ''
+    noise_entity_count = 0
+    if noise:
+        noise_features_qs = noise.features.all().order_by('-distinguishing_score')[:15]
+        noise_features = [
+            {'feature_name': f.feature_name, 'score': f.distinguishing_score, 'mean': f.mean,
+             'std': f.std, 'p25': f.p25, 'p75': f.p75}
+            for f in noise_features_qs
+        ]
+        if noise_features:
+            noise_summary = nl_describe.describe_cluster(noise_features)
+        noise_entity_count = noise.size
+
+    # LLM workflow timeline for auto runs
+    llm_timeline_json = ''
+    if run.run_type == 'auto':
+        llm_thinking = run.llm_thinking or ''
+        tool_calls = run.tool_calls_json or []
+        if tool_calls or llm_thinking:
+            llm_timeline_json = json.dumps({
+                'thinking': llm_thinking,
+                'tool_calls': tool_calls,
+            })
+
+    # Build geo scatter data from feature_json (lat/lon)
+    geo_data = []
+    geo_skipped = 0
+    for e in run.entities.all().only('feature_json', 'cluster_label', 'entity_value'):
+        fj = e.feature_json
+        if not fj:
+            continue
+        lat = _extract_lat(fj.get('avg_latitude'))
+        lon = _extract_lon(fj.get('avg_longitude'))
+        if lat is not None and lon is not None:
+            geo_data.append({
+                'x': lon,
+                'y': lat,
+                'label': e.cluster_label if e.cluster_label is not None else -1,
+                'entity': e.entity_value,
+            })
+        else:
+            geo_skipped += 1
+
+    # Cluster sizes for bar chart
+    cluster_sizes = [{'label': c.cluster_label, 'size': c.size, 'silhouette': c.silhouette_score}
+                     for c in clusters]
+
+    # Top features per cluster from ClusterFeature + NL summaries
+    top_features = {}
+    for c in clusters:
+        features_qs = c.features.all().order_by('-distinguishing_score')[:10]
+        feats = [
+            {'feature_name': f.feature_name, 'score': f.distinguishing_score, 'mean': f.mean,
+             'std': f.std}
+            for f in features_qs
+        ]
+        top_features[str(c.cluster_label)] = feats
+        # Attach features + NL summary directly to cluster object for template
+        c._top_features = feats
+        c.nl_summary = nl_describe.describe_cluster(feats) if feats else ''
+
+    return render(request, 'analysis/cluster_overview.html', {
+        'run': run,
+        'clusters': clusters,
+        'noise': noise,
+        'scatter_data_json': json.dumps(scatter_data),
+        'has_z': has_z,
+        'geo_data_json': json.dumps(geo_data),
+        'geo_skipped': geo_skipped,
+        'geo_count': len(geo_data),
+        'cluster_sizes_json': json.dumps(cluster_sizes),
+        'top_features_json': json.dumps(top_features),
+        'noise_features': noise_features,
+        'noise_summary': noise_summary,
+        'noise_entity_count': noise_entity_count,
+        'llm_timeline_json': llm_timeline_json,
+        # Globe flow data for the sidebar
+        'globe_flows_json': json.dumps(_get_globe_flows(run)),
+        'selected_ds_id': f'upload_{run.display_id}',
+    })
+
+
+def _get_globe_flows(run, max_rows=500):
+    """Extract flow data for 3D globe visualization from a run."""
+    from analysis.data_loader import load_csv_directory, load_from_db
+    try:
+        lf = None
+        if run.sqlite_table:
+            lf = load_from_db(run.sqlite_table)
+        if lf is None and run.csv_glob:
+            lf, _, _, _, _ = load_csv_directory(run.csv_glob)
+        if lf is None:
+            return []
+        df = lf.head(max_rows).collect()
+        from analysis.type_classifier import TLS_HEX_MAP
+        from analysis.geoip import lookup as geo_lookup
+        flows = []
+        src_ip_col = next((c for c in df.columns if 'src_ip' in c.lower() or ':ips' in c.lower()), None)
+        dst_ip_col = next((c for c in df.columns if 'dst_ip' in c.lower() or ':ipd' in c.lower()), None)
+        tls_col = next((c for c in df.columns if c.lower() in ('0ver', 'tls_version', '_tlsver', 'version')), None)
+        bytes_col = next((c for c in df.columns if c.lower() in ('8ack', '8pak', '8byt', 'bytes_sent', 'bytes')), None)
+        for row in df.iter_rows(named=True):
+            sip = str(row.get(src_ip_col, '')) if src_ip_col else ''
+            dip = str(row.get(dst_ip_col, '')) if dst_ip_col else ''
+            if not sip or not dip:
+                continue
+            sg = geo_lookup(sip)
+            dg = geo_lookup(dip)
+            if not sg or not dg:
+                continue
+            tls_val = str(row.get(tls_col, '') or '') if tls_col else ''
+            tls_ver = TLS_HEX_MAP.get(tls_val.replace(' ', ''), tls_val)
+            flows.append({
+                'slat': sg['lat'], 'slon': sg['lon'],
+                'dlat': dg['lat'], 'dlon': dg['lon'],
+                'tls': tls_ver,
+                'bytes': float(row.get(bytes_col, 0) or 0) if bytes_col else 0,
+            })
+        return flows
+    except Exception:
+        logger.error("unknown failed: {}".format(traceback.format_exc()))
+        return []
+
+
+def cluster_detail(request, display_id, cluster_label):
+    """Detailed view of a single cluster with raw data rows, SVD features, and NL summary."""
+    import json
+    import polars as pl
+    run = get_object_or_404(AnalysisRun, display_id=display_id)
+    try:
+        cluster_label = int(cluster_label)
+    except (ValueError, TypeError):
+        raise Http404("Invalid cluster label")
+    cluster = get_object_or_404(ClusterResult, run=run, cluster_label=cluster_label)
+    features = cluster.features.all().order_by('-distinguishing_score')[:50]
+
+    # Load raw data rows for this cluster from EP row indices
+    row_data = []
+    try:
+        from analysis.data_loader import load_csv_directory, load_from_db
+        lf = None
+        if run.sqlite_table:
+            lf = load_from_db(run.sqlite_table)
+        if lf is None and run.csv_glob:
+            lf, _, _, _, _ = load_csv_directory(run.csv_glob)
+        if lf is not None:
+            # Get row indices assigned to this cluster from EntityProfile
+            eps = cluster.entities.all().order_by('entity_value')[:100]
+            indices = []
+            for ep in eps:
+                try:
+                    idx = int(ep.entity_value.replace('row_', ''))
+                    indices.append(idx)
+                except (ValueError, AttributeError):
+                    continue
+            if indices:
+                df = lf.collect(streaming=True)
+                df_filtered = df[indices]
+                # Convert to dict list for template (limit columns for display)
+                cols = df_filtered.columns[:20]  # show first 20 columns
+                for row_idx, row in enumerate(df_filtered.select(cols).iter_rows(named=True)):
+                    entry = {'index': indices[row_idx] if row_idx < len(indices) else row_idx}
+                    for k, v in row.items():
+                        entry[k] = v
+                    row_data.append(entry)
+    except Exception as exc:
+        logger.warning(f'Failed to load raw data for cluster detail: {exc}')
+
+    nl_summary = ''
+    feats_list = [{'feature_name': f.feature_name, 'score': f.distinguishing_score,
+                   'mean': f.mean, 'std': f.std} for f in features[:10]]
+    if feats_list:
+        nl_summary = nl_describe.describe_cluster(feats_list)
+
+    return render(request, 'analysis/cluster_detail.html', {
+        'run': run,
+        'cluster': cluster,
+        'features': features,
+        'row_data': row_data,
+        'nl_summary': nl_summary,
+    })
+
+
+@profile
+def _run_clustering_pipeline(run, store, ds_id, feature_columns, algorithm,
+                              min_cluster_size, run_umap=True, head=None):
+    """Clustering pipeline on raw rows: clustering → feature extraction → UMAP → ORM save.
+
+    Operates directly on raw data rows (no entity aggregation).
+    Per-row results stored in EntityProfile with row index as identifier.
+
+    Args:
+        run: AnalysisRun ORM object.
+        store: SessionStore instance.
+        ds_id: dataset ID in SessionStore pointing to raw data.
+        feature_columns: list of feature column names (None = auto-detect all numeric).
+        algorithm: clustering algorithm.
+        min_cluster_size: int for HDBSCAN.
+        run_umap: bool, whether to compute and save UMAP-2D embeddings.
+        head: optional int, downsample to at most this many rows (min 100).
+    """
+    import warnings
+    warnings.filterwarnings('ignore', category=RuntimeWarning, module='sklearn')
+
+    try:
+        from analysis.tool_registry import _handle_run_clustering, _handle_extract_features
+        import polars as pl
+
+        entry = store.get_dataset(ds_id)
+        if entry is None:
+            run.status = 'failed'
+            run.error_message = '数据集在会话中不存在'
+            run.save(update_fields=['status', 'error_message'])
+            return
+
+        schema = entry.get('schema', {})
+
+        # ── Downsampling ──
+        if head is not None and head > 0:
+            lf = entry['lazyframe']
+            try:
+                row_count = lf.select(pl.len()).collect(streaming=True).item()
+            except Exception:
+                logger.error("unknown failed: {}".format(traceback.format_exc()))
+                row_count = 0
+
+            if row_count > head:
+                head = max(head, 100)  # ensure minimum rows for clustering
+                run.progress_msg = f'正在采样 {head}/{row_count} 行...'
+                run.save(update_fields=['progress_msg'])
+                lf = lf.head(head)
+                store.store_dataset(
+                    ds_id, lf,
+                    schema=schema,
+                    metadata=entry.get('metadata', {}),
+                )
+                entry = store.get_dataset(ds_id)
+                row_count = head
+
+        # Resolve feature columns — use actual LazyFrame dtypes (not stored schema dict
+        # which may be stale after Utf8 coercion in _background_process).
+        numeric_types = (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
+                         pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
+                         pl.Float32, pl.Float64)
+
+        lf = entry['lazyframe']
+        live_schema = lf.collect_schema()
+        live_names = live_schema.names()
+        live_dtypes = list(live_schema.dtypes())
+
+        if feature_columns:
+            feature_cols = [
+                c for c in feature_columns
+                if c in live_names and live_dtypes[live_names.index(c)] in numeric_types
+            ]
+        else:
+            feature_cols = [
+                live_names[i] for i, dt in enumerate(live_dtypes)
+                if dt in numeric_types and not live_names[i].startswith('_')
+            ]
+
+        # ── Step 1: Clustering ────────────────────────────────────────────────
+        # If no numeric columns detected, pass empty list to let
+        # _handle_run_clustering apply its Utf8→Float64 coercion fallback.
+        run.status = 'clustering'
+        run.progress_pct = 70
+        run.progress_msg = '正在进行聚类分析...'
+        run.save(update_fields=['status', 'progress_pct', 'progress_msg'])
+
+        result = asyncio.run(_handle_run_clustering(
+            dataset_id=ds_id,
+            cluster_columns=feature_cols,
+            algorithm=algorithm,
+            params={'min_cluster_size': min_cluster_size},
+            random_state=RANDOM_SEED,
+        ))
+
+        if 'error' in result:
+            err = result['error']
+            if 'numeric' in err.lower():
+                available = [
+                    f"{k}({v})" for k, v in schema.items()
+                    if v.split('(')[0].strip() in numeric_types and not k.startswith('_')
+                ]
+                err += f"。可用数值列: {available}"
+            raise Exception(err)
+
+        cluster_id = result.get('cluster_result_id', '')
+        run.cluster_count = result.get('n_clusters', 0)
+        run.save(update_fields=['cluster_count'])
+
+        # ── Step 2: Feature extraction ──────────────────────────────────────
+        run.status = 'extracting'
+        run.progress_pct = 90
+        run.progress_msg = '正在提取聚类特征...'
+        run.save(update_fields=['status', 'progress_pct', 'progress_msg'])
+
+        feat_result = asyncio.run(_handle_extract_features(
+            dataset_id=ds_id,
+            cluster_result_id=cluster_id,
+            top_k=10, method='zscore', save_to_db=True,
+            run_id=run.id,
+        ))
+
+        if 'error' in feat_result:
+            raise Exception(feat_result['error'])
+
+        # ── Step 2b: Cluster SVD feature extraction ──────────────────────
+        run.progress_msg = '正在进行聚类SVD特征提取...'
+        run.save(update_fields=['progress_msg'])
+        try:
+            from analysis.distance import cluster_svd_extract
+            lf = entry['lazyframe']
+            cluster_entry = store.get_cluster_result(cluster_id)
+            labels_list = cluster_entry.get('labels', []) if cluster_entry else []
+
+            if labels_list and feature_cols:
+                svd_results = cluster_svd_extract(lf, labels_list, feature_cols)
+
+                # Persist SVD features to ClusterFeature (method='cluster_svd')
+                from analysis.models import ClusterFeature as CF
+                from analysis.models import ClusterResult as CR
+                for label, svd_info in svd_results.items():
+                    cr = CR.objects.filter(run=run, cluster_label=label).first()
+                    if cr is None:
+                        continue
+                    # Delete old zscore entries for this cluster, replace with SVD
+                    CF.objects.filter(cluster=cr, distinguishing_method='zscore').delete()
+                    for feat_name in svd_info.get('features', [])[:10]:
+                        idx = svd_info['features'].index(feat_name) if feat_name in svd_info['features'] else -1
+                        strength = (svd_info.get('feature_strength', [])[idx]
+                                    if idx >= 0 and idx < len(svd_info.get('feature_strength', []))
+                                    else None)
+                        CF.objects.get_or_create(
+                            cluster=cr,
+                            feature_name=feat_name,
+                            defaults={
+                                'mean': None, 'std': None, 'median': None,
+                                'p25': None, 'p75': None, 'missing_rate': None,
+                                'distinguishing_score': float(strength) if strength is not None else None,
+                                'distinguishing_method': 'cluster_svd',
+                            },
+                        )
+        except Exception as svd_err:
+            logger.warning(f'Cluster SVD extraction skipped: {svd_err}')
+
+        # ── Step 2c: Save per-row cluster labels to EntityProfile ──
+        # Always persist row→cluster mapping so cluster_detail can show data rows
+        # even when UMAP is skipped or fails.
+        try:
+            cluster_entry_save = store.get_cluster_result(cluster_id)
+            labels_list_save = cluster_entry_save.get('labels', []) if cluster_entry_save else []
+            if labels_list_save:
+                from analysis.models import EntityProfile as EP_save
+                from analysis.models import ClusterResult as CR_save
+                # Delete stale EP records for this run first
+                EP_save.objects.filter(run=run).delete()
+                cr_cache_save = {}
+                row_profiles = []
+                for r_idx in range(len(labels_list_save)):
+                    lbl = labels_list_save[r_idx]
+                    ck = f'{run.id}_{lbl}'
+                    if ck not in cr_cache_save:
+                        cr_cache_save[ck] = CR_save.objects.filter(run=run, cluster_label=lbl).first()
+                    row_profiles.append(EP_save(
+                        entity_value=f'row_{r_idx}', run=run, cluster_label=lbl,
+                        cluster=cr_cache_save[ck],
+                        embedding_x=None, embedding_y=None, embedding_z=None,
+                    ))
+                if row_profiles:
+                    EP_save.objects.bulk_create(row_profiles, ignore_conflicts=True, batch_size=1000)
+                    run.entity_count = len(row_profiles)
+                    run.save(update_fields=['entity_count'])
+        except Exception as save_err:
+            logger.warning(f'Row labels save skipped: {save_err}')
+
+        # ── Step 3: UMAP-3D embedding (2D fallback) ───────────────────────
+        if run_umap:
+            try:
+                lf = entry['lazyframe']
+                # UMAP: sample max 10K for training, batch-transform rest in 1K batches
+                MAX_UMAP_TRAIN = UMAP_TRAIN_SAMPLE
+                BATCH_SIZE = UMAP_BATCH_SIZE
+                try:
+                    n_total = lf.select(pl.len()).collect(streaming=True).item()
+                except Exception:
+                    logger.error("unknown failed: {}".format(traceback.format_exc()))
+                    n_total = 0
+                from sklearn.preprocessing import StandardScaler
+                import umap
+                import numpy as np
+
+                live_schema = lf.collect_schema()
+                num_cols = [name for name, dt in zip(live_schema.names(), live_schema.dtypes())
+                            if dt in (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
+                                      pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
+                                      pl.Float32, pl.Float64) and not name.startswith('_')]
+
+                if len(num_cols) >= 2:
+                    if n_total > MAX_UMAP_TRAIN:
+                        run.progress_msg = f'UMAP 采样 {MAX_UMAP_TRAIN}/{n_total} 行训练...'
+                        run.save(update_fields=['progress_msg'])
+                        # LazyFrame.sample() is unavailable in Polars 1.42 — collect then sample
+                        df_sample = df_umap.sample(n=MAX_UMAP_TRAIN, seed=RANDOM_SEED)
+                        df_sample = df_umap
+
+                    umap_components = 3  # try 3D first
+                    coords = None
+                    reducer = None
+
+                    if len(df_umap) > 2:
+                        for attempt_n in (3, 2):
+                            try:
+                                if n_total > MAX_UMAP_TRAIN:
+                                    # Train UMAP on 10K sample, batch-transform full dataset
+                                    mat_sample = df_sample.select(num_cols).to_numpy()
+                                    scaler = StandardScaler().fit(mat_sample)
+                                    mat_sample_scaled = np.nan_to_num(
+                                        scaler.transform(mat_sample), nan=0.0)
+                                    reducer = umap.UMAP(n_components=attempt_n,
+                                                        random_state=RANDOM_SEED,
+                                                        n_neighbors=15, min_dist=0.1,
+                                                        metric='euclidean')
+                                    reducer.fit(mat_sample_scaled)
+                                    # Batch transform in 1K-row batches
+                                    coords_list = []
+                                    for start in range(0, len(df_umap), BATCH_SIZE):
+                                        end = min(start + BATCH_SIZE, len(df_umap))
+                                        mat_batch = scaler.transform(
+                                            df_umap[start:end].select(num_cols).to_numpy())
+                                        mat_batch = np.nan_to_num(mat_batch, nan=0.0)
+                                        coords_list.append(reducer.transform(mat_batch))
+                                    coords = np.vstack(coords_list)
+                                else:
+                                    mat = np.nan_to_num(StandardScaler().fit_transform(
+                                        df_umap.select(num_cols).to_numpy()), nan=0.0)
+                                    reducer = umap.UMAP(n_components=attempt_n,
+                                                        random_state=RANDOM_SEED,
+                                                        n_neighbors=15, min_dist=0.1,
+                                                        metric='euclidean')
+                                    coords = reducer.fit_transform(mat)
+                                umap_components = attempt_n
+                                break
+                            except Exception as e_umap:
+                                if attempt_n == 2:
+                                    raise
+                                logger.warning(f'UMAP 3D failed ({e_umap}), falling back to 2D')
+
+                        if coords is None:
+                            raise Exception('UMAP produced no output')
+
+                        cluster_entry = store.get_cluster_result(cluster_id)
+                        labels_list = cluster_entry.get('labels', []) if cluster_entry else []
+
+                        from analysis.models import EntityProfile as EP
+
+                        # Update existing EP records with UMAP coordinates
+                        for i in range(len(df_umap)):
+                            ev = f'row_{i}'
+                            has_z = umap_components >= 3 and coords.shape[1] >= 3
+                            update_fields = {
+                                'embedding_x': float(coords[i, 0]) if i < len(coords) else None,
+                                'embedding_y': float(coords[i, 1]) if i < len(coords) else None,
+                            }
+                            if has_z:
+                                update_fields['embedding_z'] = float(coords[i, 2]) if i < len(coords) else None
+                            EP.objects.filter(run=run, entity_value=ev).update(**update_fields)
+                        run.entity_count = len(df_umap)
+                        run.save(update_fields=['entity_count'])
+            except Exception as umap_err:
+                logger.warning(f'UMAP embedding skipped: {umap_err}')
+
+        # ── Done ──
+        run.status = 'completed'
+        run.progress_pct = 100
+        run.progress_msg = '分析完成'
+        run.save(update_fields=['status', 'progress_pct', 'progress_msg', 'entity_count'])
+
+    except Exception:
+        tb = traceback.format_exc()
+        logger.error(tb)
+        try:
+            run.status = 'failed'
+            run.error_message = tb
+            run.progress_msg = f'失败: {tb}'
+            run.run_log += f'\n[ERROR] {tb}'
+            run.save(update_fields=['status', 'error_message', 'progress_msg', 'run_log'])
+        except Exception as e:
+            logger.warning('save failed after pipeline error: %s', e)

analysis/views/config.py

@@ -0,0 +1,132 @@
+"""Configuration views: config editor and LLM connectivity test."""
+import traceback
+import json
+import time
+import urllib.request
+import urllib.error
+import logging
+
+from django.shortcuts import render
+from django.http import JsonResponse
+from django.views.decorators.csrf import csrf_exempt
+
+from config import get_config, save_config, Config
+
+logger = logging.getLogger(__name__)
+
+
+def config_view(request):
+    """Config page: GET renders current config, POST saves updates."""
+    cfg = get_config()
+
+    if request.method == 'POST':
+        # Build a new Config from POST data (fall back to current values)
+        server = Config.Server(
+            host=request.POST.get('server_host', cfg.server.host),
+            port=int(request.POST.get('server_port', cfg.server.port)),
+            debug=request.POST.get('server_debug') == 'true',
+        )
+        data = Config.Data(
+            schema_strict=request.POST.get('data_schema_strict') == 'true',
+            recursive=request.POST.get('data_recursive') == 'true',
+        )
+        clustering = Config.Clustering(
+            algorithm=request.POST.get('clustering_algorithm', cfg.clustering.algorithm),
+            min_cluster_size=int(request.POST.get('clustering_min_cluster_size', cfg.clustering.min_cluster_size)),
+            random_state=cfg.clustering.random_state,
+        )
+        llm = Config.LLM(
+            enabled=request.POST.get('llm_enabled') == 'true',
+            base_url=request.POST.get('llm_base_url', cfg.llm.base_url),
+            api_key=request.POST.get('llm_api_key', cfg.llm.api_key),
+            model=request.POST.get('llm_model', cfg.llm.model),
+        )
+        cfg = Config(server=server, data=data, clustering=clustering, llm=llm)
+        save_config(cfg)
+        # Re-render with saved flag
+        return render(request, 'tianxuan/config.html', {'config': cfg, 'saved': True})
+
+    return render(request, 'tianxuan/config.html', {'config': cfg})
+
+
+@csrf_exempt
+def llm_test(request):
+    """LLM connectivity test: POST JSON {base_url, api_key, model} → chat/completions."""
+    if request.method != 'POST':
+        return JsonResponse({'success': False, 'message': '仅支持 POST 请求', 'latency_ms': 0}, status=405)
+
+    try:
+        body = json.loads(request.body)
+    except json.JSONDecodeError:
+        return JsonResponse({'success': False, 'message': '无效的 JSON 请求体', 'latency_ms': 0})
+
+    base_url = (body.get('base_url') or '').rstrip('/')
+    api_key = body.get('api_key') or ''
+    model = body.get('model') or ''
+
+    if not base_url:
+        return JsonResponse({'success': False, 'message': 'Base URL 不能为空', 'latency_ms': 0})
+    if not api_key:
+        return JsonResponse({'success': False, 'message': 'API Key 不能为空', 'latency_ms': 0})
+    if not model:
+        return JsonResponse({'success': False, 'message': 'Model 不能为空', 'latency_ms': 0})
+
+    url = f'{base_url}/chat/completions'
+    payload = json.dumps({
+        'model': model,
+        'messages': [{'role': 'user', 'content': 'respond with ok'}],
+        'max_tokens': 5,
+    }).encode('utf-8')
+
+    req = urllib.request.Request(
+        url,
+        data=payload,
+        headers={
+            'Content-Type': 'application/json',
+            'Authorization': f'Bearer {api_key}',
+        },
+        method='POST',
+    )
+
+    start = time.monotonic()
+    try:
+        with urllib.request.urlopen(req, timeout=5) as resp:
+            elapsed = int((time.monotonic() - start) * 1000)
+            data = json.loads(resp.read().decode('utf-8'))
+            if 'choices' in data and len(data['choices']) > 0:
+                logger.info(f'[LLM_TEST] base_url={base_url} model={model} success=True latency={elapsed}ms')
+                return JsonResponse({
+                    'success': True,
+                    'message': f'连接成功，模型 {model} 返回正常',
+                    'latency_ms': elapsed,
+                })
+            else:
+                logger.info(f'[LLM_TEST] base_url={base_url} model={model} success=True latency={elapsed}ms')
+                return JsonResponse({
+                    'success': True,
+                    'message': f'连接成功，但响应中无 choices（原始响应已记录）',
+                    'latency_ms': elapsed,
+                })
+    except urllib.error.HTTPError as e:
+        elapsed = int((time.monotonic() - start) * 1000)
+        try:
+            detail = json.loads(e.read().decode('utf-8', errors='replace'))
+            msg = detail.get('error', {}).get('message', str(e))
+        except Exception:
+            logger.error("unknown failed: {}".format(traceback.format_exc()))
+            msg = str(e)
+        logger.info(f'[LLM_TEST] base_url={base_url} model={model} success=False latency={elapsed}ms')
+        return JsonResponse({'success': False, 'message': f'HTTP {e.code}: {msg}', 'latency_ms': elapsed})
+    except urllib.error.URLError as e:
+        elapsed = int((time.monotonic() - start) * 1000)
+        reason = str(e.reason) if e.reason else '连接失败'
+        logger.info(f'[LLM_TEST] base_url={base_url} model={model} success=False latency={elapsed}ms')
+        return JsonResponse({'success': False, 'message': f'连接失败: {reason}', 'latency_ms': elapsed})
+    except TimeoutError:
+        elapsed = int((time.monotonic() - start) * 1000)
+        logger.info(f'[LLM_TEST] base_url={base_url} model={model} success=False latency={elapsed}ms')
+        return JsonResponse({'success': False, 'message': '请求超时（5秒）', 'latency_ms': elapsed})
+    except Exception as e:
+        elapsed = int((time.monotonic() - start) * 1000)
+        logger.info(f'[LLM_TEST] base_url={base_url} model={model} success=False latency={elapsed}ms')
+        return JsonResponse({'success': False, 'message': f'未知错误: {str(e)}', 'latency_ms': elapsed})

analysis/views/dashboard.py

@@ -0,0 +1,131 @@
+"""Dashboard views: home, run list, run detail, run status."""
+from django.shortcuts import render, get_object_or_404
+from django.http import JsonResponse
+
+from analysis.models import AnalysisRun
+
+
+def dashboard(request):
+    """Home page: recent runs overview."""
+    runs = AnalysisRun.objects.all()[:20]
+    return render(request, 'analysis/dashboard.html', {'runs': runs})
+
+
+def run_list(request):
+    """List all analysis runs with pagination and optional ?type= filter."""
+    run_type_filter = request.GET.get('type')
+    page = int(request.GET.get('page', 1))
+    per_page = 20
+
+    qs = AnalysisRun.objects.all()
+    if run_type_filter in dict(AnalysisRun.RUN_TYPE_CHOICES):
+        qs = qs.filter(run_type=run_type_filter)
+
+    total = qs.count()
+    runs = qs[(page - 1) * per_page: page * per_page]
+    return render(request, 'analysis/run_list.html', {
+        'runs': runs,
+        'page': page,
+        'total_pages': (total + per_page - 1) // per_page if total else 1,
+        'current_type': run_type_filter or '',
+    })
+
+
+def run_detail(request, display_id):
+    """Details for a single analysis run."""
+    import json
+    run = get_object_or_404(AnalysisRun, display_id=display_id)
+    clusters = run.clusters.all().order_by('-size')
+
+    # ── Global SVD features (compute on-demand from the stored run data) ──
+    svd_features = []
+    svd_explained_var = []
+    try:
+        from analysis.data_loader import load_csv_directory, load_from_db
+        import polars as pl
+        from sklearn.decomposition import TruncatedSVD
+        import numpy as np
+        import logging
+        logger = logging.getLogger(__name__)
+
+        lf = None
+        if run.sqlite_table:
+            lf = load_from_db(run.sqlite_table)
+        if lf is None and run.csv_glob:
+            lf, _, _, _, _ = load_csv_directory(run.csv_glob)
+
+        if lf is not None:
+            schema = lf.collect_schema()
+            numeric_types = (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
+                             pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
+                             pl.Float32, pl.Float64)
+            num_cols = [n for n, dt in zip(schema.names(), schema.dtypes()) if dt in numeric_types and not n.startswith('_')]
+
+            if len(num_cols) >= 3:
+                n_rows_lf = lf.select(pl.len()).collect(streaming=True).item()
+                sample = min(int(n_rows_lf), 50000)
+                df = lf.sample(n=sample, seed=42).select(num_cols).collect(streaming=True) if n_rows_lf > sample else lf.select(num_cols).collect(streaming=True)
+                mat = np.nan_to_num(df.to_numpy(), nan=0.0)
+                from sklearn.preprocessing import StandardScaler
+                mat = StandardScaler().fit_transform(mat)
+
+                n_comp = min(20, min(mat.shape) - 1)
+                if n_comp >= 2:
+                    svd = TruncatedSVD(n_components=n_comp, random_state=42)
+                    svd.fit(mat)
+                    var_ratio = svd.explained_variance_ratio_.tolist()
+                    svd_explained_var = var_ratio
+                    for comp_idx in range(min(10, n_comp)):
+                        loadings = np.abs(svd.components_[comp_idx])
+                        top_idx = np.argsort(loadings)[-5:][::-1]
+                        features = [{'feature': num_cols[i], 'strength': round(float(loadings[i]), 4)} for i in top_idx]
+                        svd_features.append({
+                            'component': comp_idx + 1,
+                            'explained_var': round(var_ratio[comp_idx] * 100, 2),
+                            'cumulative_var': round(sum(var_ratio[:comp_idx + 1]) * 100, 2),
+                            'top_features': features,
+                        })
+    except Exception as exc:
+        logger = logging.getLogger(__name__)
+        logger.warning(f'SVD feature extraction failed for run {run.display_id}: {exc}')
+
+    # ── LLM workflow timeline ──
+    llm_timeline_json = ''
+    if run.run_type == 'auto':
+        llm_thinking = run.llm_thinking or ''
+        tool_calls = run.tool_calls_json or []
+        if tool_calls or llm_thinking:
+            llm_timeline_json = json.dumps({
+                'thinking': llm_thinking,
+                'tool_calls': tool_calls,
+            })
+
+    return render(request, 'analysis/run_detail.html', {
+        'run': run,
+        'clusters': clusters,
+        'svd_features': svd_features,
+        'svd_explained_var': svd_explained_var,
+        'llm_timeline_json': llm_timeline_json,
+    })
+
+
+def run_status_api(request, display_id):
+    """Return current run status as JSON."""
+    run = get_object_or_404(AnalysisRun, display_id=display_id)
+    return JsonResponse({
+        'id': run.display_id,
+        'status': run.status,
+        'entity_count': run.entity_count,
+        'cluster_count': run.cluster_count,
+        'error_message': run.error_message,
+        'progress_pct': run.progress_pct,
+        'progress_msg': run.progress_msg,
+        'run_log': run.run_log[-2000:],  # last 2000 chars
+        'llm_thinking': run.llm_thinking[-50000:] if run.llm_thinking else '',
+        'tool_calls': run.tool_calls_json,
+    })
+
+
+def start_analysis(request):
+    """Placeholder: API endpoint to trigger analysis via MCP."""
+    return JsonResponse({'status': 'not_implemented', 'message': 'Use MCP tools to trigger analysis'})

analysis/views/entity.py

@@ -0,0 +1,33 @@
+"""Entity profile view."""
+import json
+
+from django.shortcuts import render, get_object_or_404
+
+from analysis.models import EntityProfile
+
+
+def entity_profile(request, entity_id):
+    """Detailed profile for a single entity."""
+    entity = get_object_or_404(EntityProfile, id=entity_id)
+    run = entity.run
+
+    # Compute deviation from cluster mean
+    deviations = {}
+    if entity.feature_json and entity.cluster:
+        cluster_features = {
+            f.feature_name: {'mean': f.mean, 'std': f.std}
+            for f in entity.cluster.features.all()
+        }
+        for col, val in entity.feature_json.items():
+            if col in cluster_features and cluster_features[col].get('std') and cluster_features[col]['std'] > 1e-10:
+                deviations[col] = {
+                    'value': val,
+                    'cluster_mean': cluster_features[col]['mean'],
+                    'zscore': (val - cluster_features[col]['mean']) / cluster_features[col]['std'],
+                }
+
+    return render(request, 'analysis/entity_profile.html', {
+        'entity': entity,
+        'run': run,
+        'deviations': deviations,
+    })

analysis/views/globe.py

@@ -0,0 +1,220 @@
+"""3D Globe view and flow extraction."""
+import json
+import logging
+import traceback
+
+from django.shortcuts import render
+from django.views.decorators.clickjacking import xframe_options_exempt
+
+from analysis.constants import GLOBE_MAX_ROWS
+from analysis.models import AnalysisRun
+from .helpers import _extract_lat, _extract_lon
+
+logger = logging.getLogger(__name__)
+
+
+def _extract_flows_from_df(df, MAX_ROWS):
+    """Extract flow data (lat/lon/TLS/bytes) from a Polars DataFrame."""
+    from analysis.geoip import lookup as geo_lookup
+    from analysis.type_classifier import TLS_HEX_MAP
+
+    src_lat_col = next((c for c in df.columns if c.lower() in (
+        ':ips.latd', ':ips_latd', ':ips_lat', 'src_latitude', 'src_lat', 'source_latitude', 'source_lat')), None)
+    src_lon_col = next((c for c in df.columns if c.lower() in (
+        ':ips.lond', ':ips_lond', ':ips_lon', 'src_longitude', 'src_lon', 'source_longitude', 'source_lon')), None)
+    dst_lat_col = next((c for c in df.columns if c.lower() in (
+        ':ipd.latd', ':ipd_latd', ':ipd_lat', 'dst_latitude', 'dst_lat', 'dest_latitude', 'dest_lat')), None)
+    dst_lon_col = next((c for c in df.columns if c.lower() in (
+        ':ipd.lond', ':ipd_lond', ':ipd_lon', 'dst_longitude', 'dst_lon', 'dest_longitude', 'dest_lon')), None)
+    src_ip_col = next((c for c in df.columns if 'src_ip' in c.lower() or ':ips' in c.lower()), None)
+    dst_ip_col = next((c for c in df.columns if 'dst_ip' in c.lower() or ':ipd' in c.lower()), None)
+    tls_col = next((c for c in df.columns if c.lower() in ('0ver', 'tls_version', '_tlsver', 'version')), None)
+    bytes_col = next((c for c in df.columns if c.lower() in ('8ack', '8pak', '8byt', 'bytes_sent', 'bytes')), None)
+    time_col = next((c for c in df.columns if c.lower() in (
+        'timestamp', 'time', '_time', 'ts', 'datetime', 'epoch',
+        'created_at', 'created', 'event_time')), None)
+
+    flows = []
+    for row in df.iter_rows(named=True):
+        slat = slon = dlat = dlon = None
+
+        if src_lat_col and src_lon_col:
+            slat = _extract_lat(row.get(src_lat_col))
+            slon = _extract_lon(row.get(src_lon_col))
+        if dst_lat_col and dst_lon_col:
+            dlat = _extract_lat(row.get(dst_lat_col))
+            dlon = _extract_lon(row.get(dst_lon_col))
+
+        if (slat is None or slon is None) and src_ip_col:
+            sg = geo_lookup(str(row.get(src_ip_col, '')))
+            if sg:
+                slat, slon = sg['lat'], sg['lon']
+        if (dlat is None or dlon is None) and dst_ip_col:
+            dg = geo_lookup(str(row.get(dst_ip_col, '')))
+            if dg:
+                dlat, dlon = dg['lat'], dg['lon']
+
+        if slat is None or slon is None or dlat is None or dlon is None:
+            continue
+
+        tls_val = str(row.get(tls_col or '', ''))
+        # Check hex format first (e.g. "03 03" or "0303")
+        hex_key = tls_val.replace(' ', '').strip()
+        if hex_key in TLS_HEX_MAP:
+            tls_label = TLS_HEX_MAP[hex_key]
+        elif '1.3' in tls_val:
+            tls_label = 'TLSv1.3'
+        elif '1.2' in tls_val:
+            tls_label = 'TLSv1.2'
+        else:
+            tls_label = 'other'
+
+        bs = float(row.get(bytes_col or '', 0) or 0)
+
+        # Extract timestamp for animation ordering
+        t_raw = row.get(time_col) if time_col else None
+        t_val = None
+        if t_raw is not None:
+            if isinstance(t_raw, (int, float)):
+                t_val = float(t_raw)
+            elif hasattr(t_raw, 'timestamp'):
+                t_val = t_raw.timestamp()
+            else:
+                try:
+                    t_val = float(t_raw)
+                except (ValueError, TypeError):
+                    pass
+
+        flows.append({
+            'slat': slat, 'slon': slon,
+            'dlat': dlat, 'dlon': dlon,
+            'tls': tls_label,
+            'bytes': bs,
+            'time': t_val,
+        })
+
+    # Normalize timestamps to 0-1 and sort flows by time
+    timed = [f for f in flows if f['time'] is not None]
+    if len(timed) > 1:
+        timed.sort(key=lambda f: f['time'])
+        t_min, t_max = timed[0]['time'], timed[-1]['time']
+        t_range = max(t_max - t_min, 1)
+        for f in timed:
+            f['time'] = (f['time'] - t_min) / t_range
+        # Untimed flows get evenly spaced after timed ones
+        untimed = [f for f in flows if f['time'] is None]
+        for i, f in enumerate(untimed):
+            f['time'] = min(1.0, (len(timed) + i) / max(len(flows) - 1, 1))
+        flows = timed + untimed
+    else:
+        for i, f in enumerate(flows):
+            f['time'] = i / max(len(flows) - 1, 1)
+
+    return flows
+
+
+from django.views.decorators.clickjacking import xframe_options_exempt
+
+
+@xframe_options_exempt
+def globe_view(request):
+    from analysis.geoip import lookup as geo_lookup
+    from analysis.data_loader import load_csv_directory, load_from_db
+    from analysis.models import AnalysisRun
+    from analysis.session_store import SessionStore
+
+    MAX_ROWS_PER_RUN = GLOBE_MAX_ROWS  # Limit per run to prevent browser overload
+
+    # Check for ?data=dataset_id parameter (direct data loading from session store)
+    data_param = request.GET.get('data')
+    embed_mode = request.GET.get('embed') == '1'
+    template_name = 'tianxuan/globe_embed.html' if embed_mode else 'tianxuan/globe.html'
+
+    if data_param:
+        store = SessionStore()
+        flows = []
+        all_runs = list(AnalysisRun.objects.filter(status='completed').order_by('-id').values('display_id', 'csv_glob'))
+        try:
+            entry = store.get_dataset(data_param)
+            if entry is not None:
+                lf = entry['lazyframe']
+                df = lf.head(MAX_ROWS_PER_RUN).collect(streaming=True)
+                flows = _extract_flows_from_df(df, MAX_ROWS_PER_RUN)
+        except Exception as exc:
+            logging.getLogger(__name__).warning(f'globe ?data= param failed: {exc}')
+        return render(request, template_name, {
+            'geo_flows': json.dumps(flows),
+            'all_runs': all_runs,
+            'selected_ids': [],
+        })
+
+    # Get all completed runs for the selector
+    all_runs = list(AnalysisRun.objects.filter(status='completed').order_by('-id').values('display_id', 'csv_glob'))
+
+    # Get selected run IDs from query params (multiple checkboxes)
+    selected_ids_list = request.GET.getlist('runs')
+    selected_runs = []
+    if selected_ids_list:
+        for sid in selected_ids_list:
+            sid = sid.strip()
+            if sid.isdigit():
+                try:
+                    r = AnalysisRun.objects.get(display_id=int(sid), status='completed')
+                    selected_runs.append(r)
+                except AnalysisRun.DoesNotExist:
+                    pass
+
+    # Check if user-selected runs have valid files; if not, fall through
+    all_selected_failed = bool(selected_ids_list)
+    fallback_run = None
+
+    if not selected_runs:
+        all_selected_failed = False
+
+    flows = []
+
+    for run in selected_runs:
+        try:
+            lf = None
+            if run.sqlite_table:
+                lf = load_from_db(run.sqlite_table)
+            if lf is None:
+                lf, schema, rc, fc, mem = load_csv_directory(run.csv_glob)
+            df = lf.head(MAX_ROWS_PER_RUN).collect(streaming=True)
+            flows.extend(_extract_flows_from_df(df, MAX_ROWS_PER_RUN))
+            all_selected_failed = False
+            break  # stop after first successful load
+        except Exception as exc:
+            logger.warning('[GLOBE] Failed to load run %s (%s): %s', run.display_id, run.csv_glob, exc)
+            continue
+
+    if all_selected_failed or not flows:
+        # Fallback: latest completed run with valid data (SQLite or CSV)
+        for latest in AnalysisRun.objects.filter(status='completed').order_by('-id'):
+            if not latest.csv_glob and not latest.sqlite_table:
+                continue
+            try:
+                lf = None
+                if latest.sqlite_table:
+                    lf = load_from_db(latest.sqlite_table)
+                if lf is None:
+                    if not latest.csv_glob:
+                        continue
+                    lf, schema, rc, fc, mem = load_csv_directory(latest.csv_glob)
+                df = lf.head(MAX_ROWS_PER_RUN).collect(streaming=True)
+                flows.extend(_extract_flows_from_df(df, MAX_ROWS_PER_RUN))
+                selected_runs = [latest]
+                break
+            except Exception:
+                logger.error("unknown failed: {}".format(traceback.format_exc()))
+                continue
+
+    resp = render(request, 'tianxuan/globe.html', {
+        'geo_flows': json.dumps(flows),
+        'all_runs': all_runs,
+        'selected_ids': [r.display_id for r in selected_runs],
+    })
+    resp['Cache-Control'] = 'no-cache, no-store, must-revalidate'
+    resp['Pragma'] = 'no-cache'
+    resp['Expires'] = '0'
+    return resp

analysis/views/helpers.py

@@ -0,0 +1,75 @@
+"""Geo extraction helpers and plan utilities."""
+import traceback
+from typing import Optional
+from pathlib import Path
+import json as _json
+import logging
+logger = logging.getLogger(__name__)
+
+
+def _extract_lat(val):
+    """Try to extract a latitude from *val* (number or str)."""
+    if val is None:
+        return None
+    try:
+        v = float(val)
+        return None if v < -90 or v > 90 else v
+    except (ValueError, TypeError):
+        return None
+
+
+def _extract_lon(val):
+    """Try to extract a longitude from *val* (number or str)."""
+    if val is None:
+        return None
+    try:
+        v = float(val)
+        return None if v < -180 or v > 180 else v
+    except (ValueError, TypeError):
+        return None
+
+
+# ── Plan index utilities (shared by tools.py and auto.py) ─────────────
+
+_PLANS_DIR = Path(__file__).resolve().parent.parent.parent / '.omo' / 'plans'
+_PLANS_DIR.mkdir(parents=True, exist_ok=True)
+
+_AUTO_INDEX_PATH = _PLANS_DIR / '_auto_index.json'
+_MAX_AUTO_PLANS = 10
+
+
+def _read_auto_index():
+    """Read the auto index file, returning list of {file, pinned}."""
+    if not _AUTO_INDEX_PATH.exists():
+        return []
+    try:
+        data = _json.loads(_AUTO_INDEX_PATH.read_text(encoding='utf-8'))
+        if not isinstance(data, list):
+            return []
+        return data
+    except Exception:
+        logger.error("_read_auto_index failed: {}".format(traceback.format_exc()))
+        return []
+
+
+def _write_auto_index(index):
+    """Write the auto index list."""
+    _AUTO_INDEX_PATH.write_text(
+        _json.dumps(index, ensure_ascii=False, indent=2),
+        encoding='utf-8',
+    )
+
+
+def _add_to_auto_index(filename):
+    """Prepend filename to auto index, evict unpinned entries when >10."""
+    index = _read_auto_index()
+    # Remove if already present
+    index = [entry for entry in index if entry.get('file') != filename]
+    # Prepend
+    index.insert(0, {'file': filename, 'pinned': False})
+    # Evict unpinned entries beyond max
+    pinned_entries = [e for e in index if e.get('pinned')]
+    unpinned_entries = [e for e in index if not e.get('pinned')]
+    # Keep all pinned + fill remaining slots with unpinned
+    result = pinned_entries + unpinned_entries[:_MAX_AUTO_PLANS - len(pinned_entries)]
+    _write_auto_index(result)

analysis/views/log_viewer.py

@@ -0,0 +1,36 @@
+"""Log viewer view."""
+from django.shortcuts import render
+from django.http import HttpResponse
+
+
+def log_viewer(request):
+    """Display last 200 lines of the app log (DEBUG only).
+    With ?pos=N, returns text: line_count\\nnew_content for AJAX polling.
+    """
+    from django.conf import settings
+    if not settings.DEBUG:
+        return HttpResponse('Log viewer only available in DEBUG mode', status=403)
+    log_path = settings.BASE_DIR / 'logs' / 'tianxuan.log'
+    pos = request.GET.get('pos')
+    if pos is not None:
+        # AJAX polling mode: return new content since byte position
+        try:
+            pos = int(pos)
+        except ValueError:
+            pos = 0
+        if log_path.exists():
+            with open(log_path, 'r', encoding='utf-8') as f:
+                total = f.seek(0, 2)  # seek to end
+                if pos >= total:
+                    return HttpResponse(f'{total}\n', content_type='text/plain')
+                f.seek(pos)
+                new_content = f.read()
+                return HttpResponse(f'{total}\n{new_content}', content_type='text/plain')
+        return HttpResponse('0\n', content_type='text/plain')
+
+    # HTML page mode
+    lines = []
+    if log_path.exists():
+        with open(log_path, 'r', encoding='utf-8') as f:
+            lines = f.readlines()[-200:]
+    return render(request, 'tianxuan/log_viewer.html', {'lines': lines})

analysis/views/manual.py

@@ -0,0 +1,191 @@
+"""Manual analysis page: workflow builder and run execution."""
+import json
+import threading
+import traceback
+import logging
+
+from django.shortcuts import render, get_object_or_404
+from django.http import JsonResponse
+from django.views.decorators.csrf import csrf_exempt
+
+from analysis.models import AnalysisRun
+from .pipeline import _run_pipeline_worker
+from .clustering import _run_clustering_pipeline
+
+logger = logging.getLogger(__name__)
+
+
+def manual_page(request):
+    """Workflow builder: select dataset, add MCP tool steps, execute."""
+    from analysis.session_store import SessionStore
+    from analysis.tool_registry import get_tools_meta
+    import json as _json
+
+    # Get all tools metadata
+    tools = get_tools_meta()
+    core_names = {'profile_data', 'run_clustering', 'extract_features',
+                  'filter_data', 'preprocess_data',
+                  'detect_anomalies', 'visualize_anomalies'}
+    diag_names = {'validate_data', 'explore_distributions', 'find_outliers',
+                  'diagnose_clustering', 'compare_datasets', 'export_debug_sample',
+                  'repair_schema'}
+    analysis_names = {'analyze_patterns', 'analyze_temporal', 'analyze_tls_health',
+                      'analyze_geo_distribution', 'analyze_entity_detail'}
+    core_tools, diag_tools, analysis_tools, other_tools = [], [], [], []
+    for t in tools:
+        if t.name in core_names: core_tools.append(t)
+        elif t.name in diag_names: diag_tools.append(t)
+        elif t.name in analysis_names: analysis_tools.append(t)
+        else: other_tools.append(t)
+
+    meta_list = [{'name': t.name, 'description': t.description, 'inputSchema': t.inputSchema} for t in tools]
+
+    # Get datasets from SessionStore
+    store = SessionStore()
+    session_datasets = store.list_datasets() if hasattr(store, 'list_datasets') else []
+    session_ds_map = {ds['dataset_id']: ds for ds in session_datasets}
+
+    # Build dataset entries from ALL AnalysisRun records (upload/manual/auto, any status)
+    from analysis.models import AnalysisRun
+    all_runs = AnalysisRun.objects.all().order_by('-created_at')[:100]
+    datasets_with_status = []
+
+    for run in all_runs:
+        # Determine how the dataset was keyed in SessionStore
+        if run.run_type == 'upload':
+            ds_id = f'upload_{run.display_id}'
+        else:
+            ds_id = f'run_{run.display_id}'
+
+        session_entry = session_ds_map.get(ds_id)
+
+        if session_entry:
+            # Full data available in SessionStore — merge with run status
+            datasets_with_status.append({
+                **session_entry,
+                'dataset_id': ds_id,
+                'run_status': run.status,
+                'total_flows': run.total_flows,
+                'display_id': run.display_id,
+                'needs_reload': False,
+            })
+        elif run.csv_glob or run.sqlite_table:
+            # Data not in SessionStore but recoverable from disk/DB
+            datasets_with_status.append({
+                'dataset_id': ds_id,
+                'run_status': run.status,
+                'display_id': run.display_id,
+                'total_flows': run.total_flows,
+                'row_count': run.total_flows,
+                'file_count': None,
+                'column_count': 0,
+                'columns': [],
+                'svd_components': 0,
+                'needs_reload': True,
+                'csv_glob': run.csv_glob,
+                'sqlite_table': run.sqlite_table or '',
+            })
+
+    # Map statuses to user-facing labels for the filter tabs
+    # ready → 待分析, completed → 已分析, failed → 错误
+    # Everything else (pending/loading/profiling/aggregating/clustering/extracting) → analyzing → 分析中
+    for ds_info in datasets_with_status:
+        s = ds_info['run_status']
+        if s in ('completed',):
+            ds_info['status_label'] = '已分析'
+            ds_info['badge_class'] = 'completed'
+        elif s in ('failed',):
+            ds_info['status_label'] = '错误'
+            ds_info['badge_class'] = 'failed'
+        elif s in ('pending', 'loading', 'profiling', 'aggregating', 'clustering', 'extracting'):
+            ds_info['status_label'] = '分析中'
+            ds_info['badge_class'] = 'analyzing'
+        else:
+            ds_info['status_label'] = '待分析'
+            ds_info['badge_class'] = 'ready'
+
+    return render(request, 'tianxuan/manual.html', {
+        'core_tools': core_tools,
+        'diag_tools': diag_tools,
+        'analysis_tools': analysis_tools,
+        'other_tools': other_tools,
+        'tools_meta': _json.dumps(meta_list, ensure_ascii=False),
+        'datasets': datasets_with_status,
+        'datasets_with_status_json': _json.dumps(datasets_with_status, ensure_ascii=False),
+    })
+
+
+@csrf_exempt
+def manual_run_analysis(request):
+    """AJAX POST: run clustering pipeline on the upload dataset."""
+    if request.method != 'POST':
+        return JsonResponse({'error': 'POST required'}, status=405)
+    import json as _json
+    body = _json.loads(request.body)
+    display_id = body.get('run_id')
+    run = get_object_or_404(AnalysisRun, display_id=display_id)
+    pk = run.id
+    run.run_type = 'manual'
+    run.save(update_fields=['run_type'])
+    feature_columns = body.get('feature_columns')
+    algorithm = body.get('algorithm', 'agglomerative')
+    min_cluster_size = int(body.get('min_cluster_size', 5))
+    head = body.get('head')
+    cluster_mode = body.get('cluster_mode', 'raw')
+
+    def _analysis_fn(run, ctx):
+        from analysis.session_store import SessionStore
+
+        feature_columns = ctx['feature_columns']
+        algorithm = ctx['algorithm']
+        min_cluster_size = ctx['min_cluster_size']
+        head = ctx.get('head')
+        pk = ctx['pk']
+        cluster_mode = ctx.get('cluster_mode', 'raw')
+
+        store = SessionStore()
+        try:
+            # Use display_id for dataset key (consistent with _background_process)
+            display_id = ctx.get('display_id', pk)
+            upload_ds_id = f'upload_{display_id}'
+            entry = store.get_dataset(upload_ds_id)
+            if entry is None:
+                # Fallback: try PK-based key (old pipeline)
+                entry = store.get_dataset(f'upload_{pk}')
+            if entry is None:
+                # Fallback: try entity dataset (from older pipeline)
+                entry = store.get_dataset(f'entity_{pk}')
+            if entry is None:
+                run.error_message = '请先上传并等待预处理完成'
+                run.status = 'failed'
+                run.save(update_fields=['status', 'error_message'])
+                return
+
+            ds_id = upload_ds_id if store.get_dataset(upload_ds_id) else f'upload_{pk}'
+
+            # Use the unified clustering pipeline (clustering → extraction → UMAP)
+            _run_clustering_pipeline(
+                run=run, store=store, ds_id=ds_id,
+                feature_columns=feature_columns,
+                algorithm=algorithm, min_cluster_size=min_cluster_size,
+                run_umap=True, head=head,
+            )
+        except Exception:
+            tb = traceback.format_exc()
+            logger.error(tb)
+            run.status = 'failed'
+            run.error_message = tb
+            run.progress_msg = f'失败: {tb}'
+            run.run_log += f'\n[ERROR] {tb}'
+            run.save(update_fields=['status', 'error_message', 'progress_msg', 'run_log'])
+
+    t = threading.Thread(
+        target=_run_pipeline_worker,
+        args=(pk, _analysis_fn),
+        kwargs=dict(feature_columns=feature_columns, algorithm=algorithm,
+                     min_cluster_size=min_cluster_size, head=head, pk=pk,
+                     display_id=run.display_id, cluster_mode=cluster_mode),
+        daemon=True,
+    )
+    t.start()
+    return JsonResponse({'status': 'started', 'redirect': f'/runs/{run.display_id}/'})

analysis/views/pipeline.py

@@ -0,0 +1,278 @@
+"""Background pipeline: CSV loading, SVD, and pipeline worker."""
+import os
+import threading
+import traceback
+import logging
+
+from analysis.appdata import ensure_appdata_dir
+from analysis.constants import MAX_CLUSTER_FEATURES, RANDOM_SEED
+from analysis.models import AnalysisRun
+from analysis.profile_util import profile
+
+logger = logging.getLogger(__name__)
+
+
+def _run_pipeline_worker(run_id, analysis_fn, **ctx):
+    """Unified background worker: setup / ORM / progress / error handling."""
+    import django
+    os.environ['DJANGO_SETTINGS_MODULE'] = 'tianxuan.settings'
+    os.environ['DJANGO_ALLOW_ASYNC_UNSAFE'] = 'true'
+    django.setup()
+
+    from analysis.db_utils import retry_on_lock
+    from django.db import close_old_connections
+
+    close_old_connections()
+
+    @retry_on_lock()
+    def _get_run():
+        return AnalysisRun.objects.get(id=run_id)
+
+    run = _get_run()
+    try:
+        analysis_fn(run, ctx)
+    except Exception:
+        tb = traceback.format_exc()
+        logger.error(tb)
+        run.status = 'failed'
+        run.error_message = tb
+        run.run_log += f'\n[FATAL] {tb}'
+        retry_on_lock()(lambda: run.save(update_fields=['status', 'error_message', 'run_log']))()
+
+
+@profile
+def _background_process(run_id, upload_dir):
+    """Background: stream-load CSVs → save to SQLite → ready for analysis."""
+    import django
+    os.environ['DJANGO_SETTINGS_MODULE'] = 'tianxuan.settings'
+    django.setup()
+    from analysis.models import AnalysisRun
+    from analysis.session_store import SessionStore
+
+    run = AnalysisRun.objects.get(id=run_id)
+    store = SessionStore()
+    try:
+        # Check upload directory still exists (may have been temp-cleaned)
+        if not upload_dir.is_dir():
+            msg = '上传数据目录已不存在，请重新上传'
+            logger.error('[BACKGROUND] %s (run_id=%s, upload_dir=%s)', msg, run_id, upload_dir)
+            run.status = 'failed'
+            run.error_message = msg
+            run.save(update_fields=['status', 'error_message'])
+            return
+
+        import glob
+        import polars as pl
+        csv_paths = sorted(glob.glob(str(upload_dir / '*.csv')))
+        if not csv_paths:
+            msg = '没有找到 CSV 文件'
+            logger.error('[BACKGROUND] %s (run_id=%s)', msg, run_id)
+            run.status = 'failed'
+            run.error_message = msg
+            run.save(update_fields=['status', 'error_message'])
+            return
+
+        # ── Detect lat/lon columns for schema override ──
+        import csv as _csv
+        _LAT_LON_KW = ('latd', 'lond', 'latitude', 'longitude')
+        lat_lon_overrides = {}
+        try:
+            with open(csv_paths[0], 'r', encoding='utf-8') as _f:
+                _reader = _csv.reader(_f)
+                _headers = next(_reader, [])
+            for _col in _headers:
+                _norm = _col.lower().replace('-', '_').replace('.', '_').replace(' ', '_')
+                if any(kw in _norm for kw in _LAT_LON_KW):
+                    lat_lon_overrides[_col] = pl.Utf8
+                    logger.info('[BACKGROUND] Forcing lat/lon column %r to Utf8 for safe parsing', _col)
+        except Exception:
+            logger.warning('[BACKGROUND] Failed to detect lat/lon columns', exc_info=True)
+
+        total_files = len(csv_paths)
+        logger.info('[BACKGROUND] Found %d CSV files in %s', total_files, upload_dir)
+        # Adaptive batch size: aim for ~20 batches total, cap at 500 files per batch
+        batch_size = max(1, min(500, total_files // 20 or 1))
+        total_batches = (total_files + batch_size - 1) // batch_size
+
+        run.status = 'loading'; run.progress_pct = 10
+        run.progress_msg = f'正在流式加载 {total_files} 个文件（共 {total_batches} 批）...'
+        run.save(update_fields=['status', 'progress_pct', 'progress_msg'])
+
+        from analysis.data_loader import save_to_db, drop_sqlite_table, load_from_db, _dtype_to_sqlite
+
+        table_name = f'_data_{run_id}'
+
+        # ── Phase 1: Pre-scan all files for full union column set ──
+        all_columns = []
+        for _path in csv_paths:
+            with open(_path, 'r', encoding='utf-8') as _f:
+                _reader = _csv.reader(_f)
+                _headers = next(_reader, [])
+                for _h in _headers:
+                    if _h not in all_columns:
+                        all_columns.append(_h)
+        reference_columns = sorted(all_columns, key=str.lower)
+        schema = {col: 'Utf8' for col in all_columns}
+
+        # ── Phase 2: Create SQLite table once with all columns as TEXT ──
+        import sqlite3 as _sqlite3
+        from django.conf import settings as _dj_settings
+        _db_path = _dj_settings.DATABASES['default']['NAME']
+        _conn = _sqlite3.connect(_db_path)
+        try:
+            _col_defs = [f'"{_col}" TEXT' for _col in reference_columns]
+            _conn.execute(f'DROP TABLE IF EXISTS "{table_name}"')
+            _conn.execute(f'CREATE TABLE "{table_name}" ({", ".join(_col_defs)})')
+            _conn.commit()
+        finally:
+            _conn.close()
+
+        # ── Phase 3: Batched collection → SQLite append ──
+        schema_overrides = {col: pl.Utf8 for col in reference_columns}
+        completed_batches = 0
+        total_rows = 0
+        for i in range(0, total_files, batch_size):
+            batch = csv_paths[i:i+batch_size]
+            # Scan each file INDIVIDUALLY — batch scan fails when files have different schemas
+            dfs = []
+            for fp in batch:
+                lf = pl.scan_csv(fp, infer_schema_length=0,
+                                 schema_overrides=schema_overrides)
+                df = lf.collect(streaming=True)
+                # Fill missing columns with NULL
+                for col in reference_columns:
+                    if col not in df.columns:
+                        df = df.with_columns(pl.lit(None).alias(col))
+                # Select all reference_columns in canonical order
+                df = df.select(reference_columns)
+                dfs.append(df)
+            df_batch = pl.concat(dfs, how='diagonal_relaxed')
+
+            # Append to pre-created SQLite table
+            save_to_db(run.id, df_batch.lazy(), mode='append')
+
+            # Delete temp files after successful SQLite write
+            for f in batch:
+                try:
+                    os.unlink(f)
+                except OSError:
+                    pass
+
+            completed_batches += 1
+            run.progress_pct = int((completed_batches / total_batches) * 80 + 10)
+            run.progress_msg = f'正在加载数据... ({completed_batches}/{total_batches} 批)'
+            run.save(update_fields=['progress_pct', 'progress_msg'])
+
+        # ── Phase 3: Reload from SQLite with correct types ──
+        lf = load_from_db(table_name, schema_overrides=schema)
+        if lf is None:
+            raise RuntimeError(f'SQLite 表 {table_name} 加载失败')
+
+        # Apply _coerce_to_float on lat/lon columns (restore Float64 from Utf8 storage)
+        from analysis.data_loader import _coerce_to_float as _coerce_to_float_fn
+        for lat_lon_col in lat_lon_overrides:
+            if lat_lon_col in lf.columns:
+                lf = lf.with_columns(
+                    pl.col(lat_lon_col).map_batches(_coerce_to_float_fn, return_dtype=pl.Float64).alias(lat_lon_col)
+                )
+
+        # ── Restore column types after SQLite storage ──
+        # All columns were stored as Utf8 for SQLite.  Recover original types
+        # from the classification system — ONLY the 8 allowed types.
+        from analysis.type_classifier import classify_schema, DataType
+        type_map = classify_schema(lf)
+        # Apply Polars-level type casts based on classification
+        dtypes = {
+            DataType.FLOAT: pl.Float64,
+            DataType.INT: pl.Int64,
+            DataType.BOOL: pl.Boolean,
+            DataType.TIMESTAMP: pl.Float64,
+        }
+        casts = []
+        for col, dt in type_map.items():
+            target = dtypes.get(dt)
+            if target and col in lf.collect_schema().names():
+                casts.append(pl.col(col).cast(target, strict=False))
+                schema[col] = str(target)
+        if casts:
+            lf = lf.with_columns(casts)
+            restored = [(col, dt.value) for col, dt in type_map.items() if dt in dtypes][:15]
+            logger.info('[BACKGROUND] Type restoration: %s', restored)
+
+        # ── SVD dimensionality reduction ──
+        svd_components = 0
+        # Only operate on columns with known numeric types in the schema
+        numeric_dtypes = {'Int64', 'Float64', 'Int32', 'Float32',
+                          'Int8', 'Int16', 'UInt8', 'UInt16', 'UInt32', 'UInt64'}
+        numeric_cols = [c for c, dt in schema.items() if dt in numeric_dtypes]
+        n_numeric = len(numeric_cols)
+
+        if n_numeric >= 2:
+            n_components = min(MAX_CLUSTER_FEATURES, n_numeric - 1)
+            try:
+                from sklearn.decomposition import TruncatedSVD  # fmt: skip
+                import numpy as np  # fmt: skip
+
+                # Collect numeric data for SVD (full materialise → transform → add back)
+                df_full = lf.select(numeric_cols).collect(streaming=True)
+                X = df_full.to_numpy()
+                X = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)
+
+                svd = TruncatedSVD(n_components=n_components, random_state=RANDOM_SEED)
+                X_svd = svd.fit_transform(X)
+                svd_components = n_components
+
+                # Build a new LazyFrame that includes _svd_* columns
+                # Strategy: collect full LazyFrame once, add SVD columns, re-wrap as lazy
+                df_all = lf.collect(streaming=True)
+                for i in range(n_components):
+                    col_name = f'_svd_{i}'
+                    schema[col_name] = 'Float64'
+                    df_all = df_all.with_columns(
+                        pl.Series(col_name, X_svd[:, i])
+                    )
+                lf = df_all.lazy()
+
+                logger.info('[BACKGROUND] SVD: %d components from %d numeric columns '
+                            '(explained variance ratio sum=%.4f)',
+                            svd_components, n_numeric,
+                            float(svd.explained_variance_ratio_.sum()))
+            except Exception as e:
+                logger.warning('[BACKGROUND] SVD failed (non-fatal): %s', e)
+        elif n_numeric == 1:
+            logger.info('[BACKGROUND] SVD skipped: only 1 numeric column (%s)',
+                        numeric_cols[0])
+
+        # Get row count
+        df_count = lf.select(pl.len()).collect(streaming=True)
+        total_rows = df_count[0, 0] if df_count.height > 0 else 0
+
+        run.sqlite_table = table_name
+        run.total_flows = total_rows
+        run.save(update_fields=['sqlite_table', 'total_flows'])
+
+        ds_id = f'upload_{run.display_id}'
+        store.store_dataset(ds_id, lf, schema=schema, metadata={
+            'row_count': total_rows, 'file_count': total_files, 'upload_dir': str(upload_dir),
+            'csv_glob': str(upload_dir / '*.csv'),
+            'sqlite_table': table_name,
+            'svd_components': svd_components,
+        })
+
+        # ── Phase 3: Data ready for filtering/clustering via MCP tools ──
+        run.status = 'ready'; run.progress_pct = 60; run.progress_msg = '数据准备就绪'
+        run.save(update_fields=['status', 'progress_pct', 'progress_msg'])
+    except Exception:
+        tb = traceback.format_exc()
+        logger.error(tb)
+        # Drop partial SQLite table on failure (idempotent)
+        try:
+            from analysis.data_loader import drop_sqlite_table  # fmt: skip
+            drop_sqlite_table(f'_data_{run_id}')
+        except Exception:
+            logger.error("unknown failed: {}".format(traceback.format_exc()))
+            pass  # Expected: drop_sqlite_table may fail if table already gone
+        run.status = 'failed'
+        run.error_message = tb
+        run.run_log += f'\n[ERROR] {tb}'
+        run.save(update_fields=['status', 'error_message', 'run_log'])

analysis/views/tools.py

@@ -0,0 +1,381 @@
+"""MCP Tool Lab, plan management, and utility views."""
+import json
+import asyncio
+import threading
+import traceback
+import logging
+from pathlib import Path
+
+from django.shortcuts import render, get_object_or_404, redirect
+from django.http import JsonResponse
+from django.views.decorators.csrf import csrf_exempt
+
+from analysis.appdata import ensure_appdata_dir
+from analysis.models import AnalysisRun
+from .pipeline import _background_process
+from .helpers import (
+    _PLANS_DIR, _AUTO_INDEX_PATH, _MAX_AUTO_PLANS,
+    _read_auto_index, _write_auto_index, _add_to_auto_index,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def tool_lab(request):
+    """List all available MCP tools with descriptions and input schemas."""
+    from analysis.tool_registry import get_tools_meta
+    tools = get_tools_meta()
+    # Group tools by category
+    core_names = {'profile_data', 'build_entity_profiles', 'compute_scores',
+                  'run_clustering', 'extract_features', 'detect_anomalies',
+                  'visualize_anomalies'}
+    diag_names = {'validate_data', 'explore_distributions', 'find_outliers',
+                  'diagnose_clustering', 'compare_datasets', 'export_debug_sample',
+                  'repair_schema'}
+    analysis_names = {'analyze_patterns', 'analyze_temporal', 'analyze_tls_health',
+                      'analyze_geo_distribution', 'analyze_entity_detail'}
+
+    core_tools, diag_tools, analysis_tools, other_tools = [], [], [], []
+    for t in tools:
+        if t.name in core_names:
+            core_tools.append(t)
+        elif t.name in diag_names:
+            diag_tools.append(t)
+        elif t.name in analysis_names:
+            analysis_tools.append(t)
+        else:
+            other_tools.append(t)
+
+    # Get list of datasets for parameter autofill
+    from analysis.session_store import SessionStore
+    store = SessionStore()
+    ds_list = store.list_datasets() if hasattr(store, 'list_datasets') else []
+
+    import json as _json
+    # Serialize tools metadata for JS
+    meta_list = []
+    for t in tools:
+        meta_list.append({
+            'name': t.name,
+            'description': t.description,
+            'inputSchema': t.inputSchema,
+        })
+    return render(request, 'analysis/tool_lab.html', {
+        'core_tools': core_tools,
+        'diag_tools': diag_tools,
+        'analysis_tools': analysis_tools,
+        'other_tools': other_tools,
+        'datasets': ds_list,
+        'tools_meta': _json.dumps(meta_list, ensure_ascii=False),
+    })
+
+
+@csrf_exempt
+def tool_lab_run(request):
+    """Execute a single MCP tool via AJAX POST."""
+    if request.method != 'POST':
+        return JsonResponse({'error': 'POST required'}, status=405)
+    import json as _json
+    try:
+        body = _json.loads(request.body)
+    except Exception:
+        logger.error("tool_lab_run failed: {}".format(traceback.format_exc()))
+        return JsonResponse({'error': 'Invalid JSON body'}, status=400)
+
+    tool_name = body.get('tool')
+    params = body.get('params', {})
+    if not tool_name:
+        return JsonResponse({'error': 'Missing tool name'}, status=400)
+
+    # ── Backend validation: check required params + dataset existence ──
+    from analysis.tool_registry import handle_call, get_tools_meta
+    from analysis.session_store import SessionStore
+    tools_meta = get_tools_meta()
+    tool_schema = None
+    for t in tools_meta:
+        if t.name == tool_name:
+            tool_schema = t.inputSchema
+            break
+    if tool_schema:
+        required_fields = tool_schema.get('required', [])
+        for field in required_fields:
+            val = params.get(field)
+            if val is None or (isinstance(val, str) and val.strip() == ''):
+                return JsonResponse({'error': f'Missing required parameter: {field}'}, status=400)
+            # If field is a dataset_id, verify it exists in the store
+            if 'dataset' in field.lower():
+                store = SessionStore()
+                if not store.get_dataset(str(val)):
+                    return JsonResponse({'error': f'Dataset not found: {val}'}, status=400)
+
+    result = asyncio.run(handle_call(tool_name, params))
+    return JsonResponse({'status': 'ok', 'result': result})
+
+
+@csrf_exempt
+def reload_run_data(request):
+    """Reload a run's data from csv_glob or sqlite_table into SessionStore."""
+    if request.method != 'POST':
+        return JsonResponse({'error': 'POST required'}, status=405)
+    import json as _json
+    try:
+        body = _json.loads(request.body)
+    except Exception:
+        logger.error("reload_run_data failed: {}".format(traceback.format_exc()))
+        return JsonResponse({'error': 'Invalid JSON body'}, status=400)
+
+    display_id = body.get('display_id')
+    if not display_id:
+        return JsonResponse({'error': 'Missing display_id'}, status=400)
+
+    from analysis.models import AnalysisRun
+    from analysis.data_loader import load_csv_directory, load_from_db
+
+    run = get_object_or_404(AnalysisRun, display_id=display_id)
+
+    # Determine dataset_id (same keying as manual_page)
+    ds_id = f'upload_{display_id}' if run.run_type == 'upload' else f'run_{display_id}'
+
+    from analysis.session_store import SessionStore
+    store = SessionStore()
+    # Check if already loaded
+    existing = store.get_dataset(ds_id)
+    if existing is not None:
+        return JsonResponse({'status': 'ok', 'dataset_id': ds_id, 'reloaded': False})
+
+    # Try loading from sqlite_table first, then csv_glob
+    lf = None
+    schema = {}
+    row_count = 0
+    file_count = 0
+
+    if run.sqlite_table:
+        try:
+            lf = load_from_db(run.sqlite_table)
+            if lf is not None:
+                import polars as pl
+                row_count = lf.select(pl.len()).collect(streaming=True)[0, 0]
+        except Exception as exc:
+            logging.getLogger(__name__).warning(f'Failed to load from sqlite_table: {exc}')
+
+    if lf is None and run.csv_glob:
+        try:
+            from pathlib import Path
+            csv_path = Path(run.csv_glob)
+            if csv_path.is_file() or csv_path.is_dir() or csv_path.parent.is_dir():
+                lf, schema, row_count, file_count, _ = load_csv_directory(run.csv_glob)
+        except Exception as exc:
+            logging.getLogger(__name__).warning(f'Failed to load from csv_glob: {exc}')
+
+    if lf is None:
+        return JsonResponse({'error': 'No data source available (upload dir or sqlite table gone)'}, status=404)
+
+    store.store_dataset(ds_id, lf, schema=schema, metadata={
+        'row_count': row_count,
+        'file_count': file_count,
+        'csv_glob': run.csv_glob or '',
+        'sqlite_table': run.sqlite_table or '',
+    })
+
+    return JsonResponse({
+        'status': 'ok',
+        'dataset_id': ds_id,
+        'reloaded': True,
+        'row_count': row_count,
+        'columns': list(schema.keys()),
+    })
+
+
+@csrf_exempt
+def tool_plan(request):
+    """Save, list, load, delete, or pin/unpin an analysis plan."""
+    import json as _j
+    if request.method == 'GET':
+        name = request.GET.get('name')
+        if name:
+            # Auto plans stored with _auto_ prefix
+            if name.startswith('_auto_'):
+                path = _PLANS_DIR / f'{name}.json'
+            else:
+                path = _PLANS_DIR / f'{name}.json'
+            if not path.exists():
+                return JsonResponse({'error': f'Plan "{name}" not found'}, status=404)
+            try:
+                data = _j.loads(path.read_text(encoding='utf-8'))
+                return JsonResponse(data)
+            except Exception as e:
+                return JsonResponse({'error': str(e)}, status=500)
+
+        # List all plans (regular + auto)
+        auto_index = _read_auto_index()
+        auto_index_map = {e.get('file', ''): e.get('pinned', False) for e in auto_index}
+        auto_names = set()
+        plans = []
+        for p in sorted(_PLANS_DIR.glob('*.json')):
+            # Skip index file and auto plans (handled separately)
+            if p.name == '_auto_index.json':
+                continue
+            if p.name.startswith('_auto_'):
+                auto_names.add(p.name)
+                continue
+            try:
+                content = _j.loads(p.read_text(encoding='utf-8'))
+                steps = len(content.get('steps', []))
+                plans.append({'name': p.stem, 'steps': steps})
+            except Exception:
+                logger.error("unknown failed: {}".format(traceback.format_exc()))
+                plans.append({'name': p.stem, 'steps': 0})
+
+        # Add auto plans with markers
+        auto_plans = []
+        for auto_name in sorted(auto_names):
+            try:
+                path = _PLANS_DIR / auto_name
+                content = _j.loads(path.read_text(encoding='utf-8'))
+                steps = len(content.get('steps', []))
+                auto_plans.append({
+                    'name': auto_name.replace('.json', ''),
+                    'steps': steps,
+                    'auto': True,
+                    'pinned': auto_index_map.get(auto_name, False),
+                })
+            except Exception:
+                logger.error("unknown failed: {}".format(traceback.format_exc()))
+                auto_plans.append({
+                    'name': auto_name.replace('.json', ''),
+                    'steps': 0,
+                    'auto': True,
+                    'pinned': auto_index_map.get(auto_name, False),
+                })
+        # Interleave: auto plans first, then regular
+        return JsonResponse({'plans': auto_plans + plans})
+
+    elif request.method == 'POST':
+        try:
+            body = _j.loads(request.body)
+            # Pin/unpin toggle for auto plans
+            if 'pin' in body and body.get('name', '').startswith('_auto_'):
+                filename = body['name'].replace('.json', '') + '.json' \
+                    if not body['name'].endswith('.json') else body['name']
+                index = _read_auto_index()
+                new_pinned = bool(body['pin'])
+                found = False
+                for entry in index:
+                    if entry.get('file') == filename:
+                        entry['pinned'] = new_pinned
+                        found = True
+                        break
+                if not found and new_pinned:
+                    index.append({'file': filename, 'pinned': True})
+                _write_auto_index(index)
+                return JsonResponse({
+                    'status': 'pinned' if new_pinned else 'unpinned',
+                    'name': body['name'],
+                    'pinned': new_pinned,
+                })
+
+            # Regular save
+            name = body.get('name', '').strip()
+            steps = body.get('steps', [])
+            if not name:
+                return JsonResponse({'error': 'Missing plan name'}, status=400)
+            path = _PLANS_DIR / f'{name}.json'
+            path.write_text(_j.dumps({'name': name, 'steps': steps}, ensure_ascii=False, indent=2), encoding='utf-8')
+            return JsonResponse({'status': 'saved', 'name': name, 'steps': len(steps)})
+        except Exception as e:
+            return JsonResponse({'error': str(e)}, status=500)
+
+    elif request.method == 'DELETE':
+        name = request.GET.get('name', '').strip()
+        if not name:
+            return JsonResponse({'error': 'Missing plan name'}, status=400)
+        path = _PLANS_DIR / f'{name}.json'
+        if path.exists():
+            path.unlink()
+        # Also remove from auto index if applicable
+        if name.startswith('_auto_'):
+            filename = name + '.json' if not name.endswith('.json') else name
+            index = _read_auto_index()
+            index = [e for e in index if e.get('file') != filename]
+            _write_auto_index(index)
+        return JsonResponse({'status': 'deleted', 'name': name})
+
+    return JsonResponse({'error': 'Method not allowed'}, status=405)
+
+
+# ── Filter endpoint for manual analysis page ──────────────────────────
+
+@csrf_exempt
+def apply_filter(request):
+    """POST {dataset_id, filters, logic} → apply filter_data tool → return filtered_X id."""
+    if request.method != 'POST':
+        return JsonResponse({'error': 'POST required'}, status=405)
+    import json as _json
+    body = _json.loads(request.body)
+    dataset_id = body.get('dataset_id')
+    filters = body.get('filters', [])
+    logic = body.get('logic', 'and')
+
+    if not dataset_id:
+        return JsonResponse({'error': 'dataset_id is required'})
+    if not filters:
+        return JsonResponse({'error': 'filters is required'})
+
+    try:
+        from analysis.tool_registry import _handle_filter_data
+        result = asyncio.run(_handle_filter_data(
+            dataset_id=dataset_id,
+            filters=filters,
+            logic=logic,
+        ))
+        if 'error' in result:
+            return JsonResponse({'error': result['error']})
+
+        filtered_id = result.get('filtered_dataset_id', '')
+        row_count = result.get('row_count', 0)
+
+        # Use the unique flt_xxx ID from _handle_filter_data — never rename
+        # to filtered_{num} which would collide on repeated filters of the same source.
+        return JsonResponse({
+            'dataset_id': filtered_id,
+            'row_count': row_count,
+        })
+    except Exception as e:
+        logger.error(f'[apply_filter] {traceback.format_exc()}')
+        return JsonResponse({'error': str(e)})
+
+
+@csrf_exempt
+def retry_run(request, display_id):
+    """Reset a failed run and re-trigger based on run_type."""
+    if request.method != 'POST':
+        return JsonResponse({'error': 'POST required'}, status=405)
+
+    run = get_object_or_404(AnalysisRun, display_id=display_id)
+    if run.status != 'failed':
+        return JsonResponse({'error': '只能重试失败的分析运行'}, status=400)
+
+    run.status = 'pending'
+    run.error_message = ''
+    run.progress_pct = 0
+    run.progress_msg = ''
+    run.save(update_fields=['status', 'error_message', 'progress_pct', 'progress_msg'])
+
+    if run.run_type == 'upload' and run.csv_glob:
+        upload_dir = Path(run.csv_glob).parent
+        expected_base = ensure_appdata_dir() / 'data' / 'uploads'
+        if not str(upload_dir.resolve()).startswith(str(expected_base.resolve())):
+            return JsonResponse({'error': 'Invalid upload path'}, status=400)
+        if upload_dir.is_dir():
+            t = threading.Thread(
+                target=_background_process,
+                args=(run.id, upload_dir),
+                daemon=True,
+            )
+            t.start()
+        else:
+            run.error_message = '上传数据目录已不存在，请重新上传'
+            run.status = 'failed'
+            run.save(update_fields=['status', 'error_message'])
+
+    return redirect('analysis:run_detail', display_id=run.display_id)

analysis/views/upload.py

@@ -0,0 +1,133 @@
+"""Upload views: CSV upload, batch, finalize, delete."""
+import threading
+import logging
+import shutil
+from pathlib import Path
+
+from django.shortcuts import render, get_object_or_404
+from django.http import JsonResponse
+from django.views.decorators.csrf import csrf_exempt
+
+from analysis.appdata import ensure_appdata_dir
+from analysis.models import AnalysisRun
+from .pipeline import _background_process
+
+logger = logging.getLogger(__name__)
+
+
+def upload_page(request):
+    """CSV upload page."""
+    runs = AnalysisRun.objects.all().order_by('-created_at')[:20]
+    return render(request, 'tianxuan/upload.html', {'runs': runs})
+
+
+@csrf_exempt
+def upload_csv(request):
+    """Handle CSV upload AJAX POST."""
+    if request.method != 'POST':
+        return JsonResponse({'error': 'POST required'}, status=405)
+    files = request.FILES.getlist('files')
+    if not files:
+        return JsonResponse({'error': 'No files uploaded'})
+    logger.info('[UPLOAD] Received %d files', len(files))
+    for f in files:
+        if f.size > 500 * 1024 * 1024:
+            return JsonResponse({'warning': f'文件 {f.name} 超过 500MB，建议分割后上传'})
+    total_size = sum(f.size for f in files)
+    if total_size > 50 * 1024 * 1024 * 1024:
+        return JsonResponse({'error': '总大小超过 50GB 限制'})
+    from django.utils import timezone
+    ts = timezone.now().strftime('%Y%m%d_%H%M%S')
+    from analysis.appdata import ensure_appdata_dir
+    base = ensure_appdata_dir() / 'data' / 'uploads'
+    upload_dir = base / ts
+    upload_dir.mkdir(parents=True, exist_ok=True)
+    saved = []
+    for f in files:
+        path = upload_dir / f.name
+        with open(path, 'wb+') as dst:
+            for chunk in f.chunks():
+                dst.write(chunk)
+        saved.append(str(path))
+    # Create run with status='pending' — background processing deferred to finalize endpoint
+    run = AnalysisRun.objects.create(
+        csv_glob=str(upload_dir / '*.csv'),
+        status='pending',
+        run_type='upload',
+    )
+    # Deferred: _background_process will be started by finalize_upload endpoint
+    return JsonResponse({'run_id': run.display_id, 'files': saved, 'status': 'pending'})
+
+
+@csrf_exempt
+def upload_csv_batch(request, display_id):
+    """Append files to existing upload run. Does NOT re-trigger _background_process."""
+    run = get_object_or_404(AnalysisRun, display_id=display_id)
+    files = request.FILES.getlist('files')
+    if not files:
+        return JsonResponse({'error': 'No files uploaded'})
+    upload_dir = Path(run.csv_glob).parent if run.csv_glob else None
+    if upload_dir:
+        expected_base = ensure_appdata_dir() / 'data' / 'uploads'
+        if not str(upload_dir.resolve()).startswith(str(expected_base.resolve())):
+            return JsonResponse({'error': 'Invalid upload path'}, status=400)
+    if not upload_dir or not upload_dir.exists():
+        return JsonResponse({'error': 'Upload directory not found'})
+    saved = []
+    for f in files:
+        path = upload_dir / f.name
+        with open(path, 'wb+') as dst:
+            for chunk in f.chunks():
+                dst.write(chunk)
+        saved.append(str(path))
+    # NO background process re-trigger — first batch handles it
+    return JsonResponse({'run_id': run.display_id, 'files': len(saved), 'status': 'ok'})
+
+
+@csrf_exempt
+def finalize_upload(request, display_id):
+    """Finalize batch upload: validate upload_dir, set status='loading', start background processing."""
+    if request.method != 'POST':
+        return JsonResponse({'error': 'POST required'}, status=405)
+    run = get_object_or_404(AnalysisRun, display_id=display_id)
+    if run.status != 'pending':
+        return JsonResponse({'error': f'Run status is {run.status}, expected pending'}, status=400)
+    upload_dir = Path(run.csv_glob).parent if run.csv_glob else None
+    if not upload_dir or not upload_dir.exists():
+        return JsonResponse({'error': 'Upload directory not found'}, status=400)
+    run.status = 'loading'
+    run.save(update_fields=['status'])
+    t = threading.Thread(target=_background_process, args=(run.id, upload_dir), daemon=True)
+    t.start()
+    return JsonResponse({'run_id': run.display_id, 'status': 'loading'})
+
+
+@csrf_exempt
+def delete_upload(request, display_id):
+    """Delete uploaded files, SQLite data table, and AnalysisRun.
+
+    Handles all run statuses (loading, profiling, clustering, completed, failed).
+    Gracefully tolerates null csv_glob and missing/errored SQLite tables.
+    """
+    run = get_object_or_404(AnalysisRun, display_id=display_id)
+
+    # 1. Drop the SQLite data table if one exists
+    if run.sqlite_table:
+        try:
+            from analysis.data_loader import drop_sqlite_table  # fmt: skip
+            drop_sqlite_table(run.sqlite_table)
+        except Exception as exc:
+            logger.warning('[DELETE] failed to drop SQLite table %s: %s', run.sqlite_table, exc)
+
+    # 2. Remove the upload directory if csv_glob is set
+    if run.csv_glob:
+        p = Path(run.csv_glob).parent
+        expected_base = ensure_appdata_dir() / 'data' / 'uploads'
+        if not str(p.resolve()).startswith(str(expected_base.resolve())):
+            return JsonResponse({'error': 'Invalid upload path'}, status=400)
+        if p.exists():
+            shutil.rmtree(p, ignore_errors=True)
+
+    # 3. Delete the ORM record
+    run.delete()
+    return JsonResponse({'deleted': True})

analysis/views_backup.py

@@ -14,6 +14,8 @@ from django.views.decorators.csrf import csrf_exempt
 
 from config import get_config, save_config, Config
 from .models import AnalysisRun, ClusterResult, EntityProfile
+from . import nl_describe
+from .profile_util import profile
 
 
 def dashboard(request):
@@ -80,13 +82,43 @@ def cluster_overview(request, display_id):
     clusters = run.clusters.exclude(cluster_label=-1).order_by('-size').prefetch_related('features')
     noise = run.clusters.filter(cluster_label=-1).first()
 
-    # Build UMAP scatter data for Canvas
+    # Build UMAP scatter data for Canvas / Three.js 3D
-    entities = run.entities.exclude(embedding_x=None).exclude(embedding_y=None).only('embedding_x', 'embedding_y', 'cluster_label', 'entity_value')
+    entities = run.entities.exclude(embedding_x=None).exclude(embedding_y=None).only(
+        'embedding_x', 'embedding_y', 'embedding_z', 'cluster_label', 'entity_value'
+    )
+    has_z = any(e.embedding_z is not None for e in entities)
     scatter_data = [
-        {'x': e.embedding_x, 'y': e.embedding_y, 'label': e.cluster_label, 'entity': e.entity_value}
+        {'x': e.embedding_x, 'y': e.embedding_y, 'z': e.embedding_z if has_z else 0,
+         'label': e.cluster_label if e.cluster_label is not None else -1, 'entity': e.entity_value}
         for e in entities
     ]
 
+    # Noise cluster: features + entity detail
+    noise_features = []
+    noise_summary = ''
+    noise_entity_count = 0
+    if noise:
+        noise_features_qs = noise.features.all().order_by('-distinguishing_score')[:15]
+        noise_features = [
+            {'feature_name': f.feature_name, 'score': f.distinguishing_score, 'mean': f.mean,
+             'std': f.std, 'p25': f.p25, 'p75': f.p75}
+            for f in noise_features_qs
+        ]
+        if noise_features:
+            noise_summary = nl_describe.describe_cluster(noise_features)
+        noise_entity_count = noise.size
+
+    # LLM workflow timeline for auto runs
+    llm_timeline_json = ''
+    if run.run_type == 'auto':
+        llm_thinking = run.llm_thinking or ''
+        tool_calls = run.tool_calls_json or []
+        if tool_calls or llm_thinking:
+            llm_timeline_json = json.dumps({
+                'thinking': llm_thinking,
+                'tool_calls': tool_calls,
+            })
+
     # Build geo scatter data from feature_json (lat/lon)
     geo_data = []
     geo_skipped = 0
@@ -110,40 +142,115 @@ def cluster_overview(request, display_id):
     cluster_sizes = [{'label': c.cluster_label, 'size': c.size, 'silhouette': c.silhouette_score}
                      for c in clusters]
 
-    # Top features per cluster from ClusterFeature
+    # Top features per cluster from ClusterFeature + NL summaries
     top_features = {}
     for c in clusters:
         features_qs = c.features.all().order_by('-distinguishing_score')[:10]
-        top_features[str(c.cluster_label)] = [
+        feats = [
-            {'feature_name': f.feature_name, 'score': f.distinguishing_score, 'mean': f.mean}
+            {'feature_name': f.feature_name, 'score': f.distinguishing_score, 'mean': f.mean,
+             'std': f.std}
             for f in features_qs
         ]
+        top_features[str(c.cluster_label)] = feats
+        # Attach features + NL summary directly to cluster object for template
+        c._top_features = feats
+        c.nl_summary = nl_describe.describe_cluster(feats) if feats else ''
 
     return render(request, 'analysis/cluster_overview.html', {
         'run': run,
         'clusters': clusters,
         'noise': noise,
         'scatter_data_json': json.dumps(scatter_data),
+        'has_z': has_z,
         'geo_data_json': json.dumps(geo_data),
         'geo_skipped': geo_skipped,
         'geo_count': len(geo_data),
         'cluster_sizes_json': json.dumps(cluster_sizes),
         'top_features_json': json.dumps(top_features),
+        'noise_features': noise_features,
+        'noise_summary': noise_summary,
+        'noise_entity_count': noise_entity_count,
+        'llm_timeline_json': llm_timeline_json,
+        # Globe flow data for the sidebar
+        'globe_flows_json': json.dumps(_get_globe_flows(run)),
     })
 
 
+def _get_globe_flows(run, max_rows=500):
+    """Extract flow data for 3D globe visualization from a run."""
+    import json
+    from analysis.data_loader import load_csv_directory, load_from_db
+    try:
+        lf = None
+        if run.sqlite_table:
+            lf = load_from_db(run.sqlite_table)
+        if lf is None and run.csv_glob:
+            lf, _, _, _, _ = load_csv_directory(run.csv_glob)
+        if lf is None:
+            return []
+        df = lf.head(max_rows).collect()
+        from analysis.geoip import lookup as geo_lookup
+        flows = []
+        TLS_HEX_MAP = {'0303': 'TLSv1.2', '0304': 'TLSv1.3', '0302': 'TLSv1.1', '0301': 'TLSv1.0'}
+        src_ip_col = next((c for c in df.columns if 'src_ip' in c.lower() or ':ips' in c.lower()), None)
+        dst_ip_col = next((c for c in df.columns if 'dst_ip' in c.lower() or ':ipd' in c.lower()), None)
+        tls_col = next((c for c in df.columns if c.lower() in ('0ver', 'tls_version', '_tlsver', 'version')), None)
+        bytes_col = next((c for c in df.columns if c.lower() in ('8ack', '8pak', '8byt', 'bytes_sent', 'bytes')), None)
+        for row in df.iter_rows(named=True):
+            sip = str(row.get(src_ip_col, '')) if src_ip_col else ''
+            dip = str(row.get(dst_ip_col, '')) if dst_ip_col else ''
+            if not sip or not dip:
+                continue
+            sg = geo_lookup(sip)
+            dg = geo_lookup(dip)
+            if not sg or not dg:
+                continue
+            tls_val = str(row.get(tls_col, '') or '') if tls_col else ''
+            tls_ver = TLS_HEX_MAP.get(tls_val.replace(' ', ''), tls_val)
+            flows.append({
+                'slat': sg['lat'], 'slon': sg['lon'],
+                'dlat': dg['lat'], 'dlon': dg['lon'],
+                'tls': tls_ver,
+                'bytes': float(row.get(bytes_col, 0) or 0) if bytes_col else 0,
+            })
+        return flows
+    except Exception:
+        return []
+
+
 def cluster_detail(request, display_id, cluster_label):
-    """Detailed view of a single cluster with entity list."""
+    """Detailed view of a single cluster with entity list, SVD features, and NL summary."""
     run = get_object_or_404(AnalysisRun, display_id=display_id)
+    try:
+        cluster_label = int(cluster_label)
+    except (ValueError, TypeError):
+        raise Http404("Invalid cluster label")
     cluster = get_object_or_404(ClusterResult, run=run, cluster_label=cluster_label)
     features = cluster.features.all().order_by('-distinguishing_score')[:50]
-    entities = cluster.entities.all()[:50]
+    entities = cluster.entities.all()[:100]
+    nl_summary = ''
+    feats_list = [{'feature_name': f.feature_name, 'score': f.distinguishing_score,
+                   'mean': f.mean, 'std': f.std} for f in features[:10]]
+    if feats_list:
+        nl_summary = nl_describe.describe_cluster(feats_list)
+
+    # Build entity data with feature_json values
+    entity_data = []
+    for e in entities:
+        entry = {'id': e.id, 'value': e.entity_value,
+                 'embedding_x': e.embedding_x, 'embedding_y': e.embedding_y}
+        if e.feature_json:
+            entry['features'] = {k: round(float(v), 4) if isinstance(v, (int, float)) else str(v)
+                                 for k, v in e.feature_json.items()}
+        entity_data.append(entry)
 
     return render(request, 'analysis/cluster_detail.html', {
         'run': run,
         'cluster': cluster,
         'features': features,
         'entities': entities,
+        'entity_data_json': json.dumps(entity_data),
+        'nl_summary': nl_summary,
     })
 
 
@@ -423,6 +530,7 @@ import traceback, logging
 logger = logging.getLogger(__name__)
 
 
+@profile
 def _background_process(run_id, upload_dir):
     """Background: stream-load CSVs → save to SQLite → ready for analysis."""
     import django
@@ -709,33 +817,49 @@ def manual_page(request):
 
     # Get datasets from SessionStore
     store = SessionStore()
-    datasets = store.list_datasets() if hasattr(store, 'list_datasets') else []
+    session_datasets = store.list_datasets() if hasattr(store, 'list_datasets') else []
+    session_ds_map = {ds['dataset_id']: ds for ds in session_datasets}
 
-    # Get upload AnalysisRun statuses for dataset badges
+    # Build dataset entries from ALL AnalysisRun records (upload/manual/auto, any status)
     from analysis.models import AnalysisRun
-    upload_runs = AnalysisRun.objects.filter(run_type='upload').order_by('-created_at')[:50]
+    all_runs = AnalysisRun.objects.all().order_by('-created_at')[:100]
-    run_status_map = {
-        f'upload_{r.display_id}': {
-            'status': r.status,
-            'total_flows': r.total_flows,
-            'display_id': r.display_id,
-        }
-        for r in upload_runs
-    }
-
-    # Build datasets_with_status: each dataset annotated with run status
     datasets_with_status = []
-    for ds in datasets:
+
-        run_info = run_status_map.get(ds['dataset_id'])
+    for run in all_runs:
-        if run_info:
+        # Determine how the dataset was keyed in SessionStore
-            datasets_with_status.append({
+        if run.run_type == 'upload':
-                **ds,
+            ds_id = f'upload_{run.display_id}'
-                'run_status': run_info['status'],
-                'total_flows': run_info['total_flows'],
-                'display_id': run_info['display_id'],
-            })
         else:
-            datasets_with_status.append({**ds, 'run_status': 'ready', 'total_flows': None, 'display_id': None})
+            ds_id = f'run_{run.display_id}'
+
+        session_entry = session_ds_map.get(ds_id)
+
+        if session_entry:
+            # Full data available in SessionStore — merge with run status
+            datasets_with_status.append({
+                **session_entry,
+                'dataset_id': ds_id,
+                'run_status': run.status,
+                'total_flows': run.total_flows,
+                'display_id': run.display_id,
+                'needs_reload': False,
+            })
+        elif run.csv_glob or run.sqlite_table:
+            # Data not in SessionStore but recoverable from disk/DB
+            datasets_with_status.append({
+                'dataset_id': ds_id,
+                'run_status': run.status,
+                'display_id': run.display_id,
+                'total_flows': run.total_flows,
+                'row_count': run.total_flows,
+                'file_count': None,
+                'column_count': 0,
+                'columns': [],
+                'svd_components': 0,
+                'needs_reload': True,
+                'csv_glob': run.csv_glob,
+                'sqlite_table': run.sqlite_table or '',
+            })
 
     # Map statuses to user-facing labels for the filter tabs
     # ready → 待分析, completed → 已分析, failed → 错误
@@ -744,12 +868,16 @@ def manual_page(request):
         s = ds_info['run_status']
         if s in ('completed',):
             ds_info['status_label'] = '已分析'
+            ds_info['badge_class'] = 'completed'
         elif s in ('failed',):
             ds_info['status_label'] = '错误'
+            ds_info['badge_class'] = 'failed'
         elif s in ('pending', 'loading', 'profiling', 'aggregating', 'clustering', 'extracting'):
             ds_info['status_label'] = '分析中'
+            ds_info['badge_class'] = 'analyzing'
         else:
             ds_info['status_label'] = '待分析'
+            ds_info['badge_class'] = 'ready'
 
     return render(request, 'tianxuan/manual.html', {
         'core_tools': core_tools,
@@ -757,7 +885,7 @@ def manual_page(request):
         'analysis_tools': analysis_tools,
         'other_tools': other_tools,
         'tools_meta': _json.dumps(meta_list, ensure_ascii=False),
-        'datasets': datasets,
+        'datasets': datasets_with_status,
         'datasets_with_status_json': _json.dumps(datasets_with_status, ensure_ascii=False),
     })
 
@@ -796,6 +924,7 @@ def _run_pipeline_worker(run_id, analysis_fn, **ctx):
 
 # ── Shared clustering pipeline ─────────────────────────────────────────
 
+@profile
 def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorithm,
                               min_cluster_size, run_umap=True, head=None):
     """Unified clustering pipeline: clustering → feature extraction → UMAP → ORM save.
@@ -917,7 +1046,46 @@ def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorith
         if 'error' in feat_result:
             raise Exception(feat_result['error'])
 
-        # ── Step 3: UMAP-2D embedding (sampled training for large data) ─────────
+        # ── Step 2b: Cluster SVD feature extraction ──────────────────────
+        run.progress_msg = '正在进行聚类SVD特征提取...'
+        run.save(update_fields=['progress_msg'])
+        try:
+            from analysis.distance import cluster_svd_extract
+            lf = entry['lazyframe']
+            cluster_entry = store.get_cluster_result(cluster_id)
+            labels_list = cluster_entry.get('labels', []) if cluster_entry else []
+
+            if labels_list and feature_cols:
+                svd_results = cluster_svd_extract(lf, labels_list, feature_cols)
+
+                # Persist SVD features to ClusterFeature (method='cluster_svd')
+                from analysis.models import ClusterFeature as CF
+                from analysis.models import ClusterResult as CR
+                for label, svd_info in svd_results.items():
+                    cr = CR.objects.filter(run=run, cluster_label=label).first()
+                    if cr is None:
+                        continue
+                    # Delete old zscore entries for this cluster, replace with SVD
+                    CF.objects.filter(cluster=cr, distinguishing_method='zscore').delete()
+                    for feat_name in svd_info.get('features', [])[:10]:
+                        idx = svd_info['features'].index(feat_name) if feat_name in svd_info['features'] else -1
+                        strength = (svd_info.get('feature_strength', [])[idx]
+                                    if idx >= 0 and idx < len(svd_info.get('feature_strength', []))
+                                    else None)
+                        CF.objects.get_or_create(
+                            cluster=cr,
+                            feature_name=feat_name,
+                            defaults={
+                                'mean': None, 'std': None, 'median': None,
+                                'p25': None, 'p75': None, 'missing_rate': None,
+                                'distinguishing_score': float(strength) if strength is not None else None,
+                                'distinguishing_method': 'cluster_svd',
+                            },
+                        )
+        except Exception as svd_err:
+            logger.warning(f'Cluster SVD extraction skipped: {svd_err}')
+
+        # ── Step 3: UMAP-3D embedding (2D fallback) ───────────────────────
         if run_umap:
             try:
                 lf = entry['lazyframe']
@@ -947,34 +1115,53 @@ def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorith
                     else:
                         df_umap = lf.collect(streaming=True)
 
-                    if len(df_umap) > 2:
+                    umap_components = 3  # try 3D first
-                        if n_total > MAX_UMAP_TRAIN:
+                    coords = None
-                            # Train UMAP on 10K sample, batch-transform full dataset
+                    reducer = None
-                            mat_sample = df_sample.select(num_cols).to_numpy()
-                            scaler = StandardScaler().fit(mat_sample)
-                            mat_sample_scaled = np.nan_to_num(scaler.transform(mat_sample), nan=0.0)
-                            reducer = umap.UMAP(n_components=2, random_state=42,
-                                                n_neighbors=15, min_dist=0.1,
-                                                metric='euclidean')
-                            reducer.fit(mat_sample_scaled)
-                            # Batch transform in 1K-row batches
-                            coords_list = []
-                            for start in range(0, len(df_umap), BATCH_SIZE):
-                                end = min(start + BATCH_SIZE, len(df_umap))
-                                mat_batch = scaler.transform(
-                                    df_umap[start:end].select(num_cols).to_numpy())
-                                mat_batch = np.nan_to_num(mat_batch, nan=0.0)
-                                coords_list.append(reducer.transform(mat_batch))
-                            coords = np.vstack(coords_list)
-                        else:
-                            mat = np.nan_to_num(StandardScaler().fit_transform(
-                                df_umap.select(num_cols).to_numpy()), nan=0.0)
-                            reducer = umap.UMAP(n_components=2, random_state=42,
-                                                n_neighbors=15, min_dist=0.1,
-                                                metric='euclidean')
-                            coords = reducer.fit_transform(mat)
 
-                        non_num = [c for c in df_umap.columns if c not in num_cols and not c.startswith('_')]
+                    if len(df_umap) > 2:
+                        for attempt_n in (3, 2):
+                            try:
+                                if n_total > MAX_UMAP_TRAIN:
+                                    # Train UMAP on 10K sample, batch-transform full dataset
+                                    mat_sample = df_sample.select(num_cols).to_numpy()
+                                    scaler = StandardScaler().fit(mat_sample)
+                                    mat_sample_scaled = np.nan_to_num(
+                                        scaler.transform(mat_sample), nan=0.0)
+                                    reducer = umap.UMAP(n_components=attempt_n,
+                                                        random_state=42,
+                                                        n_neighbors=15, min_dist=0.1,
+                                                        metric='euclidean')
+                                    reducer.fit(mat_sample_scaled)
+                                    # Batch transform in 1K-row batches
+                                    coords_list = []
+                                    for start in range(0, len(df_umap), BATCH_SIZE):
+                                        end = min(start + BATCH_SIZE, len(df_umap))
+                                        mat_batch = scaler.transform(
+                                            df_umap[start:end].select(num_cols).to_numpy())
+                                        mat_batch = np.nan_to_num(mat_batch, nan=0.0)
+                                        coords_list.append(reducer.transform(mat_batch))
+                                    coords = np.vstack(coords_list)
+                                else:
+                                    mat = np.nan_to_num(StandardScaler().fit_transform(
+                                        df_umap.select(num_cols).to_numpy()), nan=0.0)
+                                    reducer = umap.UMAP(n_components=attempt_n,
+                                                        random_state=42,
+                                                        n_neighbors=15, min_dist=0.1,
+                                                        metric='euclidean')
+                                    coords = reducer.fit_transform(mat)
+                                umap_components = attempt_n
+                                break
+                            except Exception as e_umap:
+                                if attempt_n == 2:
+                                    raise
+                                logger.warning(f'UMAP 3D failed ({e_umap}), falling back to 2D')
+
+                        if coords is None:
+                            raise Exception('UMAP produced no output')
+
+                        non_num = [c for c in df_umap.columns
+                                   if c not in num_cols and not c.startswith('_')]
                         ent_col = non_num[0] if non_num else df_umap.columns[0]
 
                         cluster_entry = store.get_cluster_result(cluster_id)
@@ -991,19 +1178,23 @@ def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorith
                             lbl = labels_list[i] if i < len(labels_list) else -1
                             cache_key = f'{run.id}_{lbl}'
                             if cache_key not in cr_cache:
-                                cr_cache[cache_key] = CR.objects.filter(run=run, cluster_label=lbl).first()
+                                cr_cache[cache_key] = CR.objects.filter(
+                                    run=run, cluster_label=lbl).first()
+                            has_z = umap_components >= 3 and coords.shape[1] >= 3
                             profiles.append(EP(
                                 entity_value=ev, run=run, cluster_label=lbl,
                                 cluster=cr_cache[cache_key],
                                 embedding_x=float(coords[i, 0]) if i < len(coords) else None,
                                 embedding_y=float(coords[i, 1]) if i < len(coords) else None,
+                                embedding_z=float(coords[i, 2]) if has_z
+                                    and i < len(coords) else None,
                             ))
                         if profiles:
                             EP.objects.bulk_create(profiles, ignore_conflicts=True, batch_size=1000)
                             run.entity_count = len(profiles)
                             run.save(update_fields=['entity_count'])
             except Exception as umap_err:
-                logger.warning(f'UMAP-2D embedding skipped: {umap_err}')
+                logger.warning(f'UMAP embedding skipped: {umap_err}')
 
         # Fallback: if entity_count not set (UMAP skipped), derive from cluster sizes
         if run.entity_count is None:
@@ -1044,7 +1235,7 @@ def manual_run_analysis(request):
     run.run_type = 'manual'
     run.save(update_fields=['run_type'])
     feature_columns = body.get('feature_columns')
-    algorithm = body.get('algorithm', 'hdbscan')
+    algorithm = body.get('algorithm', 'agglomerative')
     min_cluster_size = int(body.get('min_cluster_size', 5))
     head = body.get('head')
     cluster_mode = body.get('cluster_mode', 'raw')
@@ -1145,30 +1336,10 @@ def apply_filter(request):
         filtered_id = result.get('filtered_dataset_id', '')
         row_count = result.get('row_count', 0)
 
-        # Re-store with filtered_X ID pattern
+        # Use the unique flt_xxx ID from _handle_filter_data — never rename
-        if dataset_id.startswith('upload_'):
+        # to filtered_{num} which would collide on repeated filters of the same source.
-            display_suffix = dataset_id.split('_', 1)[1]
-            new_id = f'filtered_{display_suffix}'
-        else:
-            new_id = filtered_id
-
-        from analysis.session_store import SessionStore
-        store = SessionStore()
-        entry = store.get_dataset(filtered_id)
-        if entry and new_id != filtered_id:
-            store.store_dataset(
-                dataset_id=new_id,
-                lazyframe=entry['lazyframe'],
-                schema=entry.get('schema', {}),
-                metadata=dict(entry.get('metadata', {}), parent_dataset_id=dataset_id),
-                parent_id=dataset_id,
-            )
-            new_id_to_use = new_id
-        else:
-            new_id_to_use = filtered_id
-
         return JsonResponse({
-            'dataset_id': new_id_to_use,
+            'dataset_id': filtered_id,
             'row_count': row_count,
         })
     except Exception as e:
@@ -1196,22 +1367,74 @@ def run_llm_analysis_view(request):
 
     from analysis.session_store import SessionStore
     store = SessionStore()
-    # Use display_id for dataset key (consistent with _background_process)
+
+    # Determine dataset key format (consistent with manual_page)
     upload_ds_id = f'upload_{display_id}'
+    run_ds_id = f'run_{display_id}'
     entry = store.get_dataset(upload_ds_id)
     if entry is None:
-        # Fallback: try PK-based key (old pipeline)
+        entry = store.get_dataset(run_ds_id)
-        upload_ds_id = f'upload_{pk}'
-        entry = store.get_dataset(upload_ds_id)
 
     if entry is None:
-        # Try entity dataset as fallback
+        # Try primary key fallback
-        entity_ds_id = f'entity_{pk}'
+        upload_pk_id = f'upload_{pk}'
-        entry = store.get_dataset(entity_ds_id)
+        entity_pk_id = f'entity_{pk}'
+        entry = store.get_dataset(upload_pk_id)
         if entry is None:
-            return JsonResponse({'error': '数据集不存在，请先上传并等待预处理完成'})
+            entry = store.get_dataset(entity_pk_id)
 
-    dataset_id = upload_ds_id if store.get_dataset(upload_ds_id) else f'entity_{pk}'
+    if entry is None:
+        # Auto-reload from sqlite_table or csv_glob
+        from analysis.data_loader import load_csv_directory, load_from_db
+        lf = None
+        schema = {}
+        if run.sqlite_table:
+            try:
+                lf = load_from_db(run.sqlite_table)
+            except Exception:
+                pass
+        if lf is None and run.csv_glob:
+            try:
+                lf, schema, _, _, _ = load_csv_directory(run.csv_glob)
+            except Exception:
+                pass
+        if lf is None:
+            return JsonResponse({'error': '数据集不存在，请先上传并等待预处理完成'})
+        entry_key = upload_ds_id if run.run_type == 'upload' else run_ds_id
+        store.store_dataset(entry_key, lf, schema=schema, metadata={
+            'row_count': run.total_flows,
+            'csv_glob': run.csv_glob or '',
+            'sqlite_table': run.sqlite_table or '',
+        })
+        # Re-fetch entry
+        entry = store.get_dataset(entry_key)
+
+    # Determine the active dataset_id
+    for candidate in [upload_ds_id, run_ds_id, f'upload_{pk}', f'entity_{pk}']:
+        if store.get_dataset(candidate):
+            dataset_id = candidate
+            break
+    else:
+        return JsonResponse({'error': '数据集不存在，请先上传并等待预处理完成'})
+
+    # ── Optional: apply filter before LLM pipeline ──
+    filters = body.get('filters', [])
+    if filters and len(filters) > 0:
+        logic = body.get('logic', 'and')
+        try:
+            from analysis.tool_registry import _handle_filter_data
+            filter_result = asyncio.run(_handle_filter_data(
+                dataset_id=dataset_id,
+                filters=filters,
+                logic=logic,
+            ))
+            if 'error' not in filter_result:
+                filtered_id = filter_result.get('filtered_dataset_id', '')
+                if filtered_id and store.get_dataset(filtered_id):
+                    dataset_id = filtered_id
+        except Exception:
+            import traceback
+            logger.warning(f'[LLM auto] Filter failed (non-fatal): {traceback.format_exc()}')
 
     def _analysis_fn(run, ctx):
         from tianxuan.llm_orchestrator import run_llm_pipeline, LLMConfig
@@ -1228,26 +1451,22 @@ def run_llm_analysis_view(request):
 
                 save_fields = ['progress_pct', 'progress_msg']
                 if tool_name == 'thinking':
-                    # Accumulate LLM thinking text
+                    # Accumulate LLM thinking text (appended to llm_thinking)
                     if result:
                         run.llm_thinking = (run.llm_thinking or '') + f'\n[{step}] {result}'
                     save_fields = ['progress_pct', 'progress_msg', 'llm_thinking']
                 elif tool_name == 'complete':
                     pass  # No extra tracking needed
                 else:
-                    # Real tool call — log it
+                    # Single tool call — always has result
-                    if result:
+                    if isinstance(result, dict):
-                        if isinstance(result, dict):
+                        import json as _json
-                            # Format dict as JSON, truncate large values
+                        summary = _json.dumps(result, default=str, ensure_ascii=False)[:500]
-                            import json as _json
-                            summary = _json.dumps(result, default=str, ensure_ascii=False)[:500]
-                        else:
-                            summary = str(result)[:500]
-                        run.run_log += f'[{step}] {tool_name}: {summary}...\n'
                     else:
-                        run.run_log += f'[{step}] {tool_name}\n'
+                        summary = str(result)[:500]
+                    run.run_log += f'[{step}] {tool_name}: {summary}...\n'
 
-                    # Capture structured tool call data
+                    # One entry per tool call
                     if meta and isinstance(meta, dict) and 'args' in meta:
                         tool_calls = list(run.tool_calls_json or [])
                         tool_calls.append({
@@ -1317,7 +1536,7 @@ def run_llm_analysis_view(request):
                         store=store,
                         entity_ds_id=entity_ds_id,
                         feature_columns=None,
-                        algorithm='hdbscan',
+                        algorithm='agglomerative',
                         min_cluster_size=5,
                         run_umap=True,
                         head=ctx.get('head'),
@@ -1392,17 +1611,73 @@ def run_llm_analysis_view(request):
 
 def auto_page(request):
     """LLM-driven auto analysis page."""
+    from analysis.session_store import SessionStore
+    import json as _json
+
     cfg = get_config()
-    selected_run = None
+
-    run_id = request.GET.get('run_id')
+    # Get datasets from SessionStore + ALL runs from DB
-    if run_id:
+    store = SessionStore()
-        try:
+    session_datasets = store.list_datasets() if hasattr(store, 'list_datasets') else []
-            selected_run = AnalysisRun.objects.get(display_id=int(run_id))
+    session_ds_map = {ds['dataset_id']: ds for ds in session_datasets}
-        except (ValueError, AnalysisRun.DoesNotExist):
+
-            pass
+    # Build dataset entries from ALL AnalysisRun records
+    all_runs = AnalysisRun.objects.all().order_by('-created_at')[:100]
+    datasets_with_status = []
+
+    for run in all_runs:
+        if run.run_type == 'upload':
+            ds_id = f'upload_{run.display_id}'
+        else:
+            ds_id = f'run_{run.display_id}'
+
+        session_entry = session_ds_map.get(ds_id)
+
+        if session_entry:
+            datasets_with_status.append({
+                **session_entry,
+                'dataset_id': ds_id,
+                'run_status': run.status,
+                'total_flows': run.total_flows,
+                'display_id': run.display_id,
+                'needs_reload': False,
+                'csv_glob': run.csv_glob or '',
+                'sqlite_table': run.sqlite_table or '',
+            })
+        elif run.csv_glob or run.sqlite_table:
+            datasets_with_status.append({
+                'dataset_id': ds_id,
+                'run_status': run.status,
+                'display_id': run.display_id,
+                'total_flows': run.total_flows,
+                'row_count': run.total_flows,
+                'file_count': None,
+                'columns': [],
+                'column_count': 0,
+                'svd_components': 0,
+                'needs_reload': True,
+                'csv_glob': run.csv_glob or '',
+                'sqlite_table': run.sqlite_table or '',
+            })
+
+    for ds_info in datasets_with_status:
+        s = ds_info['run_status']
+        if s in ('completed',):
+            ds_info['status_label'] = '已分析'
+            ds_info['badge_class'] = 'completed'
+        elif s in ('failed',):
+            ds_info['status_label'] = '错误'
+            ds_info['badge_class'] = 'failed'
+        elif s in ('pending', 'loading', 'profiling', 'aggregating', 'clustering', 'extracting'):
+            ds_info['status_label'] = '分析中'
+            ds_info['badge_class'] = 'analyzing'
+        else:
+            ds_info['status_label'] = '待分析'
+            ds_info['badge_class'] = 'ready'
+
     return render(request, 'tianxuan/auto.html', {
         'llm_configured': bool(cfg.llm.base_url and cfg.llm.api_key),
-        'selected_run': selected_run,
+        'datasets_with_status_json': _json.dumps(datasets_with_status, ensure_ascii=False),
     })
 
 
@@ -1729,12 +2004,105 @@ def tool_lab_run(request):
     if not tool_name:
         return JsonResponse({'error': 'Missing tool name'}, status=400)
 
-    from analysis.tool_registry import handle_call
+    # ── Backend validation: check required params + dataset existence ──
+    from analysis.tool_registry import handle_call, get_tools_meta
+    from analysis.session_store import SessionStore
+    tools_meta = get_tools_meta()
+    tool_schema = None
+    for t in tools_meta:
+        if t.name == tool_name:
+            tool_schema = t.inputSchema
+            break
+    if tool_schema:
+        required_fields = tool_schema.get('required', [])
+        for field in required_fields:
+            val = params.get(field)
+            if val is None or (isinstance(val, str) and val.strip() == ''):
+                return JsonResponse({'error': f'Missing required parameter: {field}'}, status=400)
+            # If field is a dataset_id, verify it exists in the store
+            if 'dataset' in field.lower():
+                store = SessionStore()
+                if not store.get_dataset(str(val)):
+                    return JsonResponse({'error': f'Dataset not found: {val}'}, status=400)
+
     import asyncio
     result = asyncio.run(handle_call(tool_name, params))
     return JsonResponse({'status': 'ok', 'result': result})
 
 
+@csrf_exempt
+def reload_run_data(request):
+    """Reload a run's data from csv_glob or sqlite_table into SessionStore."""
+    if request.method != 'POST':
+        return JsonResponse({'error': 'POST required'}, status=405)
+    import json as _json
+    try:
+        body = _json.loads(request.body)
+    except Exception:
+        return JsonResponse({'error': 'Invalid JSON body'}, status=400)
+
+    display_id = body.get('display_id')
+    if not display_id:
+        return JsonResponse({'error': 'Missing display_id'}, status=400)
+
+    from analysis.models import AnalysisRun
+    from analysis.data_loader import load_csv_directory, load_from_db
+
+    run = get_object_or_404(AnalysisRun, display_id=display_id)
+
+    # Determine dataset_id (same keying as manual_page)
+    ds_id = f'upload_{display_id}' if run.run_type == 'upload' else f'run_{display_id}'
+
+    store = SessionStore()
+    # Check if already loaded
+    existing = store.get_dataset(ds_id)
+    if existing is not None:
+        return JsonResponse({'status': 'ok', 'dataset_id': ds_id, 'reloaded': False})
+
+    # Try loading from sqlite_table first, then csv_glob
+    lf = None
+    schema = {}
+    row_count = 0
+    file_count = 0
+
+    if run.sqlite_table:
+        try:
+            lf = load_from_db(run.sqlite_table)
+            if lf is not None:
+                row_count = lf.select(pl.len()).collect(streaming=True)[0, 0]
+        except Exception as exc:
+            import logging
+            logging.getLogger(__name__).warning(f'Failed to load from sqlite_table: {exc}')
+
+    if lf is None and run.csv_glob:
+        try:
+            from pathlib import Path
+            csv_path = Path(run.csv_glob)
+            if csv_path.is_file() or csv_path.is_dir() or csv_path.parent.is_dir():
+                lf, schema, row_count, file_count, _ = load_csv_directory(run.csv_glob)
+        except Exception as exc:
+            import logging
+            logging.getLogger(__name__).warning(f'Failed to load from csv_glob: {exc}')
+
+    if lf is None:
+        return JsonResponse({'error': 'No data source available (upload dir or sqlite table gone)'}, status=404)
+
+    store.store_dataset(ds_id, lf, schema=schema, metadata={
+        'row_count': row_count,
+        'file_count': file_count,
+        'csv_glob': run.csv_glob or '',
+        'sqlite_table': run.sqlite_table or '',
+    })
+
+    return JsonResponse({
+        'status': 'ok',
+        'dataset_id': ds_id,
+        'reloaded': True,
+        'row_count': row_count,
+        'columns': list(schema.keys()),
+    })
+
+
 # ── Save / Load / Delete analysis plans ─────────────────────────────
 
 from pathlib import Path
@@ -1933,3 +2301,8 @@ def retry_run(request, display_id):
 
     return redirect('analysis:run_detail', display_id=run.display_id)
 
+
+# Auto-profile all public functions in this module
+from analysis.profile_util import auto_profile_module  # noqa: E402
+auto_profile_module(__name__)
+

config/config.yaml.example

@@ -1,6 +1,6 @@
 server:
   host: 0.0.0.0
-  port: 80
+  port: 8000
   debug: false
 data:
   schema_strict: false
@@ -25,7 +25,7 @@ anomaly_detection:
 llm:
   enabled: true
   base_url: 'https://api.deepseek.com'
-  api_key: 'sk-ecd33e5341ff4ec9a2c5a5c97a97699b'
+  api_key: 'sk-your-api-key-here'
   model: deepseek-v4-flash
 entity:
   subnet_masks: [24, 28]

config/loader.py

@@ -19,7 +19,7 @@ class Config(BaseModel):
         recursive: bool = False
 
     class Clustering(BaseModel):
-        algorithm: str = 'hdbscan'
+        algorithm: str = 'agglomerative'
         min_cluster_size: int = 5
         random_state: int = 42
 

data/geoip_cache.json

@@ -0,0 +1,683 @@
+{
+  "8.8.8.8": {
+    "lat": 37.422,
+    "lon": -122.084,
+    "city": "Sunnyvale",
+    "country": "US"
+  },
+  "1.1.1.1": {
+    "lat": 37.7749,
+    "lon": -122.4194,
+    "city": "San Francisco",
+    "country": "US"
+  },
+  "52.0.0.1": {
+    "lat": 39.0438,
+    "lon": -77.4874,
+    "city": "Ashburn",
+    "country": "US"
+  },
+  "104.16.1.1": {
+    "lat": 37.7749,
+    "lon": -122.4194,
+    "city": "San Francisco",
+    "country": "US"
+  },
+  "47.96.1.1": {
+    "lat": 31.2304,
+    "lon": 121.4737,
+    "city": "Shanghai",
+    "country": "CN"
+  },
+  "218.1.1.1": null,
+  "157.1.1.1": {
+    "lat": 34.6937,
+    "lon": 135.5023,
+    "city": "Japan",
+    "country": "JP"
+  },
+  "10.0.0.1": null,
+  "192.168.1.1": null,
+  "13.250.0.1": null,
+  "185.234.0.1": null,
+  "103.235.0.1": {
+    "lat": 20.0,
+    "lon": 77.0,
+    "city": "",
+    "country": "印度"
+  },
+  "anon, hosting": null,
+  "-108.418446": null,
+  "-63.303819": null,
+  "156.494253": {
+    "lat": 34.053,
+    "lon": -118.2642,
+    "city": "洛杉矶",
+    "country": "美国"
+  },
+  "174.324906": {
+    "lat": 43.6425,
+    "lon": -79.3872,
+    "city": "",
+    "country": "加拿大"
+  },
+  "91.251045": {
+    "lat": 53.55,
+    "lon": 10.0,
+    "city": "汉堡市",
+    "country": "德国"
+  },
+  "136.838644": {
+    "lat": 42.3223,
+    "lon": -83.1763,
+    "city": "迪尔伯恩",
+    "country": "美国"
+  },
+  "-135.232683": null,
+  "134.708535": {
+    "lat": 45.4763,
+    "lon": -122.6408,
+    "city": "Portland",
+    "country": "美国"
+  },
+  "172.779207": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "76.375007": {
+    "lat": 35.6507,
+    "lon": -78.4564,
+    "city": "Clayton",
+    "country": "美国"
+  },
+  "-37.054377": null,
+  "58.368656": {
+    "lat": 35.685,
+    "lon": 139.7514,
+    "city": "东京",
+    "country": "日本"
+  },
+  "-128.312338": null,
+  "-86.295682": null,
+  "-83.710197": null,
+  "-117.654327": null,
+  "-94.931072": null,
+  "-31.118326": null,
+  "41.94697": {
+    "lat": -29.0,
+    "lon": 24.0,
+    "city": "",
+    "country": "南非"
+  },
+  "-175.557379": null,
+  "None": null,
+  "90.484378": {
+    "lat": 50.6846,
+    "lon": 1.7610999999999999,
+    "city": "维尔维涅",
+    "country": "法国"
+  },
+  "91.136808": {
+    "lat": 50.8934,
+    "lon": 7.8856,
+    "city": "Freudenberg",
+    "country": "德国"
+  },
+  "98.398423": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "-37.351993": null,
+  "108.8728": {
+    "lat": 33.5363,
+    "lon": -117.0439,
+    "city": "特曼库拉",
+    "country": "美国"
+  },
+  "-40.490238": null,
+  "-108.320482": null,
+  "-42.599081": null,
+  "60.847895": {
+    "lat": 30.2936,
+    "lon": 120.1614,
+    "city": "杭州",
+    "country": "中国"
+  },
+  "-70.344735": null,
+  "-17.065978": null,
+  "-17.594761": null,
+  "3.748318": {
+    "lat": 41.1412,
+    "lon": -73.2637,
+    "city": "Fairfield",
+    "country": "美国"
+  },
+  "117.90784": {
+    "lat": 21.0333,
+    "lon": 105.85,
+    "city": "Hanoi",
+    "country": "越南"
+  },
+  "-178.325477": null,
+  "47.034714": {
+    "lat": 45.3155,
+    "lon": -75.837,
+    "city": "Ottawa",
+    "country": "加拿大"
+  },
+  "12.6356": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "-4.376563": null,
+  "117.415791": {
+    "lat": 21.0333,
+    "lon": 105.85,
+    "city": "Hanoi",
+    "country": "越南"
+  },
+  "-95.943745": null,
+  "115.811466": {
+    "lat": 37.57,
+    "lon": 126.98,
+    "city": "",
+    "country": "大韩民国"
+  },
+  "82.858229": {
+    "lat": 54.5842,
+    "lon": -1.5631,
+    "city": "Darlington",
+    "country": "英国"
+  },
+  "120.177108": {
+    "lat": 39.8897,
+    "lon": 115.275,
+    "city": "Hebei",
+    "country": "中国"
+  },
+  "12.054574": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "-112.238338": null,
+  "-161.091649": null,
+  "24.207799": {
+    "lat": 40.3698,
+    "lon": -80.634,
+    "city": "斯托本维尔",
+    "country": "美国"
+  },
+  "-148.31237": null,
+  "-132.223679": null,
+  "76.252382": {
+    "lat": 35.6513,
+    "lon": -78.8336,
+    "city": "Holly Springs",
+    "country": "美国"
+  },
+  "104.839037": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "-147.319447": null,
+  "-137.111721": null,
+  "-107.76671": null,
+  "-167.802595": null,
+  "168.500812": {
+    "lat": 29.7176,
+    "lon": -95.4188,
+    "city": "休斯敦",
+    "country": "美国"
+  },
+  "-19.640731": null,
+  "-36.115066": null,
+  "-31.172609": null,
+  "4.746775": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "-128.714202": null,
+  "-95.66714": null,
+  "-19.935247": null,
+  "52.225763": {
+    "lat": 39.5645,
+    "lon": -75.597,
+    "city": "威尔明顿",
+    "country": "美国"
+  },
+  "-39.370987": null,
+  "162.006633": {
+    "lat": 43.7807,
+    "lon": -79.2855,
+    "city": "士嘉堡",
+    "country": "加拿大"
+  },
+  "51.696746": {
+    "lat": 51.5,
+    "lon": -0.13,
+    "city": "",
+    "country": "英国"
+  },
+  "6.673336": {
+    "lat": 31.5273,
+    "lon": -110.3607,
+    "city": "谢拉维斯塔",
+    "country": "美国"
+  },
+  "102.5503": null,
+  "-144.964956": null,
+  "-104.457778": null,
+  "83.087052": null,
+  "-125.101539": null,
+  "90.262005": {
+    "lat": 48.9018,
+    "lon": 2.4893,
+    "city": "邦迪",
+    "country": "法国"
+  },
+  "-83.715609": null,
+  "29.378547": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "-57.090496": null,
+  "142.184372": {
+    "lat": 44.7314,
+    "lon": -63.6482,
+    "city": "Dartmouth",
+    "country": "加拿大"
+  },
+  "165.297813": {
+    "lat": -33.9167,
+    "lon": 18.4167,
+    "city": "Cape Town",
+    "country": "南非"
+  },
+  "105.875829": {
+    "lat": -29.0,
+    "lon": 24.0,
+    "city": "",
+    "country": "南非"
+  },
+  "160.97215": {
+    "lat": 43.286,
+    "lon": -77.6843,
+    "city": "Rochester",
+    "country": "美国"
+  },
+  "-76.327692": null,
+  "-179.768984": null,
+  "-135.649249": null,
+  "-129.397466": null,
+  "-159.604348": null,
+  "123.957119": {
+    "lat": 34.6836,
+    "lon": 113.5325,
+    "city": "郑州",
+    "country": "中国"
+  },
+  "106.508142": {
+    "lat": 28.55,
+    "lon": 115.9333,
+    "city": "南昌",
+    "country": "中国"
+  },
+  "60.737607": {
+    "lat": 45.75,
+    "lon": 126.65,
+    "city": "哈尔滨",
+    "country": "中国"
+  },
+  "12.851572": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "133.109702": {
+    "lat": 35.69,
+    "lon": 139.69,
+    "city": "",
+    "country": "日本"
+  },
+  "97.468122": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "-49.099249": null,
+  "31.114424": {
+    "lat": 52.25,
+    "lon": 21.0,
+    "city": "华沙",
+    "country": "波兰"
+  },
+  "49.613214": {
+    "lat": 37.5985,
+    "lon": 126.9783,
+    "city": "首尔特别市",
+    "country": "大韩民国"
+  },
+  "-76.937967": null,
+  "24.350375": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "-57.188778": null,
+  "-129.431676": null,
+  "-136.186649": null,
+  "-61.694727": null,
+  "-90.513611": null,
+  "91.194896": {
+    "lat": 49.5333,
+    "lon": 11.8,
+    "city": "哈恩巴赫",
+    "country": "德国"
+  },
+  "132.327773": {
+    "lat": 32.404,
+    "lon": -86.2539,
+    "city": "蒙哥马利",
+    "country": "美国"
+  },
+  "-112.197533": null,
+  "155.022955": null,
+  "162.854691": {
+    "lat": 51.5,
+    "lon": -0.13,
+    "city": "",
+    "country": "英国"
+  },
+  "-52.303734": null,
+  "-37.91953": null,
+  "144.662392": {
+    "lat": 37.3762,
+    "lon": -122.1826,
+    "city": "Palo Alto",
+    "country": "美国"
+  },
+  "17.315107": {
+    "lat": 37.323,
+    "lon": -122.0322,
+    "city": "Cupertino",
+    "country": "美国"
+  },
+  "97.359727": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "-37.814213": null,
+  "-57.066252": null,
+  "170.670964": {
+    "lat": 47.6689,
+    "lon": -117.4369,
+    "city": "斯波坎",
+    "country": "美国"
+  },
+  "19.237916": {
+    "lat": 42.3223,
+    "lon": -83.1763,
+    "city": "迪尔伯恩",
+    "country": "美国"
+  },
+  "-77.426219": null,
+  "35.273579": {
+    "lat": 42.2776,
+    "lon": -83.7409,
+    "city": "安娜堡",
+    "country": "美国"
+  },
+  "-29.56673": null,
+  "-173.438083": null,
+  "-164.691055": null,
+  "-140.463216": null,
+  "-168.663276": null,
+  "-138.543279": null,
+  "-47.142859": null,
+  "-102.389184": null,
+  "24.754033": {
+    "lat": 32.981,
+    "lon": -80.0326,
+    "city": "Goose Creek",
+    "country": "美国"
+  },
+  "-109.476174": null,
+  "92.836244": {
+    "lat": 51.5,
+    "lon": -0.13,
+    "city": "",
+    "country": "英国"
+  },
+  "2.089979": null,
+  "82.41569": {
+    "lat": 51.6578,
+    "lon": -0.3954,
+    "city": "沃特福德",
+    "country": "英国"
+  },
+  "-117.765392": null,
+  "-112.753899": null,
+  "-147.563834": null,
+  "-27.4301": null,
+  "66.656057": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "-52.329152": null,
+  "16.783619": {
+    "lat": 37.3762,
+    "lon": -122.1826,
+    "city": "Palo Alto",
+    "country": "美国"
+  },
+  "-160.668335": null,
+  "-155.782242": null,
+  "-43.415128": null,
+  "-151.682652": null,
+  "33.713437": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "151.170164": {
+    "lat": 42.8333,
+    "lon": 12.8333,
+    "city": "",
+    "country": "意大利"
+  },
+  "-2.437103": null,
+  "17.096003": null,
+  "4.372196": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "12.775114": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "-26.847855": null,
+  "-2.96053": null,
+  "-136.421502": null,
+  "156.152711": {
+    "lat": 46.2857,
+    "lon": -119.2845,
+    "city": "里奇兰",
+    "country": "美国"
+  },
+  "-121.186972": null,
+  "-126.924167": null,
+  "-101.388131": null,
+  "-101.03754": null,
+  "77.828572": {
+    "lat": 52.5167,
+    "lon": 13.4,
+    "city": "柏林",
+    "country": "德国"
+  },
+  "-34.813058": null,
+  "123.011334": {
+    "lat": 23.7,
+    "lon": 90.375,
+    "city": "",
+    "country": "孟加拉"
+  },
+  "11.265017": {
+    "lat": 39.9612,
+    "lon": -82.9988,
+    "city": "哥伦布",
+    "country": "美国"
+  },
+  "11.202764": {
+    "lat": 39.9612,
+    "lon": -82.9988,
+    "city": "哥伦布",
+    "country": "美国"
+  },
+  "70.262109": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "-70.027646": null,
+  "26.990087": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "147.506751": {
+    "lat": 47.5,
+    "lon": 19.0833,
+    "city": "布达佩斯",
+    "country": "匈牙利"
+  },
+  "-107.81862": null,
+  "141.75262": {
+    "lat": 51.0,
+    "lon": 9.0,
+    "city": "",
+    "country": "德国"
+  },
+  "30.674809": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  },
+  "120.327798": {
+    "lat": 39.8897,
+    "lon": 115.275,
+    "city": "Hebei",
+    "country": "中国"
+  },
+  "-158.088393": null,
+  "-121.790773": null,
+  "77.908193": {
+    "lat": 50.3167,
+    "lon": 7.8,
+    "city": "Nassau",
+    "country": "德国"
+  },
+  "57.252522": {
+    "lat": 47.0,
+    "lon": 8.0,
+    "city": "",
+    "country": ""
+  },
+  "-15.466886": null,
+  "-136.800708": null,
+  "-139.560022": null,
+  "155.309838": {
+    "lat": 59.3294,
+    "lon": 18.0686,
+    "city": "",
+    "country": "瑞典"
+  },
+  "158.175545": {
+    "lat": 31.5273,
+    "lon": -110.3607,
+    "city": "谢拉维斯塔",
+    "country": "美国"
+  },
+  "-38.283752": null,
+  "146.597747": {
+    "lat": 42.3755,
+    "lon": -83.0772,
+    "city": "底特律",
+    "country": "美国"
+  },
+  "-146.199812": null,
+  "-60.696207": null,
+  "141.547786": {
+    "lat": 35.8869,
+    "lon": 14.4025,
+    "city": "Mdina",
+    "country": "马耳他"
+  },
+  "10.542121": null,
+  "169.586656": {
+    "lat": 45.0059,
+    "lon": -93.1059,
+    "city": "圣保罗",
+    "country": "美国"
+  },
+  "53.588162": {
+    "lat": 51.0,
+    "lon": 9.0,
+    "city": "",
+    "country": "德国"
+  },
+  "-48.221466": null,
+  "-104.847562": null,
+  "-97.617615": null,
+  "145.767865": {
+    "lat": 51.8425,
+    "lon": 5.8528,
+    "city": "奈梅亨",
+    "country": "荷兰"
+  },
+  "-173.124365": null,
+  "-158.431061": null,
+  "-132.023148": null,
+  "-171.873785": null,
+  "-23.369992": null,
+  "53.281077": {
+    "lat": 51.0,
+    "lon": 9.0,
+    "city": "",
+    "country": "德国"
+  },
+  "28.294006": {
+    "lat": 38.0,
+    "lon": -97.0,
+    "city": "",
+    "country": "美国"
+  }
+}

data/geoip_data.txt

@@ -1,808 +0,0 @@
-# ===== Offline GeoIP Database — 天璇 (TianXuan) =====
-# Format: start_ip,end_ip,lat,lon,city,country
-# Layers: Private, DNS, AWS, Azure, GCP, Alibaba, Cloudflare, Akamai, Fastly, CN_ISP, US_ISP, EU_ISP, JP_ISP
-# Strategy: priority-ordered, overlap-resolved (first wins), gaps filled with city blocks
-
-1.0.0.0,1.0.0.0,31.2304,121.4737,Shanghai,CN
-1.0.0.1,1.0.0.1,37.7749,-122.4194,San Francisco,US
-1.0.0.2,1.1.1.0,31.2304,121.4737,Shanghai,CN
-1.1.1.1,1.1.1.1,37.7749,-122.4194,San Francisco,US
-1.1.1.2,1.255.255.255,31.2304,121.4737,Shanghai,CN
-2.0.0.0,2.15.255.255,51.5074,-0.1278,London,GB
-2.16.0.0,2.23.255.255,42.3736,-71.1097,Cambridge,US
-2.24.0.0,2.255.255.255,51.5074,-0.1278,London,GB
-3.0.0.0,3.1.255.255,1.3521,103.8198,Singapore,SG
-3.2.0.0,3.5.255.255,31.2304,121.4737,Shanghai,CN
-3.6.0.0,3.7.255.255,19.076,72.8777,Mumbai,IN
-3.8.0.0,3.15.255.255,51.5074,-0.1278,London,GB
-3.16.0.0,3.23.255.255,39.9042,116.4074,Beijing,CN
-3.24.0.0,3.27.255.255,-33.8688,151.2093,Sydney,AU
-3.28.0.0,3.33.255.255,35.6762,139.6503,Tokyo,JP
-3.34.0.0,3.35.255.255,37.5665,126.978,Seoul,KR
-3.36.0.0,3.51.255.255,37.5665,126.978,Seoul,KR
-3.52.0.0,3.63.255.255,1.3521,103.8198,Singapore,SG
-3.64.0.0,3.79.255.255,50.1109,8.6821,Frankfurt,DE
-3.80.0.0,3.95.255.255,51.5074,-0.1278,London,GB
-3.96.0.0,3.99.255.255,45.5017,-73.5673,Montreal,CA
-3.100.0.0,3.111.255.255,50.1109,8.6821,Frankfurt,DE
-3.112.0.0,3.115.255.255,35.6762,139.6503,Tokyo,JP
-3.116.0.0,3.131.255.255,40.7128,-74.006,New York,US
-3.132.0.0,3.147.255.255,37.7749,-122.4194,San Francisco,US
-3.148.0.0,3.163.255.255,-33.8688,151.2093,Sydney,AU
-3.164.0.0,3.179.255.255,19.076,72.8777,Mumbai,IN
-3.180.0.0,3.195.255.255,48.8566,2.3522,Paris,FR
-3.196.0.0,3.207.255.255,52.3676,4.9041,Amsterdam,NL
-3.208.0.0,3.215.255.255,39.0438,-77.4874,Ashburn,US
-3.216.0.0,3.223.255.255,-23.5505,-46.6333,Sao Paulo,BR
-3.224.0.0,3.239.255.255,39.0438,-77.4874,Ashburn,US
-3.240.0.0,4.111.255.255,55.7558,37.6173,Moscow,RU
-4.112.0.0,4.239.255.255,41.8781,-87.6298,Chicago,US
-4.240.0.0,4.255.255.255,32.7767,-96.797,Dallas,US
-5.0.0.0,5.255.255.255,51.5074,-0.1278,London,GB
-6.0.0.0,6.127.255.255,33.749,-84.388,Atlanta,US
-6.128.0.0,6.255.255.255,43.6532,-79.3832,Toronto,CA
-7.0.0.0,7.127.255.255,22.3193,114.1694,Hong Kong,HK
-7.128.0.0,7.255.255.255,25.033,121.5654,Taipei,TW
-8.0.0.0,8.8.4.3,40.4168,-3.7038,Madrid,ES
-8.8.4.4,8.8.4.4,37.3861,-122.0839,Mountain View,US
-8.8.4.5,8.8.8.7,45.4642,9.19,Milan,IT
-8.8.8.8,8.8.8.8,37.3861,-122.0839,Mountain View,US
-8.8.8.9,8.136.8.8,59.3293,18.0686,Stockholm,SE
-8.136.8.9,9.8.8.8,34.6937,135.5022,Osaka,JP
-9.8.8.9,9.136.8.8,23.1291,113.2644,Guangzhou,CN
-9.136.8.9,9.255.255.255,22.5431,114.0579,Shenzhen,CN
-10.0.0.0,10.255.255.255,39.9042,116.4074,Beijing,CN
-11.0.0.0,11.127.255.255,52.52,13.405,Berlin,DE
-11.128.0.0,11.255.255.255,25.2048,55.2708,Dubai,AE
-12.0.0.0,12.255.255.255,32.7767,-96.797,Dallas,US
-13.0.0.0,13.15.255.255,13.7563,100.5018,Bangkok,TH
-13.16.0.0,13.31.255.255,31.2304,121.4737,Shanghai,CN
-13.32.0.0,13.47.255.255,39.9042,116.4074,Beijing,CN
-13.48.0.0,13.51.255.255,59.3293,18.0686,Stockholm,SE
-13.52.0.0,13.55.255.255,37.7749,-122.4194,San Francisco,US
-13.56.0.0,13.59.255.255,37.7749,-122.4194,San Francisco,US
-13.60.0.0,13.63.255.255,35.6762,139.6503,Tokyo,JP
-13.64.0.0,13.95.255.255,37.7749,-122.4194,San Francisco,US
-13.96.0.0,13.111.255.255,37.5665,126.978,Seoul,KR
-13.112.0.0,13.115.255.255,35.6762,139.6503,Tokyo,JP
-13.116.0.0,13.123.255.255,1.3521,103.8198,Singapore,SG
-13.124.0.0,13.124.255.255,37.5665,126.978,Seoul,KR
-13.125.0.0,13.125.255.255,51.5074,-0.1278,London,GB
-13.126.0.0,13.127.255.255,19.076,72.8777,Mumbai,IN
-13.128.0.0,13.143.255.255,50.1109,8.6821,Frankfurt,DE
-13.144.0.0,13.159.255.255,40.7128,-74.006,New York,US
-13.160.0.0,13.175.255.255,37.7749,-122.4194,San Francisco,US
-13.176.0.0,13.191.255.255,-33.8688,151.2093,Sydney,AU
-13.192.0.0,13.207.255.255,19.076,72.8777,Mumbai,IN
-13.208.0.0,13.208.255.255,34.6937,135.5022,Osaka,JP
-13.209.0.0,13.209.255.255,37.5665,126.978,Seoul,KR
-13.210.0.0,13.211.255.255,-33.8688,151.2093,Sydney,AU
-13.212.0.0,13.215.255.255,1.3521,103.8198,Singapore,SG
-13.216.0.0,13.227.255.255,48.8566,2.3522,Paris,FR
-13.228.0.0,13.229.255.255,1.3521,103.8198,Singapore,SG
-13.230.0.0,13.231.255.255,35.6762,139.6503,Tokyo,JP
-13.232.0.0,13.243.255.255,52.3676,4.9041,Amsterdam,NL
-13.244.0.0,13.245.255.255,-33.9249,18.4241,Cape Town,ZA
-13.246.0.0,13.255.255.255,-23.5505,-46.6333,Sao Paulo,BR
-14.0.0.0,14.255.255.255,39.9042,116.4074,Beijing,CN
-15.0.0.0,15.127.255.255,55.7558,37.6173,Moscow,RU
-15.128.0.0,15.151.255.255,41.8781,-87.6298,Chicago,US
-15.152.0.0,15.152.255.255,34.6937,135.5022,Osaka,JP
-15.153.0.0,15.159.255.255,32.7767,-96.797,Dallas,US
-15.160.0.0,15.160.255.255,45.4642,9.19,Milan,IT
-15.161.0.0,15.163.255.255,33.749,-84.388,Atlanta,US
-15.164.0.0,15.164.255.255,37.5665,126.978,Seoul,KR
-15.165.0.0,15.180.255.255,43.6532,-79.3832,Toronto,CA
-15.181.0.0,15.183.255.255,22.3193,114.1694,Hong Kong,HK
-15.184.0.0,15.185.255.255,26.0667,50.5577,Bahrain,BH
-15.186.0.0,15.187.255.255,25.033,121.5654,Taipei,TW
-15.188.0.0,15.188.255.255,48.8566,2.3522,Paris,FR
-15.189.0.0,15.204.255.255,40.4168,-3.7038,Madrid,ES
-15.205.0.0,15.205.255.255,45.4642,9.19,Milan,IT
-15.206.0.0,15.206.255.255,19.076,72.8777,Mumbai,IN
-15.207.0.0,15.221.255.255,59.3293,18.0686,Stockholm,SE
-15.222.0.0,15.223.255.255,45.5017,-73.5673,Montreal,CA
-15.224.0.0,15.227.255.255,34.6937,135.5022,Osaka,JP
-15.228.0.0,15.229.255.255,-23.5505,-46.6333,Sao Paulo,BR
-15.230.0.0,15.245.255.255,23.1291,113.2644,Guangzhou,CN
-15.246.0.0,16.5.255.255,22.5431,114.0579,Shenzhen,CN
-16.6.0.0,16.15.255.255,52.52,13.405,Berlin,DE
-16.16.0.0,16.16.255.255,59.3293,18.0686,Stockholm,SE
-16.17.0.0,16.144.255.255,25.2048,55.2708,Dubai,AE
-16.145.0.0,16.169.255.255,13.7563,100.5018,Bangkok,TH
-16.170.0.0,16.171.255.255,59.3293,18.0686,Stockholm,SE
-16.172.0.0,17.43.255.255,31.2304,121.4737,Shanghai,CN
-17.44.0.0,17.171.255.255,39.9042,116.4074,Beijing,CN
-17.172.0.0,18.43.255.255,35.6762,139.6503,Tokyo,JP
-18.44.0.0,18.135.255.255,37.5665,126.978,Seoul,KR
-18.136.0.0,18.139.255.255,1.3521,103.8198,Singapore,SG
-18.140.0.0,18.155.255.255,1.3521,103.8198,Singapore,SG
-18.156.0.0,18.161.255.255,51.5074,-0.1278,London,GB
-18.162.0.0,18.163.255.255,22.3193,114.1694,Hong Kong,HK
-18.164.0.0,18.167.255.255,50.1109,8.6821,Frankfurt,DE
-18.168.0.0,18.171.255.255,51.5074,-0.1278,London,GB
-18.172.0.0,18.175.255.255,40.7128,-74.006,New York,US
-18.176.0.0,18.177.255.255,35.6762,139.6503,Tokyo,JP
-18.178.0.0,18.183.255.255,37.7749,-122.4194,San Francisco,US
-18.184.0.0,18.187.255.255,50.1109,8.6821,Frankfurt,DE
-18.188.0.0,18.193.255.255,-33.8688,151.2093,Sydney,AU
-18.194.0.0,18.195.255.255,50.1109,8.6821,Frankfurt,DE
-18.196.0.0,18.211.255.255,19.076,72.8777,Mumbai,IN
-18.212.0.0,18.227.255.255,48.8566,2.3522,Paris,FR
-18.228.0.0,18.229.255.255,-23.5505,-46.6333,Sao Paulo,BR
-18.230.0.0,18.231.255.255,52.3676,4.9041,Amsterdam,NL
-18.232.0.0,18.235.255.255,39.0438,-77.4874,Ashburn,US
-18.236.0.0,19.107.255.255,-23.5505,-46.6333,Sao Paulo,BR
-19.108.0.0,19.235.255.255,55.7558,37.6173,Moscow,RU
-19.236.0.0,19.255.255.255,41.8781,-87.6298,Chicago,US
-20.0.0.0,20.31.255.255,39.0438,-77.4874,Ashburn,US
-20.32.0.0,20.35.255.255,32.7767,-96.797,Dallas,US
-20.36.0.0,20.39.255.255,-33.8688,151.2093,Sydney,AU
-20.40.0.0,20.40.255.255,33.749,-84.388,Atlanta,US
-20.41.0.0,20.41.255.255,37.5665,126.978,Seoul,KR
-20.42.0.0,20.42.255.255,43.6532,-79.3832,Toronto,CA
-20.43.0.0,20.43.255.255,48.8566,2.3522,Paris,FR
-20.44.0.0,20.45.255.255,35.6762,139.6503,Tokyo,JP
-20.46.0.0,20.48.255.255,22.3193,114.1694,Hong Kong,HK
-20.49.0.0,20.49.255.255,51.5074,-0.1278,London,GB
-20.50.0.0,20.51.255.255,25.033,121.5654,Taipei,TW
-20.52.0.0,20.52.255.255,50.1109,8.6821,Frankfurt,DE
-20.53.0.0,20.55.255.255,40.4168,-3.7038,Madrid,ES
-20.56.0.0,20.56.255.255,52.3676,4.9041,Amsterdam,NL
-20.57.0.0,20.72.255.255,45.4642,9.19,Milan,IT
-20.73.0.0,20.88.255.255,59.3293,18.0686,Stockholm,SE
-20.89.0.0,20.104.255.255,34.6937,135.5022,Osaka,JP
-20.105.0.0,20.120.255.255,23.1291,113.2644,Guangzhou,CN
-20.121.0.0,20.136.255.255,22.5431,114.0579,Shenzhen,CN
-20.137.0.0,20.152.255.255,52.52,13.405,Berlin,DE
-20.153.0.0,20.168.255.255,25.2048,55.2708,Dubai,AE
-20.169.0.0,20.183.255.255,13.7563,100.5018,Bangkok,TH
-20.184.0.0,20.185.255.255,1.3521,103.8198,Singapore,SG
-20.186.0.0,20.191.255.255,31.2304,121.4737,Shanghai,CN
-20.192.0.0,20.192.255.255,19.076,72.8777,Mumbai,IN
-20.193.0.0,20.195.255.255,39.9042,116.4074,Beijing,CN
-20.196.0.0,20.196.255.255,22.3193,114.1694,Hong Kong,HK
-20.197.0.0,20.197.255.255,-23.5505,-46.6333,Sao Paulo,BR
-20.198.0.0,21.69.255.255,35.6762,139.6503,Tokyo,JP
-21.70.0.0,21.197.255.255,37.5665,126.978,Seoul,KR
-21.198.0.0,22.69.255.255,1.3521,103.8198,Singapore,SG
-22.70.0.0,22.197.255.255,51.5074,-0.1278,London,GB
-22.198.0.0,22.255.255.255,50.1109,8.6821,Frankfurt,DE
-23.0.0.0,23.15.255.255,42.3736,-71.1097,Cambridge,US
-23.16.0.0,23.31.255.255,40.7128,-74.006,New York,US
-23.32.0.0,23.63.255.255,42.3736,-71.1097,Cambridge,US
-23.64.0.0,23.67.255.255,42.3736,-71.1097,Cambridge,US
-23.68.0.0,23.71.255.255,37.7749,-122.4194,San Francisco,US
-23.72.0.0,23.79.255.255,42.3736,-71.1097,Cambridge,US
-23.80.0.0,23.95.255.255,-33.8688,151.2093,Sydney,AU
-23.96.0.0,23.103.255.255,41.8781,-87.6298,Chicago,US
-23.104.0.0,23.119.255.255,19.076,72.8777,Mumbai,IN
-23.120.0.0,23.135.255.255,48.8566,2.3522,Paris,FR
-23.136.0.0,23.151.255.255,52.3676,4.9041,Amsterdam,NL
-23.152.0.0,23.167.255.255,-23.5505,-46.6333,Sao Paulo,BR
-23.168.0.0,23.183.255.255,55.7558,37.6173,Moscow,RU
-23.184.0.0,23.191.255.255,41.8781,-87.6298,Chicago,US
-23.192.0.0,23.223.255.255,42.3736,-71.1097,Cambridge,US
-23.224.0.0,23.239.255.255,32.7767,-96.797,Dallas,US
-23.240.0.0,23.255.255.255,33.749,-84.388,Atlanta,US
-24.0.0.0,24.255.255.255,40.7128,-74.006,New York,US
-25.0.0.0,25.255.255.255,51.5074,-0.1278,London,GB
-26.0.0.0,26.127.255.255,43.6532,-79.3832,Toronto,CA
-26.128.0.0,26.255.255.255,22.3193,114.1694,Hong Kong,HK
-27.0.0.0,27.255.255.255,31.2304,121.4737,Shanghai,CN
-28.0.0.0,28.127.255.255,25.033,121.5654,Taipei,TW
-28.128.0.0,28.255.255.255,40.4168,-3.7038,Madrid,ES
-29.0.0.0,29.127.255.255,45.4642,9.19,Milan,IT
-29.128.0.0,29.255.255.255,59.3293,18.0686,Stockholm,SE
-30.0.0.0,30.127.255.255,34.6937,135.5022,Osaka,JP
-30.128.0.0,30.255.255.255,23.1291,113.2644,Guangzhou,CN
-31.0.0.0,31.255.255.255,51.5074,-0.1278,London,GB
-32.0.0.0,32.255.255.255,33.749,-84.388,Atlanta,US
-33.0.0.0,33.127.255.255,22.5431,114.0579,Shenzhen,CN
-33.128.0.0,33.255.255.255,52.52,13.405,Berlin,DE
-34.0.0.0,34.207.255.255,39.0438,-77.4874,Ashburn,US
-34.208.0.0,34.223.255.255,45.5152,-122.6784,Portland,US
-34.224.0.0,34.239.255.255,39.0438,-77.4874,Ashburn,US
-34.240.0.0,34.247.255.255,53.3498,-6.2603,Dublin,IE
-34.248.0.0,34.255.255.255,39.0438,-77.4874,Ashburn,US
-35.0.0.0,35.127.255.255,25.2048,55.2708,Dubai,AE
-35.128.0.0,35.151.255.255,13.7563,100.5018,Bangkok,TH
-35.152.0.0,35.152.255.255,45.4642,9.19,Milan,IT
-35.153.0.0,35.153.255.255,31.2304,121.4737,Shanghai,CN
-35.154.0.0,35.155.255.255,19.076,72.8777,Mumbai,IN
-35.156.0.0,35.159.255.255,50.1109,8.6821,Frankfurt,DE
-35.160.0.0,35.175.255.255,39.9042,116.4074,Beijing,CN
-35.176.0.0,35.177.255.255,51.5074,-0.1278,London,GB
-35.178.0.0,35.179.255.255,51.5074,-0.1278,London,GB
-35.180.0.0,35.183.255.255,48.8566,2.3522,Paris,FR
-35.184.0.0,35.191.255.255,35.6762,139.6503,Tokyo,JP
-35.192.0.0,35.195.255.255,25.033,121.5654,Taipei,TW
-35.196.0.0,35.198.255.255,37.5665,126.978,Seoul,KR
-35.199.0.0,35.199.255.255,-23.5505,-46.6333,Sao Paulo,BR
-35.200.0.0,35.203.255.255,35.6762,139.6503,Tokyo,JP
-35.204.0.0,35.207.255.255,52.3676,4.9041,Amsterdam,NL
-35.208.0.0,35.215.255.255,1.3521,103.8198,Singapore,SG
-35.216.0.0,35.219.255.255,37.5665,126.978,Seoul,KR
-35.220.0.0,35.221.255.255,32.7767,-96.797,Dallas,US
-35.222.0.0,35.223.255.255,41.8781,-87.6298,Chicago,US
-35.224.0.0,35.239.255.255,39.0438,-77.4874,Ashburn,US
-35.240.0.0,35.247.255.255,51.5074,-0.1278,London,GB
-35.248.0.0,35.255.255.255,51.5074,-0.1278,London,GB
-36.0.0.0,36.255.255.255,23.1291,113.2644,Guangzhou,CN
-37.0.0.0,37.255.255.255,48.8566,2.3522,Paris,FR
-38.0.0.0,38.127.255.255,50.1109,8.6821,Frankfurt,DE
-38.128.0.0,38.255.255.255,40.7128,-74.006,New York,US
-39.0.0.0,39.95.255.255,37.7749,-122.4194,San Francisco,US
-39.96.0.0,39.103.255.255,39.9042,116.4074,Beijing,CN
-39.104.0.0,39.231.255.255,-33.8688,151.2093,Sydney,AU
-39.232.0.0,40.63.255.255,19.076,72.8777,Mumbai,IN
-40.64.0.0,40.127.255.255,39.0438,-77.4874,Ashburn,US
-40.128.0.0,40.255.255.255,48.8566,2.3522,Paris,FR
-41.0.0.0,41.127.255.255,52.3676,4.9041,Amsterdam,NL
-41.128.0.0,41.255.255.255,-23.5505,-46.6333,Sao Paulo,BR
-42.0.0.0,42.255.255.255,39.9042,116.4074,Beijing,CN
-43.0.0.0,43.127.255.255,55.7558,37.6173,Moscow,RU
-43.128.0.0,43.255.255.255,41.8781,-87.6298,Chicago,US
-44.0.0.0,44.127.255.255,32.7767,-96.797,Dallas,US
-44.128.0.0,44.191.255.255,33.749,-84.388,Atlanta,US
-44.192.0.0,44.223.255.255,39.0438,-77.4874,Ashburn,US
-44.224.0.0,44.255.255.255,45.5152,-122.6784,Portland,US
-45.0.0.0,45.255.255.255,40.7128,-74.006,New York,US
-46.0.0.0,46.136.255.255,52.3676,4.9041,Amsterdam,NL
-46.137.0.0,46.137.127.255,1.3521,103.8198,Singapore,SG
-46.137.128.0,46.255.255.255,52.3676,4.9041,Amsterdam,NL
-47.0.0.0,47.15.255.255,43.6532,-79.3832,Toronto,CA
-47.16.0.0,47.31.255.255,22.3193,114.1694,Hong Kong,HK
-47.32.0.0,47.47.255.255,25.033,121.5654,Taipei,TW
-47.48.0.0,47.51.255.255,40.4168,-3.7038,Madrid,ES
-47.52.0.0,47.55.255.255,22.3193,114.1694,Hong Kong,HK
-47.56.0.0,47.57.255.255,22.3193,114.1694,Hong Kong,HK
-47.58.0.0,47.73.255.255,45.4642,9.19,Milan,IT
-47.74.0.0,47.75.255.255,1.3521,103.8198,Singapore,SG
-47.76.0.0,47.77.255.255,37.7749,-122.4194,San Francisco,US
-47.78.0.0,47.87.255.255,59.3293,18.0686,Stockholm,SE
-47.88.0.0,47.88.255.255,1.3521,103.8198,Singapore,SG
-47.89.0.0,47.89.255.255,22.3193,114.1694,Hong Kong,HK
-47.90.0.0,47.90.255.255,34.6937,135.5022,Osaka,JP
-47.91.0.0,47.91.255.255,35.6762,139.6503,Tokyo,JP
-47.92.0.0,47.95.255.255,39.9042,116.4074,Beijing,CN
-47.96.0.0,47.97.255.255,30.2741,120.1551,Hangzhou,CN
-47.98.0.0,47.99.255.255,30.2741,120.1551,Hangzhou,CN
-47.100.0.0,47.103.255.255,31.2304,121.4737,Shanghai,CN
-47.104.0.0,47.105.255.255,23.1291,113.2644,Guangzhou,CN
-47.106.0.0,47.106.255.255,22.5431,114.0579,Shenzhen,CN
-47.107.0.0,47.107.255.255,22.5431,114.0579,Shenzhen,CN
-47.108.0.0,47.109.255.255,30.5728,104.0668,Chengdu,CN
-47.110.0.0,47.111.255.255,52.52,13.405,Berlin,DE
-47.112.0.0,47.112.255.255,19.076,72.8777,Mumbai,IN
-47.113.0.0,47.113.255.255,25.2048,55.2708,Dubai,AE
-47.114.0.0,47.114.255.255,-33.8688,151.2093,Sydney,AU
-47.115.0.0,47.115.255.255,22.5431,114.0579,Shenzhen,CN
-47.116.0.0,47.116.255.255,50.1109,8.6821,Frankfurt,DE
-47.117.0.0,47.117.255.255,35.6762,139.6503,Tokyo,JP
-47.118.0.0,47.118.255.255,51.5074,-0.1278,London,GB
-47.119.0.0,47.119.255.255,25.2048,55.2708,Dubai,AE
-47.120.0.0,47.120.255.255,3.139,101.6869,Kuala Lumpur,MY
-47.121.0.0,47.121.255.255,13.7563,100.5018,Bangkok,TH
-47.122.0.0,47.122.255.255,-6.2088,106.8456,Jakarta,ID
-47.123.0.0,47.250.255.255,31.2304,121.4737,Shanghai,CN
-47.251.0.0,47.252.255.255,39.9042,116.4074,Beijing,CN
-47.253.0.0,47.253.255.255,37.7749,-122.4194,San Francisco,US
-47.254.0.0,47.254.255.255,50.1109,8.6821,Frankfurt,DE
-47.255.0.0,48.126.255.255,35.6762,139.6503,Tokyo,JP
-48.127.0.0,48.254.255.255,37.5665,126.978,Seoul,KR
-48.255.0.0,49.126.255.255,1.3521,103.8198,Singapore,SG
-49.127.0.0,49.254.255.255,51.5074,-0.1278,London,GB
-49.255.0.0,50.17.255.255,50.1109,8.6821,Frankfurt,DE
-50.18.0.0,50.19.255.255,37.7749,-122.4194,San Francisco,US
-50.20.0.0,50.35.255.255,40.7128,-74.006,New York,US
-50.36.0.0,50.51.255.255,37.7749,-122.4194,San Francisco,US
-50.52.0.0,50.67.255.255,-33.8688,151.2093,Sydney,AU
-50.68.0.0,50.83.255.255,19.076,72.8777,Mumbai,IN
-50.84.0.0,50.99.255.255,48.8566,2.3522,Paris,FR
-50.100.0.0,50.111.255.255,52.3676,4.9041,Amsterdam,NL
-50.112.0.0,50.112.255.255,45.5152,-122.6784,Portland,US
-50.113.0.0,50.127.255.255,-23.5505,-46.6333,Sao Paulo,BR
-50.128.0.0,50.255.255.255,41.8781,-87.6298,Chicago,US
-51.0.0.0,51.102.255.255,51.5074,-0.1278,London,GB
-51.103.0.0,51.103.255.255,50.1109,8.6821,Frankfurt,DE
-51.104.0.0,51.104.255.255,51.5074,-0.1278,London,GB
-51.105.0.0,51.105.255.255,51.5074,-0.1278,London,GB
-51.106.0.0,51.139.255.255,51.5074,-0.1278,London,GB
-51.140.0.0,51.143.255.255,51.5074,-0.1278,London,GB
-51.144.0.0,51.255.255.255,51.5074,-0.1278,London,GB
-52.0.0.0,52.1.255.255,39.0438,-77.4874,Ashburn,US
-52.2.0.0,52.3.255.255,39.0438,-77.4874,Ashburn,US
-52.4.0.0,52.7.255.255,39.0438,-77.4874,Ashburn,US
-52.8.0.0,52.15.255.255,55.7558,37.6173,Moscow,RU
-52.16.0.0,52.19.255.255,53.3498,-6.2603,Dublin,IE
-52.20.0.0,52.23.255.255,39.0438,-77.4874,Ashburn,US
-52.24.0.0,52.27.255.255,41.8781,-87.6298,Chicago,US
-52.28.0.0,52.31.255.255,50.1109,8.6821,Frankfurt,DE
-52.32.0.0,52.46.255.255,32.7767,-96.797,Dallas,US
-52.47.0.0,52.47.255.255,48.8566,2.3522,Paris,FR
-52.48.0.0,52.51.255.255,53.3498,-6.2603,Dublin,IE
-52.52.0.0,52.56.255.255,33.749,-84.388,Atlanta,US
-52.57.0.0,52.57.255.255,50.1109,8.6821,Frankfurt,DE
-52.58.0.0,52.59.255.255,43.6532,-79.3832,Toronto,CA
-52.60.0.0,52.60.255.255,45.5017,-73.5673,Montreal,CA
-52.61.0.0,52.61.255.255,22.3193,114.1694,Hong Kong,HK
-52.62.0.0,52.62.255.255,-33.8688,151.2093,Sydney,AU
-52.63.0.0,52.65.255.255,25.033,121.5654,Taipei,TW
-52.66.0.0,52.66.255.255,19.076,72.8777,Mumbai,IN
-52.67.0.0,52.73.255.255,40.4168,-3.7038,Madrid,ES
-52.74.0.0,52.74.255.255,1.3521,103.8198,Singapore,SG
-52.75.0.0,52.77.255.255,45.4642,9.19,Milan,IT
-52.78.0.0,52.78.255.255,37.5665,126.978,Seoul,KR
-52.79.0.0,52.94.255.255,59.3293,18.0686,Stockholm,SE
-52.95.0.0,52.110.255.255,34.6937,135.5022,Osaka,JP
-52.111.0.0,52.126.255.255,23.1291,113.2644,Guangzhou,CN
-52.127.0.0,52.138.255.255,22.5431,114.0579,Shenzhen,CN
-52.139.0.0,52.139.255.255,1.3521,103.8198,Singapore,SG
-52.140.0.0,52.140.255.255,52.52,13.405,Berlin,DE
-52.141.0.0,52.141.255.255,37.5665,126.978,Seoul,KR
-52.142.0.0,52.144.255.255,25.2048,55.2708,Dubai,AE
-52.145.0.0,52.145.255.255,39.0438,-77.4874,Ashburn,US
-52.146.0.0,52.146.255.255,13.7563,100.5018,Bangkok,TH
-52.147.0.0,52.147.255.255,-33.8688,151.2093,Sydney,AU
-52.148.0.0,52.163.255.255,31.2304,121.4737,Shanghai,CN
-52.164.0.0,52.179.255.255,39.9042,116.4074,Beijing,CN
-52.180.0.0,52.184.255.255,35.6762,139.6503,Tokyo,JP
-52.185.0.0,52.185.255.255,35.6762,139.6503,Tokyo,JP
-52.186.0.0,52.201.255.255,37.5665,126.978,Seoul,KR
-52.202.0.0,52.217.255.255,1.3521,103.8198,Singapore,SG
-52.218.0.0,52.223.255.255,51.5074,-0.1278,London,GB
-52.224.0.0,52.255.255.255,39.0438,-77.4874,Ashburn,US
-53.0.0.0,53.127.255.255,50.1109,8.6821,Frankfurt,DE
-53.128.0.0,53.255.255.255,40.7128,-74.006,New York,US
-54.0.0.0,54.63.255.255,37.7749,-122.4194,San Francisco,US
-54.64.0.0,54.65.255.255,35.6762,139.6503,Tokyo,JP
-54.66.0.0,54.66.255.255,-33.8688,151.2093,Sydney,AU
-54.67.0.0,54.67.255.255,-33.8688,151.2093,Sydney,AU
-54.68.0.0,54.71.255.255,45.5152,-122.6784,Portland,US
-54.72.0.0,54.79.255.255,53.3498,-6.2603,Dublin,IE
-54.80.0.0,54.95.255.255,39.0438,-77.4874,Ashburn,US
-54.96.0.0,54.111.255.255,19.076,72.8777,Mumbai,IN
-54.112.0.0,54.127.255.255,48.8566,2.3522,Paris,FR
-54.128.0.0,54.143.255.255,52.3676,4.9041,Amsterdam,NL
-54.144.0.0,54.151.255.255,-23.5505,-46.6333,Sao Paulo,BR
-54.152.0.0,54.159.255.255,39.0438,-77.4874,Ashburn,US
-54.160.0.0,54.191.255.255,39.0438,-77.4874,Ashburn,US
-54.192.0.0,54.205.255.255,55.7558,37.6173,Moscow,RU
-54.206.0.0,54.206.255.255,-33.8688,151.2093,Sydney,AU
-54.207.0.0,54.207.255.255,41.8781,-87.6298,Chicago,US
-54.208.0.0,54.209.255.255,39.0438,-77.4874,Ashburn,US
-54.210.0.0,54.223.255.255,32.7767,-96.797,Dallas,US
-54.224.0.0,54.239.255.255,39.0438,-77.4874,Ashburn,US
-54.240.0.0,54.240.255.255,33.749,-84.388,Atlanta,US
-54.241.0.0,54.241.255.255,37.7749,-122.4194,San Francisco,US
-54.242.0.0,54.243.255.255,43.6532,-79.3832,Toronto,CA
-54.244.0.0,54.245.255.255,45.5152,-122.6784,Portland,US
-54.246.0.0,55.117.255.255,22.3193,114.1694,Hong Kong,HK
-55.118.0.0,55.245.255.255,25.033,121.5654,Taipei,TW
-55.246.0.0,56.117.255.255,40.4168,-3.7038,Madrid,ES
-56.118.0.0,56.245.255.255,45.4642,9.19,Milan,IT
-56.246.0.0,57.117.255.255,59.3293,18.0686,Stockholm,SE
-57.118.0.0,57.245.255.255,34.6937,135.5022,Osaka,JP
-57.246.0.0,57.255.255.255,23.1291,113.2644,Guangzhou,CN
-58.0.0.0,58.255.255.255,22.5431,114.0579,Shenzhen,CN
-59.0.0.0,59.109.255.255,31.2304,121.4737,Shanghai,CN
-59.110.0.0,59.110.255.255,39.9042,116.4074,Beijing,CN
-59.111.0.0,59.255.255.255,31.2304,121.4737,Shanghai,CN
-60.0.0.0,60.255.255.255,39.9042,116.4074,Beijing,CN
-61.0.0.0,61.255.255.255,31.2304,121.4737,Shanghai,CN
-62.0.0.0,62.255.255.255,52.3676,4.9041,Amsterdam,NL
-63.0.0.0,63.15.255.255,22.5431,114.0579,Shenzhen,CN
-63.16.0.0,63.31.255.255,52.52,13.405,Berlin,DE
-63.32.0.0,63.35.255.255,53.3498,-6.2603,Dublin,IE
-63.36.0.0,63.163.255.255,25.2048,55.2708,Dubai,AE
-63.164.0.0,63.255.255.255,13.7563,100.5018,Bangkok,TH
-64.0.0.0,64.255.255.255,41.8781,-87.6298,Chicago,US
-65.0.0.0,65.255.255.255,37.7749,-122.4194,San Francisco,US
-66.0.0.0,66.255.255.255,32.7767,-96.797,Dallas,US
-67.0.0.0,67.159.255.255,25.7617,-80.1918,Miami,US
-67.160.0.0,67.191.255.255,39.9526,-75.1652,Philadelphia,US
-67.192.0.0,67.255.255.255,25.7617,-80.1918,Miami,US
-68.0.0.0,68.127.255.255,31.2304,121.4737,Shanghai,CN
-68.128.0.0,68.255.255.255,39.9042,116.4074,Beijing,CN
-69.0.0.0,69.135.255.255,40.7128,-74.006,New York,US
-69.136.0.0,69.143.255.255,42.3601,-71.0589,Boston,US
-69.144.0.0,69.239.255.255,40.7128,-74.006,New York,US
-69.240.0.0,69.255.255.255,47.6062,-122.3321,Seattle,US
-70.0.0.0,70.255.255.255,41.8781,-87.6298,Chicago,US
-71.0.0.0,71.127.255.255,35.6762,139.6503,Tokyo,JP
-71.128.0.0,71.159.255.255,37.5665,126.978,Seoul,KR
-71.160.0.0,71.175.255.255,40.7128,-74.006,New York,US
-71.176.0.0,71.191.255.255,1.3521,103.8198,Singapore,SG
-71.192.0.0,71.255.255.255,37.7749,-122.4194,San Francisco,US
-72.0.0.0,72.15.255.255,51.5074,-0.1278,London,GB
-72.16.0.0,72.31.255.255,50.1109,8.6821,Frankfurt,DE
-72.32.0.0,72.47.255.255,40.7128,-74.006,New York,US
-72.48.0.0,72.63.255.255,37.7749,-122.4194,San Francisco,US
-72.64.0.0,72.127.255.255,32.7767,-96.797,Dallas,US
-72.128.0.0,72.143.255.255,-33.8688,151.2093,Sydney,AU
-72.144.0.0,72.159.255.255,19.076,72.8777,Mumbai,IN
-72.160.0.0,72.175.255.255,48.8566,2.3522,Paris,FR
-72.176.0.0,72.191.255.255,52.3676,4.9041,Amsterdam,NL
-72.192.0.0,72.207.255.255,-23.5505,-46.6333,Sao Paulo,BR
-72.208.0.0,72.223.255.255,55.7558,37.6173,Moscow,RU
-72.224.0.0,72.239.255.255,41.8781,-87.6298,Chicago,US
-72.240.0.0,72.245.255.255,32.7767,-96.797,Dallas,US
-72.246.0.0,72.247.255.255,42.3736,-71.1097,Cambridge,US
-72.248.0.0,72.255.255.255,33.749,-84.388,Atlanta,US
-73.0.0.0,73.255.255.255,33.749,-84.388,Atlanta,US
-74.0.0.0,74.15.255.255,43.6532,-79.3832,Toronto,CA
-74.16.0.0,74.31.255.255,22.3193,114.1694,Hong Kong,HK
-74.32.0.0,74.47.255.255,25.033,121.5654,Taipei,TW
-74.48.0.0,74.63.255.255,40.4168,-3.7038,Madrid,ES
-74.64.0.0,74.79.255.255,45.4642,9.19,Milan,IT
-74.80.0.0,74.95.255.255,59.3293,18.0686,Stockholm,SE
-74.96.0.0,74.127.255.255,33.749,-84.388,Atlanta,US
-74.128.0.0,74.143.255.255,34.6937,135.5022,Osaka,JP
-74.144.0.0,74.159.255.255,23.1291,113.2644,Guangzhou,CN
-74.160.0.0,74.175.255.255,22.5431,114.0579,Shenzhen,CN
-74.176.0.0,74.191.255.255,52.52,13.405,Berlin,DE
-74.192.0.0,74.207.255.255,25.2048,55.2708,Dubai,AE
-74.208.0.0,74.223.255.255,13.7563,100.5018,Bangkok,TH
-74.224.0.0,74.239.255.255,31.2304,121.4737,Shanghai,CN
-74.240.0.0,74.255.255.255,39.9042,116.4074,Beijing,CN
-75.0.0.0,75.255.255.255,37.7749,-122.4194,San Francisco,US
-76.0.0.0,76.255.255.255,29.7604,-95.3698,Houston,US
-77.0.0.0,77.255.255.255,52.3676,4.9041,Amsterdam,NL
-78.0.0.0,78.255.255.255,52.52,13.405,Berlin,DE
-79.0.0.0,79.191.255.255,52.52,13.405,Berlin,DE
-79.192.0.0,79.255.255.255,50.1109,8.6821,Frankfurt,DE
-80.0.0.0,80.127.255.255,52.3676,4.9041,Amsterdam,NL
-80.128.0.0,80.159.255.255,52.52,13.405,Berlin,DE
-80.160.0.0,80.255.255.255,52.3676,4.9041,Amsterdam,NL
-81.0.0.0,81.255.255.255,51.5074,-0.1278,London,GB
-82.0.0.0,82.255.255.255,52.3676,4.9041,Amsterdam,NL
-83.0.0.0,83.255.255.255,52.3676,4.9041,Amsterdam,NL
-84.0.0.0,84.255.255.255,52.3676,4.9041,Amsterdam,NL
-85.0.0.0,85.255.255.255,52.3676,4.9041,Amsterdam,NL
-86.0.0.0,86.255.255.255,52.3676,4.9041,Amsterdam,NL
-87.0.0.0,87.127.255.255,52.52,13.405,Berlin,DE
-87.128.0.0,87.191.255.255,48.1351,11.582,Munich,DE
-87.192.0.0,87.255.255.255,52.52,13.405,Berlin,DE
-88.0.0.0,88.255.255.255,52.3676,4.9041,Amsterdam,NL
-89.0.0.0,89.255.255.255,52.3676,4.9041,Amsterdam,NL
-90.0.0.0,90.255.255.255,52.3676,4.9041,Amsterdam,NL
-91.0.0.0,91.63.255.255,50.1109,8.6821,Frankfurt,DE
-91.64.0.0,91.191.255.255,35.6762,139.6503,Tokyo,JP
-91.192.0.0,91.255.255.255,37.5665,126.978,Seoul,KR
-92.0.0.0,92.121.255.255,55.7558,37.6173,Moscow,RU
-92.122.0.0,92.123.255.255,42.3736,-71.1097,Cambridge,US
-92.124.0.0,92.255.255.255,55.7558,37.6173,Moscow,RU
-93.0.0.0,93.127.255.255,1.3521,103.8198,Singapore,SG
-93.128.0.0,93.191.255.255,51.5074,-0.1278,London,GB
-93.192.0.0,93.255.255.255,52.52,13.405,Berlin,DE
-94.0.0.0,94.255.255.255,55.7558,37.6173,Moscow,RU
-95.0.0.0,95.99.255.255,52.52,13.405,Berlin,DE
-95.100.0.0,95.101.255.255,42.3736,-71.1097,Cambridge,US
-95.102.0.0,95.255.255.255,52.52,13.405,Berlin,DE
-96.0.0.0,96.5.255.255,50.1109,8.6821,Frankfurt,DE
-96.6.0.0,96.7.255.255,42.3736,-71.1097,Cambridge,US
-96.8.0.0,96.15.255.255,40.7128,-74.006,New York,US
-96.16.0.0,96.17.255.255,42.3736,-71.1097,Cambridge,US
-96.18.0.0,96.145.255.255,37.7749,-122.4194,San Francisco,US
-96.146.0.0,96.223.255.255,-33.8688,151.2093,Sydney,AU
-96.224.0.0,96.255.255.255,40.7128,-74.006,New York,US
-97.0.0.0,97.127.255.255,19.076,72.8777,Mumbai,IN
-97.128.0.0,97.255.255.255,48.8566,2.3522,Paris,FR
-98.0.0.0,98.127.255.255,52.3676,4.9041,Amsterdam,NL
-98.128.0.0,98.191.255.255,-23.5505,-46.6333,Sao Paulo,BR
-98.192.0.0,98.255.255.255,41.8781,-87.6298,Chicago,US
-99.0.0.0,99.78.255.255,40.7128,-74.006,New York,US
-99.79.0.0,99.79.255.255,45.5017,-73.5673,Montreal,CA
-99.80.0.0,99.255.255.255,40.7128,-74.006,New York,US
-100.0.0.0,100.63.255.255,41.8781,-87.6298,Chicago,US
-100.64.0.0,100.191.255.255,55.7558,37.6173,Moscow,RU
-100.192.0.0,100.255.255.255,41.8781,-87.6298,Chicago,US
-101.0.0.0,101.131.255.255,39.9042,116.4074,Beijing,CN
-101.132.0.0,101.132.255.255,31.2304,121.4737,Shanghai,CN
-101.133.0.0,101.199.255.255,39.9042,116.4074,Beijing,CN
-101.200.0.0,101.201.255.255,39.9042,116.4074,Beijing,CN
-101.202.0.0,101.255.255.255,39.9042,116.4074,Beijing,CN
-102.0.0.0,102.127.255.255,32.7767,-96.797,Dallas,US
-102.128.0.0,102.255.255.255,33.749,-84.388,Atlanta,US
-103.0.0.0,103.21.243.255,43.6532,-79.3832,Toronto,CA
-103.21.244.0,103.21.247.255,37.7749,-122.4194,San Francisco,US
-103.21.248.0,103.22.199.255,22.3193,114.1694,Hong Kong,HK
-103.22.200.0,103.22.203.255,37.7749,-122.4194,San Francisco,US
-103.22.204.0,103.31.3.255,25.033,121.5654,Taipei,TW
-103.31.4.0,103.31.7.255,37.7749,-122.4194,San Francisco,US
-103.31.8.0,103.159.7.255,40.4168,-3.7038,Madrid,ES
-103.159.8.0,103.255.255.255,45.4642,9.19,Milan,IT
-104.0.0.0,104.15.255.255,32.7767,-96.797,Dallas,US
-104.16.0.0,104.23.255.255,37.7749,-122.4194,San Francisco,US
-104.24.0.0,104.27.255.255,37.7749,-122.4194,San Francisco,US
-104.28.0.0,104.39.255.255,32.7767,-96.797,Dallas,US
-104.40.0.0,104.47.255.255,39.0438,-77.4874,Ashburn,US
-104.48.0.0,104.63.255.255,32.7767,-96.797,Dallas,US
-104.64.0.0,104.127.255.255,42.3736,-71.1097,Cambridge,US
-104.128.0.0,104.151.255.255,32.7767,-96.797,Dallas,US
-104.152.0.0,104.153.255.255,50.1109,8.6821,Frankfurt,DE
-104.154.0.0,104.155.255.255,41.8781,-87.6298,Chicago,US
-104.156.0.0,104.195.255.255,32.7767,-96.797,Dallas,US
-104.196.0.0,104.199.255.255,39.0438,-77.4874,Ashburn,US
-104.200.0.0,104.210.255.255,32.7767,-96.797,Dallas,US
-104.211.0.0,104.211.255.255,19.076,72.8777,Mumbai,IN
-104.212.0.0,104.255.255.255,32.7767,-96.797,Dallas,US
-105.0.0.0,105.127.255.255,59.3293,18.0686,Stockholm,SE
-105.128.0.0,105.255.255.255,34.6937,135.5022,Osaka,JP
-106.0.0.0,106.13.255.255,23.1291,113.2644,Guangzhou,CN
-106.14.0.0,106.15.255.255,31.2304,121.4737,Shanghai,CN
-106.16.0.0,106.143.255.255,22.5431,114.0579,Shenzhen,CN
-106.144.0.0,106.255.255.255,52.52,13.405,Berlin,DE
-107.0.0.0,107.255.255.255,33.749,-84.388,Atlanta,US
-108.0.0.0,108.162.191.255,41.8781,-87.6298,Chicago,US
-108.162.192.0,108.162.255.255,37.7749,-122.4194,San Francisco,US
-108.163.0.0,108.255.255.255,41.8781,-87.6298,Chicago,US
-109.0.0.0,109.255.255.255,51.5074,-0.1278,London,GB
-110.0.0.0,110.255.255.255,23.1291,113.2644,Guangzhou,CN
-111.0.0.0,111.255.255.255,31.2304,121.4737,Shanghai,CN
-112.0.0.0,112.255.255.255,39.9042,116.4074,Beijing,CN
-113.0.0.0,113.255.255.255,31.2304,121.4737,Shanghai,CN
-114.0.0.0,114.255.255.255,39.9042,116.4074,Beijing,CN
-115.0.0.0,115.255.255.255,31.2304,121.4737,Shanghai,CN
-116.0.0.0,116.255.255.255,39.9042,116.4074,Beijing,CN
-117.0.0.0,117.255.255.255,31.2304,121.4737,Shanghai,CN
-118.0.0.0,118.255.255.255,39.9042,116.4074,Beijing,CN
-119.0.0.0,119.255.255.255,31.2304,121.4737,Shanghai,CN
-120.0.0.0,120.23.255.255,39.9042,116.4074,Beijing,CN
-120.24.0.0,120.27.255.255,39.9042,116.4074,Beijing,CN
-120.28.0.0,120.47.255.255,39.9042,116.4074,Beijing,CN
-120.48.0.0,120.49.255.255,39.9042,116.4074,Beijing,CN
-120.50.0.0,120.54.255.255,39.9042,116.4074,Beijing,CN
-120.55.0.0,120.55.255.255,31.2304,121.4737,Shanghai,CN
-120.56.0.0,120.75.255.255,39.9042,116.4074,Beijing,CN
-120.76.0.0,120.77.255.255,30.2741,120.1551,Hangzhou,CN
-120.78.0.0,120.79.255.255,22.5431,114.0579,Shenzhen,CN
-120.80.0.0,120.81.255.255,23.1291,113.2644,Guangzhou,CN
-120.82.0.0,120.255.255.255,39.9042,116.4074,Beijing,CN
-121.0.0.0,121.39.255.255,31.2304,121.4737,Shanghai,CN
-121.40.0.0,121.43.255.255,30.2741,120.1551,Hangzhou,CN
-121.44.0.0,121.195.255.255,31.2304,121.4737,Shanghai,CN
-121.196.0.0,121.199.255.255,30.2741,120.1551,Hangzhou,CN
-121.200.0.0,121.255.255.255,31.2304,121.4737,Shanghai,CN
-122.0.0.0,122.255.255.255,39.9042,116.4074,Beijing,CN
-123.0.0.0,123.255.255.255,31.2304,121.4737,Shanghai,CN
-124.0.0.0,124.255.255.255,39.9042,116.4074,Beijing,CN
-125.0.0.0,125.255.255.255,31.2304,121.4737,Shanghai,CN
-126.0.0.0,126.255.255.255,35.6762,139.6503,Tokyo,JP
-127.0.0.0,127.127.255.255,25.2048,55.2708,Dubai,AE
-127.128.0.0,127.255.255.255,13.7563,100.5018,Bangkok,TH
-128.0.0.0,128.255.255.255,48.8566,2.3522,Paris,FR
-129.0.0.0,129.255.255.255,48.8566,2.3522,Paris,FR
-130.0.0.0,130.255.255.255,52.52,13.405,Berlin,DE
-131.0.0.0,131.0.71.255,59.3293,18.0686,Stockholm,SE
-131.0.72.0,131.0.75.255,37.7749,-122.4194,San Francisco,US
-131.0.76.0,131.255.255.255,59.3293,18.0686,Stockholm,SE
-132.0.0.0,132.255.255.255,45.4642,9.19,Milan,IT
-133.0.0.0,133.255.255.255,35.6762,139.6503,Tokyo,JP
-134.0.0.0,134.255.255.255,52.52,13.405,Berlin,DE
-135.0.0.0,135.127.255.255,31.2304,121.4737,Shanghai,CN
-135.128.0.0,135.255.255.255,39.9042,116.4074,Beijing,CN
-136.0.0.0,136.127.255.255,35.6762,139.6503,Tokyo,JP
-136.128.0.0,136.255.255.255,37.5665,126.978,Seoul,KR
-137.0.0.0,137.255.255.255,48.8566,2.3522,Paris,FR
-138.0.0.0,138.90.255.255,45.4642,9.19,Milan,IT
-138.91.0.0,138.91.255.255,37.7749,-122.4194,San Francisco,US
-138.92.0.0,138.255.255.255,45.4642,9.19,Milan,IT
-139.0.0.0,139.195.255.255,23.1291,113.2644,Guangzhou,CN
-139.196.0.0,139.199.255.255,31.2304,121.4737,Shanghai,CN
-139.200.0.0,139.255.255.255,23.1291,113.2644,Guangzhou,CN
-140.0.0.0,140.127.255.255,1.3521,103.8198,Singapore,SG
-140.128.0.0,140.255.255.255,51.5074,-0.1278,London,GB
-141.0.0.0,141.101.63.255,52.52,13.405,Berlin,DE
-141.101.64.0,141.101.127.255,37.7749,-122.4194,San Francisco,US
-141.101.128.0,141.143.255.255,52.52,13.405,Berlin,DE
-141.144.0.0,141.159.255.255,32.7767,-96.797,Dallas,US
-141.160.0.0,141.255.255.255,52.52,13.405,Berlin,DE
-142.0.0.0,142.127.255.255,50.1109,8.6821,Frankfurt,DE
-142.128.0.0,142.255.255.255,40.7128,-74.006,New York,US
-143.0.0.0,143.255.255.255,45.4642,9.19,Milan,IT
-144.0.0.0,144.127.255.255,37.7749,-122.4194,San Francisco,US
-144.128.0.0,144.255.255.255,-33.8688,151.2093,Sydney,AU
-145.0.0.0,145.255.255.255,48.8566,2.3522,Paris,FR
-146.0.0.0,146.255.255.255,51.5074,-0.1278,London,GB
-147.0.0.0,147.255.255.255,48.8566,2.3522,Paris,FR
-148.0.0.0,148.127.255.255,19.076,72.8777,Mumbai,IN
-148.128.0.0,148.255.255.255,48.8566,2.3522,Paris,FR
-149.0.0.0,149.128.255.255,52.52,13.405,Berlin,DE
-149.129.0.0,149.129.255.255,1.3521,103.8198,Singapore,SG
-149.130.0.0,149.255.255.255,52.52,13.405,Berlin,DE
-150.0.0.0,150.191.255.255,35.6762,139.6503,Tokyo,JP
-150.192.0.0,150.255.255.255,40.4168,-3.7038,Madrid,ES
-151.0.0.0,151.100.255.255,45.4642,9.19,Milan,IT
-151.101.0.0,151.101.255.255,37.7749,-122.4194,San Francisco,US
-151.102.0.0,151.255.255.255,45.4642,9.19,Milan,IT
-152.0.0.0,152.127.255.255,52.3676,4.9041,Amsterdam,NL
-152.128.0.0,152.255.255.255,-23.5505,-46.6333,Sao Paulo,BR
-153.0.0.0,153.127.255.255,55.7558,37.6173,Moscow,RU
-153.128.0.0,153.191.255.255,34.6937,135.5022,Osaka,JP
-153.192.0.0,154.63.255.255,41.8781,-87.6298,Chicago,US
-154.64.0.0,154.191.255.255,32.7767,-96.797,Dallas,US
-154.192.0.0,154.255.255.255,33.749,-84.388,Atlanta,US
-155.0.0.0,155.255.255.255,59.3293,18.0686,Stockholm,SE
-156.0.0.0,156.255.255.255,48.8566,2.3522,Paris,FR
-157.0.0.0,157.52.63.255,35.6762,139.6503,Tokyo,JP
-157.52.64.0,157.52.127.255,37.7749,-122.4194,San Francisco,US
-157.52.128.0,157.52.255.255,37.7749,-122.4194,San Francisco,US
-157.53.0.0,157.174.255.255,35.6762,139.6503,Tokyo,JP
-157.175.0.0,157.175.255.255,26.0667,50.5577,Bahrain,BH
-157.176.0.0,157.255.255.255,35.6762,139.6503,Tokyo,JP
-158.0.0.0,158.255.255.255,59.3293,18.0686,Stockholm,SE
-159.0.0.0,159.255.255.255,48.8566,2.3522,Paris,FR
-160.0.0.0,160.255.255.255,34.6937,135.5022,Osaka,JP
-161.0.0.0,161.255.255.255,48.8566,2.3522,Paris,FR
-162.0.0.0,162.127.255.255,43.6532,-79.3832,Toronto,CA
-162.128.0.0,162.157.255.255,22.3193,114.1694,Hong Kong,HK
-162.158.0.0,162.159.255.255,37.7749,-122.4194,San Francisco,US
-162.160.0.0,162.175.255.255,25.033,121.5654,Taipei,TW
-162.176.0.0,162.191.255.255,40.4168,-3.7038,Madrid,ES
-162.192.0.0,162.207.255.255,45.4642,9.19,Milan,IT
-162.208.0.0,162.223.255.255,59.3293,18.0686,Stockholm,SE
-162.224.0.0,162.239.255.255,34.6937,135.5022,Osaka,JP
-162.240.0.0,162.255.255.255,23.1291,113.2644,Guangzhou,CN
-163.0.0.0,163.255.255.255,35.6762,139.6503,Tokyo,JP
-164.0.0.0,164.255.255.255,34.6937,135.5022,Osaka,JP
-165.0.0.0,165.127.255.255,22.5431,114.0579,Shenzhen,CN
-165.128.0.0,165.255.255.255,52.52,13.405,Berlin,DE
-166.0.0.0,166.127.255.255,25.2048,55.2708,Dubai,AE
-166.128.0.0,166.255.255.255,13.7563,100.5018,Bangkok,TH
-167.0.0.0,167.81.255.255,31.2304,121.4737,Shanghai,CN
-167.82.0.0,167.82.127.255,37.7749,-122.4194,San Francisco,US
-167.82.128.0,167.82.191.255,37.7749,-122.4194,San Francisco,US
-167.82.192.0,167.210.191.255,39.9042,116.4074,Beijing,CN
-167.210.192.0,168.82.191.255,35.6762,139.6503,Tokyo,JP
-168.82.192.0,168.210.191.255,37.5665,126.978,Seoul,KR
-168.210.192.0,169.82.191.255,1.3521,103.8198,Singapore,SG
-169.82.192.0,169.210.191.255,51.5074,-0.1278,London,GB
-169.210.192.0,170.82.191.255,50.1109,8.6821,Frankfurt,DE
-170.82.192.0,170.210.191.255,40.7128,-74.006,New York,US
-170.210.192.0,170.255.255.255,37.7749,-122.4194,San Francisco,US
-171.0.0.0,171.255.255.255,59.3293,18.0686,Stockholm,SE
-172.0.0.0,172.15.255.255,-33.8688,151.2093,Sydney,AU
-172.16.0.0,172.31.255.255,31.2304,121.4737,Shanghai,CN
-172.32.0.0,172.47.255.255,19.076,72.8777,Mumbai,IN
-172.48.0.0,172.63.255.255,48.8566,2.3522,Paris,FR
-172.64.0.0,172.71.255.255,37.7749,-122.4194,San Francisco,US
-172.72.0.0,172.87.255.255,52.3676,4.9041,Amsterdam,NL
-172.88.0.0,172.103.255.255,-23.5505,-46.6333,Sao Paulo,BR
-172.104.0.0,172.111.63.255,55.7558,37.6173,Moscow,RU
-172.111.64.0,172.111.127.255,37.7749,-122.4194,San Francisco,US
-172.111.128.0,172.239.127.255,41.8781,-87.6298,Chicago,US
-172.239.128.0,173.47.255.255,32.7767,-96.797,Dallas,US
-173.48.0.0,173.63.255.255,25.7617,-80.1918,Miami,US
-173.64.0.0,173.191.255.255,33.749,-84.388,Atlanta,US
-173.192.0.0,173.245.47.255,43.6532,-79.3832,Toronto,CA
-173.245.48.0,173.245.63.255,37.7749,-122.4194,San Francisco,US
-173.245.64.0,174.117.63.255,22.3193,114.1694,Hong Kong,HK
-174.117.64.0,174.245.63.255,25.033,121.5654,Taipei,TW
-174.245.64.0,175.117.63.255,40.4168,-3.7038,Madrid,ES
-175.117.64.0,175.245.63.255,45.4642,9.19,Milan,IT
-175.245.64.0,175.255.255.255,59.3293,18.0686,Stockholm,SE
-176.0.0.0,176.255.255.255,51.5074,-0.1278,London,GB
-177.0.0.0,177.127.255.255,34.6937,135.5022,Osaka,JP
-177.128.0.0,177.255.255.255,23.1291,113.2644,Guangzhou,CN
-178.0.0.0,178.127.255.255,22.5431,114.0579,Shenzhen,CN
-178.128.0.0,178.255.255.255,52.52,13.405,Berlin,DE
-179.0.0.0,179.127.255.255,25.2048,55.2708,Dubai,AE
-179.128.0.0,179.255.255.255,13.7563,100.5018,Bangkok,TH
-180.0.0.0,180.255.255.255,31.2304,121.4737,Shanghai,CN
-181.0.0.0,181.127.255.255,31.2304,121.4737,Shanghai,CN
-181.128.0.0,181.255.255.255,39.9042,116.4074,Beijing,CN
-182.0.0.0,182.91.255.255,39.9042,116.4074,Beijing,CN
-182.92.0.0,182.92.255.255,39.9042,116.4074,Beijing,CN
-182.93.0.0,182.255.255.255,39.9042,116.4074,Beijing,CN
-183.0.0.0,183.255.255.255,31.2304,121.4737,Shanghai,CN
-184.0.0.0,184.15.255.255,35.6762,139.6503,Tokyo,JP
-184.16.0.0,184.23.255.255,37.5665,126.978,Seoul,KR
-184.24.0.0,184.31.255.255,42.3736,-71.1097,Cambridge,US
-184.32.0.0,184.47.255.255,1.3521,103.8198,Singapore,SG
-184.48.0.0,184.63.255.255,51.5074,-0.1278,London,GB
-184.64.0.0,184.71.255.255,50.1109,8.6821,Frankfurt,DE
-184.72.0.0,184.73.255.255,37.7749,-122.4194,San Francisco,US
-184.74.0.0,184.83.255.255,40.7128,-74.006,New York,US
-184.84.0.0,184.87.255.255,42.3736,-71.1097,Cambridge,US
-184.88.0.0,184.215.255.255,37.7749,-122.4194,San Francisco,US
-184.216.0.0,184.255.255.255,-33.8688,151.2093,Sydney,AU
-185.0.0.0,185.31.15.255,51.5074,-0.1278,London,GB
-185.31.16.0,185.31.19.255,37.7749,-122.4194,San Francisco,US
-185.31.20.0,185.255.255.255,51.5074,-0.1278,London,GB
-186.0.0.0,186.127.255.255,19.076,72.8777,Mumbai,IN
-186.128.0.0,186.255.255.255,48.8566,2.3522,Paris,FR
-187.0.0.0,187.127.255.255,52.3676,4.9041,Amsterdam,NL
-187.128.0.0,187.255.255.255,-23.5505,-46.6333,Sao Paulo,BR
-188.0.0.0,188.114.95.255,51.5074,-0.1278,London,GB
-188.114.96.0,188.114.111.255,37.7749,-122.4194,San Francisco,US
-188.114.112.0,188.255.255.255,51.5074,-0.1278,London,GB
-189.0.0.0,189.127.255.255,55.7558,37.6173,Moscow,RU
-189.128.0.0,189.255.255.255,41.8781,-87.6298,Chicago,US
-190.0.0.0,190.93.239.255,32.7767,-96.797,Dallas,US
-190.93.240.0,190.93.255.255,37.7749,-122.4194,San Francisco,US
-190.94.0.0,190.221.255.255,33.749,-84.388,Atlanta,US
-190.222.0.0,191.93.255.255,43.6532,-79.3832,Toronto,CA
-191.94.0.0,191.221.255.255,22.3193,114.1694,Hong Kong,HK
-191.222.0.0,191.231.255.255,25.033,121.5654,Taipei,TW
-191.232.0.0,191.233.255.255,-23.5505,-46.6333,Sao Paulo,BR
-191.234.0.0,191.249.255.255,40.4168,-3.7038,Madrid,ES
-191.250.0.0,192.9.255.255,45.4642,9.19,Milan,IT
-192.10.0.0,192.25.255.255,59.3293,18.0686,Stockholm,SE
-192.26.0.0,192.41.255.255,34.6937,135.5022,Osaka,JP
-192.42.0.0,192.49.255.255,23.1291,113.2644,Guangzhou,CN
-192.50.0.0,192.50.255.255,35.1815,136.9066,Nagoya,JP
-192.51.0.0,192.66.255.255,22.5431,114.0579,Shenzhen,CN
-192.67.0.0,192.82.255.255,52.52,13.405,Berlin,DE
-192.83.0.0,192.98.255.255,25.2048,55.2708,Dubai,AE
-192.99.0.0,192.114.255.255,13.7563,100.5018,Bangkok,TH
-192.115.0.0,192.130.255.255,31.2304,121.4737,Shanghai,CN
-192.131.0.0,192.146.255.255,39.9042,116.4074,Beijing,CN
-192.147.0.0,192.162.255.255,35.6762,139.6503,Tokyo,JP
-192.163.0.0,192.167.255.255,37.5665,126.978,Seoul,KR
-192.168.0.0,192.168.255.255,23.1291,113.2644,Guangzhou,CN
-192.169.0.0,192.184.255.255,1.3521,103.8198,Singapore,SG
-192.185.0.0,192.200.255.255,51.5074,-0.1278,London,GB
-192.201.0.0,192.216.255.255,50.1109,8.6821,Frankfurt,DE
-192.217.0.0,192.232.255.255,40.7128,-74.006,New York,US
-192.233.0.0,192.248.255.255,37.7749,-122.4194,San Francisco,US
-192.249.0.0,192.255.255.255,-33.8688,151.2093,Sydney,AU
-193.0.0.0,193.255.255.255,52.52,13.405,Berlin,DE
-194.0.0.0,194.255.255.255,48.8566,2.3522,Paris,FR
-195.0.0.0,195.255.255.255,52.3676,4.9041,Amsterdam,NL
-196.0.0.0,196.127.255.255,19.076,72.8777,Mumbai,IN
-196.128.0.0,196.255.255.255,48.8566,2.3522,Paris,FR
-197.0.0.0,197.127.255.255,52.3676,4.9041,Amsterdam,NL
-197.128.0.0,197.234.239.255,-23.5505,-46.6333,Sao Paulo,BR
-197.234.240.0,197.234.243.255,37.7749,-122.4194,San Francisco,US
-197.234.244.0,197.250.243.255,55.7558,37.6173,Moscow,RU
-197.250.244.0,198.10.243.255,41.8781,-87.6298,Chicago,US
-198.10.244.0,198.26.243.255,32.7767,-96.797,Dallas,US
-198.26.244.0,198.41.127.255,33.749,-84.388,Atlanta,US
-198.41.128.0,198.41.255.255,37.7749,-122.4194,San Francisco,US
-198.42.0.0,198.169.255.255,43.6532,-79.3832,Toronto,CA
-198.170.0.0,199.27.71.255,22.3193,114.1694,Hong Kong,HK
-199.27.72.0,199.27.79.255,37.7749,-122.4194,San Francisco,US
-199.27.80.0,199.155.79.255,25.033,121.5654,Taipei,TW
-199.155.80.0,199.231.255.255,40.4168,-3.7038,Madrid,ES
-199.232.0.0,199.232.255.255,37.7749,-122.4194,San Francisco,US
-199.233.0.0,200.104.255.255,45.4642,9.19,Milan,IT
-200.105.0.0,200.232.255.255,59.3293,18.0686,Stockholm,SE
-200.233.0.0,201.104.255.255,34.6937,135.5022,Osaka,JP
-201.105.0.0,201.232.255.255,23.1291,113.2644,Guangzhou,CN
-201.233.0.0,201.255.255.255,22.5431,114.0579,Shenzhen,CN
-202.0.0.0,202.255.255.255,39.9042,116.4074,Beijing,CN
-203.0.0.0,203.255.255.255,39.9042,116.4074,Beijing,CN
-204.0.0.0,204.127.255.255,52.52,13.405,Berlin,DE
-204.128.0.0,204.255.255.255,25.2048,55.2708,Dubai,AE
-205.0.0.0,205.127.255.255,13.7563,100.5018,Bangkok,TH
-205.128.0.0,205.255.255.255,31.2304,121.4737,Shanghai,CN
-206.0.0.0,206.127.255.255,39.9042,116.4074,Beijing,CN
-206.128.0.0,206.255.255.255,35.6762,139.6503,Tokyo,JP
-207.0.0.0,207.127.255.255,37.5665,126.978,Seoul,KR
-207.128.0.0,207.255.255.255,1.3521,103.8198,Singapore,SG
-208.0.0.0,208.127.255.255,51.5074,-0.1278,London,GB
-208.128.0.0,208.255.255.255,50.1109,8.6821,Frankfurt,DE
-209.0.0.0,209.127.255.255,40.7128,-74.006,New York,US
-209.128.0.0,209.255.255.255,37.7749,-122.4194,San Francisco,US
-210.0.0.0,210.255.255.255,31.2304,121.4737,Shanghai,CN
-211.0.0.0,211.255.255.255,39.9042,116.4074,Beijing,CN
-212.0.0.0,212.255.255.255,51.5074,-0.1278,London,GB
-213.0.0.0,213.255.255.255,52.3676,4.9041,Amsterdam,NL
-214.0.0.0,214.127.255.255,-33.8688,151.2093,Sydney,AU
-214.128.0.0,214.255.255.255,19.076,72.8777,Mumbai,IN
-215.0.0.0,215.127.255.255,48.8566,2.3522,Paris,FR
-215.128.0.0,215.255.255.255,52.3676,4.9041,Amsterdam,NL
-216.0.0.0,216.127.255.255,-23.5505,-46.6333,Sao Paulo,BR
-216.128.0.0,216.255.255.255,55.7558,37.6173,Moscow,RU
-217.0.0.0,217.79.255.255,52.52,13.405,Berlin,DE
-217.80.0.0,217.95.255.255,48.1351,11.582,Munich,DE
-217.96.0.0,217.223.255.255,52.52,13.405,Berlin,DE
-217.224.0.0,217.255.255.255,50.1109,8.6821,Frankfurt,DE
-218.0.0.0,218.255.255.255,31.2304,121.4737,Shanghai,CN
-219.0.0.0,219.255.255.255,39.9042,116.4074,Beijing,CN
-220.0.0.0,220.255.255.255,23.1291,113.2644,Guangzhou,CN
-221.0.0.0,221.255.255.255,31.2304,121.4737,Shanghai,CN
-222.0.0.0,222.255.255.255,39.9042,116.4074,Beijing,CN
-223.0.0.0,223.255.255.255,39.9042,116.4074,Beijing,CN

docs/故障诊断手册.md

@@ -0,0 +1,502 @@
+# 天璇 (TianXuan) 故障诊断手册
+
+> 常见问题排查指南。使用 `scripts/diagnose_compare.py` 进行跨机日志对比诊断。
+
+---
+
+## 目录
+
+1. [启动问题](#1-启动问题)
+2. [数据加载问题](#2-数据加载问题)
+3. [聚类问题](#3-聚类问题)
+4. [UMAP 问题](#4-umap-问题)
+5. [数据库问题](#5-数据库问题)
+6. [LLM 集成问题](#6-llm-集成问题)
+7. [前端问题](#7-前端问题)
+8. [跨机一致性问题](#8-跨机一致性问题)
+9. [性能问题](#9-性能问题)
+10. [诊断工具使用](#10-诊断工具使用)
+
+---
+
+## 1. 启动问题
+
+### 1.1 `config` 模块找不到
+
+**症状**: `ModuleNotFoundError: No module named 'config'`
+
+**原因**: Python 路径未包含项目根目录。
+
+**解决**:
+```bat
+:: 在项目根目录下运行
+set PYTHONUTF8=1
+chcp 65001
+start /B runtime\python\python.exe manage.py runserver
+```
+
+### 1.2 首次启动无数据库
+
+**症状**: `OperationalError: no such table`
+
+**解决**:
+```bat
+runtime\python\python.exe manage.py migrate
+```
+
+`wsgi.py` 已包含 SQLite 启动自修复逻辑, 自动创建缺失的数据库文件。
+
+### 1.3 端口被占用
+
+**症状**: `Error: [WinError 10048] 通常每个套接字地址只允许使用一次`
+
+**解决**:
+```bat
+:: 查占用进程
+netstat -ano | findstr :8000
+:: 改端口 (config.yaml)
+server:
+  port: 8080
+```
+
+### 1.4 残留进程未清理
+
+**症状**: 前一次启动后进程未终止, 再次启动端口占用
+
+**解决**:
+```bat
+:: 检查 .server_pid 文件
+type .server_pid
+:: 强制终止
+taskkill /PID <pid> /F
+del .server_pid
+```
+
+---
+
+## 2. 数据加载问题
+
+### 2.1 Schema 不匹配 (多文件列名不一致)
+
+**症状**: `ValueError: schema mismatch` 或列名不一致时报错
+
+**诊断**:
+```bash
+runtime\python\python.exe scripts\column_survey.py --csv "data/*.csv"
+```
+
+**解决**:
+- 方案 A: 设置 `schema_strict=False` (默认), 自动并集合并, 缺失列填 null
+- 方案 B: 调用 `repair_schema` MCP 工具 — 自动对齐列名 (大小写合并、下划线/连字符规范化)
+- 方案 C: 在 `config.yaml` 中通过 `columns` 映射手动统一列名
+
+### 2.2 CSV 编码问题 (乱码)
+
+**症状**: 中文列名或数据出现乱码、`UnicodeDecodeError`
+
+**诊断**:
+```bash
+:: 检查文件编码
+runtime\python\python.exe -c "
+with open('data/your.csv', 'rb') as f:
+    print(f.read(10).hex())
+"
+```
+
+**解决**:
+- BOM 标记: 系统自动检测 BOM → UTF-8/UTF-16LE/UTF-16BE
+- 无 BOM: 默认 UTF-8, 可手动指定 `encoding="latin-1"` 或 `encoding="gbk"`
+- 所有命令加 `chcp 65001` + `set PYTHONUTF8=1`
+
+### 2.3 `+` 号字符串无法解析为数字
+
+**症状**: `"+"` 在数值列中导致 `cannot cast to Float64`
+
+**解决**: 已在 `_coerce_to_float()` 中修复 — `"+"` / `"-"` / 空白 / `"N/A"` 等伪值在聚合前自动转为 null。日志标记 `[CLEAN] numeric coercion applied`。
+
+### 2.4 列类型检测错误 (IP 列被识别为 String)
+
+**症状**: 某列应为 IPv4 但被检测为 STRING
+
+**诊断**:
+```bash
+:: 在 MCP 中调用
+validate_data(dataset_id="ds_...")
+```
+
+**解决**:
+- 在 `config.yaml` 中手动指定: `columns: { src_ip: ipv4, dst_ip: ipv4 }`
+- 配置优先级: config → 值优先检测 → 名称推断 → STRING 回退
+
+### 2.5 ZIP 文件解压失败
+
+**症状**: 上传 ZIP 文件后无数据
+
+**解决**: 检查 ZIP 内文件是否为 `.csv` 格式。系统仅自动解压 ZIP 中的 CSV 文件。
+
+---
+
+## 3. 聚类问题
+
+### 3.1 聚类返回 0 个簇或全噪声
+
+**症状**: `n_clusters=0` 或所有标签为 -1 (HDBSCAN 噪声)
+
+**诊断**:
+```python
+diagnose_clustering(dataset_id="...", cluster_result_id="...")
+```
+输出包含: 特征方差分析、相关矩阵、稀疏性检查、参数建议。
+
+**常见原因与解决**:
+
+| 原因 | 诊断信号 | 解决 |
+|------|---------|------|
+| 样本太少 | n < 3 | 检查数据加载是否成功; 增大数据量 |
+| min_cluster_size 太大 | 日志中 min_cluster_size > n/2 | 系统已自动调整为 n//20, 如仍失败手动降低 |
+| 全零方差特征 | 方差 < 1e-10 | 系统已自动过滤零方差列; 检查特征选择是否合理 |
+| 特征高度相关 | Pearson > 0.95 | 系统已自动去相关; 仍有问题则减少特征数 |
+| 数据全是 NaN | 日志 all-NaN columns dropped | 检查数据源; 增加样本量 |
+
+### 3.2 "mean on str" 错误
+
+**症状**: `TypeError: can't convert type 'str' to numerator/denominator`
+
+**原因**: 跨机 dtype 推断不一致 (一台推断为 Float, 另一台为 String)
+
+**解决**: 已在 `tools/entities.py::_handle_build_entity_profiles` 的聚合参数构建中修复 — 聚合前检查 dtype, 非数值列自动 cast。`_coerce_to_float()` 兜底。
+
+### 3.3 KMeans 簇数与预期不符
+
+**症状**: KMeans 产生的簇数与 `n_clusters` 参数不一致
+
+**原因**: MiniBatchKMeans 在大数据集上可能产生少于预期的簇
+
+**解决**:
+```python
+# 尝试 HDBSCAN (自动确定 k)
+run_clustering(dataset_id="...", cluster_columns=[...], algorithm="hdbscan")
+# 或增大 n_clusters
+run_clustering(..., algorithm="kmeans", params={"n_clusters": 5})
+```
+
+---
+
+## 4. UMAP 问题
+
+### 4.1 UMAP 嵌入失败 (`[UMAP_ERR]`)
+
+**症状**: 日志中出现 `[UMAP_ERR]` 但流程继续 (UMAP 计算失败不影响核心分析)
+
+**常见原因**:
+- 数据量过大 (>100K): 系统已实现训练 10K + 批量变换 1K 批次
+- 全 NaN 列: UMAP 前已有 StandardScaler + `np.nan_to_num`
+- 内存不足: UMAP 需要额外内存
+
+**解决**:
+```bash
+:: 检查 stderr
+runtime\python\python.exe manage.py run_pipeline data/*.csv 2> umap_err.log
+```
+
+### 4.2 UMAP 坐标全为 0
+
+**症状**: EntityProfile 的 embedding_x/y 全为 0.0
+
+**原因**: UMAP 计算过程中异常被 `try/except` 捕获, 静默回退
+
+**解决**:
+- 检查 stderr 是否有 `[UMAP_ERR]` 日志
+- 确认 `umap-learn` 包已安装: `runtime\python\python.exe -c "import umap; print(umap.__version__)"`
+- 减少数据量重试
+
+### 4.3 TruncatedSVD 降维导致聚类结果变化
+
+**症状**: 特征 >50 维时聚类结果与预期不同
+
+**原因**: SVD 降维会损失信息 (保留前 50 个主成分)
+
+**解决**:
+- 手动选择更少的基础特征 (<50 个), 避免触发 SVD 降维
+- 通过 `cluster_columns` 参数精确控制输入特征列
+
+---
+
+## 5. 数据库问题
+
+### 5.1 SQLite 锁 (`[DB_LOCKED]`)
+
+**症状**: 日志中出现 `[DB_LOCKED]` 警告
+
+**说明**: 系统已内置 `retry_on_lock` 装饰器 (3 次重试, 指数退避)。SQLite WAL 模式下读写并发性良好。
+
+**解决**:
+- 减少并发写入 (多个 Django 进程同时写)
+- 增大超时: `sqlite3_db.config(busy_timeout=5000)`
+- 检查是否有未关闭的长时间事务
+
+### 5.2 ClusterFeature 表无数据 (`db_saved=False`)
+
+**症状**: `extract_features` 返回 `db_saved=False`
+
+**诊断**:
+```bash
+runtime\python\python.exe scripts\debug_db.py
+```
+
+**常见原因**:
+- Django ORM sync_to_async 包装缺失 (已在 `tools/features.py` 中修复)
+- AnalysisRun 记录未找到 (csv_glob 匹配失败)
+- 权限问题 (SQLite 文件只读)
+
+### 5.3 EntityProfile 表无 embedding 坐标
+
+**症状**: EntityProfile 记录存在但 embedding_x/y 为 null
+
+**原因**: UMAP 计算失败 (见 4.1)
+
+---
+
+## 6. LLM 集成问题
+
+### 6.1 400 错误
+
+**症状**: LLM 返回 HTTP 400
+
+**原因**: API 地址错误 / Key 无效 / 模型名错误
+
+**诊断**: 配置页 → 填写 LLM 配置 → 点击「测试连通性」
+
+**解决**:
+1. 确认 `base_url` 格式: `https://api.openai.com/v1` (注意末尾的 /v1)
+2. 确认 `api_key` 无多余空格
+3. 确认 `model` 名称与 API 支持的一致
+
+### 6.2 超时
+
+**症状**: 日志中 `latency_ms` 很高, LLM 步骤无响应
+
+**解决**:
+- 增大 `max_tokens` (config.yaml)
+- 检查网络延迟
+- 32B 模型推理慢: 系统已设置 90s 超时, 如仍不足可增加
+
+### 6.3 工具调用失败
+
+**症状**: `[TOOL]` 日志后出现错误
+
+**诊断**:
+```bash
+:: 查看完整 LLM 交互日志
+type logs\tianxuan.log | findstr "[TOOL] [LLM]"
+```
+
+**解决**:
+- 检查工具返回的错误信息
+- LLM 可能生成了格式错误的工具调用 JSON → 重试
+- 重复调用检测: orchestrator 会自动跳过重复的工具调用
+
+### 6.4 分析结果为空
+
+**症状**: LLM 完成但无聚类结果
+
+**常见原因**:
+- 实体检测失败 → 手动指定 `entity_columns`
+- 数据量太小 → 增大样本
+- 实体列检测到非 IP 列 → 通过 `entity_columns` 强制指定
+
+---
+
+## 7. 前端问题
+
+### 7.1 Canvas 散点图不显示
+
+**症状**: 聚类概览页空白
+
+**诊断**:
+```bash
+:: 检查浏览器控制台 (F12)
+:: 常见错误: "Cannot read properties of undefined"
+```
+
+**解决**:
+- 确认聚类已完成 (检查 AnalysisRun.status == 'completed')
+- 确认有 UMAP/PCA 坐标数据 (EntityProfile.embedding_x 非 null)
+- 清除浏览器缓存 (Ctrl+F5)
+
+### 7.2 3D 地球空白
+
+**症状**: `/globe/` 页面渲染为空白
+
+**原因**:
+- WebGL2 不可用 (核显驱动过旧或浏览器不支持)
+- Three.js 未加载 (检查 Network tab, 603KB 的 three.min.js)
+- 地球贴图未加载 (1.4MB earth_atmos_2048.jpg)
+
+**解决**:
+- 确认浏览器支持 WebGL2: 访问 `chrome://gpu` 检查 WebGL2 状态
+- 确认 `static/tianxuan/three.min.js` 和 `earth_atmos_2048.jpg` 存在
+- 数据中优先使用 `src_latitude/src_longitude` 列, 无则 GeoIP 回退
+- 确保 GeoIP 数据文件 `data/geoip_data.txt` 存在
+
+### 7.3 地理分布图空白
+
+**症状**: 聚类概览页的地理分布 Canvas 不显示
+
+**原因**: 数据中无经纬度信息且 GeoIP 查询失败
+
+**解决**: 确保 GeoIP 数据库 `data/geoip_data.txt` 存在且格式正确。GeoIP 仅支持离线查询, 无网络依赖。
+
+---
+
+## 8. 跨机一致性问题
+
+### 8.1 核心诊断流程
+
+```bat
+:: 两台机器各自运行管道, 保存日志
+del logs\tianxuan.log
+runtime\python\python.exe manage.py run_pipeline data\complex_test.csv
+copy logs\tianxuan.log log_%COMPUTERNAME%.txt
+
+:: 对比
+runtime\python\python.exe scripts\diagnose_compare.py log_MACHINE_A.txt log_MACHINE_B.txt
+```
+
+首个 `[DIFF]` 行就是问题根源。
+
+### 8.2 常见跨机差异
+
+| 差异类型 | 日志标记 | 根因 | 检查 |
+|---------|---------|------|------|
+| 列类型不同 | `[LOAD_SCHEMA]` | Polars 版本不一致 或 PYTHONUTF8 未生效 | `polars.__version__`, `sys.flags.utf8_mode` |
+| 关键词匹配不同 | `[FIND_COL]` | frozenset 哈希随机性 | 已通过 tuple 替代 frozenset 修复 |
+| 聚合结果不同 | `[CLUSTER_INPUT]` | random_state 不一致 或 StandardScaler 差异 | 确认 random_state=42, sklearn 版本一致 |
+| 编码问题 | `UnicodeDecodeError` | 系统编码不一致 | `sys.getdefaultencoding()`, `sys.getfilesystemencoding()` |
+| dtype 推断不同 | `Utf8 vs Float64` | Polars 推断漂移 | 已通过类型安全聚合 + `_coerce_to_float` 防护 |
+
+### 8.3 运行时自检脚本
+
+```bash
+:: 在两台机器上分别运行, 对比输出
+runtime\python\python.exe -c "
+import sys, polars, numpy, sklearn
+print(f'Python: {sys.version}')
+print(f'Polars: {polars.__version__}')
+print(f'NumPy: {numpy.__version__}')
+print(f'sklearn: {sklearn.__version__}')
+print(f'defaultencoding: {sys.getdefaultencoding()}')
+print(f'fsencoding: {sys.getfilesystemencoding()}')
+print(f'utf8_mode: {sys.flags.utf8_mode}')
+print(f'hashseed: {sys.flags.hash_randomization}')
+"
+```
+
+### 8.4 已知已修复的跨机问题
+
+| 问题 | 修复方案 | 代码位置 |
+|------|---------|---------|
+| "mean on str" | frozenset → tuple + 类型安全聚合 | tools/entities.py, ip_clustering.py |
+| 编码漂移 | PYTHONUTF8=1 | run.bat |
+| DB 特征不保存 | sync_to_async 包装 | tools/features.py |
+| 图表依赖缺失 | 纯 Canvas 2D | 全部图表 |
+| 聚类结果随机 | random_state=42 | tools/clustering.py, run_pipeline.py |
+
+---
+
+## 9. 性能问题
+
+### 9.1 大数据集 OOM
+
+**症状**: 内存不足导致进程崩溃
+
+**系统已实现的防护**:
+- 可用内存 < 2GB → 自动降采样 50K (psutil 检测)
+- 样本 > 100K → 采样 100K
+- 样本 > 50K → 采样 50K
+- MiniBatchKMeans 替代 KMeans (batch_size=1024)
+- SVD 降维 (>50 特征 → 50)
+- UMAP 训练 10K + 批量变换 1K 批次
+- Polars LazyFrame + streaming 惰性执行
+
+**手动优化**:
+```python
+# 在 load_data 或 run_clustering 时降采样
+load_data(csv_glob="data/*.csv")
+# → 后续 run_clustering 会自动降采样
+# 或使用 filter_and_cluster 先过滤再聚类
+```
+
+### 9.2 SQLite 写入慢
+
+**症状**: DB 持久化耗时较长
+
+**解决**:
+- WAL 模式已开启 (synchronous=NORMAL)
+- batch_size=1000 批量写入 (bulk_create)
+- 增大 cache_size: SQLite PRAGMA cache_size=20000 (20MB)
+- 临时目录移到 SSD
+
+### 9.3 UMAP 计算慢
+
+**症状**: extract_features 步骤耗时过长
+
+**说明**: UMAP 计算分为训练 (≤10K 样本) + 批量变换 (1K 批次), 已是最优方案。
+
+**如需加速**: 减小 `MAX_UMAP_TRAIN` (`tools/features.py` 中的硬编码常量, 默认 10000)。
+
+---
+
+## 10. 诊断工具使用
+
+### 10.1 MCP 诊断工具链
+
+```
+问题 → validate_data (数据质量) → explore_distributions (分布检查) → find_outliers (异常值)
+→ diagnose_clustering (聚类诊断) → compare_datasets (处理前后对比) → export_debug_sample (导出样本)
+```
+
+所有诊断工具位于 `tools/diagnostics.py`。
+
+### 10.2 列结构调查
+
+```bash
+runtime\python\python.exe scripts\column_survey.py --csv "data/*.csv"
+# 输出: 每个 CSV 的列名/类型/空值率/分布
+```
+
+### 10.3 DB 持久化调试
+
+```bash
+runtime\python\python.exe scripts\debug_db.py
+# 期望: n_features=30 db_saved=True
+```
+
+### 10.4 跨机日志对比
+
+```bash
+runtime\python\python.exe scripts\diagnose_compare.py log_machine_a.txt log_machine_b.txt
+# 输出: 首个差异行的内容 (两侧对比)
+```
+
+### 10.5 关键日志标记汇总
+
+| 标记 | 文件 | 含义 |
+|------|------|------|
+| `[LOAD]` | data_loader.py | CSV 加载完成 |
+| `[LOAD_SCHEMA]` | data_loader.py | 每列推断类型 (最重要!) |
+| `[CLEAN]` | data_loader.py | 清洗操作 |
+| `[VALIDATE]` | data_validator.py | 数据校验 |
+| `[FIND_COL]` | tools/entities.py | 实体列关键词匹配 |
+| `[CLUSTER_INPUT]` | tools/clustering.py | 聚类输入形状 |
+| `[CLUSTER_SCALE]` | tools/clustering.py | StandardScaler |
+| `[SCORE]` | tools/entities.py | 自适应评分权重 |
+| `[UMAP_ERR]` | tools/features.py | UMAP 异常 |
+| `[DB_SAVE_ERROR]` | tools/features.py | DB 持久化失败 |
+| `[DB_LOCKED]` | db_utils.py | SQLite 锁重试 |
+| `[LLM]` | llm_orchestrator.py | LLM 调用 |
+| `[TOOL]` | llm_orchestrator.py | LLM 调用工具 |
+
+---
+
+> **相关文档**: `AGENTS.md` (架构/配置/操作流程), `docs/工作流.md` (完整分析流水线)

prototype_cluster_ui.html

@@ -0,0 +1,356 @@
+<!DOCTYPE html>
+<html lang="zh-hans">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width,initial-scale=1.0">
+<title>原型 — 聚类UI交互验证</title>
+<style>
+* { margin:0; padding:0; box-sizing:border-box; }
+body { font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif; background:#f5f7fa; color:#1a1a2e; }
+.header { background:#1a1a2e; color:#fff; padding:0.8rem 1.5rem; display:flex; justify-content:space-between; align-items:center; }
+.header h1 { font-size:1.1rem; }
+.header span { font-size:0.8rem; color:#aaa; }
+
+.main-layout { display:flex; position:relative; transition:all 0.3s; min-height:calc(100vh - 48px); }
+.left-panel { flex:1; min-width:0; transition:flex 0.3s; padding:1rem; overflow-y:auto; }
+.left-panel.globe-open { flex:0 0 55%; }
+.right-panel { flex:0 0 0; overflow:hidden; transition:flex 0.3s; }
+.right-panel.globe-open { flex:0 0 45%; }
+
+/* toggle handle */
+.toggle-handle { position:absolute; z-index:10; top:50%; width:20px; height:44px; background:#1a1a2e; color:#fff; border:none; border-radius:4px 0 0 4px; cursor:pointer; display:flex; align-items:center; justify-content:center; font-size:12px; transition:left 0.3s ease; }
+.toggle-handle:hover { background:#4361ee; }
+
+/* scatter */
+.scatter-wrap { position:relative; background:#fff; border-radius:8px; padding:0.75rem; margin-bottom:1rem; box-shadow:0 1px 3px rgba(0,0,0,0.1); }
+.scatter-wrap h3 { font-size:0.95rem; margin-bottom:0.5rem; }
+.scatter-wrap canvas { width:100%; height:360px; display:block; cursor:crosshair; border-radius:4px; background:#fafbfc; }
+
+/* pills */
+.pill-row { display:flex; flex-wrap:wrap; gap:4px; margin-bottom:0.75rem; }
+.pill { display:inline-flex; align-items:center; gap:3px; padding:2px 10px; border-radius:12px; font-size:0.75rem; cursor:pointer; border:2px solid transparent; transition:all 0.15s; color:#fff; white-space:nowrap; }
+.pill.all { background:#e8e8e8; color:#333; }
+.pill.active { border-color:#333; font-weight:700; }
+
+/* cluster cards */
+.card-grid { display:grid; grid-template-columns:1fr 1fr; gap:0.75rem; margin-bottom:1rem; }
+.cluster-card { background:#fff; border:1px solid #e0e0e0; border-radius:8px; padding:0.8rem; cursor:pointer; transition:box-shadow 0.15s; }
+.cluster-card:hover { box-shadow:0 2px 8px rgba(0,0,0,0.1); }
+.cluster-card.selected { border-color:#4361ee; box-shadow:0 0 0 2px rgba(67,97,238,0.2); }
+.cluster-card .label { font-weight:700; font-size:1rem; }
+.cluster-card .meta { font-size:0.78rem; color:#666; margin-top:0.2rem; }
+.cluster-card .desc { font-size:0.8rem; color:#333; margin-top:0.4rem; padding:0.4rem; background:#f8f9fa; border-left:3px solid #4361ee; border-radius:0 4px 4px 0; line-height:1.5; }
+
+/* detail panel */
+.detail-panel { position:fixed; bottom:0; left:0; right:0; z-index:100; transform:translateY(100%); transition:transform 0.35s cubic-bezier(.4,0,.2,1); }
+.detail-panel.open { transform:translateY(0); }
+.detail-body { background:#fff; border-radius:16px 16px 0 0; box-shadow:0 -4px 24px rgba(0,0,0,0.15); max-height:40vh; overflow-y:auto; padding:0.5rem 1.5rem 1.5rem; }
+.detail-handle { width:36px; height:4px; background:#ccc; border-radius:2px; margin:8px auto; cursor:pointer; }
+.detail-close { float:right; cursor:pointer; font-size:1.3rem; color:#999; line-height:1; border:none; background:none; }
+.detail-close:hover { color:#333; }
+.detail-body h3 { font-size:1rem; margin-bottom:0.3rem; }
+.detail-body p { font-size:0.85rem; color:#555; margin:0.2rem 0; }
+.detail-body table { width:100%; border-collapse:collapse; font-size:0.8rem; margin-top:0.5rem; }
+.detail-body th { background:#f5f5f5; padding:0.25rem 0.4rem; text-align:left; font-weight:600; border-bottom:2px solid #ddd; }
+.detail-body td { padding:0.2rem 0.4rem; border-bottom:1px solid #eee; }
+.detail-body td code { font-size:0.72rem; }
+
+/* 3D地球容器 */
+#globeContainer { width:100%; height:calc(100vh - 48px); background:#0a0a1a; position:relative; }
+#globeContainer canvas { display:block; }
+</style>
+</head>
+<body>
+
+<div class="header">
+    <h1>聚类概览 — 运行 #9</h1>
+    <span>200 条流 · 3 个簇 · 12 个噪声点</span>
+</div>
+
+<div class="main-layout" id="mainLayout">
+    <!-- Left: cluster content -->
+    <div class="left-panel" id="leftPanel">
+        <!-- Scatter -->
+        <div class="scatter-wrap">
+            <h3>UMAP 嵌入 <span style="font-weight:400;font-size:0.8rem;color:#888;" id="ptCount">加载中...</span></h3>
+            <canvas id="scatterChart"></canvas>
+        </div>
+
+        <!-- Cluster pills -->
+        <div class="pill-row" id="pillRow">
+            <span class="pill all active" data-lbl="all" onclick="clearSel()">全部</span>
+        </div>
+
+        <!-- Cluster cards -->
+        <div class="card-grid" id="cardGrid"></div>
+    </div>
+
+    <!-- Right: 3D Globe -->
+    <div class="right-panel" id="rightPanel">
+        <div id="globeContainer"></div>
+    </div>
+
+    <!-- Toggle handle -->
+    <button class="toggle-handle" id="globeHandle" onclick="toggleGlobe()">◀</button>
+</div>
+
+<!-- Bottom detail panel -->
+<div class="detail-panel" id="detailPanel">
+    <div class="detail-body" id="detailBody">
+        <button class="detail-close" onclick="closeDetail()">✕</button>
+        <div class="detail-handle" onclick="closeDetail()"></div>
+        <div id="detailContent"><p style="color:#999;">选择一个簇或数据点查看详情</p></div>
+    </div>
+</div>
+
+<script src="static/tianxuan/three.min.js"></script>
+<script>
+// ═══════════════════════════════════════════════
+//  MOCK DATA
+// ═══════════════════════════════════════════════
+const MOCK_CLUSTERS = [
+    { label:0, size:78, prop:0.39, silh:0.72, desc:'低延迟 (23ms)，比全局均值高3.1σ；大数据包 (1520 bytes)，高2.3σ；系统端口 (443)' },
+    { label:1, size:65, prop:0.33, silh:0.68, desc:'高TLS1.3比例 (92%)，比均值高1.8σ；短会话 (12s)，低1.5σ；ECDHE曲线 X25519' },
+    { label:2, size:45, prop:0.23, silh:0.54, desc:'高连接频率 (85次/min)，比均值高4.2σ；非标准端口 (8080/8443)，目标多域名' },
+    { label:-1, size:12, prop:0.06, silh:null, desc:'噪声点 — 异常TLS指纹、零散目标、低流量' },
+];
+
+const COLORS = ['#4361ee','#f72585','#7209b7','#4cc9f0','#f8961e','#43aa8b','#f94144','#577590','#e9c46a','#9b5de5'];
+function colorFor(lbl) { return lbl===-1?'rgba(150,150,150,0.7)':COLORS[Math.abs(lbl)%COLORS.length]; }
+
+// Generate mock UMAP scatter data (200 points)
+const SCATTER = [];
+const RNG = (seed) => { let s=seed%2147483647; return ()=> { s=s*16807%2147483647; return (s-1)/2147483646; }; };
+const rng = RNG(42);
+const centers = {0:[2,1.5],1:[-1.8,0.5],2:[0.5,-2], '-1':[3.5,-1.5]};
+const spreads = {0:[0.6,0.5],1:[0.7,0.6],2:[0.5,0.7], '-1':[1.2,0.8]};
+for (const c of MOCK_CLUSTERS) {
+    for (let i=0;i<c.size;i++) {
+        const cx=centers[c.label][0], cy=centers[c.label][1];
+        const sx=spreads[c.label][0], sy=spreads[c.label][1];
+        SCATTER.push({
+            x:cx+(rng()-0.5)*2*sx, y:cy+(rng()-0.5)*2*sy,
+            label:c.label, entity:`模拟主机 ${1000+i}`
+        });
+    }
+}
+
+// ═══════════════════════════════════════════════
+//  SCATTER 2D CANVAS
+// ═══════════════════════════════════════════════
+let selCluster = null, selPoint = null;
+let globeOpen = false;
+
+function drawScatter(highlight) {
+    const c=document.getElementById('scatterChart');
+    if(!c)return;
+    const W=c.parentElement.clientWidth||700, H=360;
+    c.width=W*2; c.height=H*2; c.style.width=W+'px'; c.style.height=H+'px';
+    const ctx=c.getContext('2d'); ctx.scale(2,2);
+    const pad={top:25,bottom:30,left:50,right:20};
+    const pw=W-pad.left-pad.right, ph=H-pad.top-pad.bottom;
+    const xs=SCATTER.map(d=>d.x), ys=SCATTER.map(d=>d.y);
+    const xMin=Math.min(...xs), xMax=Math.max(...xs), yMin=Math.min(...ys), yMax=Math.max(...ys);
+    const xR=xMax-xMin||1, yR=yMax-yMin||1;
+    const toX=v=>pad.left+(v-xMin)/xR*pw, toY=v=>pad.top+ph-(v-yMin)/yR*ph;
+
+    ctx.clearRect(0,0,W,H);
+    // grid
+    ctx.strokeStyle='#eee'; ctx.lineWidth=1;
+    for(let i=0;i<=5;i++){ctx.beginPath();ctx.moveTo(pad.left,pad.top+i*ph/5);ctx.lineTo(W-pad.right,pad.top+i*ph/5);ctx.stroke();}
+    // points
+    SCATTER.forEach(d=>{
+        const cx=toX(d.x),cy=toY(d.y);
+        let radius=3,color=colorFor(d.label);
+        if(highlight!==null&&highlight!==undefined){
+            if(d.label===highlight){color=colorFor(d.label);radius=5;}
+            else{color='rgba(200,200,200,0.15)';radius=2;}
+        }
+        ctx.beginPath();ctx.arc(cx,cy,radius,0,Math.PI*2);ctx.fillStyle=color;ctx.fill();
+    });
+    // labels
+    ctx.fillStyle='#999'; ctx.font='11px sans-serif'; ctx.textAlign='center';
+    ctx.fillText('UMAP-1',W/2,H-3);
+    ctx.save();ctx.translate(14,H/2);ctx.rotate(-Math.PI/2);ctx.fillText('UMAP-2',0,0);ctx.restore();
+
+    c.onclick=function(e){
+        const r=c.getBoundingClientRect();
+        const mx=e.clientX-r.left, my=e.clientY-r.top;
+        const sx=mx/W*pw, sy=my/H*ph;
+        let best=null,bestD=10;
+        for(const d of SCATTER){
+            const dx=toX(d.x)-mx, dy=toY(d.y)-my;
+            const dist=Math.sqrt(dx*dx+dy*dy);
+            if(dist<bestD){bestD=dist;best=d;}
+        }
+        if(best){selCluster=MOCK_CLUSTERS.find(c=>c.label===best.label);selPoint=best;showDetail(selCluster,selPoint);}
+    };
+    document.getElementById('ptCount').textContent=SCATTER.length+' 个点';
+}
+
+// ═══════════════════════════════════════════════
+//  CLUSTER SELECTION
+// ═══════════════════════════════════════════════
+function selectCluster(lbl){
+    selCluster=MOCK_CLUSTERS.find(c=>c.label===lbl);
+    selPoint=null;
+    drawScatter(lbl);
+    document.querySelectorAll('.pill').forEach(p=>p.classList.toggle('active',parseInt(p.dataset.lbl)===lbl));
+    document.querySelectorAll('.cluster-card').forEach(p=>p.classList.toggle('selected',parseInt(p.dataset.label)===lbl));
+    if(globeOpen)globeFilter(lbl);
+    showDetail(selCluster,null);
+}
+function clearSel(){
+    selCluster=null;selPoint=null;
+    drawScatter(null);
+    document.querySelectorAll('.pill,.cluster-card').forEach(p=>p.classList.remove('active','selected'));
+    closeDetail();
+    if(globeOpen)globeFilter(null);
+}
+
+// ═══════════════════════════════════════════════
+//  DETAIL PANEL
+// ═══════════════════════════════════════════════
+function showDetail(cluster,point){
+    const dc=document.getElementById('detailContent');
+    let html='';
+    if(point){
+        html+='<h3>数据点详情</h3><table><thead><tr><th>字段</th><th>值</th></tr></thead><tbody>'+
+            '<tr><td>标识</td><td><code>'+point.entity+'</code></td></tr>'+
+            '<tr><td>所属簇</td><td>#'+(point.label===-1?'噪声':point.label)+'</td></tr>'+
+            '<tr><td>UMAP-x</td><td>'+point.x.toFixed(4)+'</td></tr>'+
+            '<tr><td>UMAP-y</td><td>'+point.y.toFixed(4)+'</td></tr>'+
+            '</tbody></table>';
+    }else if(cluster){
+        html+='<h3>簇 #'+(cluster.label===-1?'噪声':cluster.label)+'</h3>'+
+            '<p>大小: '+cluster.size+' | 占比: '+(cluster.prop*100).toFixed(0)+'%'+
+            (cluster.silh!==null?' | 轮廓系数: '+cluster.silh.toFixed(4):'')+'</p>'+
+            '<p style="background:#f8f9fa;border-left:3px solid #4361ee;padding:0.5rem;margin:0.5rem 0;border-radius:0 4px 4px 0;font-size:0.85rem;line-height:1.6;">'+cluster.desc+'</p>'+
+            '<table><thead><tr><th>特征</th><th>区分度</th><th>均值</th><th>标准差</th></tr></thead><tbody>'+
+            '<tr><td>延迟 (4dur)</td><td style="color:#2e7d32;">+3.2</td><td>23ms</td><td>5.1</td></tr>'+
+            '<tr><td>数据包大小 (8ppk)</td><td style="color:#2e7d32;">+2.3</td><td>1520B</td><td>128</td></tr>'+
+            '<tr><td>TLS 版本 (0ver)</td><td style="color:#c62828;">-1.8</td><td>1.2</td><td>0.3</td></tr>'+
+            '</tbody></table>';
+    }
+    dc.innerHTML=html;
+    document.getElementById('detailPanel').classList.add('open');
+}
+function closeDetail(){document.getElementById('detailPanel').classList.remove('open');}
+
+// ═══════════════════════════════════════════════
+//  GLOBE SIDEBAR
+// ═══════════════════════════════════════════════
+function toggleGlobe(){
+    globeOpen=!globeOpen;
+    document.getElementById('leftPanel').classList.toggle('globe-open',globeOpen);
+    document.getElementById('rightPanel').classList.toggle('globe-open',globeOpen);
+    const h=document.getElementById('globeHandle');
+    h.innerHTML=globeOpen?'▶':'◀';
+    h.style.left=globeOpen?'calc(55% - 20px)':'calc(100% - 20px)';
+    if(globeOpen&&!window._globeInit)initGlobe3D();
+}
+
+function globeFilter(label){
+    if(!window._arcMeshes)return;
+    window._arcMeshes.forEach(m=>{
+        if(label===null){m.material.opacity=0.5;m.material.color.setHex(parseInt(m.userData.color,16));}
+        else{m.material.opacity=0.06;m.material.color.setHex(0x666666);}
+    });
+}
+
+// ═══════════════════════════════════════════════
+//  3D GLOBE (Three.js, simplified)
+// ═══════════════════════════════════════════════
+function initGlobe3D(){
+    if(window._globeInit)return;
+    const cont=document.getElementById('globeContainer');
+    if(!cont)return;
+    const W=cont.clientWidth||450, H=cont.clientHeight||500;
+
+    const scene=new THREE.Scene();scene.background=new THREE.Color(0x0a0a1a);
+    const cam=new THREE.PerspectiveCamera(45,W/H,0.1,100);cam.position.set(0,0,6);
+    const ren=new THREE.WebGLRenderer({antialias:true});ren.setSize(W,H);ren.setPixelRatio(Math.min(window.devicePixelRatio,2));
+    cont.appendChild(ren.domElement);
+
+    // earth
+    const earth=new THREE.Mesh(new THREE.SphereGeometry(2,48,48),new THREE.MeshPhongMaterial({color:0x1a3a5c,emissive:0x0a1a2a}));
+    scene.add(earth);
+    // glow
+    scene.add(new THREE.Mesh(new THREE.SphereGeometry(2.06,48,48),new THREE.MeshBasicMaterial({color:0x224488,transparent:true,opacity:0.12})));
+    // lights
+    scene.add(new THREE.AmbientLight(0x222244));
+    const dl=new THREE.DirectionalLight(0xffffff,1);dl.position.set(5,10,7);scene.add(dl);
+    scene.add(new THREE.DirectionalLight(0x4488ff,0.3));
+    // stars
+    const s=new THREE.BufferGeometry();const p=new Float32Array(600*3);
+    for(let i=0;i<600*3;i++)p[i]=(Math.random()-0.5)*100;
+    s.setAttribute('position',new THREE.BufferAttribute(p,3));
+    scene.add(new THREE.Points(s,new THREE.PointsMaterial({color:0xffffff,size:0.15})));
+
+    // mock flows → arcs
+    const tlsC={'TLSv1.3':0x4cc9f0,'TLSv1.2':0x43aa8b,'TLSv1.1':0xf8961e,'TLSv1.0':0xf94144};
+    const arcGrp=new THREE.Group();scene.add(arcGrp);
+    const arcs=[];
+    function llz(lat,lon,r){const p=(90-lat)*Math.PI/180,t=(lon+180)*Math.PI/180;return new THREE.Vector3(-r*Math.sin(p)*Math.cos(t),r*Math.cos(p),r*Math.sin(p)*Math.sin(t));}
+
+    // generate ~30 mock flows
+    const cities=[{lat:31.2,lon:121.5},{lat:39.9,lon:116.4},{lat:35.7,lon:139.7},{lat:37.6,lon:127.0},{lat:51.5,lon:-0.13},{lat:48.9,lon:2.35},{lat:40.7,lon:-74.0},{lat:37.8,lon:-122.4},{lat:1.35,lon:103.8},{lat:-33.9,lon:151.2}];
+    const tlsOpts=['TLSv1.3','TLSv1.2','TLSv1.3','TLSv1.2','TLSv1.1'];
+    for(let i=0;i<30;i++){
+        const src=cities[i%cities.length],dst=cities[(i+3)%cities.length];
+        const f=llz(src.lat,src.lon,2),t=llz(dst.lat,dst.lon,2);
+        const mid=new THREE.Vector3().addVectors(f,t).multiplyScalar(0.5).normalize().multiplyScalar(3.2);
+        const curve=new THREE.QuadraticBezierCurve3(f,mid,t);
+        const c=tlsOpts[i%tlsOpts.length];
+        const mat=new THREE.MeshBasicMaterial({color:tlsC[c],transparent:true,opacity:0.5});
+        const mesh=new THREE.Mesh(new THREE.TubeGeometry(curve,16,0.015,4,false),mat);
+        mesh.userData={color:tlsC[c].toString(16)};
+        arcGrp.add(mesh);arcs.push(mesh);
+    }
+    window._arcMeshes=arcs;
+
+    // drag
+    let isD=false,pm={x:0,y:0};
+    ren.domElement.onmousedown=e=>{isD=true;pm={x:e.clientX,y:e.clientY};};
+    window.onmouseup=()=>isD=false;
+    window.onmousemove=e=>{if(!isD)return;const dx=e.clientX-pm.x,dy=e.clientY-pm.y;earth.rotation.y+=dx*0.005;earth.rotation.x+=dy*0.005;arcGrp.rotation.y=earth.rotation.y;arcGrp.rotation.x=earth.rotation.x;pm={x:e.clientX,y:e.clientY};};
+
+    !function anim(){requestAnimationFrame(anim);if(!isD){earth.rotation.y+=0.003;arcGrp.rotation.y=earth.rotation.y;}ren.render(scene,cam);}();
+    window._globeInit=true;
+}
+
+// ═══════════════════════════════════════════════
+//  RENDER PILLS & CARDS
+// ═══════════════════════════════════════════════
+(function init(){
+    const pr=document.getElementById('pillRow');
+    MOCK_CLUSTERS.forEach(c=>{
+        if(c.label===-1)return;
+        const p=document.createElement('span');p.className='pill';p.style.background=colorFor(c.label);
+        p.dataset.lbl=c.label;p.textContent='#'+c.label+' ('+c.size+')';p.onclick=()=>selectCluster(c.label);
+        pr.appendChild(p);
+    });
+    // noise pill
+    const np=document.createElement('span');np.className='pill';np.style.background='#999';
+    np.dataset.lbl='-1';np.textContent='噪声 ('+MOCK_CLUSTERS.find(c=>c.label===-1).size+')';np.onclick=()=>selectCluster(-1);
+    pr.appendChild(np);
+
+    // cards
+    const cg=document.getElementById('cardGrid');
+    MOCK_CLUSTERS.forEach(c=>{
+        if(c.label===-1)return;
+        const card=document.createElement('div');card.className='cluster-card';card.dataset.label=c.label;
+        card.innerHTML='<div class="label">簇 #'+c.label+'</div>'+
+            '<div class="meta">'+c.size+' 个实体 · '+(c.prop*100).toFixed(0)+'%'+(c.silh!==null?' · 轮廓系数: '+c.silh.toFixed(4):'')+'</div>'+
+            '<div class="desc">'+c.desc+'</div>';
+        card.onclick=()=>selectCluster(c.label);
+        cg.appendChild(card);
+    });
+
+    drawScatter(null);
+    // set initial handle pos
+    document.getElementById('globeHandle').style.left='calc(100% - 20px)';
+})();
+</script>
+</body>
+</html>

run.bat

@@ -1,7 +1,30 @@
 @echo off
 set PYTHONUTF8=1
 cd /d "%~dp0"
-echo Starting TianXuan...
+
-start /B runtime\python\python.exe scripts/start_server.py
+:: Start server in a separate window (detached, survives parent exit)
-timeout /t 3 >nul
+:: /MIN runs minimized so it doesn't pop up a console window
-timeout /t 5 >nul
+start "" /MIN runtime\python\python.exe scripts\start_server.py
+
+:: Wait for PID file (up to 10 seconds)
+set WAIT=0
+:waitloop
+if exist .server_pid goto gotpid
+ping -n 2 127.0.0.1 >nul
+set /a WAIT+=1
+if %WAIT% lss 5 goto waitloop
+
+:gotpid
+if exist .server_pid (
+    set /p PID=<.server_pid
+)
+if defined PID (
+    echo TianXuan started. PID: %PID%
+    echo Visit: http://127.0.0.1:8000/
+    echo.
+    echo To stop: taskkill /F /PID %PID%
+) else (
+    echo TianXuan started.
+    echo Visit: http://127.0.0.1:8000/
+    echo To stop: taskkill /F /IM python.exe
+)

scripts/download_geoip.py

@@ -0,0 +1,38 @@
+"""Download latest GeoLite2-City.mmdb for offline GeoIP resolution.
+
+Usage: runtime\python\python.exe scripts\download_geoip.py
+
+Downloads the latest GeoLite2-City.mmdb from MaxMind's mirror (via
+python-geoip-geolite2) and copies it to a non-Chinese path so the
+maxminddb C extension can read it on Windows.
+
+Requirements: python-geoip-geolite2, maxminddb
+"""
+
+from pathlib import Path
+import os
+import shutil
+import subprocess
+import sys
+
+
+def main():
+    print('Installing/updating python-geoip-geolite2...')
+    subprocess.check_call([
+        sys.executable, '-m', 'pip', 'install', '--upgrade',
+        'python-geoip-geolite2', 'maxminddb',
+    ])
+
+    import _geoip_geolite2
+    src = Path(_geoip_geolite2.__file__).parent / 'GeoLite2-City.mmdb'
+    dst = Path(os.environ.get('TEMP', 'C:/temp')) / 'opencode' / 'GeoLite2-City.mmdb'
+
+    dst.parent.mkdir(parents=True, exist_ok=True)
+    shutil.copy2(str(src), str(dst))
+    print(f'Copied {src.name} ({src.stat().st_size / 1e6:.1f} MB)')
+    print(f'  to {dst}')
+    print('GeoIP database ready.')
+
+
+if __name__ == '__main__':
+    main()

scripts/gen_massive_test.py

@@ -0,0 +1,81 @@
+"""Generate 100 TLS flow CSV files (10000 rows each) matching TlsDB.csv column spec.
+
+Usage:
+    runtime\\python\\python.exe scripts\\gen_massive_test.py [--files 100] [--rows 10000] [--output-dir data/test_csvs]
+"""
+import sys
+import argparse
+from pathlib import Path
+
+# Allow importing from project root
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from scripts.gen_test_data import generate_row, COLUMNS, COVERAGE
+import random
+import csv
+from datetime import datetime
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Generate 100 massive TLS flow CSV files (10,000 rows each)'
+    )
+    parser.add_argument('--files', type=int, default=100,
+                        help='Number of CSV files (default: 100)')
+    parser.add_argument('--rows', type=int, default=10000,
+                        help='Rows per file (default: 10000)')
+    parser.add_argument('--output-dir', type=str, default='data/test_csvs',
+                        help='Output directory (default: data/test_csvs)')
+    parser.add_argument('--seed', type=int, default=42,
+                        help='Random seed for reproducibility (default: 42)')
+    args = parser.parse_args()
+
+    random.seed(args.seed)
+    out_dir = Path(args.output_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    base_time = datetime(2026, 6, 1, 0, 0, 0)
+
+    total_rows = 0
+    for file_idx in range(args.files):
+        file_seed = args.seed + file_idx
+        file_rng = random.Random(file_seed)
+
+        # Generate all rows for this file
+        rows = []
+        for row_i in range(args.rows):
+            row = generate_row(total_rows + row_i, base_time)
+            rows.append(row)
+
+        # Auto-drop columns that are ALL blank
+        non_blank_cols = set()
+        for row in rows:
+            for col in COLUMNS:
+                val = row.get(col, '')
+                if val != '' and val != '+':
+                    non_blank_cols.add(col)
+        file_columns = [c for c in COLUMNS if c in non_blank_cols]
+
+        # Randomize column order per file
+        shuffled = file_columns[:]
+        file_rng.shuffle(shuffled)
+
+        # Write: test_0000.csv .. test_0099.csv
+        out_path = out_dir / f'test_{file_idx:04d}.csv'
+        with open(out_path, 'w', newline='', encoding='utf-8') as f:
+            writer = csv.DictWriter(f, fieldnames=shuffled, extrasaction='ignore')
+            writer.writeheader()
+            for row in rows:
+                writer.writerow(row)
+
+        total_rows += args.rows
+
+        if (file_idx + 1) % 10 == 0 or file_idx == 0 or file_idx == args.files - 1:
+            print(f'  [{file_idx + 1}/{args.files}] {total_rows} rows, '
+                  f'{len(shuffled)} cols, seed={file_seed}')
+
+    print(f'\nDone: {args.files} files x {args.rows} rows = {total_rows} total rows')
+    print(f'Output: {out_dir.resolve()}')
+
+
+if __name__ == '__main__':
+    main()

scripts/rewrite_type_classifier.py

@@ -0,0 +1,8 @@
+"""Rewrite type_classifier.py as thin re-export wrapper."""
+content = '''"""Re-export from new types module for backward compatibility."""
+from analysis.types import ColumnType, TYPE_REGISTRY, TYPE_ALIASES, get_type, classify_column, classify_schema
+TLS_HEX_MAP = {"0303": "TLSv1.2", "0304": "TLSv1.3", "0302": "TLSv1.1", "0301": "TLSv1.0"}
+'''
+with open('analysis/type_classifier.py', 'w', encoding='utf-8') as f:
+    f.write(content)
+print('Rewritten OK')

scripts/start_server.py

@@ -1,13 +1,10 @@
 """Start the development server with host and port from config.yaml.
 
-Reads server.host and server.port from config/config.yaml via the project's
+All output (stdout + stderr) is redirected to log files so the process
-config loader, runs ``manage.py runserver <host>:<port>``, and opens a browser
+can run in the background without blocking any terminal.
-to http://127.0.0.1:<port> .
-
-Usage (by run.bat):
-    start /B runtime\\python\\python.exe scripts\\start_server.py
 """
 
+import os
 import subprocess
 import sys
 import webbrowser
@@ -22,21 +19,38 @@ from config import get_config
 
 def main() -> None:
     cfg = get_config()
-
     host = cfg.server.host or '0.0.0.0'
-    port = cfg.server.port or 80
+    port = cfg.server.port or 8000
-
     addr = f'{host}:{port}'
-    manage_py = Path(__file__).resolve().parent.parent / 'manage.py'
+    manage_py = _project_root / 'manage.py'
 
-    # Open browser — use 127.0.0.1 regardless of bind address so the user's
+    # Log files — capture ALL output so nothing goes to console
-    # browser always reaches the local machine.
+    stdout_log = _project_root / 'server_stdout.txt'
+    stderr_log = _project_root / 'server_stderr.txt'
+    pid_file = _project_root / '.server_pid'
+
+    # Write PID so run.bat can print it
+    pid_file.write_text(str(os.getpid()), encoding='utf-8')
+
+    # Open browser
     webbrowser.open(f'http://127.0.0.1:{port}/')
 
-    # Run the server.  subprocess.call is blocking, so this script stays alive
+    # Run server with ALL output redirected to files
-    # as long as the dev server does — the same effect as calling manage.py
+    with open(stdout_log, 'a', encoding='utf-8') as out_fh, \
-    # directly under start /B.
+         open(stderr_log, 'a', encoding='utf-8') as err_fh:
-    sys.exit(subprocess.call([sys.executable, str(manage_py), 'runserver', addr, '--noreload']))
+        rc = subprocess.call(
+            [sys.executable, str(manage_py), 'runserver', addr, '--noreload'],
+            stdout=out_fh,
+            stderr=err_fh,
+        )
+
+    # Clean up PID file on exit
+    try:
+        pid_file.unlink(missing_ok=True)
+    except Exception:
+        pass
+
+    sys.exit(rc)
 
 
 if __name__ == '__main__':

server_stderr.txt

@@ -0,0 +1,143 @@
+Restored 1 datasets from disk
+[2026-07-24 12:30:18,755] INFO analysis.data_loader: [LOAD] files=1 total_rows_est=200
+[2026-07-24 12:30:18,742] INFO django: Restored 1 datasets from disk
+[2026-07-24 12:30:18,811] INFO analysis.data_loader: [LOAD_SCHEMA] col=0cph dtype=String
+[2026-07-24 12:30:18,811] INFO analysis.data_loader: [LOAD_SCHEMA] col=0crv dtype=String
+[2026-07-24 12:30:18,811] INFO analysis.data_loader: [LOAD_SCHEMA] col=0rnd dtype=String
+[2026-07-24 12:30:18,811] INFO analysis.data_loader: [LOAD_SCHEMA] col=0rnt dtype=String
+[2026-07-24 12:30:18,811] INFO analysis.data_loader: [LOAD_SCHEMA] col=0ver dtype=String
+[2026-07-24 12:30:18,811] INFO analysis.data_loader: [LOAD_SCHEMA] col=1ipp dtype=Int64
+[2026-07-24 12:30:18,811] INFO analysis.data_loader: [LOAD_SCHEMA] col=2tmo dtype=Float64
+[2026-07-24 12:30:18,811] INFO analysis.data_loader: [LOAD_SCHEMA] col=4dbn dtype=Int64
+[2026-07-24 12:30:18,817] INFO analysis.data_loader: [LOAD_SCHEMA] col=4dur dtype=Float64
+[2026-07-24 12:30:18,817] INFO analysis.data_loader: [LOAD_SCHEMA] col=4ksz dtype=Int64
+[2026-07-24 12:30:18,817] INFO analysis.data_loader: [LOAD_SCHEMA] col=4srs dtype=Int64
+[2026-07-24 12:30:18,817] INFO analysis.data_loader: [LOAD_SCHEMA] col=8ack dtype=Int64
+[2026-07-24 12:30:18,817] INFO analysis.data_loader: [LOAD_SCHEMA] col=8byt dtype=Int64
+[2026-07-24 12:30:18,817] INFO analysis.data_loader: [LOAD_SCHEMA] col=8dbd dtype=Int64
+[2026-07-24 12:30:18,818] INFO analysis.data_loader: [LOAD_SCHEMA] col=8did dtype=Int64
+[2026-07-24 12:30:18,818] INFO analysis.data_loader: [LOAD_SCHEMA] col=8pak dtype=Int64
+[2026-07-24 12:30:18,818] INFO analysis.data_loader: [LOAD_SCHEMA] col=8ppk dtype=Int64
+[2026-07-24 12:30:18,818] INFO analysis.data_loader: [LOAD_SCHEMA] col=8seq dtype=Float64
+[2026-07-24 12:30:18,818] INFO analysis.data_loader: [LOAD_SCHEMA] col=8ses dtype=Float64
+[2026-07-24 12:30:18,818] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ipd dtype=String
+[2026-07-24 12:30:18,818] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ipd.anon dtype=String
+[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ipd.city dtype=String
+[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ipd.doma dtype=String
+[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ipd.ispn dtype=String
+[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ipd.latd dtype=String
+[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ipd.lond dtype=String
+[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ipd.orgn dtype=String
+[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ips dtype=String
+[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ips.anon dtype=String
+[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ips.city dtype=String
+[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ips.doma dtype=String
+[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ips.ispn dtype=String
+[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ips.latd dtype=String
+[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ips.lond dtype=String
+[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ips.orgn dtype=String
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=:prd dtype=Int64
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=:prs dtype=Int64
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=cipher-suite dtype=String
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=client-ip dtype=String
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=cnam dtype=String
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=crcc dtype=String
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=dcnt dtype=String
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=ecdhe-named-curve dtype=String
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=name dtype=String
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=orga dtype=String
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=orgu dtype=Int64
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=scnt dtype=String
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=server-ip dtype=String
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=snam dtype=String
+[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=source-node dtype=String
+[2026-07-24 12:30:18,832] INFO analysis.data_loader: [LOAD_SCHEMA] col=tabl dtype=String
+[2026-07-24 12:30:18,832] INFO analysis.data_loader: [LOAD_SCHEMA] col=time dtype=String
+[2026-07-24 12:30:18,832] INFO analysis.data_loader: [LOAD_SCHEMA] col=timestamp dtype=Float64
+[2026-07-24 12:30:18,840] INFO analysis.data_loader: [CLEAN] Generic numeric coercion applied: [':ipd.lond', ':ipd.doma', ':ipd.ispn', 'cnam', ':ips.ispn', ':ips.doma', 'orga', 'snam', 'time', ':ips.orgn', ':ipd.orgn', 'crcc', ':ips.latd', ':ipd.latd', ':ips.city', ':ipd.city', ':ips.lond']
+[2026-07-24 12:30:18,840] INFO analysis.data_loader: [CLEAN] HEX columns added hex_pairs_count: ['0cph', '0crv', '0ver', '0rnt', '0rnd']
+[24/Jul/2026 12:30:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:31:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:31:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:32:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:32:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:33:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:33:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:34:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:34:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:35:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:35:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:36:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:36:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:37:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:37:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:38:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:38:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:39:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:39:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:40:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:40:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:41:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:41:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:42:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:42:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:43:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:43:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:44:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:44:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:45:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:45:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:46:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:46:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:47:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:47:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:48:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:48:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:49:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:49:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:50:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:50:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:51:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:51:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:52:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:52:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:53:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:53:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:54:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:54:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:55:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:55:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:56:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:56:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:57:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:57:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:58:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:58:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:59:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 12:59:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:00:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:00:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:01:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:01:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:02:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:02:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:03:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:03:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:04:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:04:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:05:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:05:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:06:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:06:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:07:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:07:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:08:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:08:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:09:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:09:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:10:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:10:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:11:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:11:37] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:12:07] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 13:12:37] "HEAD / HTTP/1.1" 200 0

server_stdout.txt

@@ -0,0 +1,8 @@
+Performing system checks...
+
+System check identified no issues (0 silenced).
+July 24, 2026 - 12:30:18
+Django version 4.2.30, using settings 'tianxuan.settings'
+Starting development server at http://127.0.0.1:8765/
+Quit the server with CTRL-BREAK.
+

simple_analysis/apps.py

@@ -1,7 +0,0 @@
-from django.apps import AppConfig
-
-
-class SimpleAnalysisConfig(AppConfig):
-    default_auto_field = 'django.db.models.BigAutoField'
-    name = 'simple_analysis'
-    label = 'simple_analysis'

simple_analysis/static/simple_analysis/simple_analysis.js

@@ -1,8 +0,0 @@
-/* 天璇 简单分析模块 — 前端辅助逻辑
- * 主要逻辑内嵌在 simple_analysis.html 中，
- * 此文件供未来扩展或拆分使用。
- */
-(function() {
-    'use strict';
-    console.log('简单分析模块已加载');
-})();

simple_analysis/static/simple_analysis/vendor/leaflet.css

@@ -1,661 +0,0 @@
-/* required styles */
-
-.leaflet-pane,
-.leaflet-tile,
-.leaflet-marker-icon,
-.leaflet-marker-shadow,
-.leaflet-tile-container,
-.leaflet-pane > svg,
-.leaflet-pane > canvas,
-.leaflet-zoom-box,
-.leaflet-image-layer,
-.leaflet-layer {
-	position: absolute;
-	left: 0;
-	top: 0;
-	}
-.leaflet-container {
-	overflow: hidden;
-	}
-.leaflet-tile,
-.leaflet-marker-icon,
-.leaflet-marker-shadow {
-	-webkit-user-select: none;
-	   -moz-user-select: none;
-	        user-select: none;
-	  -webkit-user-drag: none;
-	}
-/* Prevents IE11 from highlighting tiles in blue */
-.leaflet-tile::selection {
-	background: transparent;
-}
-/* Safari renders non-retina tile on retina better with this, but Chrome is worse */
-.leaflet-safari .leaflet-tile {
-	image-rendering: -webkit-optimize-contrast;
-	}
-/* hack that prevents hw layers "stretching" when loading new tiles */
-.leaflet-safari .leaflet-tile-container {
-	width: 1600px;
-	height: 1600px;
-	-webkit-transform-origin: 0 0;
-	}
-.leaflet-marker-icon,
-.leaflet-marker-shadow {
-	display: block;
-	}
-/* .leaflet-container svg: reset svg max-width decleration shipped in Joomla! (joomla.org) 3.x */
-/* .leaflet-container img: map is broken in FF if you have max-width: 100% on tiles */
-.leaflet-container .leaflet-overlay-pane svg {
-	max-width: none !important;
-	max-height: none !important;
-	}
-.leaflet-container .leaflet-marker-pane img,
-.leaflet-container .leaflet-shadow-pane img,
-.leaflet-container .leaflet-tile-pane img,
-.leaflet-container img.leaflet-image-layer,
-.leaflet-container .leaflet-tile {
-	max-width: none !important;
-	max-height: none !important;
-	width: auto;
-	padding: 0;
-	}
-
-.leaflet-container img.leaflet-tile {
-	/* See: https://bugs.chromium.org/p/chromium/issues/detail?id=600120 */
-	mix-blend-mode: plus-lighter;
-}
-
-.leaflet-container.leaflet-touch-zoom {
-	-ms-touch-action: pan-x pan-y;
-	touch-action: pan-x pan-y;
-	}
-.leaflet-container.leaflet-touch-drag {
-	-ms-touch-action: pinch-zoom;
-	/* Fallback for FF which doesn't support pinch-zoom */
-	touch-action: none;
-	touch-action: pinch-zoom;
-}
-.leaflet-container.leaflet-touch-drag.leaflet-touch-zoom {
-	-ms-touch-action: none;
-	touch-action: none;
-}
-.leaflet-container {
-	-webkit-tap-highlight-color: transparent;
-}
-.leaflet-container a {
-	-webkit-tap-highlight-color: rgba(51, 181, 229, 0.4);
-}
-.leaflet-tile {
-	filter: inherit;
-	visibility: hidden;
-	}
-.leaflet-tile-loaded {
-	visibility: inherit;
-	}
-.leaflet-zoom-box {
-	width: 0;
-	height: 0;
-	-moz-box-sizing: border-box;
-	     box-sizing: border-box;
-	z-index: 800;
-	}
-/* workaround for https://bugzilla.mozilla.org/show_bug.cgi?id=888319 */
-.leaflet-overlay-pane svg {
-	-moz-user-select: none;
-	}
-
-.leaflet-pane         { z-index: 400; }
-
-.leaflet-tile-pane    { z-index: 200; }
-.leaflet-overlay-pane { z-index: 400; }
-.leaflet-shadow-pane  { z-index: 500; }
-.leaflet-marker-pane  { z-index: 600; }
-.leaflet-tooltip-pane   { z-index: 650; }
-.leaflet-popup-pane   { z-index: 700; }
-
-.leaflet-map-pane canvas { z-index: 100; }
-.leaflet-map-pane svg    { z-index: 200; }
-
-.leaflet-vml-shape {
-	width: 1px;
-	height: 1px;
-	}
-.lvml {
-	behavior: url(#default#VML);
-	display: inline-block;
-	position: absolute;
-	}
-
-
-/* control positioning */
-
-.leaflet-control {
-	position: relative;
-	z-index: 800;
-	pointer-events: visiblePainted; /* IE 9-10 doesn't have auto */
-	pointer-events: auto;
-	}
-.leaflet-top,
-.leaflet-bottom {
-	position: absolute;
-	z-index: 1000;
-	pointer-events: none;
-	}
-.leaflet-top {
-	top: 0;
-	}
-.leaflet-right {
-	right: 0;
-	}
-.leaflet-bottom {
-	bottom: 0;
-	}
-.leaflet-left {
-	left: 0;
-	}
-.leaflet-control {
-	float: left;
-	clear: both;
-	}
-.leaflet-right .leaflet-control {
-	float: right;
-	}
-.leaflet-top .leaflet-control {
-	margin-top: 10px;
-	}
-.leaflet-bottom .leaflet-control {
-	margin-bottom: 10px;
-	}
-.leaflet-left .leaflet-control {
-	margin-left: 10px;
-	}
-.leaflet-right .leaflet-control {
-	margin-right: 10px;
-	}
-
-
-/* zoom and fade animations */
-
-.leaflet-fade-anim .leaflet-popup {
-	opacity: 0;
-	-webkit-transition: opacity 0.2s linear;
-	   -moz-transition: opacity 0.2s linear;
-	        transition: opacity 0.2s linear;
-	}
-.leaflet-fade-anim .leaflet-map-pane .leaflet-popup {
-	opacity: 1;
-	}
-.leaflet-zoom-animated {
-	-webkit-transform-origin: 0 0;
-	    -ms-transform-origin: 0 0;
-	        transform-origin: 0 0;
-	}
-svg.leaflet-zoom-animated {
-	will-change: transform;
-}
-
-.leaflet-zoom-anim .leaflet-zoom-animated {
-	-webkit-transition: -webkit-transform 0.25s cubic-bezier(0,0,0.25,1);
-	   -moz-transition:    -moz-transform 0.25s cubic-bezier(0,0,0.25,1);
-	        transition:         transform 0.25s cubic-bezier(0,0,0.25,1);
-	}
-.leaflet-zoom-anim .leaflet-tile,
-.leaflet-pan-anim .leaflet-tile {
-	-webkit-transition: none;
-	   -moz-transition: none;
-	        transition: none;
-	}
-
-.leaflet-zoom-anim .leaflet-zoom-hide {
-	visibility: hidden;
-	}
-
-
-/* cursors */
-
-.leaflet-interactive {
-	cursor: pointer;
-	}
-.leaflet-grab {
-	cursor: -webkit-grab;
-	cursor:    -moz-grab;
-	cursor:         grab;
-	}
-.leaflet-crosshair,
-.leaflet-crosshair .leaflet-interactive {
-	cursor: crosshair;
-	}
-.leaflet-popup-pane,
-.leaflet-control {
-	cursor: auto;
-	}
-.leaflet-dragging .leaflet-grab,
-.leaflet-dragging .leaflet-grab .leaflet-interactive,
-.leaflet-dragging .leaflet-marker-draggable {
-	cursor: move;
-	cursor: -webkit-grabbing;
-	cursor:    -moz-grabbing;
-	cursor:         grabbing;
-	}
-
-/* marker & overlays interactivity */
-.leaflet-marker-icon,
-.leaflet-marker-shadow,
-.leaflet-image-layer,
-.leaflet-pane > svg path,
-.leaflet-tile-container {
-	pointer-events: none;
-	}
-
-.leaflet-marker-icon.leaflet-interactive,
-.leaflet-image-layer.leaflet-interactive,
-.leaflet-pane > svg path.leaflet-interactive,
-svg.leaflet-image-layer.leaflet-interactive path {
-	pointer-events: visiblePainted; /* IE 9-10 doesn't have auto */
-	pointer-events: auto;
-	}
-
-/* visual tweaks */
-
-.leaflet-container {
-	background: #ddd;
-	outline-offset: 1px;
-	}
-.leaflet-container a {
-	color: #0078A8;
-	}
-.leaflet-zoom-box {
-	border: 2px dotted #38f;
-	background: rgba(255,255,255,0.5);
-	}
-
-
-/* general typography */
-.leaflet-container {
-	font-family: "Helvetica Neue", Arial, Helvetica, sans-serif;
-	font-size: 12px;
-	font-size: 0.75rem;
-	line-height: 1.5;
-	}
-
-
-/* general toolbar styles */
-
-.leaflet-bar {
-	box-shadow: 0 1px 5px rgba(0,0,0,0.65);
-	border-radius: 4px;
-	}
-.leaflet-bar a {
-	background-color: #fff;
-	border-bottom: 1px solid #ccc;
-	width: 26px;
-	height: 26px;
-	line-height: 26px;
-	display: block;
-	text-align: center;
-	text-decoration: none;
-	color: black;
-	}
-.leaflet-bar a,
-.leaflet-control-layers-toggle {
-	background-position: 50% 50%;
-	background-repeat: no-repeat;
-	display: block;
-	}
-.leaflet-bar a:hover,
-.leaflet-bar a:focus {
-	background-color: #f4f4f4;
-	}
-.leaflet-bar a:first-child {
-	border-top-left-radius: 4px;
-	border-top-right-radius: 4px;
-	}
-.leaflet-bar a:last-child {
-	border-bottom-left-radius: 4px;
-	border-bottom-right-radius: 4px;
-	border-bottom: none;
-	}
-.leaflet-bar a.leaflet-disabled {
-	cursor: default;
-	background-color: #f4f4f4;
-	color: #bbb;
-	}
-
-.leaflet-touch .leaflet-bar a {
-	width: 30px;
-	height: 30px;
-	line-height: 30px;
-	}
-.leaflet-touch .leaflet-bar a:first-child {
-	border-top-left-radius: 2px;
-	border-top-right-radius: 2px;
-	}
-.leaflet-touch .leaflet-bar a:last-child {
-	border-bottom-left-radius: 2px;
-	border-bottom-right-radius: 2px;
-	}
-
-/* zoom control */
-
-.leaflet-control-zoom-in,
-.leaflet-control-zoom-out {
-	font: bold 18px 'Lucida Console', Monaco, monospace;
-	text-indent: 1px;
-	}
-
-.leaflet-touch .leaflet-control-zoom-in, .leaflet-touch .leaflet-control-zoom-out  {
-	font-size: 22px;
-	}
-
-
-/* layers control */
-
-.leaflet-control-layers {
-	box-shadow: 0 1px 5px rgba(0,0,0,0.4);
-	background: #fff;
-	border-radius: 5px;
-	}
-.leaflet-control-layers-toggle {
-	background-image: url(images/layers.png);
-	width: 36px;
-	height: 36px;
-	}
-.leaflet-retina .leaflet-control-layers-toggle {
-	background-image: url(images/layers-2x.png);
-	background-size: 26px 26px;
-	}
-.leaflet-touch .leaflet-control-layers-toggle {
-	width: 44px;
-	height: 44px;
-	}
-.leaflet-control-layers .leaflet-control-layers-list,
-.leaflet-control-layers-expanded .leaflet-control-layers-toggle {
-	display: none;
-	}
-.leaflet-control-layers-expanded .leaflet-control-layers-list {
-	display: block;
-	position: relative;
-	}
-.leaflet-control-layers-expanded {
-	padding: 6px 10px 6px 6px;
-	color: #333;
-	background: #fff;
-	}
-.leaflet-control-layers-scrollbar {
-	overflow-y: scroll;
-	overflow-x: hidden;
-	padding-right: 5px;
-	}
-.leaflet-control-layers-selector {
-	margin-top: 2px;
-	position: relative;
-	top: 1px;
-	}
-.leaflet-control-layers label {
-	display: block;
-	font-size: 13px;
-	font-size: 1.08333em;
-	}
-.leaflet-control-layers-separator {
-	height: 0;
-	border-top: 1px solid #ddd;
-	margin: 5px -10px 5px -6px;
-	}
-
-/* Default icon URLs */
-.leaflet-default-icon-path { /* used only in path-guessing heuristic, see L.Icon.Default */
-	background-image: url(images/marker-icon.png);
-	}
-
-
-/* attribution and scale controls */
-
-.leaflet-container .leaflet-control-attribution {
-	background: #fff;
-	background: rgba(255, 255, 255, 0.8);
-	margin: 0;
-	}
-.leaflet-control-attribution,
-.leaflet-control-scale-line {
-	padding: 0 5px;
-	color: #333;
-	line-height: 1.4;
-	}
-.leaflet-control-attribution a {
-	text-decoration: none;
-	}
-.leaflet-control-attribution a:hover,
-.leaflet-control-attribution a:focus {
-	text-decoration: underline;
-	}
-.leaflet-attribution-flag {
-	display: inline !important;
-	vertical-align: baseline !important;
-	width: 1em;
-	height: 0.6669em;
-	}
-.leaflet-left .leaflet-control-scale {
-	margin-left: 5px;
-	}
-.leaflet-bottom .leaflet-control-scale {
-	margin-bottom: 5px;
-	}
-.leaflet-control-scale-line {
-	border: 2px solid #777;
-	border-top: none;
-	line-height: 1.1;
-	padding: 2px 5px 1px;
-	white-space: nowrap;
-	-moz-box-sizing: border-box;
-	     box-sizing: border-box;
-	background: rgba(255, 255, 255, 0.8);
-	text-shadow: 1px 1px #fff;
-	}
-.leaflet-control-scale-line:not(:first-child) {
-	border-top: 2px solid #777;
-	border-bottom: none;
-	margin-top: -2px;
-	}
-.leaflet-control-scale-line:not(:first-child):not(:last-child) {
-	border-bottom: 2px solid #777;
-	}
-
-.leaflet-touch .leaflet-control-attribution,
-.leaflet-touch .leaflet-control-layers,
-.leaflet-touch .leaflet-bar {
-	box-shadow: none;
-	}
-.leaflet-touch .leaflet-control-layers,
-.leaflet-touch .leaflet-bar {
-	border: 2px solid rgba(0,0,0,0.2);
-	background-clip: padding-box;
-	}
-
-
-/* popup */
-
-.leaflet-popup {
-	position: absolute;
-	text-align: center;
-	margin-bottom: 20px;
-	}
-.leaflet-popup-content-wrapper {
-	padding: 1px;
-	text-align: left;
-	border-radius: 12px;
-	}
-.leaflet-popup-content {
-	margin: 13px 24px 13px 20px;
-	line-height: 1.3;
-	font-size: 13px;
-	font-size: 1.08333em;
-	min-height: 1px;
-	}
-.leaflet-popup-content p {
-	margin: 17px 0;
-	margin: 1.3em 0;
-	}
-.leaflet-popup-tip-container {
-	width: 40px;
-	height: 20px;
-	position: absolute;
-	left: 50%;
-	margin-top: -1px;
-	margin-left: -20px;
-	overflow: hidden;
-	pointer-events: none;
-	}
-.leaflet-popup-tip {
-	width: 17px;
-	height: 17px;
-	padding: 1px;
-
-	margin: -10px auto 0;
-	pointer-events: auto;
-
-	-webkit-transform: rotate(45deg);
-	   -moz-transform: rotate(45deg);
-	    -ms-transform: rotate(45deg);
-	        transform: rotate(45deg);
-	}
-.leaflet-popup-content-wrapper,
-.leaflet-popup-tip {
-	background: white;
-	color: #333;
-	box-shadow: 0 3px 14px rgba(0,0,0,0.4);
-	}
-.leaflet-container a.leaflet-popup-close-button {
-	position: absolute;
-	top: 0;
-	right: 0;
-	border: none;
-	text-align: center;
-	width: 24px;
-	height: 24px;
-	font: 16px/24px Tahoma, Verdana, sans-serif;
-	color: #757575;
-	text-decoration: none;
-	background: transparent;
-	}
-.leaflet-container a.leaflet-popup-close-button:hover,
-.leaflet-container a.leaflet-popup-close-button:focus {
-	color: #585858;
-	}
-.leaflet-popup-scrolled {
-	overflow: auto;
-	}
-
-.leaflet-oldie .leaflet-popup-content-wrapper {
-	-ms-zoom: 1;
-	}
-.leaflet-oldie .leaflet-popup-tip {
-	width: 24px;
-	margin: 0 auto;
-
-	-ms-filter: "progid:DXImageTransform.Microsoft.Matrix(M11=0.70710678, M12=0.70710678, M21=-0.70710678, M22=0.70710678)";
-	filter: progid:DXImageTransform.Microsoft.Matrix(M11=0.70710678, M12=0.70710678, M21=-0.70710678, M22=0.70710678);
-	}
-
-.leaflet-oldie .leaflet-control-zoom,
-.leaflet-oldie .leaflet-control-layers,
-.leaflet-oldie .leaflet-popup-content-wrapper,
-.leaflet-oldie .leaflet-popup-tip {
-	border: 1px solid #999;
-	}
-
-
-/* div icon */
-
-.leaflet-div-icon {
-	background: #fff;
-	border: 1px solid #666;
-	}
-
-
-/* Tooltip */
-/* Base styles for the element that has a tooltip */
-.leaflet-tooltip {
-	position: absolute;
-	padding: 6px;
-	background-color: #fff;
-	border: 1px solid #fff;
-	border-radius: 3px;
-	color: #222;
-	white-space: nowrap;
-	-webkit-user-select: none;
-	-moz-user-select: none;
-	-ms-user-select: none;
-	user-select: none;
-	pointer-events: none;
-	box-shadow: 0 1px 3px rgba(0,0,0,0.4);
-	}
-.leaflet-tooltip.leaflet-interactive {
-	cursor: pointer;
-	pointer-events: auto;
-	}
-.leaflet-tooltip-top:before,
-.leaflet-tooltip-bottom:before,
-.leaflet-tooltip-left:before,
-.leaflet-tooltip-right:before {
-	position: absolute;
-	pointer-events: none;
-	border: 6px solid transparent;
-	background: transparent;
-	content: "";
-	}
-
-/* Directions */
-
-.leaflet-tooltip-bottom {
-	margin-top: 6px;
-}
-.leaflet-tooltip-top {
-	margin-top: -6px;
-}
-.leaflet-tooltip-bottom:before,
-.leaflet-tooltip-top:before {
-	left: 50%;
-	margin-left: -6px;
-	}
-.leaflet-tooltip-top:before {
-	bottom: 0;
-	margin-bottom: -12px;
-	border-top-color: #fff;
-	}
-.leaflet-tooltip-bottom:before {
-	top: 0;
-	margin-top: -12px;
-	margin-left: -6px;
-	border-bottom-color: #fff;
-	}
-.leaflet-tooltip-left {
-	margin-left: -6px;
-}
-.leaflet-tooltip-right {
-	margin-left: 6px;
-}
-.leaflet-tooltip-left:before,
-.leaflet-tooltip-right:before {
-	top: 50%;
-	margin-top: -6px;
-	}
-.leaflet-tooltip-left:before {
-	right: 0;
-	margin-right: -12px;
-	border-left-color: #fff;
-	}
-.leaflet-tooltip-right:before {
-	left: 0;
-	margin-left: -12px;
-	border-right-color: #fff;
-	}
-
-/* Printing */
-
-@media print {
-	/* Prevent printers from removing background-images of controls. */
-	.leaflet-control {
-		-webkit-print-color-adjust: exact;
-		print-color-adjust: exact;
-		}
-	}

simple_analysis/urls.py

@@ -1,11 +0,0 @@
-from django.urls import path
-from . import views
-
-app_name = 'simple_analysis'
-urlpatterns = [
-    path('', views.index, name='index'),
-    path('upload/', views.upload_csv, name='upload'),
-    path('filter/', views.filter_data, name='filter'),
-    path('cluster/', views.run_clustering, name='cluster'),
-    path('cluster/<int:label>/', views.cluster_detail, name='cluster_detail'),
-]

simple_analysis/views.py

@@ -1,796 +0,0 @@
-"""简单分析模块 — 独立的上传→筛选→聚类→地图可视化工作流。
-
-不与天璇原有分析模块共享数据库或 session store，所有中间数据
-存储在模块级内存缓存（SimpleAnalysisCache）或临时目录中。
-"""
-
-import json
-import logging
-import os
-import re
-import shutil
-import tempfile
-import uuid
-from pathlib import Path
-from collections import Counter
-
-import numpy as np
-import polars as pl
-from django.conf import settings
-from django.http import JsonResponse
-from django.shortcuts import render
-from django.views.decorators.csrf import csrf_exempt
-from django.views.decorators.http import require_POST, require_GET
-from sklearn.cluster import HDBSCAN
-from sklearn.preprocessing import StandardScaler
-from sklearn.cluster import DBSCAN
-
-logger = logging.getLogger(__name__)
-
-# ── In-memory cache ──────────────────────────────────────────────────────────
-# Thread-safe for single-user desktop usage; each user session gets a unique key.
-_cache: dict[str, dict] = {}
-_TEMP_DIR = Path(settings.FILE_UPLOAD_TEMP_DIR) / 'simple_analysis'
-_TEMP_DIR.mkdir(parents=True, exist_ok=True)
-
-
-def _new_id() -> str:
-    return uuid.uuid4().hex[:12]
-
-
-def _cached(key: str) -> dict | None:
-    return _cache.get(key)
-
-
-def _store(key: str, data: dict) -> None:
-    _cache[key] = data
-
-
-# ── Column-name helpers ──────────────────────────────────────────────────────
-
-def _norm(name: str) -> str:
-    """Normalise column name: lower-case, strip whitespace, replace hyphens."""
-    return name.strip().lower().replace('-', '_').replace(' ', '_')
-
-
-def _find_col(col_names: list[str], *patterns: str) -> str | None:
-    """Find first column matching any pattern (exact or suffix after stripping)."""
-    normed = {c: _norm(c) for c in col_names}
-    for p in patterns:
-        np_norm = _norm(p)
-        for orig, n in normed.items():
-            if n == np_norm or n.endswith('_' + np_norm) or n.startswith(np_norm + '_'):
-                return orig
-    return None
-
-
-# ── Step 1: Upload ───────────────────────────────────────────────────────────
-
-@csrf_exempt
-@require_POST
-def upload_csv(request):
-    """Upload one or more CSV files, merge, return column info + cnam frequency."""
-    files = request.FILES.getlist('files')
-    if not files:
-        return JsonResponse({'error': '未上传任何文件'}, status=400)
-
-    session_id = _new_id()
-    upload_dir = _TEMP_DIR / session_id
-    upload_dir.mkdir(parents=True, exist_ok=True)
-
-    saved_paths = []
-    for f in files:
-        dest = upload_dir / f.name
-        with open(dest, 'wb') as out:
-            for chunk in f.chunks():
-                out.write(chunk)
-        saved_paths.append(str(dest))
-
-    # Load & merge CSV files using Polars
-    try:
-        lf = pl.scan_csv(
-            saved_paths[0] if len(saved_paths) == 1 else str(upload_dir / '*.csv'),
-            infer_schema_length=10000,
-            try_parse_dates=True,
-        )
-        if len(saved_paths) > 1:
-            frames = [pl.scan_csv(p, infer_schema_length=10000) for p in saved_paths]
-            lf = pl.concat(frames, how='diagonal_relaxed')
-
-        df = lf.collect(streaming=True)
-    except Exception as e:
-        shutil.rmtree(upload_dir, ignore_errors=True)
-        return JsonResponse({'error': f'CSV 解析失败: {e}'}, status=400)
-
-    col_names = df.columns
-    total_rows = len(df)
-
-    # Detect key columns — expanded patterns to match actual CSV column names
-    cnam_col = _find_col(col_names, 'cnam', 'snam', 'class_name', 'category', 'target', 'name')
-    cnrs_col = _find_col(col_names, 'cnrs', 'conn_success', 'result', '4srs')
-    isrs_col = _find_col(col_names, 'isrs', 'is_resolved', 'resolved')
-    src_lat_col = _find_col(col_names, ':ips.latd', 'src_latitude', 'src_lat', 'source_lat')
-    src_lon_col = _find_col(col_names, ':ips.lond', 'src_longitude', 'src_lon', 'source_lon')
-    dst_lat_col = _find_col(col_names, ':ipd.latd', 'dst_latitude', 'dst_lat', 'dest_lat')
-    dst_lon_col = _find_col(col_names, ':ipd.lond', 'dst_longitude', 'dst_lon', 'dest_lon')
-    ips_col = _find_col(col_names, ':ips', 'src_ip', 'source_ip', 'ip_src')
-    ipd_col = _find_col(col_names, ':ipd', 'dst_ip', 'dest_ip', 'destination_ip', 'ip_dst')
-
-    # CNAM frequency
-    cnam_freq = []
-    top_cnam = None
-    if cnam_col:
-        try:
-            cnam_series = df[cnam_col].cast(pl.Utf8).fill_null('')
-            counts = Counter(cnam_series.to_list())
-            total_cnam = sum(counts.values())
-            for val, cnt in counts.most_common():
-                cnam_freq.append({
-                    'value': val if val else '(空)',
-                    'count': cnt,
-                    'pct': round(cnt / total_cnam * 100, 1),
-                })
-            if cnam_freq:
-                top_cnam = cnam_freq[0]
-        except Exception as e:
-            logger.warning('cnam frequency failed: %s', e)
-
-    # Store metadata in cache
-    meta = {
-        'session_id': session_id,
-        'upload_dir': str(upload_dir),
-        'total_rows': total_rows,
-        'col_names': col_names,
-        'cnam_col': cnam_col,
-        'cnrs_col': cnrs_col,
-        'isrs_col': isrs_col,
-        'src_lat_col': src_lat_col,
-        'src_lon_col': src_lon_col,
-        'dst_lat_col': dst_lat_col,
-        'dst_lon_col': dst_lon_col,
-        'ips_col': ips_col,
-        'ipd_col': ipd_col,
-    }
-    _store(session_id, {'meta': meta, 'df': df})
-
-    return JsonResponse({
-        'session_id': session_id,
-        'total_rows': total_rows,
-        'columns': col_names[:50],
-        'detected_columns': {
-            'cnam': cnam_col,
-            'cnrs': cnrs_col,
-            'isrs': isrs_col,
-            'src_lat': src_lat_col,
-            'src_lon': src_lon_col,
-            'dst_lat': dst_lat_col,
-            'dst_lon': dst_lon_col,
-            'ips': ips_col,
-            'ipd': ipd_col,
-        },
-        'cnam_frequency': cnam_freq[:30],
-        'top_cnam': top_cnam,
-    })
-
-
-# ── Step 2: Filter ───────────────────────────────────────────────────────────
-
-@csrf_exempt
-@require_POST
-def filter_data(request):
-    """Apply cnam + cnrs + isrs filters; return preview rows."""
-    body = json.loads(request.body)
-    session_id = body.get('session_id')
-    selected_cnam = body.get('selected_cnam', '')
-
-    entry = _cached(session_id)
-    if entry is None:
-        return JsonResponse({'error': '会话已过期，请重新上传'}, status=400)
-
-    df: pl.DataFrame = entry['df']
-    meta = entry['meta']
-    cnam_col = meta['cnam_col']
-    cnrs_col = meta['cnrs_col']
-    isrs_col = meta['isrs_col']
-
-    # Validate required columns
-    missing = []
-    if not cnam_col:
-        missing.append('cnam/snam (目标类别)')
-
-    if missing:
-        return JsonResponse({
-            'error': f'缺少关键列: {", ".join(missing)}。请检查CSV文件格式。',
-            'missing_columns': missing,
-        }, status=400)
-
-    # Apply filter — cnrs and isrs are optional; only apply if column exists
-    try:
-        cond = (pl.col(cnam_col).cast(pl.Utf8) == selected_cnam)
-        if cnrs_col:
-            cond = cond & (pl.col(cnrs_col).cast(pl.Utf8).str.strip_chars() == '+')
-        if isrs_col:
-            cond = cond & (pl.col(isrs_col).cast(pl.Utf8).str.strip_chars() == '+')
-        filtered = df.filter(cond)
-    except Exception as e:
-        return JsonResponse({'error': f'筛选失败: {e}'}, status=400)
-
-    filtered_rows = len(filtered)
-    meta['filtered_cnam'] = selected_cnam
-    meta['filtered_rows'] = filtered_rows
-    entry['filtered_df'] = filtered
-
-    # Preview data (first 20 rows)
-    preview_cols = [c for c in filtered.columns if not c.startswith('_')][:30]
-    preview_rows = []
-    for row in filtered.head(20).select(preview_cols).iter_rows(named=True):
-        clean = {}
-        for k, v in row.items():
-            if v is None or (isinstance(v, float) and np.isnan(v)):
-                clean[k] = None
-            else:
-                clean[k] = str(v)[:120]
-        preview_rows.append(clean)
-
-    return JsonResponse({
-        'filtered_rows': filtered_rows,
-        'preview_columns': preview_cols,
-        'preview_rows': preview_rows,
-        'total_before_filter': len(df),
-    })
-
-
-# ── Step 3: Cluster ──────────────────────────────────────────────────────────
-
-@csrf_exempt
-@require_POST
-def run_clustering(request):
-    """Run HDBSCAN geo-clustering or IP-subnet soft-clustering."""
-    body = json.loads(request.body)
-    session_id = body.get('session_id')
-    mode = body.get('mode', 'geo')  # 'geo' or 'subnet'
-    min_cluster_size = int(body.get('min_cluster_size', 5))
-    cluster_selection_epsilon = float(body.get('cluster_selection_epsilon', 0.005))
-    min_records_per_subnet = int(body.get('min_records_per_subnet', 3))
-    max_cluster_size = int(body.get('max_cluster_size', 100))
-
-    entry = _cached(session_id)
-    if entry is None:
-        return JsonResponse({'error': '会话已过期，请重新上传'}, status=400)
-
-    filtered = entry.get('filtered_df')
-    if filtered is None:
-        return JsonResponse({'error': '请先执行筛选（步骤二）'}, status=400)
-
-    meta = entry['meta']
-    src_lat_col = meta['src_lat_col']
-    src_lon_col = meta['src_lon_col']
-    ips_col = meta['ips_col']
-
-    if mode == 'geo':
-        result = _cluster_geo(entry, filtered, meta, min_cluster_size, cluster_selection_epsilon)
-    else:
-        result = _cluster_subnet(entry, filtered, meta, min_records_per_subnet, max_cluster_size)
-    # Update cache with any modifications made by cluster functions
-    _store(session_id, entry)
-    return result
-
-
-def _cluster_geo(entry: dict, df: pl.DataFrame, meta: dict,
-                 min_cluster_size: int, cluster_selection_epsilon: float = 0.005):
-    """HDBSCAN clustering on source IP latitude/longitude using Haversine distance."""
-    src_lat_col = meta['src_lat_col']
-    src_lon_col = meta['src_lon_col']
-    ips_col = meta['ips_col']
-
-    if not src_lat_col or not src_lon_col:
-        return JsonResponse({'error': '缺少源经纬度列（:ips.latd / :ips.lond），无法执行地理聚类'}, status=400)
-
-    # Drop rows with missing lat/lon
-    clean = df.filter(
-        pl.col(src_lat_col).is_not_null()
-        & pl.col(src_lon_col).is_not_null()
-    )
-
-    if len(clean) == 0:
-        return JsonResponse({'error': '所有记录均缺失经纬度，无法聚类'}, status=400)
-
-    # Clean lat/lon: cast to string, filter out '+'/'-' artifacts
-    try:
-        lat_raw = clean[src_lat_col].cast(pl.Utf8)
-        lon_raw = clean[src_lon_col].cast(pl.Utf8)
-        # Build validity mask: exclude '+', '-', empty, and None
-        valid_mask = (
-            lat_raw.is_not_null() & lon_raw.is_not_null()
-            & (lat_raw != '+') & (lat_raw != '-')
-            & (lon_raw != '+') & (lon_raw != '-')
-            & (lat_raw != '') & (lon_raw != '')
-        )
-        clean = clean.filter(valid_mask)
-        if len(clean) == 0:
-            return JsonResponse({'error': '无有效经纬度记录（排除符号值后）'}, status=400)
-        lat_vals = clean[src_lat_col].cast(pl.Float64, strict=False).to_numpy()
-        lon_vals = clean[src_lon_col].cast(pl.Float64, strict=False).to_numpy()
-        # Final NaN check
-        lat_vals = np.where(np.isnan(lat_vals), 0.0, lat_vals)
-        lon_vals = np.where(np.isnan(lon_vals), 0.0, lon_vals)
-    except Exception as e:
-        return JsonResponse({'error': f'经纬度列转换失败: {e}'}, status=400)
-
-    # Filter to valid lat/lon ranges
-    valid_range_mask = (
-        (lat_vals >= -90) & (lat_vals <= 90)
-        & (lon_vals >= -180) & (lon_vals <= 180)
-        & ~np.isnan(lat_vals) & ~np.isnan(lon_vals)
-    )
-    lat_vals = lat_vals[valid_range_mask]
-    lon_vals = lon_vals[valid_range_mask]
-    clean = clean.filter(
-        pl.Series(valid_range_mask)
-    )
-
-    if len(clean) < min_cluster_size:
-        return JsonResponse({
-            'error': f'有效经纬度记录数（{len(clean)}）少于最小聚类规模（{min_cluster_size}）',
-        }, status=400)
-
-    # ── HDBSCAN with Haversine distance ────────────────────────────────────
-    # Convert degrees to radians for haversine metric
-    lat_rad = np.radians(lat_vals)
-    lon_rad = np.radians(lon_vals)
-    coords = np.column_stack([lat_rad, lon_rad])
-
-    clusterer = HDBSCAN(
-        min_cluster_size=min_cluster_size,
-        min_samples=min(3, min_cluster_size),
-        metric='haversine',
-        cluster_selection_epsilon=cluster_selection_epsilon,
-    )
-    labels = clusterer.fit_predict(coords)
-
-    n_clusters = int(len(set(labels)) - (1 if -1 in labels else 0))
-    n_noise = int((labels == -1).sum())
-    noise_ratio = n_noise / len(labels) if len(labels) > 0 else 0
-
-    # Check for degenerate clustering (fallback suggestion)
-    fallback_suggested = False
-    if noise_ratio > 0.8 or n_clusters <= 1:
-        fallback_suggested = True
-
-    # Build cluster overview
-    cluster_map = {}
-    unique_labels = sorted(set(labels))
-    data_points_by_cluster = {}
-    for lbl in unique_labels:
-        mask = labels == lbl
-        idxs = np.where(mask)[0]
-        cluster_lats = lat_vals[idxs]
-        cluster_lons = lon_vals[idxs]
-        center_lat = float(np.median(cluster_lats))
-        center_lon = float(np.median(cluster_lons))
-
-        # Unique IPs
-        if ips_col:
-            ips_in_cluster = clean.filter(pl.Series(mask))[ips_col].cast(pl.Utf8).unique().to_list()
-            n_unique_ips = len(ips_in_cluster)
-        else:
-            ips_in_cluster = []
-            n_unique_ips = 0
-
-        cluster_map[int(lbl)] = {
-            'label': int(lbl),
-            'size': int(mask.sum()),
-            'center_lat': round(float(center_lat), 4),
-            'center_lon': round(float(center_lon), 4),
-            'n_unique_ips': int(n_unique_ips),
-            'is_noise': bool(lbl == -1),
-        }
-
-        # Collect individual data points for map rendering (lat, lon, index)
-        pts = []
-        for pt_idx, i in enumerate(idxs):
-            pts.append({
-                'lat': round(float(cluster_lats[pt_idx]), 4),
-                'lon': round(float(cluster_lons[pt_idx]), 4),
-                'idx': int(i),
-            })
-        data_points_by_cluster[int(lbl)] = pts
-
-    # Annotate original filtered DataFrame with cluster labels
-    label_series = np.full(len(df), -1, dtype=int)
-    # Map back: only the valid rows got labels
-    valid_indices = np.where(valid_range_mask)[0]
-    for i, idx in enumerate(valid_indices):
-        label_series[idx] = int(labels[i])
-
-    entry['cluster_labels'] = label_series.tolist()
-    entry['cluster_map'] = cluster_map
-    entry['cluster_mode'] = 'geo'
-    entry['n_clusters'] = n_clusters
-    entry['n_noise'] = n_noise
-
-    # Update filtered_df with cluster label column
-    df_with_labels = df.with_columns(pl.Series('_cluster_label', label_series))
-    entry['filtered_df'] = df_with_labels
-
-    return JsonResponse({
-        'n_clusters': n_clusters,
-        'n_noise': n_noise,
-        'n_valid_points': len(clean),
-        'clusters': cluster_map,
-        'labels': label_series.tolist(),
-        'data_points': data_points_by_cluster,
-        'fallback_suggested': fallback_suggested,
-    })
-
-
-def _cluster_subnet(entry: dict, df: pl.DataFrame, meta: dict,
-                     min_records_per_subnet: int = 3, max_cluster_size: int = 100):
-    """Soft clustering by /24 subnet prefix.
-
-    Subnets with fewer than *min_records_per_subnet* records are marked noise (-1).
-    Subnets with more than *max_cluster_size* records are auto-split using geo
-    sub-clustering (HDBSCAN + haversine) on the available lat/lon within the subnet.
-    """
-    ips_col = meta['ips_col']
-    src_lat_col = meta['src_lat_col']
-    src_lon_col = meta['src_lon_col']
-    if not ips_col:
-        return JsonResponse({'error': '缺少源IP列（:ips），无法执行子网聚类'}, status=400)
-
-    try:
-        ip_series = df[ips_col].cast(pl.Utf8)
-    except Exception as e:
-        return JsonResponse({'error': f'IP列转换失败: {e}'}, status=400)
-
-    # Extract /24 subnet
-    def _subnet24(ip: str) -> str | None:
-        if not ip or ip == 'None' or ip == '':
-            return None
-        parts = ip.split('.')
-        if len(parts) < 4:
-            return None
-        return '.'.join(parts[:3]) + '.0/24'
-
-    subnets = [None if v is None else _subnet24(str(v)) for v in ip_series.to_list()]
-    subnet_count = Counter(s for s in subnets if s is not None)
-
-    # Assign cluster labels based on subnet frequency
-    labels = []
-    cluster_map: dict[int, dict] = {}
-    next_label = 0
-    subnet_to_label: dict[str, int] = {}
-    # Track which records belong to large subnets for geo splitting
-    large_subnet_indices: dict[str, list[int]] = {}
-
-    for row_idx, s in enumerate(subnets):
-        if s is None:
-            labels.append(-1)
-        else:
-            cnt = subnet_count.get(s, 0)
-            if cnt < min_records_per_subnet:
-                labels.append(-1)  # noise (too few records)
-            elif cnt > max_cluster_size:
-                # Large subnet → will apply geo sub-clustering later
-                if s not in subnet_to_label:
-                    subnet_to_label[s] = next_label
-                    cluster_map[next_label] = {
-                        'label': next_label,
-                        'size': 0,
-                        'subnet': s,
-                        'is_noise': False,
-                        '_is_large': True,
-                        '_large_subnet_name': s,
-                    }
-                    next_label += 1
-                    large_subnet_indices[s] = []
-                lbl = subnet_to_label[s]
-                labels.append(lbl)
-                cluster_map[lbl]['size'] = cluster_map[lbl].get('size', 0) + 1
-                large_subnet_indices[s].append(row_idx)
-            else:
-                if s not in subnet_to_label:
-                    subnet_to_label[s] = next_label
-                    cluster_map[next_label] = {
-                        'label': next_label,
-                        'size': 0,
-                        'subnet': s,
-                        'is_noise': False,
-                    }
-                    next_label += 1
-                lbl = subnet_to_label[s]
-                labels.append(lbl)
-                cluster_map[lbl]['size'] = cluster_map[lbl].get('size', 0) + 1
-
-    # ── Geo sub-clustering for large subnets ────────────────────────────────
-    if large_subnet_indices and src_lat_col and src_lon_col:
-        for subnet_name, indices in large_subnet_indices.items():
-            old_label = subnet_to_label[subnet_name]
-            # Remove the temporary large-subnet cluster entry
-            cluster_map.pop(old_label, None)
-
-            # Get lat/lon for records in this subnet
-            subnet_df = df[pl.Series(indices)]
-            try:
-                sub_lats = subnet_df[src_lat_col].cast(pl.Float64).to_numpy()
-                sub_lons = subnet_df[src_lon_col].cast(pl.Float64).to_numpy()
-                valid = (
-                    ~np.isnan(sub_lats) & ~np.isnan(sub_lons)
-                    & (sub_lats >= -90) & (sub_lats <= 90)
-                    & (sub_lons >= -180) & (sub_lons <= 180)
-                )
-                sub_lats = sub_lats[valid]
-                sub_lons = sub_lons[valid]
-                if len(sub_lats) >= 3:
-                    # Run HDBSCAN with haversine on this subnet's points
-                    coords = np.column_stack([np.radians(sub_lats), np.radians(sub_lons)])
-                    sub_cluster = HDBSCAN(
-                        min_cluster_size=min(3, len(sub_lats) // 5),
-                        min_samples=2,
-                        metric='haversine',
-                        cluster_selection_epsilon=0.005,
-                    )
-                    sub_labels = sub_cluster.fit_predict(coords)
-
-                    # Create sub-clusters
-                    sub_label_map: dict[int, int] = {}
-                    valid_idx_iter = 0
-                    for orig_rank, orig_idx in enumerate(indices):
-                        if not valid[orig_rank - valid_idx_iter]:
-                            # Was invalid, check if we skipped
-                            pass
-
-                    # Re-map: skip invalid points (mark noise)
-                    valid_counter = 0
-                    for orig_rank, orig_idx in enumerate(indices):
-                        if not valid[orig_rank]:
-                            labels[orig_idx] = -1  # no lat/lon → noise
-                        else:
-                            sl = int(sub_labels[valid_counter])
-                            valid_counter += 1
-                            if sl == -1:
-                                labels[orig_idx] = -1
-                            else:
-                                sub_cluster_name = f'{subnet_name}/sub{sl}'
-                                if sub_cluster_name not in subnet_to_label:
-                                    subnet_to_label[sub_cluster_name] = next_label
-                                    cluster_map[next_label] = {
-                                        'label': next_label,
-                                        'size': 0,
-                                        'subnet': sub_cluster_name,
-                                        'is_noise': False,
-                                    }
-                                    next_label += 1
-                                new_lbl = subnet_to_label[sub_cluster_name]
-                                labels[orig_idx] = new_lbl
-                                cluster_map[new_lbl]['size'] = cluster_map[new_lbl].get('size', 0) + 1
-                else:
-                    # Not enough geo data → keep as single cluster
-                    cluster_map[old_label] = {
-                        'label': old_label,
-                        'size': len(indices),
-                        'subnet': subnet_name,
-                        'is_noise': False,
-                    }
-                    subnet_to_label[subnet_name] = old_label
-            except Exception as e:
-                logger.warning('Subnet geo-splitting failed for %s: %s', subnet_name, e)
-                # Keep the original cluster
-                cluster_map[old_label] = {
-                    'label': old_label,
-                    'size': len(indices),
-                    'subnet': subnet_name,
-                    'is_noise': False,
-                }
-                subnet_to_label[subnet_name] = old_label
-
-    # Update cluster maps with centers and unique IP counts
-    for lbl in list(cluster_map.keys()):
-        mask = np.array(labels) == lbl
-        ips_in_cluster = df.filter(pl.Series(mask))[ips_col].cast(pl.Utf8).unique().to_list()
-        cluster_map[lbl]['n_unique_ips'] = len(ips_in_cluster)
-        # Try to get geo center if lat/lon available
-        if src_lat_col and src_lon_col:
-            try:
-                clat = df.filter(pl.Series(mask))[src_lat_col].cast(pl.Float64).drop_nulls()
-                clon = df.filter(pl.Series(mask))[src_lon_col].cast(pl.Float64).drop_nulls()
-                if len(clat) > 0 and len(clon) > 0:
-                    cluster_map[lbl]['center_lat'] = round(float(clat.median()), 4)
-                    cluster_map[lbl]['center_lon'] = round(float(clon.median()), 4)
-            except Exception:
-                cluster_map[lbl]['center_lat'] = None
-                cluster_map[lbl]['center_lon'] = None
-        else:
-            cluster_map[lbl]['center_lat'] = None
-            cluster_map[lbl]['center_lon'] = None
-
-    n_noise = labels.count(-1)
-    n_clusters = len(cluster_map)
-
-    # Build data_points for map
-    data_points_by_cluster: dict[int, list] = {}
-    for lbl in list(cluster_map.keys()):
-        mask = np.array(labels) == lbl
-        pts = []
-        if src_lat_col and src_lon_col:
-            try:
-                sub = df.filter(pl.Series(mask))
-                lats = sub[src_lat_col].cast(pl.Float64).to_numpy()
-                lons = sub[src_lon_col].cast(pl.Float64).to_numpy()
-                for i in range(len(sub)):
-                    if not np.isnan(lats[i]) and not np.isnan(lons[i]):
-                        pts.append({'lat': round(float(lats[i]), 4),
-                                    'lon': round(float(lons[i]), 4),
-                                    'idx': int(i)})
-            except Exception:
-                pass
-        data_points_by_cluster[lbl] = pts
-
-    # Also collect noisy points' coordinates
-    noise_mask = np.array(labels) == -1
-    noise_points = []
-    if src_lat_col and src_lon_col:
-        try:
-            noise_sub = df.filter(pl.Series(noise_mask))
-            nlats = noise_sub[src_lat_col].cast(pl.Float64).to_numpy()
-            nlons = noise_sub[src_lon_col].cast(pl.Float64).to_numpy()
-            for i in range(len(noise_sub)):
-                if not np.isnan(nlats[i]) and not np.isnan(nlons[i]):
-                    noise_points.append({'lat': round(float(nlats[i]), 4),
-                                         'lon': round(float(nlons[i]), 4),
-                                         'idx': int(i)})
-        except Exception:
-            pass
-    if noise_points:
-        data_points_by_cluster[-1] = noise_points
-
-    # Annotate DataFrame
-    df_with_labels = df.with_columns(pl.Series('_cluster_label', labels))
-    entry['cluster_labels'] = labels
-    entry['cluster_map'] = cluster_map
-    entry['cluster_mode'] = 'subnet'
-    entry['filtered_df'] = df_with_labels
-    entry['n_clusters'] = n_clusters
-    entry['n_noise'] = n_noise
-
-    return JsonResponse({
-        'n_clusters': n_clusters,
-        'n_noise': n_noise,
-        'n_valid_points': len(df),
-        'clusters': cluster_map,
-        'labels': labels,
-        'data_points': data_points_by_cluster,
-    })
-
-
-# ── Step 4: Cluster detail ───────────────────────────────────────────────────
-
-@require_GET
-def cluster_detail(request, label: int):
-    """Return detailed profile for a specific cluster."""
-    session_id = request.GET.get('session_id')
-    if not session_id:
-        return JsonResponse({'error': '缺少 session_id'}, status=400)
-
-    entry = _cached(session_id)
-    if entry is None:
-        return JsonResponse({'error': '会话已过期'}, status=400)
-
-    df = entry.get('filtered_df')
-    meta = entry.get('meta', {})
-    cluster_map = entry.get('cluster_map', {})
-
-    if df is None:
-        return JsonResponse({'error': '数据未就绪'}, status=400)
-
-    if '_cluster_label' not in df.columns:
-        return JsonResponse({'error': '未执行聚类'}, status=400)
-
-    cluster_df = df.filter(pl.col('_cluster_label') == label)
-    rows = len(cluster_df)
-    if rows == 0:
-        return JsonResponse({'error': f'簇 {label} 无数据'}, status=404)
-
-    cluster_info = cluster_map.get(label, {})
-
-    # Helper: get unique values for a column
-    def _top_values(col_name: str, top_n: int = 5) -> list:
-        if col_name not in df.columns:
-            return []
-        try:
-            vals = (
-                cluster_df[col_name]
-                .cast(pl.Utf8)
-                .fill_null('(空)')
-                .value_counts()
-                .sort('count', descending=True)
-                .head(top_n)
-            )
-            total = vals['count'].sum()
-            return [
-                {'value': v['counts'] if isinstance(v, dict) else str(row[0]),
-                 'count': int(row[1]),
-                 'pct': round(int(row[1]) / total * 100, 1)}
-                for row in vals.iter_rows()
-            ]
-        except Exception:
-            return []
-
-    # Source IPs
-    ips_col = meta.get('ips_col')
-    unique_ips = []
-    if ips_col and ips_col in cluster_df.columns:
-        try:
-            ip_vals = cluster_df[ips_col].cast(pl.Utf8).unique().to_list()
-            unique_ips = sorted(str(v) for v in ip_vals if v not in (None, '', 'None'))
-        except Exception:
-            pass
-
-    # Detect useful columns for profiling
-    col_names = df.columns
-    dst_ip_col = _find_col(col_names, ':ipd', 'dst_ip', 'dest_ip', 'destination_ip')
-    dst_port_col = _find_col(col_names, 'dst_port', 'dest_port', 'port', 'dport')
-    tls_ver_col = _find_col(col_names, '0ver', 'tls_version', 'version', 'tlsver')
-    cipher_col = _find_col(col_names, '0cph', 'cipher_suite', 'cipher', 'tls_cipher')
-    sni_col = _find_col(col_names, 'sni', 'server_name', 'tls_sni')
-    bytes_col = _find_col(col_names, '8byt', 'bytes_sent', 'bytes', 'src_bytes')
-    dur_col = _find_col(col_names, '4dur', 'duration', 'dur', 'flow_duration')
-    proto_col = _find_col(col_names, 'proto', 'protocol', 'l4_proto')
-
-    profile = {
-        'label': label,
-        'size': rows,
-        'cluster_info': cluster_info,
-        'unique_ips': unique_ips[:100],
-        'unique_ips_count': len(unique_ips),
-        'top_dst_ips': _top_values(dst_ip_col) if dst_ip_col else [],
-        'top_dst_ports': _top_values(dst_port_col) if dst_port_col else [],
-        'top_tls_versions': _top_values(tls_ver_col) if tls_ver_col else [],
-        'top_ciphers': _top_values(cipher_col) if cipher_col else [],
-        'top_sni': _top_values(sni_col) if sni_col else [],
-        'top_protocols': _top_values(proto_col) if proto_col else [],
-        # Traffic stats
-        'traffic': {},
-    }
-
-    # Traffic stats
-    if bytes_col and bytes_col in cluster_df.columns:
-        try:
-            total_bytes = cluster_df[bytes_col].cast(pl.Float64).sum()
-            profile['traffic']['total_bytes'] = round(total_bytes, 2) if total_bytes else 0
-        except Exception:
-            pass
-    if dur_col and dur_col in cluster_df.columns:
-        try:
-            total_dur = cluster_df[dur_col].cast(pl.Float64).sum()
-            avg_dur = cluster_df[dur_col].cast(pl.Float64).mean()
-            profile['traffic']['total_duration'] = round(total_dur, 2) if total_dur else 0
-            profile['traffic']['avg_duration'] = round(avg_dur, 2) if avg_dur else 0
-        except Exception:
-            pass
-
-    # Geo bounding box (if lat/lon available)
-    src_lat_col = meta.get('src_lat_col')
-    src_lon_col = meta.get('src_lon_col')
-    if src_lat_col and src_lon_col and src_lat_col in cluster_df.columns:
-        try:
-            lats = cluster_df[src_lat_col].cast(pl.Float64).drop_nulls()
-            lons = cluster_df[src_lon_col].cast(pl.Float64).drop_nulls()
-            if len(lats) > 0 and len(lons) > 0:
-                profile['geo_bounds'] = {
-                    'lat_min': round(float(lats.min()), 4),
-                    'lat_max': round(float(lats.max()), 4),
-                    'lon_min': round(float(lons.min()), 4),
-                    'lon_max': round(float(lons.max()), 4),
-                }
-        except Exception:
-            pass
-
-    return JsonResponse(profile)
-
-
-# ── Main page ────────────────────────────────────────────────────────────────
-
-def index(request):
-    """Render the simple analysis workbench page."""
-    return render(request, 'simple_analysis/simple_analysis.html')

templates/analysis/cluster_detail.html

@@ -1,62 +1,141 @@
-{% extends 'base.html' %}
+{% extends 'base.html' %}
-{% block title %}Cluster #{{ cluster.cluster_label }} - Run #{{ run.display_id }}{% endblock %}
+{% load static %}
+{% block title %}簇 #{{ cluster.cluster_label }} — 运行 #{{ run.display_id }}{% endblock %}
 {% block content %}
+<style>
+.feat-table { width:100%; border-collapse:collapse; font-size:0.8rem; }
+.feat-table th { background:#f5f5f5; padding:0.3rem 0.5rem; text-align:left; font-weight:600; border-bottom:2px solid #ddd; white-space:nowrap; }
+.feat-table td { padding:0.25rem 0.5rem; border-bottom:1px solid #eee; }
+.feat-table td code { font-size:0.72rem; }
+.feat-positive { color:#2e7d32; }
+.feat-negative { color:#c62828; }
+</style>
+
 <div class="card">
-    <a href="{% url 'analysis:cluster_overview' run.display_id %}" style="color:#4361ee;text-decoration:none;font-size:0.9rem;">← Back to overview</a>
+    <a href="{% url 'analysis:cluster_overview' run.display_id %}" style="color:#4361ee;text-decoration:none;font-size:0.9rem;">← 聚类概览</a>
-    <h2 style="margin-top:0.5rem;">Cluster #{{ cluster.cluster_label }}</h2>
+    <div style="display:flex;justify-content:space-between;align-items:center;flex-wrap:wrap;gap:0.5rem;margin-top:0.5rem;">
-    <p>{{ cluster.size }} entities ({{ cluster.proportion|floatformat:2 }}% of total)</p>
+        <div>
-    <p>Silhouette Score: {{ cluster.silhouette_score|floatformat:4|default:"-" }}</p>
+            <h2 style="margin:0;">簇 #{{ cluster.cluster_label }}
+                {% if cluster.cluster_label == -1 %}<span class="badge badge-warning">噪声</span>{% endif %}
+            </h2>
+            <p style="margin:0.25rem 0 0;color:#666;font-size:0.9rem;">
+                {{ cluster.size }} 个行
+                {% if cluster.proportion %} — {{ cluster.proportion|floatformat:1 }}% 占总{% endif %}
+                {% if cluster.silhouette_score is not None %} — 轮廓系数: {{ cluster.silhouette_score|floatformat:4 }}{% endif %}
+            </p>
+        </div>
+    </div>
+    {% if nl_summary %}
+    <div style="background:#f8f9fa;border-left:3px solid #4361ee;padding:0.6rem;margin:0.75rem 0;border-radius:0 4px 4px 0;font-size:0.85rem;line-height:1.6;color:#333;">{{ nl_summary }}</div>
+    {% endif %}
 </div>
 
-<div class="grid-2">
+<!-- ── Feature Analysis ── -->
-    <div class="card">
+<div class="card">
-        <h2>Feature Summary (top 50)</h2>
+    <h3>📊 特征分析 <span style="font-weight:400;font-size:0.8rem;color:#888;">{{ features|length }} 个特征</span></h3>
-        <table>
+    {% if features %}
-            <thead>
+    <table class="feat-table">
-                <tr>
+        <thead>
-                    <th>Feature</th>
+            <tr>
-                    <th>Mean</th>
+                <th>#</th>
-                    <th>Std</th>
+                <th>特征名</th>
-                    <th>Median</th>
+                <th>区分度</th>
-                    <th>Dist. Score</th>
+                <th>均值</th>
-                </tr>
+                <th>标准差</th>
-            </thead>
+                <th>中位数</th>
-            <tbody>
+                <th>P25</th>
-                {% for f in features %}
+                <th>P75</th>
-                <tr>
+            </tr>
-                    <td><code>{{ f.feature_name }}</code></td>
+        </thead>
-                    <td>{{ f.mean|floatformat:3|default:"-" }}</td>
+        <tbody>
-                    <td>{{ f.std|floatformat:3|default:"-" }}</td>
+            {% for f in features %}
-                    <td>{{ f.median|floatformat:3|default:"-" }}</td>
+            <tr>
-                    <td>{{ f.distinguishing_score|floatformat:3|default:"-" }}</td>
+                <td>{{ forloop.counter }}</td>
-                </tr>
+                <td><code>{{ f.feature_name }}</code></td>
-                {% empty %}
+                <td><span class="{% if f.distinguishing_score > 0 %}feat-positive{% else %}feat-negative{% endif %}">{{ f.distinguishing_score|floatformat:3 }}</span></td>
-                <tr><td colspan="5">No features</td></tr>
+                <td class="ts-val" data-col="{{ f.feature_name }}">{{ f.mean|floatformat:3|default:"-" }}</td>
-                {% endfor %}
+                <td>{{ f.std|floatformat:3|default:"-" }}</td>
-            </tbody>
+                <td class="ts-val" data-col="{{ f.feature_name }}">{{ f.median|floatformat:3|default:"-" }}</td>
-        </table>
+                <td class="ts-val" data-col="{{ f.feature_name }}">{{ f.p25|floatformat:3|default:"-" }}</td>
-    </div>
+                <td class="ts-val" data-col="{{ f.feature_name }}">{{ f.p75|floatformat:3|default:"-" }}</td>
+            </tr>
+            {% endfor %}
+        </tbody>
+    </table>
+    {% else %}
+    <div class="empty-state"><p>该簇无特征数据</p></div>
+    {% endif %}
+</div>
 
-    <div class="card">
+<!-- ── SVD Singular Values ── -->
-        <h2>Entities (top 50)</h2>
+{% if cluster.cluster_label != -1 %}
-        <table>
+{# SVD-extracted features (from _run_clustering_pipeline cluster_svd_extract) have higher distinguishing scores #}
-            <thead>
+{% with svd_features=features|slice:":5" %}
-                <tr>
+{% if svd_features %}
-                    <th>Entity</th>
+<div class="card">
-                    <th></th>
+    <h3>🔬 SVD 关键特征 <span style="font-weight:400;font-size:0.8rem;color:#888;">Top 5 按区分度</span></h3>
-                </tr>
+    <div style="display:grid;grid-template-columns:1fr 1fr;gap:0.5rem;">
-            </thead>
+    {% for f in svd_features %}
-            <tbody>
+        <div style="background:#f8f9fa;border-radius:6px;padding:0.5rem;border:1px solid #e0e0e0;">
-                {% for e in entities %}
+            <div style="font-weight:600;font-size:0.8rem;">{{ f.feature_name }}</div>
-                <tr>
+            <div style="font-size:0.75rem;color:#666;margin-top:0.2rem;">
-                    <td><code>{{ e.entity_value }}</code></td>
+                Score: <span class="{% if f.distinguishing_score > 0 %}feat-positive{% else %}feat-negative{% endif %}">{{ f.distinguishing_score|floatformat:2 }}</span>
-                    <td><a href="{% url 'analysis:entity_profile' e.id %}" class="btn btn-primary">Profile</a></td>
+                | 均值: {{ f.mean|floatformat:2 }}
-                </tr>
+                | 标准差: {{ f.std|floatformat:2 }}
-                {% empty %}
+            </div>
-                <tr><td colspan="2">No entities</td></tr>
+        </div>
-                {% endfor %}
+    {% endfor %}
-            </tbody>
-        </table>
     </div>
 </div>
+{% endif %}
+{% endwith %}
+{% endif %}
+
+<!-- ── Raw Data Rows ── -->
+<div class="card">
+    <h3>📋 数据行 <span style="font-weight:400;font-size:0.8rem;color:#888;">{{ row_data|length }} 条</span></h3>
+    {% if row_data %}
+    <div style="overflow-x:auto;">
+    <table class="feat-table" style="font-size:0.75rem;">
+        <thead>
+            <tr>
+                <th>#</th>
+                {% for k, v in row_data.0.items %}
+                <th>{{ k }}</th>
+                {% endfor %}
+            </tr>
+        </thead>
+        <tbody>
+            {% for row in row_data %}
+            <tr>
+                <td>{{ forloop.counter }}</td>
+                {% for k, v in row.items %}
+                <td><code>{{ v|default:"-" }}</code></td>
+                {% endfor %}
+            </tr>
+            {% endfor %}
+        </tbody>
+    </table>
+    </div>
+    {% else %}
+    <div class="empty-state"><p>该簇无数据行</p></div>
+    {% endif %}
+</div>
 {% endblock %}
+
+<script>
+(function(){
+var tsCols=["timestamp","time","epoch","ts","8dbd","created_at","updated_at","event_time","datetime"];
+document.querySelectorAll(".ts-val").forEach(function(el){
+var col=el.dataset.col,isTs=false;
+for(var i=0;i<tsCols.length;i++){if(col.toLowerCase().indexOf(tsCols[i])>=0){isTs=true;break;}}
+if(!isTs)return;
+var raw=parseFloat(el.textContent);
+if(isNaN(raw)||raw<1e8||raw>2e12)return;
+var d=new Date(raw*1000);
+if(d.getFullYear()<2000||d.getFullYear()>2100)return;
+el.textContent=d.toISOString().replace("T"," ").substring(0,16);
+el.title="epoch: "+raw;
+});
+})();
+</script>

templates/analysis/cluster_overview.html

@@ -1,334 +1,413 @@
-{% extends 'base.html' %}
+{% extends 'base.html' %}
 {% load static %}
-{% block title %}Cluster Overview - Run #{{ run.display_id }}{% endblock %}
+{% block title %}聚类概览 — 运行 #{{ run.display_id }}{% endblock %}
 {% block content %}
 <style>
-    .feature-chart { width: 100%; display: block; }
+/* ── 布局 ── */
-    #featureChartsContainer canvas { max-height: 400px; }
+.main-layout { display:flex; gap:0; transition:all 0.3s; position:relative; }
-    .chart-wrapper { overflow-x: auto; }
+.left-panel { flex:1; min-width:0; transition:flex 0.3s; }
-    .chart-container { position: relative; }
+.left-panel.globe-open { flex:0 0 55%; }
-    .canvas-legend {
+.right-panel { flex:0 0 0; overflow:hidden; transition:flex 0.3s; border-left:1px solid #e0e0e0; }
-        max-height: 110px; overflow-y: auto;
+.right-panel.globe-open { flex:0 0 45%; }
-        display: flex; flex-wrap: wrap; gap: 4px 14px;
+.globe-toggle-handle { position:absolute; z-index:10; top:50%; left:calc(100% - 24px); transform:translateY(-50%); width:24px; height:48px; background:#1a1a2e; color:#fff; border:none; border-radius:6px 0 0 6px; cursor:pointer; display:flex; align-items:center; justify-content:center; font-size:14px; transition:left 0.3s, background 0.15s; }
-        padding: 6px 10px; font-size: 11px;
+.globe-toggle-handle:hover { background:#4361ee; }
-        border-top: 1px solid #eee; margin-top: 4px;
+.globe-toggle-handle.open { left:calc(55% - 24px); border-radius:6px 0 0 6px; }
-    }
+/* ── Scatter ── */
-    .canvas-legend .legend-item {
+.scatter-wrapper { position:relative; cursor:crosshair; }
-        display: inline-flex; align-items: center; gap: 4px;
+.scatter-wrapper canvas { width:100%; height:400px; display:block; }
-        white-space: nowrap;
+#scatter3D { width:100%; height:450px; }
-    }
+/* ── Detail Panel ── */
-    .canvas-legend .legend-swatch {
+.detail-overlay { position:fixed; bottom:0; left:0; right:0; z-index:100; transform:translateY(100%); transition:transform 0.35s cubic-bezier(.4,0,.2,1); }
-        width: 10px; height: 10px; display: inline-block;
+.detail-overlay.open { transform:translateY(0); }
-        border-radius: 2px; flex-shrink: 0;
+.detail-content { background:#fff; border-radius:16px 16px 0 0; box-shadow:0 -4px 24px rgba(0,0,0,0.15); max-height:45vh; overflow-y:auto; padding:1rem 2rem 2rem; }
-    }
+.detail-handle { width:40px; height:4px; background:#ccc; border-radius:2px; margin:0.5rem auto; cursor:pointer; }
+.detail-handle:hover { background:#999; }
+.detail-close { float:right; cursor:pointer; font-size:1.5rem; color:#999; line-height:1; }
+.detail-close:hover { color:#333; }
+/* ── Globe ── */
+#globeContainer { width:100%; height:calc(100vh - 120px); position:sticky; top:0; }
+/* ── Legend pills ── */
+.legend-pills { display:flex; flex-wrap:wrap; gap:4px 8px; padding:6px 0; }
+.pill { display:inline-flex; align-items:center; gap:4px; padding:2px 10px; border-radius:12px; font-size:0.75rem; cursor:pointer; border:2px solid transparent; transition:all 0.15s; white-space:nowrap; }
+.pill:hover { opacity:0.8; }
+.pill.active { border-color:#333; font-weight:600; }
+.cluster-card { border:1px solid #e0e0e0; border-radius:8px; padding:1rem; background:#fff; cursor:pointer; transition:box-shadow 0.15s; }
+.cluster-card:hover { box-shadow:0 2px 8px rgba(0,0,0,0.1); }
+.cluster-card.selected { border-color:#4361ee; box-shadow:0 0 0 2px #4361ee33; }
+.nl-summary { background:#f8f9fa; border-left:3px solid #4361ee; padding:0.6rem; margin:0.5rem 0; border-radius:0 4px 4px 0; font-size:0.85rem; line-height:1.6; color:#333; }
+.feat-positive { color:#2e7d32; }
+.feat-negative { color:#c62828; }
 </style>
-<div class="card">
+<script src="{% static 'tianxuan/three.min.js' %}"></script>
-    <h2>Cluster Overview — Run #{{ run.display_id }}</h2>
-    <p>{{ run.total_flows }} flows, {{ run.cluster_count }} clusters</p>
-</div>
 
 <div class="card">
-    <h2>UMAP-2D Embedding</h2>
+    <div style="display:flex;justify-content:space-between;align-items:center;flex-wrap:wrap;gap:0.5rem;">
-    <div class="chart-wrapper">
+        <div>
-        <div class="chart-container">
+            <h2 style="margin:0;">聚类概览</h2>
-            <canvas id="scatterChart" class="scatter-canvas"></canvas>
+            <p style="margin:0.25rem 0 0;color:#666;font-size:0.9rem;">
-            <div id="scatterLegend" class="canvas-legend"></div>
+                运行 #{{ run.display_id }} — {{ run.total_flows }} 条流, {{ run.cluster_count }} 个簇
+                {% if noise %}, {{ noise.size }} 个噪声点{% endif %}
+            </p>
+        </div>
+        <div style="display:flex;gap:0.5rem;">
+            <a href="{% url 'analysis:run_detail' run.display_id %}" class="btn btn-sm">← 运行详情</a>
         </div>
     </div>
 </div>
 
+<div class="main-layout">
+<div class="left-panel" id="leftPanel">
+
+<!-- ── UMAP 2D / 3D ── -->
+<div class="card">
+    <div style="display:flex;justify-content:space-between;align-items:center;flex-wrap:wrap;gap:0.5rem;">
+        <h3 style="margin:0;">UMAP 嵌入 <span style="font-weight:400;font-size:0.8rem;color:#888;" id="pointCount">计算中...</span></h3>
+        <span id="viewToggle" style="display:none;">
+            <button id="btn2D" class="btn btn-sm" style="font-weight:bold;padding:2px 10px;">2D</button>
+            <button id="btn3D" class="btn btn-sm btn-outline" style="padding:2px 10px;">3D</button>
+        </span>
+    </div>
+    <div class="scatter-wrapper" style="margin-top:0.5rem;">
+        <canvas id="scatterChart" style="width:100%;height:400px;"></canvas>
+        <div id="scatterLegend" class="legend-pills"></div>
+    </div>
+    <div id="scatter3DContainer" style="display:none;">
+        <div id="scatter3D"></div>
+        <div id="scatter3DLegend" class="legend-pills"></div>
+    </div>
+</div>
+
+<!-- ── Noise Analysis ── -->
+{% if noise %}
+<div class="card" style="background:#fff8e1;border-color:#ffe082;">
+    <div style="display:flex;justify-content:space-between;align-items:center;">
+        <h3>⚠️ 噪声点 <span style="font-weight:400;font-size:0.8rem;">{{ noise.size }} 个实体</span></h3>
+        <a href="{% url 'analysis:cluster_detail' run.display_id -1 %}" class="btn btn-sm">详情 →</a>
+    </div>
+    {% if noise_summary %}<div class="nl-summary" style="border-left-color:#e65100;background:#fff3e0;">{{ noise_summary }}</div>{% endif %}
+</div>
+{% endif %}
+
+<!-- ── Cluster Grid ── -->
+<div class="legend-pills" style="margin-bottom:0.5rem;">
+    <span class="pill" style="background:#e8e8e8;" onclick="clearClusterFilter()">全部</span>
+    {% for c in clusters %}
+    <span class="pill" id="pill-{{ c.cluster_label }}" data-label="{{ c.cluster_label }}" onclick="selectCluster({{ c.cluster_label }})">#{{ c.cluster_label }} ({{ c.size }})</span>
+    {% endfor %}
+    {% if noise %}<span class="pill" style="background:#999;color:#fff;" onclick="selectCluster(-1)">Noise ({{ noise.size }})</span>{% endif %}
+</div>
+
+<div style="display:grid;grid-template-columns:1fr 1fr;gap:0.75rem;">
+{% for c in clusters %}
+<div class="cluster-card" data-label="{{ c.cluster_label }}" onclick="selectCluster({{ c.cluster_label }})">
+    <div style="font-weight:700;font-size:1rem;">簇 #{{ c.cluster_label }}</div>
+    <div style="font-size:0.8rem;color:#666;margin-top:0.2rem;">{{ c.size }} 个实体{% if c.proportion %} · {{ c.proportion|floatformat:1 }}%{% endif %}</div>
+    {% if c.nl_summary %}<div class="nl-summary">{{ c.nl_summary }}</div>{% endif %}
+</div>
+{% empty %}
+<div class="card"><div class="empty-state"><p>No clusters</p></div></div>
+{% endfor %}
+</div>
+
 {% if geo_count > 0 %}
-<div class="card">
+<div class="card" style="margin-top:1rem;">
-    <h2>地理位置分布 <span class="badge badge-info">{{ geo_count }} entities</span></h2>
+    <h3>🌍 地理分布</h3>
-    {% if geo_skipped > 0 %}
+    <div class="scatter-wrapper"><canvas id="geoChart" style="width:100%;height:300px;"></canvas><div id="geoLegend" class="legend-pills"></div></div>
-    <p style="color:#888;font-size:0.85rem;">{{ geo_skipped }} entities excluded (missing lat/lon)</p>
-    {% endif %}
-    <div class="chart-wrapper">
-        <div class="chart-container">
-            <canvas id="geoChart" class="scatter-canvas"></canvas>
-            <div id="geoLegend" class="canvas-legend"></div>
-        </div>
-    </div>
 </div>
 {% endif %}
 
-<div class="card" id="clusterSizesCard">
+</div><!-- /left-panel -->
-    <h2>Cluster Size Distribution</h2>
+
-    <div class="chart-wrapper">
+<!-- ── Right Panel: Globe ── -->
-        <canvas id="clusterSizesChart" class="scatter-canvas"></canvas>
+<div class="right-panel" id="rightPanel">
+    <iframe id="globeIframe" src="/globe/?embed=1&data={{ selected_ds_id|default:'' }}" style="width:100%;height:100%;border:none;"></iframe>
+</div>
+<button class="globe-toggle-handle" id="globeHandle" onclick="toggleGlobe()">◀</button>
+</div><!-- /main-layout -->
+
+<!-- ── Bottom Detail Panel ── -->
+<div class="detail-overlay" id="detailPanel">
+    <div class="detail-content">
+        <div class="detail-handle" onclick="closeDetail()"></div>
+        <span class="detail-close" onclick="closeDetail()">✕</span>
+        <div id="detailBody"></div>
     </div>
 </div>
 
-<div class="card" id="silhouetteCard">
-    <h2>Silhouette Score Comparison</h2>
-    <div class="chart-wrapper">
-        <canvas id="silhouetteChart" class="scatter-canvas"></canvas>
-    </div>
-</div>
-
-<div class="card" id="topFeaturesCard">
-    <h2>Top Distinguishing Features per Cluster</h2>
-    <div id="featureChartsContainer"></div>
-</div>
-
-<div class="grid-2">
-    {% for c in clusters %}
-    <div class="card">
-        <h3>Cluster #{{ c.cluster_label }} <span class="badge badge-info">{{ c.size }} entities</span></h3>
-        <p>Proportion: {{ c.proportion|floatformat:3 }}</p>
-        <p>Silhouette: {{ c.silhouette_score|floatformat:4|default:"-" }}</p>
-        <h4 style="margin-top:0.75rem;font-size:0.9rem;">Top Distinguishing Features</h4>
-        <table>
-            <thead>
-                <tr><th>Feature</th><th>Score</th><th>Mean</th></tr>
-            </thead>
-            <tbody>
-                {% for f in c.features.all|slice:":5" %}
-                <tr>
-                    <td>{{ f.feature_name }}</td>
-                    <td>{{ f.distinguishing_score|floatformat:3|default:"-" }}</td>
-                    <td>{{ f.mean|floatformat:3|default:"-" }}</td>
-                </tr>
-                {% empty %}
-                <tr><td colspan="3">No features</td></tr>
-                {% endfor %}
-            </tbody>
-        </table>
-        <a href="{% url 'analysis:cluster_detail' run.display_id c.cluster_label %}" class="btn btn-primary" style="margin-top:0.75rem;">Detail</a>
-    </div>
-    {% empty %}
-    <div class="card">
-        <div class="empty-state"><p>No clusters found.</p></div>
-    </div>
-    {% endfor %}
-</div>
-
-{% if noise %}
-<div class="card">
-    <h3>Noise (Cluster -1)</h3>
-    <p>{{ noise.size }} entities classified as noise</p>
-</div>
-{% endif %}
-
 <script>
-// Pure Canvas scatter plot — no Chart.js needed (offline compatible)
 const scatterData = {{ scatter_data_json|safe }};
-const colors = ['#4361ee','#f72585','#7209b7','#4cc9f0','#f8961e','#43aa8b','#f94144','#577590','#e9c46a','#9b5de5'];
+const scatterCount = scatterData.length;
+document.getElementById('pointCount').textContent = scatterCount + ' 个点';
+const hasZ = {{ has_z|yesno:"true,false" }};
+const colors = ['#4361ee','#f72585','#7209b7','#4cc9f0','#f8961e','#43aa8b','#f94144','#577590','#e9c46a','#9b5de5',
+    '#1f77b4','#ff7f0e','#2ca02c','#d62728','#9467bd','#8c564b','#e377c2','#7f7f7f','#bcbd22','#17becf',
+    '#aec7e8','#ffbb78','#98df8a','#ff9896','#c5b0d5','#c49c94','#f7b6d2','#c7c7c7','#dbdb8d','#9edae5',
+    '#393b79','#637939','#8c6d31','#843c39','#7b4173','#5254a3','#8ca252','#bd9e39','#ad494a','#a55194'];
 
-function drawScatter(canvasId, data, xLabel, yLabel) {
+let selectedCluster = null, selectedPoint = null;
+let globeOpen = false;
+const clusterSizes = {{ cluster_sizes_json|safe }};
+
+// Color pills after DOM ready
+(function() {
+    clusterSizes.forEach(function(c) {
+        var el = document.getElementById('pill-' + c.label);
+        if (el) { el.style.background = colorFor(c.label); el.style.color = '#fff'; }
+    });
+    var noisePill = document.querySelector('.pill[data-label="-1"]');
+    if (noisePill) { noisePill.style.background = '#999'; noisePill.style.color = '#fff'; }
+})();
+
+function colorFor(label) { return label === -1 ? 'rgba(150,150,150,0.8)' : colors[Math.abs(label) % colors.length]; }
+
+// ══════════════════════════════════════════════════════════
+//  SCATTER — 2D Canvas with click interaction
+// ══════════════════════════════════════════════════════════
+
+function drawScatter(canvasId, data, xLabel, yLabel, highlightLabel) {
     const canvas = document.getElementById(canvasId);
     if (!canvas || data.length === 0) return;
-    const ctx = canvas.getContext('2d');
+    const rect = canvas.parentElement.getBoundingClientRect();
-    const W = canvas.parentElement.clientWidth || 600;
+    const W = rect.width || 600, H = 400;
-    const H = 400;
     canvas.width = W; canvas.height = H;
-    const pad = { top: 30, bottom: 50, left: 60, right: 30 };
+    const ctx = canvas.getContext('2d');
-    const plotW = W - pad.left - pad.right;
+    const pad = { top: 30, bottom: 45, left: 60, right: 30 };
-    const plotH = H - pad.top - pad.bottom;
+    const pw = W - pad.left - pad.right, ph = H - pad.top - pad.bottom;
-
+    const xs = data.map(d => d.x), ys = data.map(d => d.y);
-    // Compute bounds
-    const xs = data.map(d => d.x); const ys = data.map(d => d.y);
     const xMin = Math.min(...xs), xMax = Math.max(...xs);
     const yMin = Math.min(...ys), yMax = Math.max(...ys);
-    const xRange = xMax - xMin || 1; const yRange = yMax - yMin || 1;
+    const xR = xMax - xMin || 1, yR = yMax - yMin || 1;
+    const toX = v => pad.left + (v - xMin) / xR * pw;
+    const toY = v => pad.top + ph - (v - yMin) / yR * ph;
 
-    const toX = v => pad.left + (v - xMin) / xRange * plotW;
-    const toY = v => pad.top + plotH - (v - yMin) / yRange * plotH;
-
-    // Clear
     ctx.clearRect(0, 0, W, H);
+    ctx.strokeStyle = '#f0f0f0'; ctx.lineWidth = 1;
+    for (let i = 0; i <= 5; i++) { const y = pad.top + i * ph / 5; ctx.beginPath(); ctx.moveTo(pad.left, y); ctx.lineTo(W - pad.right, y); ctx.stroke(); }
+    ctx.fillStyle = '#999'; ctx.font = '12px sans-serif'; ctx.textAlign = 'center';
+    ctx.fillText(xLabel, W / 2, H - 3);
+    ctx.save(); ctx.translate(16, H / 2); ctx.rotate(-Math.PI / 2); ctx.fillText(yLabel, 0, 0); ctx.restore();
 
-    // Grid lines
-    ctx.strokeStyle = '#eee'; ctx.lineWidth = 1;
-    for (let i = 0; i <= 5; i++) {
-        const y = pad.top + i * plotH / 5;
-        ctx.beginPath(); ctx.moveTo(pad.left, y); ctx.lineTo(W - pad.right, y); ctx.stroke();
-    }
-
-    // Axes
-    ctx.strokeStyle = '#333'; ctx.lineWidth = 2;
-    ctx.beginPath(); ctx.moveTo(pad.left, pad.top); ctx.lineTo(pad.left, H - pad.bottom); ctx.stroke();
-    ctx.beginPath(); ctx.moveTo(pad.left, H - pad.bottom); ctx.lineTo(W - pad.right, H - pad.bottom); ctx.stroke();
-
-    // Labels
-    ctx.fillStyle = '#666'; ctx.font = '13px sans-serif'; ctx.textAlign = 'center';
-    ctx.fillText(xLabel, W / 2, H - 5);
-    ctx.save(); ctx.translate(15, H / 2); ctx.rotate(-Math.PI / 2); ctx.fillText(yLabel, 0, 0); ctx.restore();
-
-    // Points
     data.forEach(d => {
         const cx = toX(d.x), cy = toY(d.y);
-        const color = d.label === -1 ? 'rgba(150,150,150,0.5)' : colors[Math.abs(d.label) % colors.length];
+        let color = d.label === -1 ? 'rgba(150,150,150,0.4)' : colors[Math.abs(d.label) % colors.length];
-        const r = d.label === -1 ? 2 : 4;
+        let radius = d.label === -1 ? 2 : 4;
-        ctx.beginPath(); ctx.arc(cx, cy, r, 0, Math.PI * 2);
+        if (highlightLabel !== null && highlightLabel !== undefined) {
+            if (d.label === highlightLabel) { color = d.label === -1 ? 'rgba(150,150,150,0.9)' : colors[Math.abs(d.label) % colors.length]; radius = 6; }
+            else { color = 'rgba(200,200,200,0.2)'; radius = 2; }
+        }
+        ctx.beginPath(); ctx.arc(cx, cy, radius, 0, Math.PI * 2);
         ctx.fillStyle = color; ctx.fill();
     });
 
-    // Tooltip on hover
+    // Click handler
-    canvas.onmousemove = function(e) {
+    canvas.onclick = function(e) {
-        const rect = canvas.getBoundingClientRect();
+        const rr = canvas.getBoundingClientRect();
-        const mx = e.clientX - rect.left, my = e.clientY - rect.top;
+        const mx = e.clientX - rr.left, my = e.clientY - rr.top;
-        const hit = data.find(d => Math.abs(toX(d.x) - mx) < 6 && Math.abs(toY(d.y) - my) < 6);
+        // Find closest point within 8px
-        if (hit) {
+        let best = null, bestDist = 8;
-            canvas.title = `${hit.entity} (${hit.label === -1 ? 'Noise' : 'Cluster ' + hit.label})`;
+        for (let i = 0; i < data.length; i++) {
+            const d = data[i];
+            const dx = toX(d.x) - mx, dy = toY(d.y) - my;
+            const dist = Math.sqrt(dx * dx + dy * dy);
+            if (dist < bestDist) { bestDist = dist; best = d; }
+        }
+        if (best) {
+            if (best.label === -1) selectCluster(-1);
+            else selectCluster(best.label);
+            showEntityDetail(best);
         }
     };
 
-    // Legend — HTML overlay (supports overflow-y: auto via .canvas-legend)
     const legendEl = document.getElementById(canvasId.replace('Chart', 'Legend'));
     if (legendEl) {
         legendEl.innerHTML = '';
-        const uniqueLabels = [...new Set(data.map(d => d.label))];
+        var seen = {};
-        uniqueLabels.forEach(label => {
+        data.forEach(d => { if (!seen[d.label]) { seen[d.label] = true; legendEl.innerHTML += '<span class="pill" style="background:' + colorFor(d.label) + ';color:#fff;cursor:pointer;" onclick="selectCluster(' + d.label + ')">' + (d.label === -1 ? 'Noise' : '#' + d.label) + '</span>'; } });
-            const color = label === -1 ? 'rgba(150,150,150,0.8)' : colors[Math.abs(label) % colors.length];
-            const item = document.createElement('span');
-            item.className = 'legend-item';
-            item.innerHTML = '<span class="legend-swatch" style="background:' + color + '"></span>' + (label === -1 ? 'Noise' : 'Cluster ' + label);
-            legendEl.appendChild(item);
-        });
     }
 }
 
-drawScatter('scatterChart', scatterData, 'UMAP-1', 'UMAP-2');
+function showEntityDetail(point) {
+    const body = document.getElementById('detailBody');
+    body.innerHTML = '<h3>实体: ' + point.entity + '</h3>' +
+        '<p>所属簇: ' + (point.label === -1 ? '噪声' : '#' + point.label) + '</p>' +
+        '<p>UMAP 坐标: (' + point.x.toFixed(4) + ', ' + point.y.toFixed(4) + ')</p>' +
+        '<a href="?entity=' + encodeURIComponent(point.entity) + '" class="btn btn-sm btn-primary">查看画像</a>';
+    document.getElementById('detailPanel').classList.add('open');
+}
+
+// ══════════════════════════════════════════════════════════
+//  CLUSTER SELECTION
+// ══════════════════════════════════════════════════════════
+
+function selectCluster(label) {
+    selectedCluster = label;
+    selectedPoint = null;
+    // Highlight cards
+    document.querySelectorAll('.cluster-card').forEach(el => el.classList.toggle('selected', parseInt(el.dataset.label) === label));
+    // Highlight scatter
+    drawScatter('scatterChart', scatterData, 'UMAP-1', 'UMAP-2', label);
+    // Focus legend pills
+    document.querySelectorAll('.pill').forEach(el => el.classList.toggle('active', parseInt(el.dataset.label) === label));
+    // Show cluster detail in bottom panel
+    const clusterInfo = clusterSizes.find(c => c.label === label);
+    if (clusterInfo) {
+        const body = document.getElementById('detailBody');
+        body.innerHTML = '<h3>簇 #' + label + '</h3>' +
+            '<p>大小: ' + clusterInfo.size + ' | 轮廓系数: ' + (clusterInfo.silhouette != null ? clusterInfo.silhouette.toFixed(4) : '-') + '</p>' +
+            '<a href="/clusters/{{ run.display_id }}/' + label + '/" class="btn btn-sm btn-primary">详细分析 →</a>';
+        document.getElementById('detailPanel').classList.add('open');
+    }
+    // Gray out globe if open
+    if (globeOpen) {
+        var ifr = document.getElementById('globeIframe');
+        if (ifr && ifr.contentWindow) ifr.contentWindow.postMessage({ type: 'globe_filter', cluster_label: label }, '*');
+    }
+}
+
+function clearClusterFilter() {
+    selectedCluster = null;
+    drawScatter('scatterChart', scatterData, 'UMAP-1', 'UMAP-2', null);
+    document.querySelectorAll('.cluster-card').forEach(el => el.classList.remove('selected'));
+    document.querySelectorAll('.pill').forEach(el => el.classList.remove('active'));
+    document.getElementById('detailPanel').classList.remove('open');
+    if (globeOpen) {
+        var ifr = document.getElementById('globeIframe');
+        if (ifr && ifr.contentWindow) ifr.contentWindow.postMessage({ type: 'globe_filter', cluster_label: null }, '*');
+    }
+}
+
+function closeDetail() { document.getElementById('detailPanel').classList.remove('open'); }
+
+function toggleGlobe() {
+    globeOpen = !globeOpen;
+    document.getElementById('leftPanel').classList.toggle('globe-open', globeOpen);
+    document.getElementById('rightPanel').classList.toggle('globe-open', globeOpen);
+    var handle = document.getElementById('globeHandle');
+    handle.innerHTML = globeOpen ? '▶' : '◀';
+    handle.classList.toggle('open', globeOpen);
+}
+
+// ══════════════════════════════════════════════════════════
+//  3D Three.js SCATTER (for 3D UMAP toggle)
+// ══════════════════════════════════════════════════════════
+
+var scatter3DRenderer = null, scatter3DScene = null, scatter3DCamera = null, scatter3DPoints = null;
+var scatter3DInitialized = false, is3DMode = false, scatterData3D = scatterData;
+
+function hasWebGL2() { try { var c = document.createElement('canvas'); return !!c.getContext('webgl2'); } catch(e) { return false; } }
+
+(function() {
+    var supports3D = hasWebGL2();
+    if (supports3D && hasZ) document.getElementById('viewToggle').style.display = 'inline';
+
+    var btn2D = document.getElementById('btn2D'), btn3D = document.getElementById('btn3D');
+    var c2d = document.querySelector('#scatterChart').parentElement;
+    var c3d = document.getElementById('scatter3DContainer');
+
+    function show2D() {
+        is3DMode = false; c2d.style.display = ''; c3d.style.display = 'none';
+        btn2D.style.fontWeight = 'bold'; btn2D.className = 'btn btn-sm';
+        btn3D.style.fontWeight = 'normal'; btn3D.className = 'btn btn-sm btn-outline';
+        drawScatter('scatterChart', scatterData, 'UMAP-1', 'UMAP-2', selectedCluster);
+    }
+    function show3D() {
+        is3DMode = true; c2d.style.display = 'none'; c3d.style.display = '';
+        btn3D.style.fontWeight = 'bold'; btn3D.className = 'btn btn-sm';
+        btn2D.style.fontWeight = 'normal'; btn2D.className = 'btn btn-sm btn-outline';
+        if (!scatter3DInitialized) initScatter3D();
+    }
+    btn2D.addEventListener('click', show2D);
+    btn3D.addEventListener('click', show3D);
+    window.addEventListener('resize', function() {
+        if (!scatter3DRenderer || !is3DMode) return;
+        var cont = document.getElementById('scatter3D');
+        if (!cont) return;
+        var W = cont.clientWidth || 600;
+        scatter3DRenderer.setSize(W, 450);
+        scatter3DCamera.aspect = W / 450;
+        scatter3DCamera.updateProjectionMatrix();
+    });
+})();
+
+function initScatter3D() {
+    if (scatter3DInitialized) return;
+    var container = document.getElementById('scatter3D');
+    if (!container || scatterData.length === 0) return;
+    var W = container.clientWidth || 600, H = 450;
+    var xs = scatterData.map(d => d.x), ys = scatterData.map(d => d.y), zs = scatterData.map(d => d.z || 0);
+    var xMin = Math.min(...xs), xMax = Math.max(...xs), yMin = Math.min(...ys), yMax = Math.max(...ys), zMin = Math.min(...zs), zMax = Math.max(...zs);
+    var cx = (xMin + xMax) / 2, cy = (yMin + yMax) / 2, cz = (zMin + zMax) / 2;
+    var range = Math.max(xMax - xMin, yMax - yMin, zMax - zMin) || 1;
+    function norm(v, c, r) { return ((v - c) / r) * 3; }
+
+    var scene = new THREE.Scene(); scene.background = new THREE.Color(0xf8f9fa);
+    var camera = new THREE.PerspectiveCamera(50, W / H, 0.1, 50); camera.position.set(4.5, 3.5, 6); camera.lookAt(0, 0, 0);
+    var renderer = new THREE.WebGLRenderer({ antialias: true }); renderer.setSize(W, H); renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
+    container.appendChild(renderer.domElement);
+    scatter3DRenderer = renderer; scatter3DScene = scene; scatter3DCamera = camera;
+
+    var positions = new Float32Array(scatterData.length * 3);
+    var pointColors = new Float32Array(scatterData.length * 3);
+    for (var i = 0; i < scatterData.length; i++) {
+        var d = scatterData[i];
+        positions[i*3] = norm(d.x, cx, range); positions[i*3+1] = norm(d.y, cy, range); positions[i*3+2] = d.z != null ? norm(d.z, cz, range) : 0;
+        var c3 = new THREE.Color(d.label === -1 ? '#999999' : colors[Math.abs(d.label) % colors.length]);
+        pointColors[i*3] = c3.r; pointColors[i*3+1] = c3.g; pointColors[i*3+2] = c3.b;
+    }
+    var geo = new THREE.BufferGeometry();
+    geo.setAttribute('position', new THREE.BufferAttribute(positions, 3));
+    geo.setAttribute('color', new THREE.BufferAttribute(pointColors, 3));
+    var mat = new THREE.PointsMaterial({ size: 0.15, vertexColors: true, sizeAttenuation: true, depthWrite: true, transparent: true, opacity: 0.85 });
+    var points = new THREE.Points(geo, mat);
+    scene.add(points);
+    scatter3DPoints = points;
+
+    var isDragging = false, prevMouse = { x: 0, y: 0 };
+    renderer.domElement.addEventListener('mousedown', function(e) { isDragging = true; prevMouse = { x: e.clientX, y: e.clientY }; });
+    window.addEventListener('mouseup', function() { isDragging = false; });
+    window.addEventListener('mousemove', function(e) {
+        if (!isDragging || !is3DMode) return;
+        var dx = e.clientX - prevMouse.x, dy = e.clientY - prevMouse.y;
+        points.rotation.y += dx * 0.005; points.rotation.x += dy * 0.005;
+        points.rotation.x = Math.max(-Math.PI/2, Math.min(Math.PI/2, points.rotation.x));
+        prevMouse = { x: e.clientX, y: e.clientY };
+    });
+    renderer.domElement.addEventListener('wheel', function(e) { if (!is3DMode) return; e.preventDefault(); camera.position.z += e.deltaY * 0.008; camera.position.z = Math.max(1.5, Math.min(15, camera.position.z)); }, { passive: false });
+
+    function animate() { if (!is3DMode) { requestAnimationFrame(animate); return; } requestAnimationFrame(animate); renderer.render(scene, camera); }
+    animate();
+    scatter3DInitialized = true;
+}
+
+// ══════════════════════════════════════════════════════════
+//  GEO SCATTER
+// ══════════════════════════════════════════════════════════
 
 {% if geo_count > 0 %}
 const geoData = {{ geo_data_json|safe }};
-drawScatter('geoChart', geoData, 'Longitude', 'Latitude');
+(function() {
+    const canvas = document.getElementById('geoChart');
+    if (!canvas) return;
+    const W = canvas.parentElement.clientWidth || 600, H = 300;
+    canvas.width = W; canvas.height = H;
+    const ctx = canvas.getContext('2d');
+    const pad = { top: 20, bottom: 30, left: 50, right: 20 };
+    const pw = W - pad.left - pad.right, ph = H - pad.top - pad.bottom;
+    const xs = geoData.map(d => d.x), ys = geoData.map(d => d.y);
+    const xMin = Math.min(...xs), xMax = Math.max(...xs), yMin = Math.min(...ys), yMax = Math.max(...ys);
+    const toX = v => pad.left + (v - xMin) / (xMax - xMin || 1) * pw;
+    const toY = v => pad.top + ph - (v - yMin) / (yMax - yMin || 1) * ph;
+    ctx.clearRect(0, 0, W, H);
+    geoData.forEach(d => {
+        ctx.beginPath(); ctx.arc(toX(d.x), toY(d.y), d.label === -1 ? 2 : 4, 0, Math.PI * 2);
+        ctx.fillStyle = colorFor(d.label); ctx.fill();
+    });
+    var leg = document.getElementById('geoLegend');
+    if (leg) { leg.innerHTML = ''; var seen = {}; geoData.forEach(d => { if (!seen[d.label]) { seen[d.label] = true; leg.innerHTML += '<span class="pill" style="background:' + colorFor(d.label) + ';color:#fff;">' + (d.label === -1?'Noise':'#'+d.label) + '</span>'; } }); }
+})();
 {% endif %}
 
-// ── Cluster size horizontal bar chart ──
+// Initial draw
-const clusterSizes = {{ cluster_sizes_json|safe }};
+drawScatter('scatterChart', scatterData, 'UMAP-1', 'UMAP-2', null);
-const topFeaturesAll = {{ top_features_json|safe }};
-const chartColors = ['#4361ee','#f72585','#7209b7','#4cc9f0','#f8961e','#43aa8b','#f94144','#577590','#e9c46a','#9b5de5'];
-
-if (clusterSizes.length > 0) {
-    // Cluster size bar chart
-    (function() {
-        const canvas = document.getElementById('clusterSizesChart');
-        if (!canvas) return;
-        const ctx = canvas.getContext('2d');
-        var maxClusters = 50;
-        var shownClusters = clusterSizes.slice(0, maxClusters);
-        var note = '';
-        if (clusterSizes.length > maxClusters) {
-            note = ' (top ' + maxClusters + ' of ' + clusterSizes.length + ' clusters)';
-        }
-        var W = canvas.parentElement.clientWidth || 600;
-        var H = Math.max(200, shownClusters.length * 36 + 60);
-        canvas.width = W; canvas.height = H;
-        var pad = { top: 20, bottom: 30, left: 80, right: 30 };
-        var plotW = W - pad.left - pad.right;
-        var plotH = H - pad.top - pad.bottom;
-        var maxSize = Math.max(1, ...shownClusters.map(function(d) { return d.size; }));
-        ctx.clearRect(0, 0, W, H);
-        if (note) {
-            ctx.fillStyle = '#999'; ctx.font = '11px sans-serif'; ctx.textAlign = 'left';
-            ctx.fillText(note, pad.left, 15);
-        }
-        shownClusters.forEach(function(d, i) {
-            var barH = Math.min(32, plotH / shownClusters.length * 0.7);
-            var y = pad.top + i * (barH + 4);
-            var bw = Math.max(2, (d.size / maxSize) * plotW);
-            ctx.fillStyle = chartColors[i % chartColors.length];
-            ctx.fillRect(pad.left, y, bw, barH);
-            ctx.fillStyle = '#333';
-            ctx.font = '12px sans-serif';
-            ctx.textAlign = 'right';
-            ctx.fillText('#' + d.label, pad.left - 5, y + barH / 2 + 4);
-            ctx.textAlign = 'left';
-            ctx.fillStyle = '#666';
-            ctx.fillText(d.size, pad.left + bw + 5, y + barH / 2 + 4);
-        });
-    })();
-
-    // Silhouette score bar chart
-    (function() {
-        const canvas = document.getElementById('silhouetteChart');
-        if (!canvas) return;
-        const silData = clusterSizes.filter(d => d.silhouette !== null && d.silhouette !== undefined);
-        if (silData.length === 0) {
-            document.getElementById('silhouetteCard').style.display = 'none';
-            return;
-        }
-        const ctx = canvas.getContext('2d');
-        const W = canvas.parentElement.clientWidth || 600;
-        const H = Math.max(200, silData.length * 40 + 60);
-        canvas.width = W; canvas.height = H;
-        const pad = { top: 20, bottom: 30, left: 80, right: 40 };
-        const plotW = W - pad.left - pad.right;
-        const plotH = H - pad.top - pad.bottom;
-        const maxScore = Math.max(1, ...silData.map(d => Math.abs(d.silhouette)));
-        ctx.clearRect(0, 0, W, H);
-        silData.forEach((d, i) => {
-            const barH = Math.min(40, plotH / silData.length * 0.6);
-            const y = pad.top + i * (barH + 4);
-            const bw = (d.silhouette / maxScore) * plotW * 0.5;
-            const x = pad.left + plotW / 2;
-            ctx.fillStyle = d.silhouette >= 0 ? '#43aa8b' : '#f94144';
-            ctx.fillRect(x, y, d.silhouette >= 0 ? bw : -bw, barH);
-            ctx.fillStyle = '#333';
-            ctx.font = '12px sans-serif';
-            ctx.textAlign = 'right';
-            ctx.fillText('#' + d.label, pad.left - 5, y + barH / 2 + 4);
-            ctx.fillStyle = '#666';
-            if (d.silhouette >= 0) {
-                ctx.textAlign = 'left';
-                ctx.fillText(d.silhouette.toFixed(4), x + bw + 5, y - 2);
-            } else {
-                ctx.textAlign = 'right';
-                ctx.fillText(d.silhouette.toFixed(4), x - Math.abs(bw) - 5, y + barH + 12);
-            }
-        });
-        // Zero line
-        ctx.strokeStyle = '#999'; ctx.lineWidth = 1;
-        ctx.beginPath(); ctx.moveTo(pad.left + plotW / 2, pad.top); ctx.lineTo(pad.left + plotW / 2, H - pad.bottom); ctx.stroke();
-        ctx.fillStyle = '#999'; ctx.font = '11px sans-serif'; ctx.textAlign = 'center';
-        ctx.fillText('0', pad.left + plotW / 2, H - 5);
-        ctx.fillText('+1', W - pad.right, H - 5);
-        ctx.fillText('-1', pad.left, H - 5);
-    })();
-
-    // Top features per cluster horizontal bar charts
-    (function() {
-        const container = document.getElementById('featureChartsContainer');
-        if (!container) return;
-        clusterSizes.forEach(d => {
-            const label = d.label;
-            const features = topFeaturesAll[String(label)] || [];
-            if (features.length === 0) return;
-            const card = document.createElement('div');
-            card.style.marginBottom = '1.5rem';
-            card.innerHTML = '<h4 style="margin-bottom:0.5rem;">Cluster #' + label + ' - Top Features</h4><canvas class="feature-chart" data-label="' + label + '" style="width:100%;height:' + Math.max(120, features.length * 28 + 30) + 'px;"></canvas>';
-            container.appendChild(card);
-            const canvas = card.querySelector('canvas');
-            if (!canvas) return;
-            const ctx = canvas.getContext('2d');
-            const W2 = canvas.parentElement.clientWidth || 600;
-            const H2 = Math.max(120, features.length * 28 + 30);
-            canvas.width = W2; canvas.height = H2;
-            const pad2 = { top: 10, bottom: 20, left: 100, right: 60 };
-            const plotW2 = W2 - pad2.left - pad2.right;
-            const plotH2 = H2 - pad2.top - pad2.bottom;
-            const maxScore = Math.max(0.01, ...features.map(f => Math.abs(f.score || 0)));
-            ctx.clearRect(0, 0, W2, H2);
-            features.forEach((f, i) => {
-                const barH = Math.min(30, plotH2 / features.length * 0.6);
-                const y = pad2.top + i * (barH + 3);
-                const score = f.score || 0;
-                const bw = (score / maxScore) * plotW2;
-                ctx.fillStyle = chartColors[label % chartColors.length];
-                ctx.fillRect(pad2.left, y, bw, barH);
-                ctx.fillStyle = '#333';
-                ctx.font = '11px sans-serif';
-                ctx.textAlign = 'right';
-                const name = f.feature_name.length > 18 ? f.feature_name.slice(0, 16) + '..' : f.feature_name;
-                ctx.fillText(name, pad2.left - 5, y + barH / 2 + 4);
-                ctx.textAlign = 'left';
-                ctx.fillStyle = '#666';
-                ctx.fillText(score.toFixed(3), pad2.left + bw + 5, y + barH / 2 + 4);
-            });
-        });
-    })();
-}
 </script>
 {% endblock %}

templates/analysis/run_detail.html

@@ -1,38 +1,91 @@
 {% extends 'base.html' %}
-{% block title %}Run #{{ run.display_id }} - TLS Analyzer{% endblock %}
+{% block title %}运行 #{{ run.display_id }} - 天璇{% endblock %}
 {% block content %}
+<style>
+.svd-card { background:#f8f9fa; border:1px solid #e0e0e0; border-radius:8px; padding:0.8rem; }
+.svd-card h4 { margin:0 0 0.3rem; font-size:0.9rem; }
+.var-bar { height:8px; border-radius:4px; background:#4361ee; margin:0.2rem 0; transition:width 0.3s; }
+.feat-tag { display:inline-block; background:#e8f0fe; padding:0.1rem 0.4rem; border-radius:3px; font-size:0.75rem; margin:0.1rem; }
+</style>
+
 <div class="card">
-    <h2>Run #{{ run.display_id }} — {{ run.created_at|date:"Y-m-d H:i" }}</h2>
+    <div style="display:flex;justify-content:space-between;align-items:center;flex-wrap:wrap;gap:0.5rem;">
-    <p>Status: <span class="badge {% if run.status == 'completed' %}badge-success{% elif run.status == 'failed' %}badge-danger{% else %}badge-info{% endif %}">{{ run.get_status_display }}</span></p>
+        <div>
-    <p>CSV: <code>{% if run.sqlite_table %}已存储至SQLite: {{ run.sqlite_table }}{% else %}{{ run.csv_glob }}{% endif %}</code></p>
+            <h2 style="margin:0;">运行 #{{ run.display_id }}</h2>
+            <p style="margin:0.25rem 0 0;color:#666;font-size:0.9rem;">
+                {{ run.created_at|date:"Y-m-d H:i" }}
+                — 状态: <span class="badge {% if run.status == 'completed' %}badge-success{% elif run.status == 'failed' %}badge-danger{% else %}badge-info{% endif %}">{{ run.get_status_display }}</span>
+            </p>
+        </div>
+        <a href="{% url 'analysis:cluster_overview' run.display_id %}" class="btn btn-primary" style="font-size:0.85rem;">聚类概览 →</a>
+    </div>
+    {% if run.error_message %}
+    <pre style="background:#1a1a2e;color:#e0e0e0;padding:0.5rem;border-radius:4px;font-size:0.78rem;margin-top:0.5rem;max-height:200px;overflow:auto;"><code>{{ run.error_message }}</code></pre>
+    {% endif %}
 </div>
 
 <div class="grid-3">
     <div class="card" style="text-align:center;">
         <div class="stat">{{ run.total_flows|default:"0" }}</div>
-        <div class="stat-label">Total Flows</div>
+        <div class="stat-label">总流数</div>
     </div>
     <div class="card" style="text-align:center;">
         <div class="stat">{{ run.entity_count|default:"0" }}</div>
-        <div class="stat-label">Entities</div>
+        <div class="stat-label">行数</div>
     </div>
     <div class="card" style="text-align:center;">
         <div class="stat">{{ run.cluster_count|default:"0" }}</div>
-        <div class="stat-label">Clusters</div>
+        <div class="stat-label">聚类数</div>
     </div>
 </div>
 
+<!-- ── Global SVD Features ── -->
+{% if svd_features %}
 <div class="card">
-    <h2>Clusters</h2>
+    <div style="display:flex;justify-content:space-between;align-items:center;">
-    <a href="{% url 'analysis:cluster_overview' run.display_id %}" class="btn btn-primary" style="margin-bottom:1rem;">View Cluster Overview</a>
+        <h3>🔬 全域 SVD 特征分析</h3>
+        <span style="font-size:0.8rem;color:#888;">基于 {{ run.total_flows }} 行数据</span>
+    </div>
+    <p style="font-size:0.82rem;color:#666;margin-top:0.3rem;">奇异值分解 (TruncatedSVD) 降维后的各主成分解释方差比例与贡献特征。</p>
+    <div style="display:grid;grid-template-columns:1fr 1fr;gap:0.75rem;margin-top:0.75rem;">
+    {% for svd in svd_features %}
+        <div class="svd-card">
+            <h4>PC{{ svd.component }}</h4>
+            <div style="display:flex;justify-content:space-between;font-size:0.78rem;color:#555;">
+                <span>解释方差: {{ svd.explained_var }}%</span>
+                <span>累积: {{ svd.cumulative_var }}%</span>
+            </div>
+            <div class="var-bar" style="width:{{ svd.explained_var }}%;"></div>
+            <div style="margin-top:0.4rem;">
+            {% for f in svd.top_features %}
+                <span class="feat-tag">{{ f.feature }}: {{ f.strength }}</span>
+            {% endfor %}
+            </div>
+        </div>
+    {% endfor %}
+    </div>
+</div>
+{% endif %}
+
+<!-- ── LLM 自动分析流程 ── -->
+{% if run.run_type == 'auto' and llm_timeline_json %}
+<div class="card">
+    <h3>🧠 LLM 自动分析流程</h3>
+    <div id="llmTimeline"></div>
+</div>
+{% endif %}
+
+<!-- ── 聚类列表 ── -->
+<div class="card">
+    <h3>📊 聚类结果</h3>
     {% if clusters %}
     <table>
         <thead>
             <tr>
-                <th>Cluster</th>
+                <th>簇</th>
-                <th>Size</th>
+                <th>大小</th>
-                <th>Proportion</th>
+                <th>比例</th>
-                <th>Silhouette</th>
+                <th>轮廓系数</th>
                 <th></th>
             </tr>
         </thead>
@@ -43,13 +96,83 @@
                 <td>{{ c.size }}</td>
                 <td>{{ c.proportion|floatformat:2 }}</td>
                 <td>{{ c.silhouette_score|floatformat:4|default:"-" }}</td>
-                <td><a href="{% url 'analysis:cluster_detail' run.display_id c.cluster_label %}" class="btn btn-primary">Detail</a></td>
+                <td><a href="{% url 'analysis:cluster_detail' run.display_id c.cluster_label %}" class="btn btn-primary">详情</a></td>
             </tr>
             {% endfor %}
         </tbody>
     </table>
     {% else %}
-    <div class="empty-state"><p>No clusters yet.</p></div>
+    <div class="empty-state"><p>无聚类结果</p></div>
     {% endif %}
 </div>
-{% endblock %}
+
+{% if run.run_type == 'auto' and llm_timeline_json %}
+<script>
+const llmData = {{ llm_timeline_json|safe }};
+(function() {
+    const container = document.getElementById('llmTimeline');
+    if (!container) return;
+
+    function escapeHtml(s) { if (s==null) return ''; return String(s).replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;').replace(/"/g,'&quot;'); }
+    function prettyJson(o) { try { return JSON.stringify(o,null,2); } catch(e) { return String(o); } }
+
+    var thinkingByStep = {}, thinkOrder = [];
+    if (llmData.thinking) {
+        var parts = llmData.thinking.split(/\n(?=\[\d+\])/);
+        for (var pi = 0; pi < parts.length; pi++) {
+            var m = parts[pi].match(/^\[(\d+)\]\s*(.*)/s);
+            if (m) {
+                var step = parseInt(m[1]);
+                if (!(step in thinkingByStep)) thinkOrder.push(step);
+                thinkingByStep[step] = (thinkingByStep[step]||'')+(thinkingByStep[step]?'\n':'')+m[2].trim();
+            }
+        }
+    }
+    var toolCalls = llmData.tool_calls || [];
+    var toolMap = {};
+    for (var ti = 0; ti < toolCalls.length; ti++) {
+        var tc = toolCalls[ti];
+        if (!toolMap[tc.step]) toolMap[tc.step] = [];
+        toolMap[tc.step].push(tc);
+    }
+    var allSteps = new Set(thinkOrder.concat(Object.keys(toolMap).map(Number)));
+    var sortedSteps = Array.from(allSteps).sort(function(a,b){return a-b;});
+
+    for (var si = 0; si < sortedSteps.length; si++) {
+        var stp = sortedSteps[si];
+        var hdr = document.createElement('div');
+        hdr.style.cssText = 'font-weight:700;font-size:0.85rem;color:#4361ee;padding:0.6rem 0 0.3rem;border-top:1px solid #eee;';
+        hdr.textContent = '步骤 ' + stp;
+        container.appendChild(hdr);
+        if (thinkingByStep[stp] !== undefined) {
+            var tdiv = document.createElement('div');
+            tdiv.style.cssText = 'background:#eef2ff;padding:0.5rem;border-radius:6px;font-size:0.78rem;line-height:1.6;color:#333;max-height:300px;overflow:auto;margin-bottom:0.3rem;border:1px solid #d0d8f0;';
+            tdiv.innerHTML = '<span style="font-weight:600;color:#4361ee;font-size:0.75rem;display:block;margin-bottom:0.25rem;">💭 思考</span>' + escapeHtml(thinkingByStep[stp]);
+            container.appendChild(tdiv);
+        }
+        if (toolMap[stp]) {
+            for (var tj = 0; tj < toolMap[stp].length; tj++) {
+                var tc2 = toolMap[stp][tj];
+                var card = document.createElement('div');
+                card.style.cssText = 'border:1px solid #e0e0e0;border-radius:6px;margin-bottom:0.3rem;overflow:hidden;background:#fff;';
+                var inputStr = prettyJson(tc2.input);
+                var outputStr = prettyJson(tc2.output);
+                card.innerHTML =
+                    '<div onclick="this.classList.toggle(\'open\');var b=this.nextElementSibling;if(b)b.classList.toggle(\'open\');" style="padding:0.4rem 0.8rem;cursor:pointer;display:flex;justify-content:space-between;align-items:center;background:#f8f9fa;user-select:none;">' +
+                    '<span><span style="background:#43aa8b;color:#fff;border-radius:3px;padding:0 5px;font-size:0.7rem;font-weight:600;margin-right:6px;">🔧</span>' +
+                    '<span style="font-weight:600;font-size:0.8rem;">' + escapeHtml(tc2.name||'?') + '</span></span>' +
+                    '<span style="font-size:0.75rem;color:#888;">▶</span></div>' +
+                    '<div style="display:none;padding:0.5rem 0.8rem;background:#fafbfc;">' +
+                    '<div style="font-weight:600;font-size:0.72rem;color:#555;margin-bottom:0.2rem;">输入:</div>' +
+                    '<pre style="background:#1a1a2e;color:#e0e0e0;padding:0.5rem;border-radius:4px;font-size:0.7rem;max-height:200px;overflow:auto;margin:0 0 0.3rem;"><code>' + escapeHtml(inputStr) + '</code></pre>' +
+                    '<div style="font-weight:600;font-size:0.72rem;color:#555;margin-bottom:0.2rem;">输出:</div>' +
+                    '<pre style="background:#1a1a2e;color:#e0e0e0;padding:0.5rem;border-radius:4px;font-size:0.7rem;max-height:200px;overflow:auto;margin:0;"><code>' + escapeHtml(outputStr) + '</code></pre>' +
+                    '</div>';
+                container.appendChild(card);
+            }
+        }
+    }
+})();
+</script>
+{% endif %}
+{% endblock %}