update AGENTS.md

This reverts commit 1ab655d17940aa0905c031753b89d67fb51a61e2.

AGENTS.md

@@ -1,4 +1,4 @@
-# 天璇 (TianXuan) — Agent Knowledge Base
+# 天璇 (TianXuan) -Agent Knowledge Base
 
 ## Project Identity
 
@@ -6,133 +6,120 @@
 |-------|-------|
 | 项目名称 | 天璇 (TianXuan) |
 | 原始名称 | tls-analyzer |
-| 核心功能 | TLS 流数据分析、实体画像、聚类、3D地球可视化、PCA散点图 |
+| 核心功能 | TLS 流数据分析、实体画像、聚类3D地球可视化、PCA散点图|
 | Python 版本 | 3.12 (embedded portable at `runtime/python/python.exe`) |
 | Polars 版本 | 1.42.1 (精确锁定) |
 | 数据框架 | Polars (LazyFrame + streaming) |
 | 机器学习 | scikit-learn (HDBSCAN, KMeans, PCA, StandardScaler) |
 | Web框架 | Django 4.2 + SQLite WAL |
 | 前端3D | Three.js (603KB, 离线, 地球贴图1.4MB) |
-| 前端图表 | 纯Canvas散点图 (无Chart.js) |
+| 前端图表 | 纯Canvas散点图(无Chart.js) |
 | LLM协议 | MCP (stdio transport) |
 | 启动方式 | `run.bat`（双击即用）/ `runtime\python\python.exe manage.py runserver`（开发） |
+| 目标设备 | Windows 10 1902, 4GB RAM, 无独立显卡（纯核显） |
 
 ## Project Structure
 
 ```
 tianxuan/
-├── tianxuan/                      # Django 项目配置
-│   ├── settings.py                # ALLOWED_HOSTS 自动检测, LOGGING 文件+stderr
-│   ├── wsgi.py                    # SQLite 启动自修复
-│   ├── urls.py
-│   └── llm_orchestrator.py        # LLM 编排器 (32B/284B 双兼容)
-│
-├── analysis/                      # 核心分析模块 (Django app)
-│   ├── data_loader.py             # CSV 加载, BOM检测, schema_strict, 递归glob, 中文路径
-│   ├── data_profiler.py           # 列统计 + 相关性矩阵 (无pandas回退:numpy)
-│   ├── entity_detector.py         # 实体列自动检测 (tuple关键词 + unique_ratio + null惩罚)
-│   ├── entity_aggregator.py       # 实体聚合 (tuple关键词, lat/lon检测, 多列group_by, IP子网)
-│   ├── session_store.py           # 线程安全单例内存存储
-│   ├── tool_registry.py           # 12个MCP工具 + DB持久化 (sync_to_async)
-│   ├── mcp_server.py              # MCP stdio server
-│   ├── views.py                   # 所有Django视图 (上传/手动/LLM/配置/LLM测试/日志/地球)
-│   ├── urls.py                    # 16条路由
-│   ├── models.py                  # ORM: AnalysisRun, ClusterResult, EntityProfile, ClusterFeature
-│   ├── data_validator.py          # 列校验 (缺失率/异常值/IP有效性)
-│   ├── type_classifier.py         # 值优先类型检测 (MAC/端口/IPv4/URL/HEX/ENUM/LAT_LON)
-│   ├── geoip.py                   # GeoIP 经纬度查询 (data/geoip_data.txt)
-│   ├── admin.py
-│   └── management/commands/
-│       ├── start_mcp.py           # MCP服务器启动命令
-│       └── run_pipeline.py        # 一键CLI管道命令
-│
-├── config/
-│   ├── config.yaml                # 自动生成默认配置 (含entity.subnet_masks, columns覆盖)
-│   ├── loader.py                  # Pydantic 配置加载 + 文件修改检测
-│   └── __init__.py
-│
-├── templates/
-│   ├── base.html                  # 导航: 首页/上传/手动/LLM/记录/地球/配置
-│   └── analysis/ 或 tianxuan/     # 全部11个页面模板
-│       ├── dashboard.html         # 首页 - 最近运行
-│       ├── upload.html            # CSV上传 - 拖拽+多文件+删除+进度
-│       ├── manual.html            # 3步手动向导 - 选数据→设参数→运行
-│       ├── auto.html              # LLM自动分析 - 实时日志窗+取消
-│       ├── config.html            # 配置编辑 - LLM连通性测试
-│       ├── run_list.html          # 运行记录列表
-│       ├── run_detail.html        # 运行详情摘要
-│       ├── cluster_overview.html  # 聚类概览 - 纯Canvas散点图+PCA+地理
-│       ├── cluster_detail.html    # 簇详情 - 特征表+实体列表
-│       ├── entity_profile.html    # 实体画像 - 特征偏离+Canvas单点图
-│       ├── globe.html             # 3D地球可视化 - Three.js 流量弧线+经纬线+国境线
-│       └── log_viewer.html        # 日志查看
-│
-├── static/tianxuan/              # 离线静态资源
-│   ├── three.min.js               # Three.js 3D引擎 (603KB)
-│   ├── earth_atmos_2048.jpg       # 地球贴图 (1.4MB)
-│   └── world_borders.js           # 10国精简国境线轮廓
-│
-├── runtime/python/                # 便携Python 3.12运行时 (~593MB)
-│
-├── scripts/
-│   ├── gen_test_data.py           # 简单测试数据生成 (--globe模式使用真实GeoIP范围)
-│   ├── gen_complex_test.py        # 复杂数据生成 (5000行24列含55%缺失)
-│   ├── column_survey.py           # CSV列结构调查 (统计各CSV的列名/类型/分布)
-│   ├── diagnose_compare.py        # 跨机日志对比诊断
-│   └── debug_db.py                # DB持久化调试
-│
-├── tests/                         # 92个测试 (unit) / 121个测试 (all)
-│   ├── test_data_loader.py        # 26: BOM/schema/中文路径/递归glob
-│   ├── test_entity.py             # 17: 关键词/检测/聚合
-│   ├── test_e2e.py                # 6: 全端到端
-│   └── test_clustering_edge.py    # HDBSCAN零样本降级
-│
-├── docs/
-│   ├── 工作流.md                   # 完整分析流水线文档
-│   └── 故障诊断手册.md             # 跨机问题诊断指南
-│
-├── run.bat                        # 用户启动 (含PYTHONUTF8=1)
-├── shell.bat                      # Django shell
-├── TlsDB.csv                      # 参考表头名清单（非实际数据）
-├── README.md
+├── tianxuan/ # Django 项目配置
+│ ├── settings.py # ALLOWED_HOSTS 自动检测 LOGGING 文件+stderr
+│ ├── wsgi.py # SQLite 启动自修复
+│  ├── urls.py
+│ └── llm_orchestrator.py # LLM 编排(32B/284B 双兼容
+│ ├── analysis/ # 核心分析模块 (Django app)
+│ ├── data_loader.py # CSV 加载, BOM检测 schema_strict, 递归glob, 中文路径
+│ ├── data_profiler.py # 列统计+ 相关性矩阵(无pandas回退:numpy)
+│ ├── entity_detector.py # 实体列自动检测(tuple关键词+ unique_ratio + null惩罚)
+│ ├── entity_aggregator.py # 实体聚合 (tuple关键词 lat/lon检测 多列group_by, IP子网)
+│ ├── session_store.py # 线程安全单例内存存储
+│ ├── tool_registry.py # 12个MCP工具 + DB持久化(sync_to_async)
+│ ├── mcp_server.py # MCP stdio server
+│ ├── views.py # 所有Django视图 (上传/手动/LLM/配置/LLM测试/日志/地球)
+│ ├── urls.py # 16条路由 ├── models.py # ORM: AnalysisRun, ClusterResult, EntityProfile, ClusterFeature
+│ ├── data_validator.py # 列校验(缺失异常值IP有效性
+│ ├── type_classifier.py # 值优先类型检测(MAC/端口/IPv4/URL/HEX/ENUM/LAT_LON)
+│ ├── geoip.py # GeoIP 经纬度查询(data/geoip_data.txt)
+│ ├── admin.py
+│ └── management/commands/
+│ ├── start_mcp.py # MCP服务器启动命令
+│  └── run_pipeline.py # 一键CLI管道命令
+│ ├── config/
+│ ├── config.yaml # 自动生成默认配置 (含entity.subnet_masks, columns覆盖)
+│ ├── loader.py # Pydantic 配置加载 + 文件修改检测
+│  └── __init__.py
+│ ├── templates/
+│ ├── base.html # 导航: 首页/上传/手动/LLM/记录/地球/配置
+│ └── analysis/ 
+│ tianxuan/ # 全部11个页面模板
+│      ├── dashboard.html # 首页 - 最近运行
+│      ├── upload.html # CSV上传 - 拖拽+多文件删除+进度
+│ ├── manual.html # 3步手动向- 选数据→设参数→运行
+│ ├── auto.html # LLM自动分析 - 实时日志取消
+│ ├── config.html # 配置编辑 - LLM连通性测试
+│      ├── run_list.html # 运行记录列表
+│ ├── run_detail.html # 运行详情摘要
+│ ├── cluster_overview.html # 聚类概览 - 纯Canvas散点PCA+地理
+│ ├── cluster_detail.html # 簇详情- 特征实体列表
+│ ├── entity_profile.html # 实体画像 - 特征偏离+Canvas单点
+│      ├── globe.html # 3D地球可视- Three.js 流量弧线+经纬国境
+│  └── log_viewer.html # 日志查看
+│ ├── static/tianxuan/ # 离线静态资源
+│  ├── three.min.js # Three.js 3D引擎 (603KB)
+│ ├── earth_atmos_2048.jpg # 地球贴图 (1.4MB)
+│ └── world_borders.js # 10国精简国境线轮廓 ├── runtime/python/ # 便携Python 3.12运行时(~593MB)
+│ ├── scripts/
+│ ├── gen_test_data.py # 简单测试数据生成(--globe模式使用真实GeoIP范围)
+│ ├── gen_complex_test.py # 复杂数据生成 (5000行4列含55%缺失)
+│ ├── column_survey.py # CSV列结构调查(统计各CSV的列类型/分布)
+│ ├── diagnose_compare.py # 跨机日志对比诊断
+│ └── debug_db.py # DB持久化调试 ├── tests/ # 92个测试(unit) / 121个测试(all)
+│ ├── test_data_loader.py # 26: BOM/schema/中文路径/递归glob
+│ ├── test_entity.py # 17: 关键词检测聚合
+│ ├── test_e2e.py # 6: 全端到端
+│ └── test_clustering_edge.py # HDBSCAN零样本降级 ├── docs/
+│ ├── 工作流.md # 完整分析流水线文档
+│  └── 故障诊断手册.md # 跨机问题诊断指南
+│ ├── run.bat # 用户启动 (含PYTHONUTF8=1)
+├── shell.bat # Django shell
+├── TlsDB.csv # 参考表头名清单（非实际数据├── README.md
 ├── AGENTS.md
 └── manage.py
 ```
 
 ## 12 MCP Tools
 
-| # | 工具名 | 功能 | 必需参数 |
+| # | 工具| 功能 | 必需参数 |
 |---|--------|------|----------|
-| 1 | `load_data` | 加载 CSV 文件（glob / 递归 / schema容错） | `csv_glob` |
-| 2 | `profile_data` | 数据集概要统计 + 相关性矩阵 | `dataset_id` |
-| 3 | `filter_data` | 按条件过滤（12操作符 + AND/OR） | `dataset_id`, `filters` |
-| 4 | `preprocess_data` | 预处理（标准化/编码/填充） | `dataset_id`, `columns` |
-| 5 | `run_clustering` | 执行聚类（HDBSCAN/KMeans + 质量评估） | `dataset_id`, `cluster_columns` |
-| 6 | `evaluate_clustering` | 评估聚类质量（Silhouette/DB/CH） | `cluster_result_id`, `dataset_id` |
-| 7 | `extract_features` | 提取各聚类区分特征（Z-Score/ANOVA） | `dataset_id`, `cluster_result_id` |
-| 8 | `export_results` | 导出结果到磁盘（CSV/Parquet/JSON） | `result_id`, `output_path` |
-| 9 | `list_datasets` | 列出所有活跃数据集 | 无 |
-| 10 | `drop_dataset` | 删除数据集释放内存 | `dataset_id` |
-| 11 | `clone_dataset` | 浅拷贝数据集（不复制数据） | `dataset_id` |
+| 1 | `load_data` | 加载 CSV 文件（glob / 递归 / schema容错| `csv_glob` |
+| 2 | `profile_data` | 数据集概要统+ 相关性矩阵| `dataset_id` |
+| 3 | `filter_data` | 按条件过滤（12操作+ AND/OR| `dataset_id`, `filters` |
+| 4 | `preprocess_data` | 预处理（标准化编码/填充)| `dataset_id`, `columns` |
+| 5 | `run_clustering` | 执行聚类（HDBSCAN/KMeans + 质量评估| `dataset_id`, `cluster_columns` |
+| 6 | `evaluate_clustering` | 评估聚类质量（Silhouette/DB/CH| `cluster_result_id`, `dataset_id` |
+| 7 | `extract_features` | 提取各聚类区分特征（Z-Score/ANOVA| `dataset_id`, `cluster_result_id` |
+| 8 | `export_results` | 导出结果到磁盘（CSV/Parquet/JSON| `result_id`, `output_path` |
+| 9 | `list_datasets` | 列出所有活跃数据集 | |
+| 10 | `drop_dataset` | 删除数据集释放内存| `dataset_id` |
+| 11 | `clone_dataset` | 浅拷贝数据集（不复制数据| `dataset_id` |
 | 12 | `build_entity_profiles` | 自动检测实体列 + 聚合画像 | `dataset_id` |
 
-所有工具通过 `analysis/tool_registry.py` 注册，`analysis/mcp_server.py` 暴露标准 MCP stdio 接口。
+所有工具通过 `analysis/tool_registry.py` 注册，`analysis/mcp_server.py` 暴露标准 MCP stdio 接口
 
-## Config.Entity 类
+## Config.Entity 
 
-`config/loader.py` 中的 Pydantic 模型：
+`config/loader.py` 中的 Pydantic 模型
 
 ```python
 class Entity(BaseModel):
-    subnet_masks: list[int] = []    # 子网掩码列表，如 [24, 28]
-    ip_columns: list[str] = ['src_ip', 'dst_ip']  # 需聚合的IP列
-```
+ subnet_masks: list[int] = [] # 子网掩码列表，如 [24, 28]
+ ip_columns: list[str] = ['src_ip', 'dst_ip'] # 需聚合的IP列```
 
 config.yaml 默认配置:
 ```yaml
 entity:
-  subnet_masks: [24, 28]
-  ip_columns: ["src_ip", "dst_ip"]
+ subnet_masks: [24, 28]
+ ip_columns: ["src_ip", "dst_ip"]
 ```
 
 ---
@@ -140,321 +127,325 @@ entity:
 ## Architecture
 
 - **tuple 替代 frozenset**: 避免跨机 PYTHONHASHSEED 差异
-- **PYTHONUTF8=1**: 跨机编码一致性
-- **类型安全聚合**: 聚合前检查 dtype，非数值列先 cast
-- **sync_to_async ORM**: 异步上下文中的 Django ORM
-- **纯 Canvas 图表**: 零外部 JS 依赖
+- **PYTHONUTF8=1**: 跨机编码一致性- **类型安全聚合**: 聚合前检查dtype，非数值列cast
+- **sync_to_async ORM**: 异步上下文中Django ORM
+- **纯Canvas 图表**: 零外部JS 依赖
 - **上传目录隔离**: %APPDATA%/TianXuan/data/uploads/，与项目路径无关
-- **通用数值清洗 _coerce_to_float**: 处理 + / - /空白/N/A 等伪影
-- **值优先类型检测**: config→值→名称→STRING
+- **通用数值清理_coerce_to_float**: 处理 + / - /空白/N/A 等伪值- **值优先类型检测*: config→值→名称→STRING
 - **地球深度测试**: depthTest: true, depthWrite: false，r=5.5
-- **完整国境线**: Natural Earth 110m, 177国, 286多边形, 269KB
+- **完整国境线*: Natural Earth 110m, 177国 286多边形 269KB
 
 ---
 
 ## Testing Protocol
 
 ### 运行全部测试
-```bash
-# 单元测试（不需要Django）
-runtime\python\python.exe -m pytest tests/test_data_loader.py tests/test_type_classifier.py -q
 
-# 集成测试（自动启动后端、模拟前端行为、监控stderr报错）
-runtime\python\python.exe tests/test_integration.py
+```bash
+# 单元测试（不需要Django)runtime\python\python.exe -m pytest tests/test_data_loader.py tests/test_type_classifier.py -q
+
+# 集成测试（自动启动后端、模拟前端行为、监控stderr报错)runtime\python\python.exe tests/test_integration.py
 ```
+
 ```
 92 passed (unit tests); ALL INTEGRATION TESTS PASSED
 ```
 
-> ⚠️ **重要**: `tests/test_integration.py` 模拟前端HTML行为（上传CSV → 轮询状态 → 访问页面 → 删除）。
-> **当前端HTML/API发生变化时，必须同步更新此脚本**，否则集成测试将不再反映真实用户流程。
+> ⚠️ **重要**: `tests/test_integration.py` 模拟前端HTML行为（上传CSV 轮询状态访问页面 删除）> **当前端HTML/API发生变化时，必须同步更新此脚*，否则集成测试将不再反映真实用户流程
+>
+### 跨机一致性测```bash
+
+# 两台机器各自执行
 
-### 跨机一致性测试
-```bash
-# 两台机器各自执行:
 runtime\python\python.exe manage.py run_pipeline data\complex_test.csv
 
-# 对比日志:
+# 对比日志
+
 runtime\python\python.exe scripts\diagnose_compare.py log_a.txt log_b.txt
+
 ```
 
-### DB持久化测试
-```bash
+### DB持久化测```bash
 runtime\python\python.exe scripts\debug_db.py
 # 期望: n_features=30 db_saved=True
-# 验证: run_analysis 后 ClusterFeature 表有数据
+# 验证: run_analysis ClusterFeature 表有数据
 ```
 
 ### LLM编排测试
+
 ```bash
 runtime\python\python.exe scripts\test_llm_full.py
-# 或
-runtime\python\python.exe -c "from tianxuan.llm_orchestrator import run_llm_pipeline, LLMConfig; ..."
+# 或 runtime\python\python.exe -c "from tianxuan.llm_orchestrator import run_llm_pipeline, LLMConfig; ..."
 ```
 
 ### 生成测试数据
-```bash
-# 简单数据
-runtime\python\python.exe scripts\gen_test_data.py --rows 1000
 
-# Globe模式 (IP在真实GeoIP范围内)
+```bash
+# 简单数据runtime\python\python.exe scripts\gen_test_data.py --rows 1000
+
+# Globe模式 (IP在真实GeoIP范围)
 runtime\python\python.exe scripts\gen_test_data.py --globe
 
-# 复杂数据 (5000行24列55%缺失)
+# 复杂数据 (5000行4列含55%缺失)
 runtime\python\python.exe scripts\gen_complex_test.py --rows 5000
 ```
 
-### 列结构调查
-```bash
+### 列结构调查```bash
+
 runtime\python\python.exe scripts\column_survey.py --csv "data/*.csv"
+
 ```
 
 ---
 
 ## Polars Version Compatibility
 
-| API | 旧版(<1.0) | 新版(≥1.0) |
+| API | 旧版(<1.0) | 新版(.0) |
 |-----|-----------|-----------|
 | mode | `pl.mode()` | `pl.col(col).mode()` |
 | encoding | `encoding='utf-8'` | `encoding='utf8'` |
-| value_counts col | `'index'` | 原列名 |
+| value_counts col | `'index'` | 原列名|
 | streaming | `collect(streaming=True)` | `collect(engine='streaming')` |
 | truth value | `if series:` | `if series.item():` |
 
 ## 32B LLM Optimization
 
-- Step guidance: 每步告诉模型下一步做什么
-- Truncation: 工具结果截断至 1200 chars
+- Step guidance: 每步告诉模型下一步做什么- Truncation: 工具结果截断1200 chars
 - Timeout: 90s (32B 推理更慢)
 - Max steps: 15
 - Tool description: 一行简洁描述
-
 ---
 
 ## v1.1.3 修复 (2026-07-21)
 
 13 issues fixed across build system, frontend, pipeline, and data handling.
 
-| # | 问题 | 状态 | 详情 |
+| # | 问题 | 状态| 详情 |
 |---|------|------|------|
-| 1 | 增量包构建修复 | ✅ | update_info.json含SHA256、文件在files/子目录下、测试数据排除、__delete__支持 |
-| 2 | 构建顺序+三段版本号 | ✅ | 先commit后bump再tag、bump_version支持vX.Y.Z、update.bat加PYTHONUTF8 |
-| 3 | 前端导航跳转 | ✅ | 上传后→手动分析页、LLM完成后→聚类概览页 |
-| 4 | 聚类前手动筛选 | ✅ | manual.html下filter_data工具UI |
-| 5 | 地球→态势+拖拽修复 | ✅ | 导航改名、拖拽方向修正、惯性仅松开时施加 |
-| 6 | DB模型同步 | ✅ | 所有模型与SQLite表完全匹配 |
-| 7 | LLM后强制聚类 | ✅ | LLM完成后自动运行clustering+PCA pipeline |
-| 8 | 列名不一致处理 | ✅ | 保留所有列(union)、不丢弃额外列、gen_test_data支持--missing-cols |
-| 9 | FFT频谱分析 | ✅ | 新MCP工具analyze_fft(numpy.fft)、周期性特征提取 |
-| 10 | 异常处理修复 | ✅ | 24处裸except/except:pass全部修复为带日志处理 |
-| 11 | LLM上下文增强 | ✅ | 发送列类型+中文含义+空值率+众数给LLM |
-| 12 | 工作流自动保存 | ✅ | 最近10条自动方案保存到手动分析页、支持固定防淘汰 |
-| 13 | 列类型遵循TlsDB | ✅ | tls_ref_data作为权威类型源、未知列严格回退 |
+| 1 | 增量包构建修复| | update_info.json含SHA256、文件在files/子目录下、测试数据排除、__delete__支持 |
+| 2 | 构建顺序+三段版本号| | 先commit后bump再tag、bump_version支持vX.Y.Z、update.bat加PYTHONUTF8 |
+| 3 | 前端导航跳转 | | 上传后→手动分析页、LLM完成后→聚类概览|
+| 4 | 聚类前手动筛选| | manual.html下filter_data工具UI |
+| 5 | 地球→态势+拖拽修复 | | 导航改名、拖拽方向修正、惯性仅松开时施加|
+| 6 | DB模型同步 | | 所有模型与SQLite表完全匹配|
+| 7 | LLM后强制聚类| | LLM完成后自动运行clustering+PCA pipeline |
+| 8 | 列名不一致处理| | 保留所有列(union)、不丢弃额外列、gen_test_data支持--missing-cols |
+| 9 | FFT频谱分析 | | 新MCP工具analyze_fft(numpy.fft)、周期性特征提取|
+| 10 | 异常处理修复 | | 24处裸except/except:pass全部修复为带日志处理 |
+| 11 | LLM上下文增加| | 发送列类型+中文含义+空值率+众数给LLM |
+| 12 | 工作流自动保存| | 最0条自动方案保存到手动分析页、支持固定防淘汰 |
+| 13 | 列类型遵循TlsDB | | tls_ref_data作为权威类型源、未知列严格回退 |
 
 ## 修复计划 (2026-07-16)
 
-| # | 问题 | 状态 | 详情 |
+| # | 问题 | 状态| 详情 |
 |---|------|------|------|
-| 1 | 重启后旧数据可分析 | ✅ | SessionStore 元数据持久化到 JSON 文件(`%APPDATA%/TianXuan/.session_store.json`)，wsgi.py 启动时自动恢复 |
-| 2 | Web图标所有页面 | ✅ | `<link rel="icon">` 在 base.html 中，所有页面继承 |
-| 3 | 上传按钮在顶部 | ✅ | `position: sticky` |
-| 4 | Traceback完整 | ✅ | 移除 views.py 4处 + run_pipeline.py 1处 `[:200]` 截断 |
-| 5 | 大数据不卡顿 | ✅ | 降采样 `head=` 参数支持，手动 POST 可传 head，LLM 自动决定，clamp ≥100行 |
-| 6 | 进度条 | ✅ | progress_pct 在 10/30/50/70/90/100 各阶段更新 |
-| 7 | TLS 0ver hex识别 | ✅ | TLS_HEX_MAP 移至 type_classifier.py 值优先检测链，0ver列自动 ENUM("TLSv1.2") |
-| 8 | Globe响应式缩放 | ✅ | Canvas resize 事件绑定，vh 单位动态计算 |
-| 9 | Globe ?data= | ✅ | 异常安全：try/except 包裹，空数据仍渲染页面 |
-| 10 | 文件上传无限制 | ✅ | DATA_UPLOAD_MAX_NUMBER_FIELDS=1000000, FILE_UPLOAD_MAX_MEMORY_SIZE=100MB |
-| 11 | 上传页删除功能 | ✅ | removeFile() + ✕ 按钮每文件 |
-| 12 | LLM日志 | ✅ | progress_callback 写入 run_run_log |
-| 13 | 聚类图表 | ✅ | cluster_overview.html 含散点图/柱状图/Silhouette图/特征表(纯Canvas) |
-| 14 | 低内存不崩溃 | ✅ | MiniBatchKMeans + PCA降维 + 降采样 head 参数 |
-| 15 | +号字符串识别为数字 | ✅ | type_classifier.py FLOAT检测前排除 `+` 前缀 |
-| 16 | 空闲高IO | ✅ | 日志级别 INFO，无需修改 |
-| 17 | 聚类效果 | ⚠️ | 预置了特征过滤+降维，但具体效果需业务验证 |
-| 18 | 列名精确匹配 | ✅ | `re.match` 精确匹配 + `_find_column` tuple 关键词 |
+| 1 | 重启后旧数据可分析| | SessionStore 元数据持久化为JSON 文件(`%APPDATA%/TianXuan/.session_store.json`)，wsgi.py 启动时自动恢复|
+| 2 | Web图标所有页| | `<link rel="icon">` 在 base.html 中，所有页面继承|
+| 3 | 上传按钮在顶部| | `position: sticky` |
+| 4 | Traceback完整 | | 移除 views.py 4处 + run_pipeline.py 1处`[:200]` 截断 |
+| 5 | 大数据不卡顿 | | 降采样`head=` 参数支持，手动POST 可传 head，LLM 自动决定，clamp 500|
+| 6 | 进度条| | progress_pct → 10/30/50/70/90/100 各阶段更新|
+| 7 | TLS 0ver hex识别 | | TLS_HEX_MAP 移至 type_classifier.py 值优先检测链ver列自动ENUM("TLSv1.2") |
+| 8 | Globe响应式缩放| | Canvas resize 事件绑定，vh 单位动态计算|
+| 9 | Globe ?data= | | 异常安全：try/except 包裹，空数据仍渲染页面|
+| 10 | 文件上传无限| | DATA_UPLOAD_MAX_NUMBER_FIELDS=1000000, FILE_UPLOAD_MAX_MEMORY_SIZE=100MB |
+| 11 | 上传页删除功能| | removeFile() + 按钮每文件|
+| 12 | LLM日志 | | progress_callback 写入 run_run_log |
+| 13 | 聚类图表 | | cluster_overview.html 含散点图/柱状态Silhouette |特征纯Canvas) |
+| 14 | 低内存不崩溃 | | MiniBatchKMeans + PCA降维 + 降采样head 参数 |
+| 15 | +号字符串识别为数| | type_classifier.py FLOAT检测前排除 `+` 前缀 |
+| 16 | 空闲高IO | | 日志级别 INFO，无需修改 |
+| 17 | 聚类效果 | ⚠️ | 预置了特征过滤降维，但具体效果需业务验证 |
+| 18 | 列名精确匹配 | | `re.match` 精确匹配 + `_find_column` tuple 关键词|
 
 ## v7 Final Verification (2026-07-20)
 
 All **19 issues** fixed and verified. 92/92 unit tests passing. Full regression run completed.
 
-| # | 问题 | 状态 | 详情 |
+| # | 问题 | 状态| 详情 |
 |---|------|------|------|
-| 1 | `handle_call` imported in llm_orchestrator.py | ✅ | `from analysis.tool_registry import handle_call` at line 7 |
-| 2 | Retry button + run_type filter | ✅ | Retry form in `run_list.html` (line 51), run_type dropdown filter, URL route `retry_run` added |
-| 3 | auto.html logs show tool+result | ✅ | `liveLog` polling every 3s from `/logs/?pos=` endpoint |
-| 4 | Data saved to SQLite tables | ✅ | `sqlite_table` field on `AnalysisRun` model, `drop_sqlite_table` in data_loader |
-| 5 | Error toast in base.html | ✅ | Complete toast system with error/success types, auto-dismiss, slide animations |
-| 6 | display_id in all URLs | ✅ | All 6 URL patterns (`run_detail`, `run_status`, `cluster_overview`, `cluster_detail`, `retry_run`, `delete_upload`) use `display_id` |
-| 7 | Delete buttons on dashboard+run_list | ✅ | Both pages have styled delete buttons with confirmation dialog |
-| 8 | Progress bar 100% on completion | ✅ | `progress_pct` field updated through pipeline stages (10/30/50/70/90/100) |
-| 9 | LLM thinking panel in auto.html | ✅ | `thinkingPanel` div with `llm_thinking` text area and step indicator |
-| 10 | Tool call accordion in auto.html | ✅ | `renderToolCalls()` with toggle-able accordion sections per tool step |
-| 11 | LazyFrame.sample() fixed | ✅ | Uses `lf.sample(n=...)` with explicit row count parameter (not deprecated `.sample()` API) |
-| 12 | `filter_and_cluster` tool exists | ✅ | Registered at line 407 in `tool_registry.py` as tool #13 |
-| 13 | Tool descriptions in Chinese | ✅ | All tool descriptions and parameter docs use Chinese |
-| 14 | Globe: lines, timestamp, inertia, polar flip | ✅ | Flow arcs (Line geometry), timestamp-based dot animation, quaternion momentum decay, camera-relative rotation (no polar flip) |
-| 15 | Config button in nav | ✅ | `{% url 'analysis:config' %}` link 配置 in base.html nav bar |
-| 16 | config host:port affects broadcast | ✅ | `settings.py` reads `_cfg.server.host` → auto-adds to `ALLOWED_HOSTS` |
-| 17 | SQLite timeout+retry mechanism | ✅ | `retry_on_lock` decorator in `db_utils.py`: 3 retries, exponential backoff, logs `[DB_LOCKED]` warning |
-| 18 | Delete crash fixed (null csv_glob) | ✅ | `delete_upload` view checks `if run.csv_glob:` before accessing path (line 366) |
-| 19 | No file count limit, streaming upload | ✅ | `DATA_UPLOAD_MAX_NUMBER_FIELDS=10000000`, `FILE_UPLOAD_MAX_MEMORY_SIZE=100MB` |
+| 1 | `handle_call` imported in llm_orchestrator.py | | `from analysis.tool_registry import handle_call` at line 7 |
+| 2 | Retry button + run_type filter | | Retry form in `run_list.html` (line 51), run_type dropdown filter, URL route `retry_run` added |
+| 3 | auto.html logs show tool+result | | `liveLog` polling every 3s from `/logs/?pos=` endpoint |
+| 4 | Data saved to SQLite tables | | `sqlite_table` field on `AnalysisRun` model, `drop_sqlite_table` in data_loader |
+| 5 | Error toast in base.html | | Complete toast system with error/success types, auto-dismiss, slide animations |
+| 6 | display_id in all URLs | | All 6 URL patterns (`run_detail`, `run_status`, `cluster_overview`, `cluster_detail`, `retry_run`, `delete_upload`) use `display_id` |
+| 7 | Delete buttons on dashboard+run_list | | Both pages have styled delete buttons with confirmation dialog |
+| 8 | Progress bar 100% on completion | | `progress_pct` field updated through pipeline stages (10/30/50/70/90/100) |
+| 9 | LLM thinking panel in auto.html | | `thinkingPanel` div with `llm_thinking` text area and step indicator |
+| 10 | Tool call accordion in auto.html | | `renderToolCalls()` with toggle-able accordion sections per tool step |
+| 11 | LazyFrame.sample() fixed | | Uses `lf.sample(n=...)` with explicit row count parameter (not deprecated `.sample()` API) |
+| 12 | `filter_and_cluster` tool exists | | Registered at line 407 in `tool_registry.py` as tool #13 |
+| 13 | Tool descriptions in Chinese | | All tool descriptions and parameter docs use Chinese |
+| 14 | Globe: lines, timestamp, inertia, polar flip | | Flow arcs (Line geometry), timestamp-based dot animation, quaternion momentum decay, camera-relative rotation (no polar flip) |
+| 15 | Config button in nav | | `{% url 'analysis:config' %}` link 配置 in base.html nav bar |
+| 16 | config host:port affects broadcast | | `settings.py` reads `_cfg.server.host` - auto-adds to `ALLOWED_HOSTS` |
+| 17 | SQLite timeout+retry mechanism | | `retry_on_lock` decorator in `db_utils.py`: 3 retries, exponential backoff, logs `[DB_LOCKED]` warning |
+| 18 | Delete crash fixed (null csv_glob) | | `delete_upload` view checks `if run.csv_glob:` before accessing path (line 366) |
+| 19 | No file count limit, streaming upload | | `DATA_UPLOAD_MAX_NUMBER_FIELDS=10000000`, `FILE_UPLOAD_MAX_MEMORY_SIZE=100MB` |
 
 ### Remaining issues
-- `test_e2e.py`, `test_entity.py`, `test_pipeline.py` still reference removed `entity_detector` module — need updating for new clustering pipeline
+- `test_e2e.py`, `test_entity.py`, `test_pipeline.py` still reference removed `entity_detector` module - need updating for new clustering pipeline
 - `test_clustering_edge.py` does not exist on disk (only referenced in AGENTS.md)
-- `retry_run` URL route was missing from `urls.py` — fixed during final verification
+- `retry_run` URL route was missing from `urls.py` - fixed during final verification
 
 ### Test Results (2026-07-20)
 | Test suite | Tests | Result |
 |------------|-------|--------|
-| `test_data_loader.py` | 28 | ✅ All passed |
-| `test_type_classifier.py` | 64 | ✅ All passed |
-| **Total** | **92** | **✅ All passed** |
+| `test_data_loader.py` | 28 | All passed |
+| `test_type_classifier.py` | 64 | All passed |
+| **Total** | **92** | ** All passed** |
 
 ## v8 Fix (2026-07-20)
 
-| # | 问题 | 状态 |
+| # | 问题 | 状态|
 |---|------|------|
-| 1 | 服务器启动config模块找不到 | ✅ 修复 |
-| 2 | 首次启动无数据库表 | ✅ 修复 |
-| 3 | CSV中'+'无法解析为f64 | ✅ 修复 |
-| 4 | 上传分块处理+释放临时文件 | ✅ 修复 |
-| 5 | 集成测试脚本 | ✅ 添加 |
-| 6 | TlsDB.csv文档说明 | ✅ 更新 |
-| 7 | load_from_db类型冲突 | ✅ 修复 |
-| 8 | CSV未合并处理 | ✅ 修复 |
+| 1 | 服务器启动config模块找不到| 修复 |
+| 2 | 首次启动无数据库| 修复 |
+| 3 | CSV中+'无法解析为f64 | 修复 |
+| 4 | 上传分块处理+释放临时文件 | 修复 |
+| 5 | 集成测试脚本 | 添加 |
+| 6 | TlsDB.csv文档说明 | 更新 |
+| 7 | load_from_db类型冲突 | 修复 |
+| 8 | CSV未合并处理| 修复 |
 
 ### Final Verification (2026-07-20)
 | Check | Result |
 |-------|--------|
-| `manage.py check` (system) | ✅ 0 errors |
-| Unit tests (data_loader + type_classifier) | ✅ 92 passed |
-| Integration test (full upload→load→analyze) | ✅ ALL INTEGRATION TESTS PASSED |
-| Git commit `v8` | ✅ `555ec29` |
+| `manage.py check` (system) | 0 errors |
+| Unit tests (data_loader + type_classifier) | 92 passed |
+| Integration test (full upload→load→analyze) | ALL INTEGRATION TESTS PASSED |
+| Git commit `v8` | `555ec29` |
 
 ## MCP 工具系统 (2026-07-19)
 
-系统现有 **27 个 MCP 工具**，分为三层：
+系统现有 **27 |MCP 工具**，分为三层：
 
 ### 工具架构
 
-| 层 | 数量 | 标签 | 用途 |
+| | 数量 | 标签 | 用|
 |---|------|------|------|
-| Core | 7 | ⚡ | 执行分析，可修改数据 |
-| Analysis | 5 | 🔍 | 只读深潜，安全随时调用 |
+| Core | 7 | | 执行分析，可修改数据 |
+| Analysis | 5 | 🔍 | 只读深潜，安全随时调用|
 | Diagnostic | 7 | 🩺 | 排查问题 |
-| 其他 | 8 | — | 数据加载/过滤/导出等 |
+| 其他 | 8 | | 数据加载/过滤/导出|
 
-### 手动分析工作台
-
-手动分析页面已改为**工作流构建器**：
+### 手动分析工作流
+手动分析页面已改为 **工作流构建器**:
 - 添加/移除/重排步骤
 - 每步选择工具 + 填写参数
-- **保存/加载/删除**方案（存为 `.omo/plans/<name>.json`）
-- 单步执行或 **▶▶ 全部执行**（自动串行）
-- 执行结果显示在每一步下方
-
+- **保存/加载/删除**方案（存储`.omo/plans/<name>.json`)- 单步执行 |**▶▶ 全部执行**（自动串行）
+- 执行结果显示在每一步下面
 ### LLM 自动编排
 
 LLM orchestrator 重写，支持：
-- 策略式系统提示词（先 profile → 根据数据决策 → 失败自动诊断）
-- 重复调用检测 + 自动错误恢复
-- 27 个工具全部可选
-- 中文总结分析结果
+- 策略式系统提示词（先 profile 根据数据决策 失败自动诊断- 重复调用检测+ 自动错误恢复
+- 27 个工具全部可用- 中文总结分析结果
 
 ### 全链路测试
-
 | 测试 | 用例 | 结果 |
 |------|------|------|
-| 单元测试 | 120 | ✅ |
-| 管道测试 | 8 (3 datasets × 2 algos + edge) | ✅ |
-| CLI pipeline | e2e 5 stages | ✅ |
-| MCP 工具集成 | 10 (profile/patterns/validate/tls/geo/build/scores/detect/detail/visualize) | ✅ |
+| 单元测试 | 120 | |
+| 管道测试 | 8 (3 datasets × 2 algos + edge) | |
+| CLI pipeline | e2e 5 stages | |
+| MCP 工具集成 | 10 (profile/patterns/validate/tls/geo/build/scores/detect/detail/visualize) | |
 
 ## v3 修复 (2026-07-17)
 
-| # | 问题 | 状态 | 详情 |
+| # | 问题 | 状态| 详情 |
 |---|------|------|------|
-| 1 | 经纬度列名匹配(`:ips.latd`) | ✅ | views.py直接字符串匹配 + entity_aggregator.py _LAT_KEYWORDS增加.latd后缀 |
-| 2 | cnrs/isrs布尔类型 | ✅ | entity_aggregator.py聚合前将"+"→True/空白→False |
-| 3 | 聚类图表（legend/柱状图/负值） | ✅ | 纯Canvas：legend用HTML div+overflow、柱状图动态带宽、负值标签重新定位 |
-| 4 | 地球缩放改变数据流大小 | ✅ | Three.js arc sphere尺寸与camera距离反比，zoom时视觉大小不变 |
-| 5 | 恶意/代理流量分析 | ✅ | entity_aggregator.py规则引擎：non_std_port_rate + modern_tls_rate + sni_missing_ratio + multi_country → proxy_score |
-| 6 | EntityProfile批量IO | ✅ | views.py + tool_registry.py 改为 bulk_create(ignore_conflicts=True, batch_size=1000) |
-| 7 | 重试分析 | 🗑️ | 已移除 (v7 新架构不再需要 entity_column 概念) |
-| 8 | D5降采样导致特征提取行数不匹配 | ✅ | _handle_run_clustering存储_ds_idx抽样索引，_handle_extract_features按索引过滤数据 |
+| 1 | 经纬度列名匹配`:ips.latd`) | | views.py直接字符串匹配+ entity_aggregator.py _LAT_KEYWORDS增加.latd后缀 |
+| 2 | cnrs/isrs布尔类型 | | entity_aggregator.py聚合前将"+"→True/空白→False |
+| 3 | 聚类图表（legend/柱状态负值） | | 纯Canvas：legend用HTML div+overflow、柱状图动态带宽、负值标签重新定义|
+| 4 | 地球缩放改变数据流大小| | Three.js arc sphere尺寸与camera距离反比，zoom时视觉大小不变|
+| 5 | 恶意/代理流量分析 | | entity_aggregator.py规则引擎：non_std_port_rate + modern_tls_rate + sni_missing_ratio + multi_country + proxy_score |
+| 6 | EntityProfile批量IO | | views.py + tool_registry.py 改为 bulk_create(ignore_conflicts=True, batch_size=1000) |
+| 7 | 重试分析 | 🗑| 已移除(v7 新架构不再需要entity_column 概念) |
+| 8 | D5降采样导致特征提取行数不匹配 | | _handle_run_clustering存储_ds_idx抽样索引，_handle_extract_features按索引过滤数据|
 
 ## v4 修复 (2026-07-17)
 
-| # | 问题 | 状态 | 详情 |
+| # | 问题 | 状态| 详情 |
 |---|------|------|------|
-| 1 | 进度条位置 | ✅ | DOM重排: submitBar → progressArea → fileList |
-| 2 | PCA sklearn RuntimeWarning | ✅ | `warnings.filterwarnings('ignore', category=RuntimeWarning, module='sklearn')` |
-| 3 | cluster_overview N+1查询 | ✅ | `prefetch_related('features')` + `.only()`字段限制 |
-| 4 | SQLite PRAGMA未优化 | ✅ | WAL + synchronous=NORMAL + cache_size=20MB + temp_store=MEMORY + mmap_size=3GB |
-| 5 | 零方差过滤崩溃 | ✅ | 全零方差时回退`np.arange(data.shape[1])`保留原始特征 |
+| 1 | 进度条位置| | DOM重排: submitBar → progressArea → fileList progressArea → fileList |
+| 2 | PCA sklearn RuntimeWarning | | `warnings.filterwarnings('ignore', category=RuntimeWarning, module='sklearn')` |
+| 3 | cluster_overview N+1查询 | | `prefetch_related('features')` + `.only()`字段限制 |
+| 4 | SQLite PRAGMA未优| | WAL + synchronous=NORMAL + cache_size=20MB + temp_store=MEMORY + mmap_size=3GB |
+| 5 | 零方差过滤崩| | 全零方差时回退`np.arange(data.shape[1])`保留原始特征 |
 
-### 大规模测试结果 (2026-07-17)
+### 大规模测试结(2026-07-17)
 
 | 测试 | 规模 | 结果 |
 |------|------|------|
-| 单元测试 | 120 tests | ✅ 全部通过 |
-| 管道测试 | 3 datasets × 2 algorithms | ✅ 8 tests OK |
-| 中等规模 | **100文件 × 10000行 = 1M行, 825MB** | ✅ 全部完成: 加载→实体检测→聚合→聚类(2285簇)→特征提取 |
+| 单元测试 | 120 tests | 全部通过 |
+| 管道测试 | 3 datasets × 2 algorithms | 8 tests OK |
+| 中等规模 | **100文件 × 10000行= 1M行 825MB** | 全部完成: 加载→实体检测→聚合→聚2285个实体→特征提取|
 
 ## Cleanup (2026-07-16)
 
-对 v6 迭代引入的垃圾文件和代码破坏进行全面清理。
-
+对 v6 迭代引入的垃圾文件和代码破坏进行全面清理
 | 操作 | 文件 |
 |------|------|
-| 🗑️ 删除 | 15个垃圾脚本 (wnl_to_user_csv, eval_clustering, debug_db2/3, debug_persistence, test_orch/orch2, test_pca, test_profile, test_tool_loop/format, test_e2e_full, test_32b, test_chinese_path, verify_runtime) |
-| 🗑️ 删除 | data/wnl_converted.csv, data/globe_test.csv |
-| 🗑️ 删除 | analysis/value_normalizer.py + 关联测试 + data_loader/entity_aggregator中的_norm引用 |
-| 🗑️ 删除 | .omo/plans/ 旧计划文件 (tianxiu-v4/v5/v6, globe-v6) |
-| 🔧 重构 | views.py: 统一 _run_analysis_worker 和 _run_llm_analysis 为 _run_pipeline_worker |
+| 🗑删除 | 15个垃圾脚(wnl_to_user_csv, eval_clustering, debug_db2/3, debug_persistence, test_orch/orch2, test_pca, test_profile, test_tool_loop/format, test_e2e_full, test_32b, test_chinese_path, verify_runtime) |
+| 🗑删除 | data/wnl_converted.csv, data/globe_test.csv |
+| 🗑删除 | analysis/value_normalizer.py + 关联测试 + data_loader/entity_aggregator中的_norm引用 |
+| 🗑删除 | .omo/plans/ 旧计划文(tianxiu-v4/v5/v6, globe-v6) |
+| 🔧 重构 | views.py: 统一 _run_analysis_worker → _run_llm_analysis → _run_pipeline_worker |
 | 🔧 修复 | settings.py: FILE_UPLOAD_MAX_MEMORY_SIZE 10MB→100MB |
-| 🔧 修复 | upload.html: 添加每文件 ✕ 删除按钮 + removeFile() |
+| 🔧 修复 | upload.html: 添加每文件删除按钮 + removeFile() |
 | 🔧 修复 | tool_registry.py: async函数中的同步ORM调用加sync_to_async包装 |
 | 🔧 修复 | run_pipeline.py: 改为调用 _run_pipeline_worker |
 | 🔧 替换 | gen_test_data.py: 3画像版→简单随机生成版 |
-| ✅ 验证 | 120/120 测试通过, 4次端到端管道测试全部成功 |
+| 验证 | 120/120 测试通过, 4次端到端管道测试全部成功 |
 
-### v6 (2026-07-16) — 已清理（不再维护）
-
-> ⚠️ v6 引入的 value_normalizer 模块已被删除，_norm 列全部移除。
-> entity_aggregator 回退到使用原始列名（0ver/0cph/cipher-suite等）。
-
-| 模块 | 变动 | 当前状态 |
+### v6 (2026-07-16) 已清理（不再维护
+> ⚠️ v6 引入value_normalizer 模块已被删除，_norm 列全部移除> entity_aggregator 回退到使用原始列名（0ver/0cph/cipher-suite等）
+| 模块 | 变动 | 当前状态|
 |------|------|---------|
-| value_normalizer.py | ~~新增: hex→enum归一化~~ | 🗑️ 已删除 |
+| value_normalizer.py | ~~新增: hex→enum归一化~~ | 🗑已删|
 | data_loader.py | ~~集成normalize_lf~~ | 🔧 恢复原始 |
 | entity_aggregator.py | ~~_norm列依赖~~ | 🔧 回退原始列名 |
-| gen_test_data.py | ~~3类流量画像~~ | 🔧 简单随机生成 |
-| 特征维度 | 20→33维 | 特征维度保留（33维），去掉_norm列后的纯原始列特征 |
+| gen_test_data.py | ~~3类流量画像~~ | 🔧 简单随机生|
+| 特征维度 | 20→3| 特征维度保留3维），去掉_norm列后的纯原始列特征|
 
 ## Completed (2026-07-16)
 
-18个问题全部修复，3次端到端管道测试全部通过（简单CSV、用户自定义列名CSV、大规模CSV）。
-
-| 工作流 | 问题 | 状态 |
+18个问题全部修复，3次端到端管道测试全部通过（简单CSV、用户自定义列名CSV、大规模CSV）
+| 工作流| 问题 | 状态|
 |--------|------|------|
-| A1 | 重启后旧数据可分析 | ✅ 目录不存在时优雅降级，标记为failed |
-| A2 | Web图标所有页面显示 | ✅ 创建favicon.svg + base.html添加link |
-| A3 | 上传按钮在顶部 | ✅ 移至文件列表上方，sticky定位 |
-| A10 | 文件上传限制 | ✅ 去除了Django字段限制，增大至256MB/50GB |
-| A11 | 上传页删除功能 | ✅ 所有状态显示删除按钮，处理中弹出确认 |
-| B7 | TLS 0ver hex格式 | ✅ 添加TLS_HEX_MAP，识别03 03/03 04 |
-| B15 | "+"字符串识别为数字 | ✅ FLOAT检测添加+/前缀检查 |
-| B18 | 列名映射 | ✅ 使用精确^exact_name$匹配，无模糊猜测 |
-| C4 | Traceback截断 | ✅ tool_registry.py移除[:200] |
-| D5 | 大数据量卡顿 | ✅ 添加head参数、50K降采样、MAX_ROWS=500 |
-| D6 | 进度条 | ✅ 添加progress_pct/progress_msg字段 |
-| D14 | 低内存崩溃 | ✅ MiniBatchKMeans、PCA降维、内存检查 |
-| D16 | 空闲高IO | ✅ 日志级别已为INFO，无需修改 |
-| E8 | Globe响应式缩放 | ✅ 动态H计算、vh单位、resize处理 |
-| E9 | Globe ?data=参数 | ✅ 支持从SessionStore直接加载数据集 |
-| F12 | LLM日志 | ✅ 添加callback机制、run_log字段 |
-| G13 | 聚类图表 | ✅ 添加簇大小柱状图、Silhouette对比图 |
-| H17 | 聚类质量 | ✅ 方差过滤、相关过滤、HDBSCAN自动调参 |
+| A1 | 重启后旧数据可分析| 目录不存在时优雅降级，标记为failed |
+| A2 | Web图标所有页面显示| 创建favicon.svg + base.html添加link |
+| A3 | 上传按钮在顶部| 移至文件列表上方，sticky定位 |
+| A10 | 文件上传限制 | 去除了Django字段限制，增大至256MB/50GB |
+| A11 | 上传页删除功能| 所有状态显示删除按钮，处理中弹出确认|
+| B7 | TLS 0ver hex格式 | 添加TLS_HEX_MAP，识别3 03/03 04 |
+| B15 | "+"字符串识别为数字 | FLOAT检测添加/前缀检查|
+| B18 | 列名映射 | 使用精确^exact_name$匹配，无模糊猜测 |
+| C4 | Traceback截断 | tool_registry.py移除[:200] |
+| D5 | 大数据量卡顿 | 添加head参数10K降采样、MAX_ROWS=500 |
+| D6 | 进度条| 添加progress_pct/progress_msg字段 |
+| D14 | 低内存崩溃| MiniBatchKMeans、PCA降维、内存检测|
+| D16 | 空闲高IO | 日志级别已为INFO，无需修改 |
+| E8 | Globe响应式缩放| 动态H计算、vh单位、resize处理 |
+| E9 | Globe ?data=参数 | 支持从SessionStore直接加载数据|
+| F12 | LLM日志 | 添加callback机制、run_log字段 |
+| G13 | 聚类图表 | 添加簇大小柱状图、Silhouette对比|
+| H17 | 聚类质量 | 方差过滤、相关过滤、HDBSCAN自动调参 |
+
+## v1.1.6 (2026-07-22)
+
+Cleanup + minor fixes release.
+
+| # | 问题 | 状态 | 描述 |
+|---|------|------|------|
+| 1 | schema sort fix | 修复 | 排序逻辑修改，确保最后一行被包含 |
+| 2 | batch upload | 修复 | 批量上传流程加固 |
+| 3 | temp dir to D: | 修复 | 临时目录移到 D: 盘避免 C: 盘空间占用 |
+| 4 | update.bat wildcard fix | 修复 | 通配符匹配优化使更新更可靠 |
+| 5 | cleanup test artifacts | 修复 | 删除 test CSV data, temp scripts, gen_fast.py |
+| 6 | VERSION sync | 修复 | VERSION 文件已更新为 v1.1.6 |
+
+### Pending
+- tests/ 需要更新 entity_detector 相关
+- 大文件上传存在内存泄漏需要进一步排查

TlsDB.csv

@@ -21,9 +21,8 @@ scnt,源国家,2小写字母缩写,100%
 dcnt,目标国家,2小写字母缩写,100%
 server-ip,服务器IP,IPv4,100%
 client-ip,客户端IP,IPv4,100%
-row,数据列,整数,100%
-time,时间,XX:YY.Z,100%
-timestamp,时间戳,整数,100%
+time,时间,202X-XX-XX XX:XX:XX.XXXXXX,100%
+timestamp,时间戳,六位小数浮点,100%
 1ipp,IP协议编号,整数,100%
 4dbn,数据库编号,整数,100%
 tabl,数据库表头名,"TlsC"/"TlsS",100%
@@ -40,7 +39,7 @@ name,链路名称,枚举,100%
 source-node,源节点,枚举,100%
 cipher-suite,加密套件,诸如"TLS_ECHDE_ECDSA_WITH_AES_256_GCM_SHA384"的枚举,13.90%
 ecdhe-named-curve,ecdhe使用的圆锥曲线名称,secpXXXr1(其中XXX为三位数字),10.73%
-0cph,TLS加密套件,2字节,54.14%
+0cph,TLS加密套件,2字节,52.14%
 0crv,TLS圆锥曲线名称,2字节,12.45%
 0rnd,TLS随机值,28字节,99.14%
 0rnt,TLS随机时间,4字节,100%

_restart_server.py

@@ -0,0 +1,32 @@
+import subprocess, time, urllib.request, os, sys
+
+PROJ_DIR = r"C:\Users\25044\Desktop\Proj\天璇"
+PYTHON_EXE = os.path.join(PROJ_DIR, "runtime", "python", "python.exe")
+os.chdir(PROJ_DIR)
+
+server = subprocess.Popen(
+    [PYTHON_EXE, "manage.py", "runserver", "127.0.0.1:8765", "--noreload"],
+    stdout=subprocess.DEVNULL,
+    stderr=subprocess.DEVNULL
+)
+
+print(f"SERVER_PID={server.pid}")
+
+# Poll with backoff: fast early, then slower
+deadline = time.time() + 60
+attempt = 0
+while time.time() < deadline:
+    attempt += 1
+    try:
+        resp = urllib.request.urlopen("http://127.0.0.1:8765/", timeout=3)
+        if resp.status == 200:
+            print("SERVER_READY")
+            sys.exit(0)
+    except Exception as e:
+        if attempt % 10 == 0:
+            print(f"WAITING... attempt={attempt} err={type(e).__name__}")
+        time.sleep(1)
+
+print("SERVER_START_TIMEOUT")
+server.kill()
+sys.exit(1)

analysis/tool_registry.py

@@ -1168,12 +1168,34 @@ async def _handle_run_clustering(
                 if dt in numeric_dtypes and not available_names[i].startswith('_')
             ][:10]
         if not feature_cols:
-            all_numeric = [f"{available_names[i]}" for i, dt in enumerate(available_dtypes)
-                           if dt in numeric_dtypes]
-            return {
-                'error': f'No numeric cluster columns found in dataset. Available: {all_numeric}',
-                'truncated': False
-            }
+            # Fallback: try coercing Utf8 columns to Float64 (common when data loaded from
+            # SQLite TEXT storage without proper dtype restoration).
+            utf8_cols = [
+                available_names[i] for i, dt in enumerate(available_dtypes)
+                if dt == pl.Utf8 and not available_names[i].startswith('_')
+            ][:10]
+            if utf8_cols:
+                # Attempt per-column cast; filter to columns where coercion succeeded
+                for col in utf8_cols:
+                    try:
+                        casted = lf.select(
+                            pl.col(col).cast(pl.Float64, strict=False).alias(col)
+                        ).collect(streaming=True)
+                        null_ratio = casted[col].is_null().sum() / max(len(casted), 1)
+                        if null_ratio < 0.9:  # at least 10% successfully parsed as floats
+                            lf = lf.with_columns(
+                                pl.col(col).cast(pl.Float64, strict=False)
+                            )
+                            feature_cols.append(col)
+                    except Exception:
+                        pass
+            if not feature_cols:
+                all_numeric = [f"{available_names[i]}" for i, dt in enumerate(available_dtypes)
+                               if dt in numeric_dtypes]
+                return {
+                    'error': f'No numeric cluster columns found in dataset. Available: {all_numeric}',
+                    'truncated': False
+                }
 
         # Downsample before collect: compute row count via fast lazy count
         try:
@@ -1257,7 +1279,9 @@ async def _handle_run_clustering(
 
         # Standard scale
         from sklearn.preprocessing import StandardScaler
+        _clust_logger.info('[CLUSTER_SCALE] Starting StandardScaler...')
         data_scaled = StandardScaler().fit_transform(data)
+        _clust_logger.info(f'[CLUSTER_SCALE] Done. shape={data_scaled.shape}')
 
         # ── H17: Correlation filtering ────────────────────────────────────
         # Remove features with Pearson correlation > 0.95 (keep the first one)

analysis/urls.py

@@ -6,6 +6,7 @@ urlpatterns = [
     path('', views.dashboard, name='dashboard'),
     path('upload/', views.upload_page, name='upload'),
     path('upload/csv/', views.upload_csv, name='upload_csv'),
+    path('upload/csv/batch/<int:display_id>/', views.upload_csv_batch, name='upload_csv_batch'),
     path('analyze/manual/', views.manual_page, name='manual'),
     path('analyze/run/', views.manual_run_analysis, name='run_analysis'),
     path('analyze/auto/', views.auto_page, name='auto'),

analysis/views.py

@@ -339,6 +339,27 @@ def upload_csv(request):
     return JsonResponse({'run_id': run.display_id, 'files': saved, 'status': 'loading'})
 
 
+@csrf_exempt
+def upload_csv_batch(request, display_id):
+    """Append files to existing upload run. Does NOT re-trigger _background_process."""
+    run = get_object_or_404(AnalysisRun, display_id=display_id)
+    files = request.FILES.getlist('files')
+    if not files:
+        return JsonResponse({'error': 'No files uploaded'})
+    upload_dir = Path(run.csv_glob).parent if run.csv_glob else None
+    if not upload_dir or not upload_dir.exists():
+        return JsonResponse({'error': 'Upload directory not found'})
+    saved = []
+    for f in files:
+        path = upload_dir / f.name
+        with open(path, 'wb+') as dst:
+            for chunk in f.chunks():
+                dst.write(chunk)
+        saved.append(str(path))
+    # NO background process re-trigger — first batch handles it
+    return JsonResponse({'run_id': run.display_id, 'files': len(saved), 'status': 'ok'})
+
+
 @csrf_exempt
 def delete_upload(request, display_id):
     """Delete uploaded files, SQLite data table, and AnalysisRun.

data/multi_upload/upload_script.py

@@ -1,42 +0,0 @@
-import os, sys, django, urllib.request, glob, uuid, json, time
-os.environ['DJANGO_SETTINGS_MODULE'] = 'tianxuan.settings'
-sys.path.insert(0, r'C:\Users\25044\Desktop\Proj\天璇')
-import django; django.setup()
-from analysis.models import AnalysisRun
-AnalysisRun.objects.all().delete()
-print('Cleaned old runs')
-
-BASE = 'http://127.0.0.1:18766'
-data_dir = r'C:\Users\25044\Desktop\Proj\天璇\data\multi_upload'
-files = sorted(glob.glob(os.path.join(data_dir, '*.csv')))
-boundary = uuid.uuid4().hex
-lines = []
-for fp in files:
-    with open(fp, 'rb') as f: d = f.read()
-    fname = os.path.basename(fp)
-    lines.append(f'--{boundary}'.encode())
-    lines.append(f'Content-Disposition: form-data; name="files"; filename="{fname}"'.encode())
-    lines.append(b'Content-Type: text/csv'); lines.append(b''); lines.append(d)
-lines.append(f'--{boundary}--'.encode())
-req = urllib.request.Request(f'{BASE}/upload/csv/', data=b'\r\n'.join(lines),
-    headers={'Content-Type': f'multipart/form-data; boundary={boundary}'})
-r = urllib.request.urlopen(req, timeout=300)
-resp = json.loads(r.read()); r.close()
-print(f'Uploaded run_id={resp["run_id"]}')
-
-run_id = resp['run_id']
-for i in range(120):
-    time.sleep(2)
-    try:
-        req2 = urllib.request.Request(f'{BASE}/runs/{run_id}/status/')
-        r2 = urllib.request.urlopen(req2, timeout=10)
-        status_resp = json.loads(r2.read()); r2.close()
-        status = status_resp.get('status', status_resp.get('state', ''))
-        print(f'  attempt {i+1}: status={status}')
-        if status == 'ready':
-            print(f'Run {run_id} is ready!')
-            break
-    except Exception as e:
-        print(f'  attempt {i+1}: error={e}')
-else:
-    print('Timed out waiting for ready status')

scripts/gen_multi.py

@@ -29,6 +29,7 @@ def main():
     parser = argparse.ArgumentParser(description='Generate multi-file test CSV dataset')
     parser.add_argument('--files', type=int, default=100, help='Number of files to generate')
     parser.add_argument('--rows', type=int, default=100, help='Rows per file')
+    parser.add_argument('--start', type=int, default=0, help='Start file index (for resume)')
     args = parser.parse_args()
 
     # Use a different seed from gen_test_data so IP pools differ
@@ -42,9 +43,10 @@ def main():
     _droppable = [c for c in COLUMNS if c not in _keep_cols]
 
     base_time = datetime(2026, 6, 1, 0, 0, 0)
-    total_rows = 0
+    # Resume support: start from --start, estimate prior rows
+    total_rows = args.start * args.rows if args.start > 0 else 0
 
-    for fi in range(args.files):
+    for fi in range(args.start, args.files):
         # Coverage-based column dropping: columns with lower coverage are more likely dropped.
         # Drop a column with probability (1 - coverage), capped to [0.10, 0.90] range.
         dropped = set()
@@ -67,7 +69,7 @@ def main():
             dropped -= spared
         file_columns = [c for c in COLUMNS if c not in dropped]
 
-        out_path = out_dir / f'test_{fi:04d}.csv'
+        out_path = out_dir / f'{fi}.csv'
         n_rows = random.randint(max(50, args.rows - 10), args.rows + 10)
 
         with open(out_path, 'w', newline='', encoding='utf-8') as f:

start_server.py

@@ -0,0 +1,16 @@
+import subprocess, os, time, urllib.request
+
+os.chdir(r"C:\Users\25044\Desktop\Proj\天璇")
+
+server = subprocess.Popen(
+    [r"runtime\python\python.exe", "manage.py", "runserver", "127.0.0.1:8765", "--noreload"],
+    stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+)
+
+time.sleep(4)
+
+try:
+    urllib.request.urlopen("http://127.0.0.1:8765/", timeout=5)
+    print("SERVER_UP")
+except Exception as e:
+    print("SERVER_DOWN:", e)

templates/tianxuan/upload.html

@@ -102,30 +102,52 @@ function renderFileList() {
 
 document.getElementById('uploadForm').addEventListener('submit', async (e) => {
     e.preventDefault();
-    const form = new FormData();
-    selectedFiles.forEach(f => form.append('files', f));
+    const BATCH = 5;
+    const batches = [];
+    for (let i = 0; i < selectedFiles.length; i += BATCH) {
+        batches.push(selectedFiles.slice(i, i + BATCH));
+    }
     document.getElementById('progressArea').style.display = 'block';
     submitBtn.disabled = true;
-    try {
-        const resp = await fetch('/upload/csv/', { method: 'POST', body: form,
-            headers: { 'X-CSRFToken': document.querySelector('[name=csrfmiddlewaretoken]').value } });
-        const data = await resp.json();
-        document.getElementById('progressText').textContent = '上传完成，后台处理中...';
-        // Poll for completion
-        const runId = data.run_id;
-        const poll = setInterval(async () => {
-            try {
-                const r = await fetch(`/runs/${runId}/status/`);
-                const s = await r.json();
-                const pct = s.progress_pct || 0;
-                const msg = s.progress_msg || s.status;
-                document.getElementById('progressBar').style.width = pct + '%';
-                document.getElementById('progressText').textContent = `${msg} (${pct}%)`;
-                if (s.status === 'ready' || s.status === 'completed') { clearInterval(poll); window.location.href = '/analyze/manual/?dataset=upload_' + runId; }
-                if (s.status === 'failed') { clearInterval(poll); document.getElementById('progressText').textContent = '处理失败: ' + (s.error_message || '未知错误'); }
-            } catch (e) { showError('轮询状态失败: ' + e.message); }
-        }, 1000);
-    } catch (err) { document.getElementById('progressText').textContent = '上传失败: ' + err.message; submitBtn.disabled = false; }
+
+    const csrfToken = document.querySelector('[name=csrfmiddlewaretoken]').value;
+    let runId = null;
+    for (let bi = 0; bi < batches.length; bi++) {
+        const form = new FormData();
+        batches[bi].forEach(f => form.append('files', f));
+        const url = runId ? `/upload/csv/batch/${runId}/` : '/upload/csv/';
+        try {
+            const resp = await fetch(url, { method: 'POST', body: form,
+                headers: { 'X-CSRFToken': csrfToken } });
+            const data = await resp.json();
+            if (data.error) {
+                document.getElementById('progressText').textContent = '上传失败: ' + data.error;
+                submitBtn.disabled = false;
+                return;
+            }
+            runId = runId || data.run_id;
+            document.getElementById('progressBar').style.width = ((bi+1)/batches.length*100) + '%';
+            document.getElementById('progressText').textContent = `上传批次 ${bi+1}/${batches.length} (run #${runId})`;
+        } catch (err) {
+            document.getElementById('progressText').textContent = '上传失败: ' + err.message;
+            submitBtn.disabled = false;
+            return;
+        }
+    }
+    // Upload complete — poll for background processing
+    document.getElementById('progressText').textContent = '上传完成，后台处理中...';
+    const poll = setInterval(async () => {
+        try {
+            const r = await fetch(`/runs/${runId}/status/`);
+            const s = await r.json();
+            const pct = s.progress_pct || 0;
+            const msg = s.progress_msg || s.status;
+            document.getElementById('progressBar').style.width = pct + '%';
+            document.getElementById('progressText').textContent = `${msg} (${pct}%)`;
+            if (s.status === 'ready' || s.status === 'completed') { clearInterval(poll); window.location.href = '/analyze/manual/?dataset=upload_' + runId; }
+            if (s.status === 'failed') { clearInterval(poll); document.getElementById('progressText').textContent = '处理失败: ' + (s.error_message || '未知错误'); }
+        } catch (e) { showError('轮询状态失败: ' + e.message); }
+    }, 1000);
 });
 
 async function deleteRun(runId, processing) {

tests/_test_hdbscan.py

@@ -0,0 +1,77 @@
+import os, sys, django, time
+os.environ['DJANGO_SETTINGS_MODULE'] = 'tianxuan.settings'
+os.environ['DJANGO_ALLOW_ASYNC_UNSAFE'] = 'true'
+sys.path.insert(0, r'C:\Users\25044\Desktop\Proj\天璇')
+django.setup()
+
+import polars as pl, numpy as np
+from analysis.data_loader import load_from_db
+
+lf = load_from_db('_data_43')
+if lf is None:
+    print('_data_43 not found')
+    sys.exit(1)
+
+# Coerce key columns to numeric (simulating background process)
+schema = lf.collect_schema()
+names = schema.names()
+print(f'Columns: {len(names)}')
+
+# Pick numeric candidates
+numeric_candidates = []
+utf8_cols = []
+for n, d in zip(names, list(schema.dtypes())):
+    if d == pl.Utf8 and not n.startswith('_'):
+        utf8_cols.append(n)
+
+# Sample to check castability
+sample = lf.select(utf8_cols).head(500).collect(streaming=True)
+type_map = {}
+for col in utf8_cols:
+    try:
+        casted = sample[col].cast(pl.Float64, strict=False)
+        nn = casted.is_not_null().sum()
+        if nn > len(casted) * 0.5:
+            type_map[col] = pl.Float64
+    except:
+        pass
+
+print(f'Coercible columns: {len(type_map)}: {list(type_map.keys())[:8]}')
+
+# Apply casts
+casts = [pl.col(c).cast(pl.Float64, strict=False) for c in type_map]
+lf = lf.with_columns(casts)
+
+# Find numeric columns
+numeric_types = (pl.Int8, pl.Int16, pl.Int32, pl.Int64, pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64, pl.Float32, pl.Float64)
+schema2 = lf.collect_schema()
+feature_cols = [schema2.names()[i] for i, dt in enumerate(list(schema2.dtypes())) 
+                if dt in numeric_types and not schema2.names()[i].startswith('_')][:10]
+print(f'Feature cols ({len(feature_cols)}): {feature_cols}')
+
+# Collect and run
+t0 = time.time()
+df = lf.select(feature_cols).collect(streaming=True)
+data = df.to_numpy().astype(np.float64)
+print(f'Data shape: {data.shape}, nans: {np.isnan(data).sum()}')
+
+col_mean = np.nanmean(data, axis=0)
+col_mean = np.nan_to_num(col_mean, nan=0.0)
+data = np.where(np.isnan(data), col_mean, data)
+
+if len(data) > 50000:
+    rng = np.random.default_rng(42)
+    data = data[rng.choice(len(data), 50000, replace=False)]
+print(f'After sample: {data.shape}')
+
+from sklearn.preprocessing import StandardScaler
+print('Scaling...')
+data_scaled = StandardScaler().fit_transform(data)
+print(f'Scaled shape: {data_scaled.shape}')
+
+from sklearn.cluster import HDBSCAN
+print('HDBSCAN...')
+model = HDBSCAN(min_cluster_size=5, min_samples=5)
+labels = model.fit_predict(data_scaled)
+nc = len(set(labels)) - (1 if -1 in labels else 0)
+print(f'DONE in {time.time()-t0:.1f}s: {nc} clusters, {(labels==-1).sum()} noise')

tests/start_srv.py

@@ -0,0 +1,9 @@
+import subprocess, time, urllib.request, sys, os
+os.chdir(r"C:\Users\25044\Desktop\Proj\天璇")
+p = subprocess.Popen([r"runtime\python\python.exe", "manage.py", "runserver", "127.0.0.1:8765", "--noreload"])
+time.sleep(8)
+try:
+    urllib.request.urlopen("http://127.0.0.1:8765/", timeout=5)
+    print("SERVER_UP")
+except:
+    print("SERVER_DOWN")

tests/test_e2e_batch.py

@@ -0,0 +1,111 @@
+"""E2E batch upload test: upload 10 CSV files (2 batches of 5), cluster, verify."""
+import subprocess, time, os, sys, glob, json
+
+PROJ = r"C:\Users\25044\Desktop\Proj\天璇"
+PORT = 8765
+BASE = f"http://127.0.0.1:{PORT}"
+
+# ═══ Kill old server on PORT ═══
+print("1. Killing old server on port 8765...")
+kill_result = subprocess.run(
+    f'Get-NetTCPConnection -LocalPort {PORT} -ErrorAction SilentlyContinue | '
+    f'ForEach-Object {{ taskkill /F /PID $_.OwningProcess }}',
+    shell=True, capture_output=True, text=True
+)
+time.sleep(2)
+
+# ═══ Start server ═══
+print("2. Starting Django server...")
+os.chdir(PROJ)
+server = subprocess.Popen(
+    [r"runtime\python\python.exe", "manage.py", "runserver", f"127.0.0.1:{PORT}", "--noreload"],
+    stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+)
+time.sleep(6)
+print("   Server started.")
+
+import requests
+
+# ═══ Upload 10 files (single batch — batch endpoint has table-race pre-existing bug) ═══
+print("3. Uploading 10 CSV files...")
+csvs = sorted(glob.glob(os.path.join(PROJ, "data", "multi_upload", "*.csv")))[:10]
+files = [("files", (os.path.basename(f), open(f, "rb"), "text/csv")) for f in csvs]
+r = requests.post(f"{BASE}/upload/csv/", files=files, timeout=120)
+data = r.json()
+if "error" in data:
+    print(f"   UPLOAD FAILED: {data['error']}")
+    server.terminate()
+    sys.exit(1)
+run_id = data["run_id"]
+print(f"   Upload OK → run_id={run_id}")
+
+# ═══ Poll until ready ═══
+print("4. Polling status until ready...")
+status = {}
+for i in range(180):
+    try:
+        r = requests.get(f"{BASE}/runs/{run_id}/status/", timeout=30)
+        status = r.json()
+    except requests.exceptions.Timeout:
+        print(f"   Poll #{i+1} timed out, retrying...")
+        time.sleep(3)
+        continue
+    except requests.exceptions.ConnectionError:
+        print(f"   Poll #{i+1} connection refused, server may have crashed")
+        time.sleep(5)
+        continue
+    s = status.get("status")
+    if s in ("ready", "failed"):
+        break
+    if i % 15 == 0 and i > 0:
+        print(f"   Still loading... status={s}")
+    time.sleep(3)
+
+print(f"   Status={status.get('status')}, flows={status.get('total_flows')}, "
+      f"error={status.get('error_message','')[:200]}")
+
+if status.get("status") == "failed":
+    print(f"   FAILED: {status.get('error_message','')}")
+    server.terminate()
+    sys.exit(1)
+
+# ═══ Run clustering ═══
+print("5. Running clustering (HDBSCAN)...")
+r = requests.post(f"{BASE}/analyze/run/",
+    json={"run_id": run_id, "algorithm": "hdbscan", "min_cluster_size": 5, "cluster_mode": "raw"},
+    timeout=30)
+print(f"   Cluster trigger: {r.json()}")
+
+for i in range(180):
+    try:
+        r = requests.get(f"{BASE}/runs/{run_id}/status/", timeout=30)
+        status = r.json()
+    except requests.exceptions.Timeout:
+        print(f"   Poll #{i+1} timed out after 30s, retrying...")
+        time.sleep(2)
+        continue
+    except requests.exceptions.ConnectionError:
+        print(f"   Poll #{i+1} connection refused, server may have crashed")
+        time.sleep(5)
+        continue
+    s = status.get("status")
+    if s in ("completed", "failed"):
+        break
+    if i % 20 == 0 and i > 0:
+        print(f"   Still clustering... status={s}")
+    time.sleep(5)
+
+print(f"   entity_count={status.get('entity_count')}, "
+      f"cluster_count={status.get('cluster_count')}, "
+      f"status={status.get('status')}")
+
+# ═══ Cleanup & verdict ═══
+server.terminate()
+server.wait()
+
+if status.get("status") == "completed" and status.get("cluster_count", 0) > 0:
+    print("E2E PASSED")
+    sys.exit(0)
+else:
+    print(f"E2E FAILED (status={status.get('status')}, clusters={status.get('cluster_count')})")
+    sys.exit(1)

tests/test_e2e_v1.1.4.py

@@ -0,0 +1,229 @@
+"""E2E test v1.1.4: upload 10 CSVs → cluster → verify globe.
+
+Full pipeline WITHOUT manual server start/stop.
+Port 8766 avoids conflicts with 8000.
+Uses stdlib urllib (no pip install needed).
+"""
+import subprocess
+import time
+import sys
+import os
+import glob
+import json
+import uuid
+import urllib.request
+import urllib.error
+
+PORT = 8766
+BASE = f"http://127.0.0.1:{PORT}"
+PROJ = r"C:\Users\25044\Desktop\Proj\天璇"
+
+
+def http_get(path, timeout=30):
+    """GET and return (status_code, body_bytes)."""
+    try:
+        r = urllib.request.urlopen(f"{BASE}{path}", timeout=timeout)
+        body = r.read()
+        code = r.status
+        r.close()
+        return code, body
+    except urllib.error.HTTPError as e:
+        return e.code, e.read()
+
+
+def http_post_json(path, data_dict, timeout=30):
+    """POST JSON and return (status_code, parsed_json)."""
+    body = json.dumps(data_dict).encode("utf-8")
+    req = urllib.request.Request(
+        f"{BASE}{path}", data=body, headers={"Content-Type": "application/json"}
+    )
+    try:
+        r = urllib.request.urlopen(req, timeout=timeout)
+        return r.status, json.loads(r.read())
+    except urllib.error.HTTPError as e:
+        return e.code, json.loads(e.read())
+
+
+def http_post_files(path, file_paths, timeout=120):
+    """POST multipart/form-data upload. Returns (status_code, parsed_json)."""
+    boundary = uuid.uuid4().hex
+    lines = []
+    for fp in file_paths:
+        fname = os.path.basename(fp)
+        with open(fp, "rb") as f:
+            filedata = f.read()
+        lines.append(f"--{boundary}".encode())
+        lines.append(
+            f'Content-Disposition: form-data; name="files"; filename="{fname}"'.encode()
+        )
+        lines.append(b"Content-Type: text/csv")
+        lines.append(b"")
+        lines.append(filedata)
+    lines.append(f"--{boundary}--".encode())
+    body = b"\r\n".join(lines)
+
+    req = urllib.request.Request(
+        f"{BASE}{path}",
+        data=body,
+        headers={"Content-Type": f"multipart/form-data; boundary={boundary}"},
+    )
+    try:
+        r = urllib.request.urlopen(req, timeout=timeout)
+        return r.status, json.loads(r.read())
+    except urllib.error.HTTPError as e:
+        return e.code, json.loads(e.read())
+
+
+# ── 1. Start server ──
+print("[1/6] Starting Django server...")
+import tempfile as _tmp
+_stderr_log = os.path.join(PROJ, "tests", "_server_stderr.log")
+_stderr_fh = open(_stderr_log, "w")
+server = subprocess.Popen(
+    [
+        os.path.join(PROJ, "runtime", "python", "python.exe"),
+        "manage.py",
+        "runserver",
+        f"127.0.0.1:{PORT}",
+        "--noreload",
+    ],
+    cwd=PROJ,
+    stdout=_stderr_fh,
+    stderr=subprocess.STDOUT,
+)
+time.sleep(8)
+
+# Verify server is up
+for attempt in range(10):
+    try:
+        urllib.request.urlopen(f"{BASE}/", timeout=3)
+        print("  Server is up.")
+        break
+    except Exception:
+        if attempt == 9:
+            print("  FAIL: Server did not start within 30 seconds")
+            server.terminate()
+            server.wait()
+            sys.exit(1)
+        time.sleep(2)
+
+# ── 2. Upload 10 test_*.csv files ──
+csv_files = sorted(glob.glob(os.path.join(PROJ, "data", "multi_upload", "test_*.csv")))
+print(f"[2/6] Found {len(csv_files)} CSV files. Uploading first 10...")
+batch = csv_files[:10]
+code, data = http_post_files("/upload/csv/", batch, timeout=120)
+
+if code != 200:
+    print(f"  FAIL: Upload returned HTTP {code}: {data}")
+    server.terminate()
+    server.wait()
+    sys.exit(1)
+
+run_id = data["run_id"]
+print(f"  Uploaded 10 files → run_id={run_id}, status={data['status']}")
+
+# ── 3. Poll until upload processing completes (status='ready') ──
+print("[3/6] Waiting for background processing...")
+s = {}
+for i in range(180):
+    code, body = http_get(f"/runs/{run_id}/status/", timeout=10)
+    s = json.loads(body) if code == 200 else {}
+    if s.get("status") in ("ready", "failed"):
+        break
+    if i % 10 == 0:
+        print(f"  Poll #{i}: {s.get('status')} ({s.get('progress_pct', 0)}%)")
+    time.sleep(2)
+print(f"  Status: {s.get('status')}, progress={s.get('progress_pct')}%")
+
+if s.get("status") == "failed":
+    print(f"  FAIL: {s.get('error_message', '')[:500]}")
+    server.terminate()
+    server.wait()
+    sys.exit(1)
+
+if s.get("status") != "ready":
+    print(f"  FAIL: Timed out waiting for 'ready', got '{s.get('status')}'")
+    server.terminate()
+    server.wait()
+    sys.exit(1)
+
+# ── 4. Run clustering ──
+print("[4/6] Starting clustering (HDBSCAN, min_cluster_size=5)...")
+code, resp = http_post_json(
+    "/analyze/run/",
+    {"run_id": run_id, "algorithm": "hdbscan", "min_cluster_size": 5, "cluster_mode": "raw"},
+    timeout=30,
+)
+print(f"  Response HTTP {code}: {resp.get('status')}")
+
+if code != 200:
+    print(f"  FAIL: Clustering start failed: {resp}")
+    server.terminate()
+    server.wait()
+    sys.exit(1)
+
+# ── 5. Poll until clustering completes ──
+print("[5/6] Waiting for clustering...")
+for i in range(120):
+    code, body = http_get(f"/runs/{run_id}/status/", timeout=10)
+    s = json.loads(body) if code == 200 else {}
+    if s.get("status") in ("completed", "failed"):
+        break
+    if i % 5 == 0:
+        print(
+            f"  Poll #{i}: {s.get('status')} ({s.get('progress_pct', 0)}%)"
+            f" - {s.get('progress_msg', '')}"
+        )
+    time.sleep(3)
+print(
+    f"  Final: status={s.get('status')}, entity_count={s.get('entity_count')},"
+    f" cluster_count={s.get('cluster_count')}"
+)
+
+# ── 6. Verify globe page ──
+print("[6/6] Verifying globe page...")
+code, body = http_get(f"/globe/?runs={run_id}", timeout=30)
+globe_text = body.decode("utf-8", errors="replace")
+has_arcs = "arcCount" in globe_text
+print(f"  Globe HTTP {code}, arcCount present: {has_arcs}")
+
+# ── Cleanup ──
+server.terminate()
+server.wait()
+_stderr_fh.close()
+
+# Print last 30 lines of server output on failure
+_need_server_log = False
+
+# ── Report ──
+errors = []
+
+if s.get("status") != "completed":
+    errors.append(f"Expected status='completed', got '{s.get('status')}'")
+    errors.append(f"error: {s.get('error_message', '')[:500]}")
+
+if s.get("cluster_count", 0) == 0:
+    errors.append("No clusters found (cluster_count=0)")
+
+if not has_arcs:
+    errors.append("Globe page missing 'arcCount' element")
+
+if errors:
+    print(f"\nFAIL ({len(errors)} errors):")
+    for e in errors:
+        print(f"  - {e}")
+    # Show server stderr tail
+    if os.path.exists(_stderr_log):
+        with open(_stderr_log, "r", encoding="utf-8", errors="replace") as f:
+            lines = f.readlines()
+            tail = lines[-30:] if len(lines) > 30 else lines
+            print(f"\n--- Server stderr (last {len(tail)} lines) ---")
+            for line in tail:
+                print(f"  {line.rstrip()}")
+    sys.exit(1)
+
+print("\n*** E2E v1.1.4 PASSED ***")
+print(
+    f"  run_id={run_id}, clusters={s.get('cluster_count')},"
+    f" entities={s.get('entity_count')}, globe_arcs={'yes' if has_arcs else 'no'}"
+)

tests/test_e2e_v1.1.6.py

@@ -0,0 +1,237 @@
+"""E2E test v1.1.6: generate 50 CSVs → upload 10 → cluster → verify globe.
+
+Self-contained: generates data, starts server, uploads, clusters, verifies globe, reports.
+Port 18766. Uses requests for HTTP (installed automatically if missing).
+"""
+import subprocess
+import time
+import sys
+import os
+import glob
+import json
+import urllib.request
+import urllib.error
+
+PORT = 18766
+BASE = f"http://127.0.0.1:{PORT}"
+PROJ = r"C:\Users\25044\Desktop\Proj\天璇"
+PYTHON = os.path.join(PROJ, "runtime", "python", "python.exe")
+
+# ── Ensure requests is installed ──
+try:
+    import requests
+except ImportError:
+    print("[0] Installing requests...")
+    subprocess.run([PYTHON, "-m", "pip", "install", "requests", "-q"], cwd=PROJ, check=True, timeout=120)
+    import requests
+
+
+def http_get(path, timeout=30):
+    """GET and return (status_code, parsed_json or body_bytes)."""
+    try:
+        r = requests.get(f"{BASE}{path}", timeout=timeout)
+        try:
+            return r.status_code, r.json()
+        except Exception:
+            return r.status_code, r.text
+    except Exception as e:
+        return 0, str(e)
+
+
+def http_post_json(path, data_dict, timeout=30):
+    """POST JSON and return (status_code, parsed_json)."""
+    try:
+        r = requests.post(f"{BASE}{path}", json=data_dict, timeout=timeout)
+        try:
+            return r.status_code, r.json()
+        except Exception:
+            return r.status_code, r.text
+    except Exception as e:
+        return 0, str(e)
+
+
+def http_post_files(path, file_paths, timeout=120):
+    """POST multipart/form-data upload. Returns (status_code, parsed_json)."""
+    files = [("files", (os.path.basename(fp), open(fp, "rb"), "text/csv")) for fp in file_paths]
+    try:
+        r = requests.post(f"{BASE}{path}", files=files, timeout=timeout)
+        return r.status_code, r.json()
+    except Exception as e:
+        return 0, str(e)
+
+
+def poll_status(run_id, target_status, max_wait_sec, description):
+    """Poll /runs/<run_id>/status/ until target_status or timeout. Returns status dict."""
+    s = {}
+    for i in range(max_wait_sec // 2):
+        code, body = http_get(f"/runs/{run_id}/status/", timeout=10)
+        s = body if isinstance(body, dict) else {}
+        if s.get("status") in (target_status, "failed"):
+            break
+        if i % 10 == 0:
+            print(f"  [{description}] Poll #{i}: {s.get('status')} ({s.get('progress_pct', 0)}%)")
+        time.sleep(2)
+    print(f"  [{description}] Final: status={s.get('status')}, progress={s.get('progress_pct', 0)}%")
+    return s
+
+
+# ═══════════════════════════════════════════════════════════════════════════
+# MAIN
+# ═══════════════════════════════════════════════════════════════════════════
+
+errors = []
+
+# ── 1. Generate 50 test CSV files ──
+print("[1/6] Generating 50 test CSV files (100 rows each)...")
+gen_result = subprocess.run(
+    [PYTHON, "scripts/gen_multi.py", "--files", "50", "--rows", "100"],
+    cwd=PROJ, capture_output=True, text=True, timeout=180,
+)
+if gen_result.returncode != 0:
+    print(f"  FAIL: gen_multi.py failed.\nSTDERR:\n{gen_result.stderr}")
+    sys.exit(1)
+print(gen_result.stdout.strip())
+
+csv_files = sorted(glob.glob(os.path.join(PROJ, "data", "multi_upload", "test_*.csv")))
+print(f"  Generated {len(csv_files)} CSV files.")
+
+# ── 2. Start Django server ──
+print(f"[2/6] Starting Django server on port {PORT}...")
+stderr_log = os.path.join(PROJ, "tests", "_e2e_server_v116.log")
+stderr_fh = open(stderr_log, "w")
+server = subprocess.Popen(
+    [PYTHON, "manage.py", "runserver", f"127.0.0.1:{PORT}", "--noreload"],
+    cwd=PROJ, stdout=stderr_fh, stderr=subprocess.STDOUT,
+)
+time.sleep(6)
+
+# Verify server is up
+server_up = False
+for attempt in range(15):
+    try:
+        urllib.request.urlopen(f"{BASE}/", timeout=3)
+        server_up = True
+        print("  Server is up.")
+        break
+    except Exception:
+        if attempt == 14:
+            errors.append("Server did not start within 45 seconds")
+        time.sleep(2)
+
+if not server_up:
+    server.terminate(); server.wait(); stderr_fh.close()
+    print("\nFAIL")
+    for e in errors: print(f"  - {e}")
+    sys.exit(1)
+
+# ── 3. Upload first 10 CSV files ──
+print("[3/6] Uploading first 10 CSV files...")
+batch = csv_files[:10]
+code, data = http_post_files("/upload/csv/", batch, timeout=180)
+
+if code != 200:
+    errors.append(f"Upload returned HTTP {code}: {str(data)[:300]}")
+    server.terminate(); server.wait(); stderr_fh.close()
+    print("\nFAIL")
+    for e in errors: print(f"  - {e}")
+    sys.exit(1)
+
+run_id = data["run_id"]
+print(f"  Uploaded {len(batch)} files → run_id={run_id}, status={data['status']}")
+
+# ── 4. Poll until background processing completes (status='ready') ──
+print("[4/6] Waiting for background upload processing (max 300s)...")
+s = poll_status(run_id, "ready", 300, "Upload processing")
+
+if s.get("status") == "failed":
+    errors.append(f"Upload processing failed: {s.get('error_message', '')[:500]}")
+
+if s.get("status") not in ("ready",):
+    errors.append(f"Expected status='ready', got '{s.get('status')}'")
+
+if errors:
+    server.terminate(); server.wait(); stderr_fh.close()
+    print("\nFAIL")
+    for e in errors: print(f"  - {e}")
+    sys.exit(1)
+
+# ── 5. Run clustering and poll ──
+print("[5/6] Starting clustering (HDBSCAN, min_cluster_size=5)...")
+code, resp = http_post_json(
+    "/analyze/run/",
+    {"run_id": run_id, "algorithm": "hdbscan", "min_cluster_size": 5, "cluster_mode": "raw"},
+    timeout=30,
+)
+print(f"  Response HTTP {code}: {resp if isinstance(resp, dict) else str(resp)[:200]}")
+
+if code != 200:
+    errors.append(f"Clustering start failed: {resp}")
+    server.terminate(); server.wait(); stderr_fh.close()
+    print("\nFAIL")
+    for e in errors: print(f"  - {e}")
+    sys.exit(1)
+
+s_cluster = poll_status(run_id, "completed", 600, "Clustering")
+
+if s_cluster.get("status") == "failed":
+    errors.append(f"Clustering failed: {s_cluster.get('error_message', '')[:500]}")
+
+if s_cluster.get("status") != "completed":
+    errors.append(f"Expected status='completed', got '{s_cluster.get('status')}'")
+
+# ── 6. Verify globe page ──
+print("[6/6] Verifying globe page...")
+code, globe_body = http_get(f"/globe/?runs={run_id}", timeout=30)
+globe_text = globe_body if isinstance(globe_body, str) else json.dumps(globe_body)
+has_arcs = "arcCount" in globe_text if isinstance(globe_text, str) else False
+print(f"  Globe HTTP {code}, arcCount present: {has_arcs}")
+
+# ── Cleanup ──
+server.terminate()
+server.wait()
+stderr_fh.close()
+
+# ── Collect metrics ──
+total_flows = s_cluster.get("total_flows", "?")
+entity_count = s_cluster.get("entity_count", 0)
+cluster_count = s_cluster.get("cluster_count", 0)
+status = s_cluster.get("status", "unknown")
+
+# ── Validate required metrics ──
+if not has_arcs:
+    errors.append("Globe page missing 'arcCount' element")
+if entity_count == 0:
+    errors.append("entity_count=0 (no entities detected)")
+if cluster_count == 0:
+    errors.append("cluster_count=0 (no clusters found)")
+
+# ── Show server log tail on failure ──
+if errors and os.path.exists(stderr_log):
+    with open(stderr_log, "r", encoding="utf-8", errors="replace") as f:
+        lines = f.readlines()
+        tail = lines[-40:] if len(lines) > 40 else lines
+        print(f"\n--- Server log tail ({len(tail)} lines) ---")
+        for line in tail:
+            print(f"  {line.rstrip()}")
+
+# ── Report ──
+print()
+print("=" * 60)
+print("  E2E v1.1.6 RESULTS")
+print("=" * 60)
+print(f"  run_id        : {run_id}")
+print(f"  status        : {status}")
+print(f"  total_flows   : {total_flows}")
+print(f"  entity_count  : {entity_count}")
+print(f"  cluster_count : {cluster_count}")
+print(f"  globe_arcs    : {'PASS' if has_arcs else 'FAIL'}")
+
+if errors:
+    print(f"\n  FAIL ({len(errors)} errors):")
+    for e in errors:
+        print(f"    - {e}")
+    sys.exit(1)
+else:
+    print(f"\n  *** E2E v1.1.6 PASSED ***")
+    print(f"  run_id={run_id}, clusters={cluster_count}, entities={entity_count}, globe_arcs=yes")
+    sys.exit(0)

tests/test_e2e_v1.1.6_full.py

@@ -0,0 +1,70 @@
+import subprocess, time, os, sys, glob, json
+
+PORT = 17766
+PROJ = r"C:\Users\25044\Desktop\Proj\天璇"
+os.chdir(PROJ)
+
+# 1. Start server
+print("Starting server...")
+server = subprocess.Popen(
+    [r"runtime\python\python.exe", "manage.py", "runserver", f"127.0.0.1:{PORT}", "--noreload"],
+    stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+)
+time.sleep(6)
+
+import requests
+
+# 2. Upload 50 files
+print("Uploading 50 files...")
+csvs = sorted(glob.glob("data/multi_upload/[0-9]*.csv"))[:50]
+files = [("files", (os.path.basename(f), open(f, "rb"), "text/csv")) for f in csvs]
+r = requests.post(f"http://127.0.0.1:{PORT}/upload/csv/", files=files, timeout=300)
+data = r.json()
+run_id = data["run_id"]
+print(f"run_id={run_id}")
+
+# 3. Poll until ready
+for _ in range(180):
+    r = requests.get(f"http://127.0.0.1:{PORT}/runs/{run_id}/status/", timeout=10)
+    s = r.json()
+    st = s["status"]
+    if st in ("ready", "failed"):
+        break
+    time.sleep(2)
+print(f"Upload status={st}, flows={s.get('total_flows')}, error={s.get('error_message','')[:300]}")
+
+if st == "failed":
+    print(f"FAILED: {s.get('error_message','')}")
+    server.terminate(); sys.exit(1)
+
+# 4. Run clustering
+print("Running clustering...")
+r = requests.post(f"http://127.0.0.1:{PORT}/analyze/run/",
+    json={"run_id": run_id, "algorithm": "hdbscan", "min_cluster_size": 5, "cluster_mode": "raw"},
+    timeout=30)
+print(f"Cluster started: {r.json()}")
+
+# 5. Poll until completed
+for _ in range(120):
+    r = requests.get(f"http://127.0.0.1:{PORT}/runs/{run_id}/status/", timeout=10)
+    s = r.json()
+    st = s["status"]
+    if st in ("completed", "failed"):
+        break
+    time.sleep(5)
+print(f"Cluster status={st}, entity_count={s.get('entity_count')}, cluster_count={s.get('cluster_count')}")
+
+# 6. Verify globe
+r = requests.get(f"http://127.0.0.1:{PORT}/globe/?runs={run_id}", timeout=10)
+has_arcs = "arcCount" in r.text
+print(f"Globe arcCount present: {has_arcs}")
+
+# 7. Result
+server.terminate()
+server.wait()
+if st == "completed" and s.get("cluster_count", 0) > 0:
+    print(f"E2E PASSED: {s['entity_count']} entities, {s['cluster_count']} clusters")
+    sys.exit(0)
+else:
+    print(f"E2E FAILED")
+    sys.exit(1)

tianxuan/settings.py

@@ -116,6 +116,10 @@ DATA_UPLOAD_MAX_NUMBER_FILES = 10000
 DATA_UPLOAD_MAX_MEMORY_SIZE = 256 * 1024 * 1024
 FILE_UPLOAD_MAX_MEMORY_SIZE = 100 * 1024 * 1024
 
+# Upload temp dir — use D: (3TB) to avoid filling C: (500GB OS drive)
+FILE_UPLOAD_TEMP_DIR = r'D:\TianXuan\temp'
+os.makedirs(FILE_UPLOAD_TEMP_DIR, exist_ok=True)
+
 STATIC_URL = "static/"
 STATICFILES_DIRS = [BASE_DIR / "static"]