From c54917791ecd4387384d6b7458bfbb44eea99bb4 Mon Sep 17 00:00:00 2001 From: PM-pinou <2504420230@qq.com> Date: Fri, 24 Jul 2026 13:43:49 +0800 Subject: [PATCH] cleanup: remove dead backup files, orphaned static files, fix merge conflict markers --- analysis/tool_registry_backup.py | 3368 ------------------------------ analysis/views/clustering.py | 3 +- analysis/views_backup.py | 2308 -------------------- scripts/fix_merge_conflict.py | 11 + static/tianxuan/cluster.css | 133 -- static/tianxuan/globe_embed.js | 129 -- static/tianxuan/markdown.js | 43 - static/tianxuan/timeline.js | 162 -- 8 files changed, 12 insertions(+), 6145 deletions(-) delete mode 100644 analysis/tool_registry_backup.py delete mode 100644 analysis/views_backup.py create mode 100644 scripts/fix_merge_conflict.py delete mode 100644 static/tianxuan/cluster.css delete mode 100644 static/tianxuan/globe_embed.js delete mode 100644 static/tianxuan/markdown.js delete mode 100644 static/tianxuan/timeline.js diff --git a/analysis/tool_registry_backup.py b/analysis/tool_registry_backup.py deleted file mode 100644 index cb4e805..0000000 --- a/analysis/tool_registry_backup.py +++ /dev/null @@ -1,3368 +0,0 @@ -"""Tool definitions and handler functions for the TLS Analyzer MCP server. - -Each tool is defined by a :class:`Tool ` metadata object and -an async handler function. Handlers interact with the :class:`SessionStore` -singleton and the Django ORM. - -All 12 tools: - 1. load_data 7. filter_and_cluster - 2. profile_data 8. build_entity_profiles - 3. filter_data 9. export_results - 4. preprocess_data 10. list_datasets - 5. run_clustering 11. drop_dataset - 6. evaluate_clustering 12. clone_dataset -""" -import json -import os -from pathlib import Path -from typing import Any, Optional - -import numpy as np -import polars as pl -from mcp.types import Tool - -from .data_loader import load_csv_directory -from .data_profiler import profile_dataset as _profile_dataset -from .session_store import SessionStore, _generate_id -from .profile_util import profile - - -# ═══════════════════════════════════════════════════════════════════════ -# Tool metadata -# ═══════════════════════════════════════════════════════════════════════ - -def _json_size(obj: Any) -> int: - return len(json.dumps(obj, default=str, ensure_ascii=False)) - - -_MAX_RESPONSE_KB = 64 -_MAX_RESPONSE_BYTES = _MAX_RESPONSE_KB * 1024 - - -def _truncate_response(result: dict, max_kb: int = _MAX_RESPONSE_KB) -> dict: - """Truncate large lists/keys in *result* to stay under *max_kb*.""" - if _json_size(result) <= max_kb * 1024: - result.setdefault('truncated', False) - return result - - # Drop verbose keys first - for costly_key in ('column_stats', 'features', 'correlation_matrix', - 'labels', 'cluster_labels'): - if costly_key in result and isinstance(result[costly_key], dict): - result[costly_key] = {'__truncated__': True, - 'key_count': len(result[costly_key])} - elif costly_key in result and isinstance(result[costly_key], list): - lst = result[costly_key] - if len(lst) > 10: - result[costly_key] = lst[:10] + [{'__truncated__': True, - 'omitted': len(lst) - 10}] - result['truncated'] = True - return result - - -def get_tools_meta() -> list[Tool]: - """Return the list of Tool metadata objects for MCP server registration. - - Call this from :meth:`TLSAnalyzerMCPServer._setup_tools`. - """ - return [ - Tool( - name="load_data", - description=( - "加载匹配glob模式的CSV文件。自动检测每个文件的BOM，验证schema一致性，" - "纵向合并，并将结果作为惰性数据集存储在会话存储中。" - ), - inputSchema={ - "type": "object", - "properties": { - "csv_glob": { - "type": "string", - "description": "Glob pattern for CSV files (e.g. 'data/*.csv')", - }, - "config_path": { - "type": "string", - "description": "Optional path to a YAML config file", - }, - "encoding": { - "type": "string", - "description": "Fallback encoding when no BOM is found", - "default": "utf-8", - }, - "delimiter": { - "type": "string", - "description": "Column delimiter", - "default": ",", - }, - "schema_strict": { - "type": "boolean", - "description": "When True, raise on column mismatch; when False, merge leniently with nulls", - "default": False, - }, - "recursive": { - "type": "boolean", - "description": "When True, use recursive glob for **/*.csv patterns", - "default": False, - }, - }, - "required": ["csv_glob"], - }, - ), - Tool( - name="profile_data", - description=( - "计算数据集的每列统计信息和数值相关矩阵。结果自动截断至约16 KB以" - "节省LLM上下文窗口空间。" - ), - inputSchema={ - "type": "object", - "properties": { - "dataset_id": { - "type": "string", - "description": "Dataset identifier from a previous load_data call", - }, - "sample_size": { - "type": "integer", - "description": "Maximum rows to profile (sampled if larger)", - "default": 1000, - }, - "columns": { - "type": "array", - "items": {"type": "string"}, - "description": "Columns to profile (default: all)", - }, - }, - "required": ["dataset_id"], - }, - ), - Tool( - name="filter_data", - description=( - "对数据集应用列级过滤器，将结果存储为新的派生数据集。支持eq/neq/gt/gte/" - "lt/lte/contains/not_contains/in/not_in/is_null/is_not_null运算符。" - ), - inputSchema={ - "type": "object", - "properties": { - "dataset_id": { - "type": "string", - "description": "Source dataset identifier", - }, - "filters": { - "type": "array", - "items": { - "type": "object", - "properties": { - "column": {"type": "string"}, - "op": { - "type": "string", - "enum": [ - "eq", "neq", "gt", "gte", "lt", "lte", - "contains", "not_contains", - "in", "not_in", - "is_null", "is_not_null", - ], - }, - "value": {"description": "Operand value"}, - }, - "required": ["column", "op"], - }, - "description": "List of filter expressions", - }, - "logic": { - "type": "string", - "enum": ["and", "or"], - "description": "Combine filters with AND or OR logic", - "default": "and", - }, - }, - "required": ["dataset_id", "filters"], - }, - ), - Tool( - name="preprocess_data", - description=( - "预处理选定列：标准化（StandardScaler）、独热编码、缺失值填充和列删除。" - ), - inputSchema={ - "type": "object", - "properties": { - "dataset_id": { - "type": "string", - "description": "Source dataset identifier", - }, - "columns": { - "type": "array", - "items": {"type": "string"}, - "description": "Columns to preprocess", - }, - "config": { - "type": "object", - "properties": { - "standardize": { - "type": "boolean", - "description": "Apply StandardScaler to numeric columns", - "default": False, - }, - "onehot": { - "type": "boolean", - "description": "One-hot encode categorical columns", - "default": False, - }, - "fillna": { - "type": "string", - "enum": ["mean", "median", "zero", "drop"], - "description": "Missing value strategy", - }, - "drop": { - "type": "array", - "items": {"type": "string"}, - "description": "Columns to drop entirely", - }, - }, - }, - }, - "required": ["dataset_id", "columns"], - }, - ), - Tool( - name="run_clustering", - description=( - "对选定列执行聚类。使用HDBSCAN（默认）或KMeans。特征在聚类前进行" - "标准化缩放。返回聚类标签和质量指标。" - ), - inputSchema={ - "type": "object", - "properties": { - "dataset_id": { - "type": "string", - "description": "Dataset identifier", - }, - "cluster_columns": { - "type": "array", - "items": {"type": "string"}, - "description": "Columns to use as clustering features", - }, - "algorithm": { - "type": "string", - "enum": ["agglomerative", "hdbscan", "kmeans"], - "description": "Clustering algorithm (default: agglomerative/Ward)", - "default": "agglomerative", - }, - "params": { - "type": "object", - "description": ( - "Algorithm parameters. Agglomerative: n_clusters (5), metric ('euclidean'), " - "linkage ('ward'). HDBSCAN: min_cluster_size (5), " - "min_samples (None), metric ('euclidean'). " - "KMeans: n_clusters (3), random_state (42)." - ), - }, - "random_state": { - "type": "integer", - "description": "Random seed for reproducibility", - "default": 42, - }, - }, - "required": ["dataset_id", "cluster_columns"], - }, - ), - Tool( - name="evaluate_clustering", - description=( - "评估数据集的聚类结果。当前支持轮廓系数（silhouette）、Davies-Bouldin指数、" - "Calinski-Harabasz指数、噪声比（noise_ratio）和簇大小（cluster_sizes）指标。" - ), - inputSchema={ - "type": "object", - "properties": { - "cluster_result_id": { - "type": "string", - "description": "Result ID from run_clustering", - }, - "dataset_id": { - "type": "string", - "description": "Original dataset for metric computation", - }, - "metrics": { - "type": "array", - "items": {"type": "string"}, - "description": ( - "Metrics to compute. Options: silhouette, " - "davies_bouldin, calinski_harabasz, noise_ratio, " - "cluster_sizes (default: all)" - ), - }, - }, - "required": ["cluster_result_id", "dataset_id"], - }, - ), - Tool( - name="extract_features", - description=( - "提取每个聚类的区分特征并自动保存到Django ORM（ClusterFeature模型）。" - "使用z-score或ANOVA对每个聚类的特征重要性进行排序。" - ), - inputSchema={ - "type": "object", - "properties": { - "dataset_id": { - "type": "string", - "description": "Dataset containing the feature columns", - }, - "cluster_result_id": { - "type": "string", - "description": "Result ID from run_clustering", - }, - "top_k": { - "type": "integer", - "description": "Top distinguishing features per cluster", - "default": 10, - }, - "method": { - "type": "string", - "enum": ["zscore", "anova"], - "description": "Method to rank feature importance", - "default": "zscore", - }, - "save_to_db": { - "type": "boolean", - "description": "Save features to Django ORM if True", - "default": True, - }, - }, - "required": ["dataset_id", "cluster_result_id"], - }, - ), - Tool( - name="export_results", - description=( - "将数据集或聚类结果以CSV或Parquet格式导出到磁盘。数据集物化后写入；" - "聚类结果序列化为JSON。" - ), - inputSchema={ - "type": "object", - "properties": { - "result_id": { - "type": "string", - "description": "Dataset ID or cluster result ID to export", - }, - "output_path": { - "type": "string", - "description": "Output file path or directory", - }, - "format": { - "type": "string", - "enum": ["csv", "parquet", "json"], - "description": "Output format", - "default": "csv", - }, - "overwrite": { - "type": "boolean", - "description": "Overwrite existing file", - "default": False, - }, - }, - "required": ["result_id", "output_path"], - }, - ), - Tool( - name="list_datasets", - description="列出会话存储中的所有活跃数据集和结果。", - inputSchema={ - "type": "object", - "properties": {}, - }, - ), - Tool( - name="drop_dataset", - description=( - "从会话存储中移除数据集（或结果）并通过垃圾回收释放内存。" - ), - inputSchema={ - "type": "object", - "properties": { - "dataset_id": { - "type": "string", - "description": "Dataset or result identifier to remove", - }, - }, - "required": ["dataset_id"], - }, - ), - Tool( - name="clone_dataset", - description=( - "浅拷贝数据集。由于LazyFrame是查询计划，不会复制实际数据。适用于分支分析。" - ), - inputSchema={ - "type": "object", - "properties": { - "dataset_id": { - "type": "string", - "description": "Source dataset identifier", - }, - }, - "required": ["dataset_id"], - }, - ), - Tool( - name="filter_and_cluster", - description=( - "一步完成：先应用列级过滤器，再对过滤结果执行聚类。支持eq/neq/gt/gte/lt/lte/" - "contains/not_contains/in/not_in/is_null/is_not_null运算符和HDBSCAN/KMeans聚类。" - "自动选择数值列作为聚类特征。将过滤后的数据集和聚类结果都存储在会话中。" - ), - inputSchema={ - "type": "object", - "properties": { - "dataset_id": { - "type": "string", - "description": "Source dataset identifier", - }, - "filters": { - "type": "array", - "items": { - "type": "object", - "properties": { - "column": {"type": "string"}, - "op": { - "type": "string", - "enum": [ - "eq", "neq", "gt", "gte", "lt", "lte", - "contains", "not_contains", - "in", "not_in", - "is_null", "is_not_null", - ], - }, - "value": {"description": "Operand value"}, - }, - "required": ["column", "op"], - }, - "description": "List of filter expressions", - }, - "logic": { - "type": "string", - "enum": ["and", "or"], - "description": "Combine filters with AND or OR logic", - "default": "and", - }, - "algorithm": { - "type": "string", - "enum": ["agglomerative", "hdbscan", "kmeans"], - "description": "Clustering algorithm (default: agglomerative/Ward)", - "default": "agglomerative", - }, - "params": { - "type": "object", - "description": ( - "Algorithm parameters. Agglomerative: n_clusters (5), metric ('euclidean'), " - "linkage ('ward'). HDBSCAN: min_cluster_size (5), " - "min_samples (None), metric ('euclidean'). " - "KMeans: n_clusters (3), random_state (42)." - ), - }, - }, - "required": ["dataset_id", "filters"], - }, - ), - Tool( - name="build_entity_profiles", - description=( - "按实体列（源IP、SNI等）分组原始流数据。自动检测实体列（如 src_ip, " - "dst_ip, sni, host, mac, domain），按列分组并计算每个实体的聚合特征：" - "流数量、唯一端口数、TLS版本分布、字节总量、唯一目标数。" - "生成的数据集每个实体一行。结果存储在会话中，可供后续 compute_scores 使用。" - ), - inputSchema={ - "type": "object", - "properties": { - "dataset_id": { - "type": "string", - "description": "Dataset ID from load_data or profile_data", - }, - "entity_column": { - "type": "string", - "description": "Specific entity column to group by (e.g. 'src_ip')", - }, - "entity_columns": { - "type": "array", - "items": {"type": "string"}, - "description": "Multiple entity columns to group by", - }, - "auto_detect": { - "type": "boolean", - "description": "Auto-detect entity columns (IP, SNI, host, MAC, domain)", - "default": True, - }, - }, - "required": ["dataset_id"], - }, - ), - Tool( - name="compute_scores", - description=( - "在实体数据上计算自适应代理/异常/威胁评分。从数据自动学习" - "特征权重，计算加权代理评分（proxy_score）、基于IQR的风险等级" - "（normal/watch/suspicious/critical）和威胁评分（threat_score）。" - "返回评分摘要与阈值。" - ), - inputSchema={ - "type": "object", - "properties": { - "dataset_id": { - "type": "string", - "description": "Entity dataset ID from filter_and_cluster or a scored dataset", - }, - }, - "required": ["dataset_id"], - }, - ), - Tool( - name="detect_anomalies", - description=( - "对已评分的实体数据运行孤立森林（Isolation Forest）异常检测。识别偏离" - "正常模式的实体。分块处理以应对大数据集。返回异常实体ID列表。" - ), - inputSchema={ - "type": "object", - "properties": { - "dataset_id": { - "type": "string", - "description": "Scored entity dataset ID (after compute_scores)", - }, - "contamination": { - "type": "number", - "description": "Expected proportion of anomalies (0.0-0.5)", - "default": 0.05, - }, - }, - "required": ["dataset_id"], - }, - ), - Tool( - name="visualize_anomalies", - description=( - "为高风险实体生成UMAP-2D嵌入和聚类可视化数据。按风险等级和聚类着色。" - "返回用于前端渲染的散点图JSON数据。" - ), - inputSchema={ - "type": "object", - "properties": { - "dataset_id": { - "type": "string", - "description": "Scored entity dataset ID (after compute_scores)", - }, - }, - "required": ["dataset_id"], - }, - ), - # ── Edge-case / Diagnostic tools (unexpected situations) ──────── - Tool( - name="validate_data", - description=( - "诊断工具：检查数据质量问题——文件间schema一致性、缺失值比例、列类型冲突、" - "编码问题和空列。当流水线步骤因schema错误失败，或怀疑CSV数据损坏/格式异常时使用。" - ), - inputSchema={ - "type": "object", - "properties": { - "dataset_id": {"type": "string", "description": "Dataset ID from load_data"}, - "fix_issues": {"type": "boolean", "description": "Auto-fix schema inconsistencies", "default": False}, - }, - "required": ["dataset_id"], - }, - ), - Tool( - name="explore_distributions", - description=( - "诊断工具：计算每列分布统计（唯一值计数、空值比例、最小/最大/均值/标准差、" - "前10高频值、直方图分箱）。当聚类产生意外结果、实体检测失败，或需要在选择" - "分析策略前了解数据时使用。" - ), - inputSchema={ - "type": "object", - "properties": { - "dataset_id": {"type": "string", "description": "Dataset ID from load_data or entity data"}, - "columns": {"type": "array", "items": {"type": "string"}, - "description": "Specific columns to analyze (default: auto-detect numeric)"}, - "max_sample": {"type": "integer", "description": "Max rows to sample", "default": 10000}, - }, - "required": ["dataset_id"], - }, - ), - Tool( - name="find_outliers", - description=( - "诊断工具：使用IQR方法检测数值列中的统计异常值。返回任何数值列超过中位数3倍IQR的行。" - "当怀疑数据质量问题或极端值偏离分析时使用。" - ), - inputSchema={ - "type": "object", - "properties": { - "dataset_id": {"type": "string", "description": "Dataset ID from load_data"}, - "columns": {"type": "array", "items": {"type": "string"}, - "description": "Columns to check (default: all numeric)"}, - "threshold": {"type": "number", "description": "IQR multiplier (default 3.0)", "default": 3.0}, - }, - "required": ["dataset_id"], - }, - ), - Tool( - name="diagnose_clustering", - description=( - "诊断工具：当聚类产生太少/太多聚类、全部为噪声或轮廓系数低时——诊断原因。" - "检查：特征方差、相关矩阵、特征降维分析、数据稀疏性，并建议参数调整" - "（min_cluster_size、算法选择、特征选择）。" - ), - inputSchema={ - "type": "object", - "properties": { - "dataset_id": {"type": "string", "description": "Entity dataset ID"}, - "cluster_result_id": {"type": "string", "description": "Cluster result ID from previous run_clustering"}, - }, - "required": ["dataset_id", "cluster_result_id"], - }, - ), - Tool( - name="compare_datasets", - description=( - "诊断工具：比较两个数据集（如处理前/后，或两次不同运行）。报告schema差异、" - "行数变化、列值分布偏移。用于验证处理步骤未损坏数据或比较不同运行的结果。" - ), - inputSchema={ - "type": "object", - "properties": { - "dataset_id_a": {"type": "string", "description": "First dataset ID"}, - "dataset_id_b": {"type": "string", "description": "Second dataset ID"}, - }, - "required": ["dataset_id_a", "dataset_id_b"], - }, - ), - Tool( - name="export_debug_sample", - description=( - "诊断工具：导出原始数据样本为JSON用于外部调试。当流水线深处发生错误且需要" - "检查实际数据值时使用。返回前N行所有列。" - ), - inputSchema={ - "type": "object", - "properties": { - "dataset_id": {"type": "string", "description": "Dataset ID"}, - "max_rows": {"type": "integer", "description": "Rows to export", "default": 50}, - }, - "required": ["dataset_id"], - }, - ), - Tool( - name="repair_schema", - description=( - "诊断工具：通过对齐列名（不区分大小写合并、下划线/连字符规范化）并用空值填充" - "缺失列，尝试修复文件间的schema不匹配。当load_data因CSV文件具有不同列集而" - "失败时使用。" - ), - inputSchema={ - "type": "object", - "properties": { - "dataset_id": {"type": "string", "description": "Failed dataset ID from load_data"}, - }, - "required": ["dataset_id"], - }, - ), - # ── Read-only analysis tools (LLM deep-dive, no side effects) ──── - Tool( - name="analyze_patterns", - description=( - "分析工具：检测流量模式——top源/目标IP、端口分布、TLS版本分布、协议混合。" - "返回汇总统计。只读，不修改数据。用于了解高层流量结构。" - ), - inputSchema={ - "type": "object", - "properties": { - "dataset_id": {"type": "string", "description": "Dataset ID from load_data"}, - "top_n": {"type": "integer", "description": "Number of top items per category", "default": 10}, - }, - "required": ["dataset_id"], - }, - ), - Tool( - name="analyze_temporal", - description=( - "分析工具：分析随时间变化的流量——小时/日流量计数、繁忙时段、空闲时段。" - "需要时间戳列。只读。用于了解时间相关模式和周期性。" - ), - inputSchema={ - "type": "object", - "properties": { - "dataset_id": {"type": "string", "description": "Dataset ID from load_data"}, - "timestamp_column": {"type": "string", "description": "Column name for timestamps (auto-detected if omitted)"}, - }, - "required": ["dataset_id"], - }, - ), - Tool( - name="analyze_fft", - description=( - "FFT频谱分析——从时间戳列提取周期模式和频率特征。适用于检测时间周期性" - "（如每日/每周流量模式）。返回频率、幅度、top周期、频谱质心、周期性评分。" - "只读，无副作用。" - ), - inputSchema={ - "type": "object", - "properties": { - "dataset_id": { - "type": "string", - "description": "Dataset ID from load_data", - }, - "timestamp_column": { - "type": "string", - "description": "Column name containing timestamps or numeric time values", - }, - "top_n": { - "type": "integer", - "description": "Number of top frequency peaks to return", - "default": 5, - }, - }, - "required": ["dataset_id", "timestamp_column"], - }, - ), - Tool( - name="analyze_tls_health", - description=( - "分析工具：TLS安全态势评估——检查过时的TLS版本（1.0/1.1）、弱加密套件、" - "证书问题、SNI异常。只读。用于评估流量的TLS安全性。" - ), - inputSchema={ - "type": "object", - "properties": { - "dataset_id": {"type": "string", "description": "Dataset ID from load_data"}, - }, - "required": ["dataset_id"], - }, - ), - Tool( - name="analyze_geo_distribution", - description=( - "分析工具：流量地理分布——top源/目标国家、异常地点对（如不可能的行程）、" - "ISP多样性。只读。需要国家或经纬度列。用于检测地理异常。" - ), - inputSchema={ - "type": "object", - "properties": { - "dataset_id": {"type": "string", "description": "Dataset ID from load_data"}, - }, - "required": ["dataset_id"], - }, - ), - Tool( - name="analyze_entity_detail", - description=( - "分析工具：深入分析单个实体——其所有流、使用的TLS版本、目标多样性、时间模式、" - "异常评分。只读。在compute_scores后调查特定可疑实体时使用。" - ), - inputSchema={ - "type": "object", - "properties": { - "dataset_id": {"type": "string", "description": "Entity dataset ID (after filter_and_cluster or compute_scores)"}, - "entity_value": {"type": "string", "description": "Entity value to investigate (e.g. an IP address or SNI)"}, - }, - "required": ["dataset_id", "entity_value"], - }, - ), - Tool( - name="compute_distance_matrix", - description=( - "LLM驱动工具：对数据集的每一行执行用户编写的Python函数，返回距离/相似度评分摘要。" - "LLM提供函数体，签名格式为`def distance_fn(row: dict) -> float:`。执行器将其包装，" - "在受限命名空间（仅math、numpy）中逐行运行，返回得分的最小/最大/均值/标准差以及" - "评分行样本。用于现有工具无法表达的一次性度量、过滤或评分任务。" - ), - inputSchema={ - "type": "object", - "properties": { - "dataset_id": { - "type": "string", - "description": "Source dataset identifier", - }, - "python_function": { - "type": "string", - "description": ( - "Body of a Python function with signature `def distance_fn(row: dict) -> float:`. " - "Example: \"return abs(row.get('col1', 0)) + row.get('col2', 0)\". " - "Available namespace: math, numpy (as np). No dangerous modules." - ), - }, - "columns": { - "type": "array", - "items": {"type": "string"}, - "description": "Columns to pass to the function (default: auto-detect numeric columns)", - }, - "max_sample": { - "type": "integer", - "description": "Maximum rows to evaluate (default 10000 for performance)", - "default": 10000, - }, - }, - "required": ["dataset_id", "python_function"], - }, - ), - ] - - -# ═══════════════════════════════════════════════════════════════════════ -# Handler implementations -# ═══════════════════════════════════════════════════════════════════════ - -async def handle_call(name: str, arguments: dict) -> dict: - """Dispatch a tool call to the appropriate handler. - - Args: - name: Tool name (must match one of the 11 tools). - arguments: Tool-specific parameters. - - Returns: - A JSON-serialisable dict. - - Raises: - ValueError: If *name* is not a recognised tool. - """ - _handlers = { - 'load_data': _handle_load_data, - 'profile_data': _handle_profile_data, - 'filter_data': _handle_filter_data, - 'preprocess_data': _handle_preprocess_data, - 'run_clustering': _handle_run_clustering, - 'evaluate_clustering': _handle_evaluate_clustering, - 'extract_features': _handle_extract_features, - 'export_results': _handle_export_results, - 'list_datasets': _handle_list_datasets, - 'drop_dataset': _handle_drop_dataset, - 'clone_dataset': _handle_clone_dataset, - 'build_entity_profiles': _handle_build_entity_profiles, - 'compute_scores': _handle_compute_scores, - 'filter_and_cluster': _handle_filter_and_cluster, - 'detect_anomalies': _handle_detect_anomalies, - 'visualize_anomalies': _handle_visualize_anomalies, - 'validate_data': _handle_validate_data, - 'explore_distributions': _handle_explore_distributions, - 'find_outliers': _handle_find_outliers, - 'diagnose_clustering': _handle_diagnose_clustering, - 'compare_datasets': _handle_compare_datasets, - 'export_debug_sample': _handle_export_debug_sample, - 'repair_schema': _handle_repair_schema, - 'analyze_patterns': _handle_analyze_patterns, - 'analyze_temporal': _handle_analyze_temporal, - 'analyze_fft': _handle_analyze_fft, - 'analyze_tls_health': _handle_analyze_tls_health, - 'analyze_geo_distribution': _handle_analyze_geo_distribution, - 'analyze_entity_detail': _handle_analyze_entity_detail, - 'compute_distance_matrix': _handle_compute_distance_matrix, - } - handler = _handlers.get(name) - if handler is None: - raise ValueError(f"Unknown tool: '{name}'") - result = await handler(**arguments) - # Always ensure truncated key exists - if 'truncated' not in result: - result['truncated'] = False - return _truncate_response(result) - - -# ── 1. load_data ──────────────────────────────────────────────────────── - -async def _handle_load_data( - csv_glob: str, - config_path: Optional[str] = None, - encoding: str = 'utf-8', - delimiter: str = ',', - schema_strict: bool = False, - recursive: bool = False, -) -> dict: - """Load, validate, merge CSVs, and store in session.""" - try: - lf, schema, row_count, file_count, memory_mb = load_csv_directory( - glob_pattern=csv_glob, - encoding=encoding, - delimiter=delimiter, - config_path=config_path, - sample_rows=10000, - schema_strict=schema_strict, - recursive=recursive, - ) - except FileNotFoundError as e: - return {'error': f'File not found: {e}', 'truncated': False} - except ValueError as e: - return {'error': f'Schema validation failed: {e}', 'truncated': False} - - store = SessionStore() - dataset_id = _generate_id('ds') - store.store_dataset( - dataset_id=dataset_id, - lazyframe=lf, - schema=schema, - metadata={ - 'row_count': row_count, - 'file_count': file_count, - 'memory_mb': round(memory_mb, 2), - 'csv_glob': csv_glob, - 'encoding': encoding, - }, - ) - - return { - 'dataset_id': dataset_id, - 'schema': schema, - 'row_count': row_count, - 'file_count': file_count, - 'memory_mb': round(memory_mb, 2), - } - - -# ── 2. profile_data ───────────────────────────────────────────────────── - -async def _handle_profile_data( - dataset_id: str, - sample_size: int = 1000, - columns: Optional[list[str]] = None, -) -> dict: - """Profile dataset columns and correlations.""" - store = SessionStore() - entry = store.get_dataset(dataset_id) - if entry is None: - return {'error': f'Dataset not found: {dataset_id}', 'truncated': False} - - lf: pl.LazyFrame = entry['lazyframe'] - profile = _profile_dataset(lf, sample_size=sample_size, columns=columns) - return profile - - -# ── 3. filter_data ────────────────────────────────────────────────────── - -def _build_filter_expr(column: str, op: str, value: Any) -> pl.Expr: - """Build a single Polars expression from a filter descriptor.""" - col = pl.col(column) - op_map = { - 'eq': col == value, - 'neq': col != value, - 'gt': col > value, - 'gte': col >= value, - 'lt': col < value, - 'lte': col <= value, - 'contains': col.str.contains(value) if value is not None else col.cast(str).str.contains(''), - 'not_contains': ~(col.str.contains(value) if value is not None else col.cast(str).str.contains('')), - 'in': col.is_in(value) if isinstance(value, (list, tuple)) else col.is_in([value]), - 'not_in': ~(col.is_in(value) if isinstance(value, (list, tuple)) else col.is_in([value])), - 'is_null': col.is_null(), - 'is_not_null': col.is_not_null(), - } - expr = op_map.get(op) - if expr is None: - raise ValueError(f"Unsupported filter operator: {op}") - return expr - - -async def _handle_filter_data( - dataset_id: str, - filters: list[dict], - logic: str = 'and', -) -> dict: - """Apply filters and store filtered dataset.""" - store = SessionStore() - entry = store.get_dataset(dataset_id) - if entry is None: - return {'error': f'Dataset not found: {dataset_id}', 'truncated': False} - - lf: pl.LazyFrame = entry['lazyframe'] - full_schema = entry.get('schema', {}) - - exprs = [_build_filter_expr( - f['column'], f.get('op', 'eq'), f.get('value'), - ) for f in filters] - - if not exprs: - return {'error': 'No filter expressions could be built', 'truncated': False} - - if logic == 'and': - combined = exprs[0] - for e in exprs[1:]: - combined = combined & e - else: - combined = exprs[0] - for e in exprs[1:]: - combined = combined | e - - filtered_lf = lf.filter(combined) - - # Estimate row count (fast streaming count) - try: - row_count = filtered_lf.select(pl.len()).collect(streaming=True).item() - except Exception: - row_count = None - - new_id = _generate_id('flt') - store.store_dataset( - dataset_id=new_id, - lazyframe=filtered_lf, - schema=full_schema, - metadata={ - 'row_count': row_count, - 'parent_dataset_id': dataset_id, - 'filter_count': len(filters), - 'logic': logic, - }, - parent_id=dataset_id, - ) - - return { - 'filtered_dataset_id': new_id, - 'row_count': row_count, - 'column_deltas': {'kept': list(full_schema.keys()), 'dropped': []}, - } - - -# ── 4. preprocess_data ────────────────────────────────────────────────── - -async def _handle_preprocess_data( - dataset_id: str, - columns: list[str], - config: Optional[dict] = None, -) -> dict: - """Preprocess selected columns (scale, encode, impute, drop).""" - store = SessionStore() - entry = store.get_dataset(dataset_id) - if entry is None: - return {'error': f'Dataset not found: {dataset_id}', 'truncated': False} - - lf: pl.LazyFrame = entry['lazyframe'] - cfg = config or {} - column_mapping: dict[str, str] = {} - - df = lf.collect(streaming=True) - - # Resolve columns that actually exist - available = set(df.columns) - target_cols = [c for c in columns if c in available] - if not target_cols: - return {'error': 'None of the specified columns exist in the dataset', - 'truncated': False} - - # Drop columns - drop_cols = cfg.get('drop', []) - if drop_cols: - df = df.drop([c for c in drop_cols if c in df.columns]) - for c in drop_cols: - if c in available: - column_mapping[c] = f'' - - # Fill missing values - fill_strategy = cfg.get('fillna') - if fill_strategy and fill_strategy != 'drop': - for col in target_cols: - if col not in df.columns: - continue - if df[col].null_count() == 0: - continue - if fill_strategy == 'mean': - mean_val = df[col].drop_nulls().mean() - if mean_val is not None: - df = df.with_columns(pl.col(col).fill_null(mean_val)) - elif fill_strategy == 'median': - med_val = df[col].drop_nulls().median() - if med_val is not None: - df = df.with_columns(pl.col(col).fill_null(med_val)) - elif fill_strategy == 'zero': - df = df.with_columns(pl.col(col).fill_null(0)) - elif fill_strategy == 'drop': - df = df.drop_nulls(subset=target_cols) - - # Standard scaling (z-score) - if cfg.get('standardize'): - from sklearn.preprocessing import StandardScaler - num_cols = [c for c in target_cols if c in df.columns - and df[c].dtype in (pl.Float32, pl.Float64, pl.Int32, pl.Int64)] - if num_cols: - scaler = StandardScaler() - scaled = scaler.fit_transform(df.select(num_cols).to_numpy()) - for i, col in enumerate(num_cols): - col_scaled = f'{col}_scaled' - df = df.with_columns(pl.Series(col_scaled, scaled[:, i])) - column_mapping[col] = col_scaled - - # One-hot encoding - if cfg.get('onehot'): - str_cols = [c for c in target_cols if c in df.columns - and df[c].dtype in (pl.Utf8, pl.String, pl.Categorical)] - if str_cols: - df = df.to_dummies(columns=str_cols, drop_first=False) - for c in str_cols: - column_mapping[c] = f'' - - new_id = _generate_id('pp') - new_lf = df.lazy() - new_schema = {name: str(dtype) for name, dtype in - zip(new_lf.collect_schema().names(), new_lf.collect_schema().dtypes())} - - store.store_dataset( - dataset_id=new_id, - lazyframe=new_lf, - schema=new_schema, - metadata={ - 'row_count': len(df), - 'parent_dataset_id': dataset_id, - 'preprocessing_config': cfg, - }, - parent_id=dataset_id, - ) - - return { - 'preprocessed_dataset_id': new_id, - 'column_mapping': column_mapping, - 'row_count': len(df), - 'column_count': len(new_schema), - } - - -# ── 5. run_clustering ─────────────────────────────────────────────────── - -async def _handle_run_clustering( - dataset_id: str, - cluster_columns: list[str], - algorithm: str = 'agglomerative', - params: Optional[dict] = None, - random_state: int = 42, -) -> dict: - """Run HDBSCAN or KMeans clustering on selected columns (async).""" - store = SessionStore() - entry = store.get_dataset(dataset_id) - if entry is None: - return {'error': f'Dataset not found: {dataset_id}', 'truncated': False} - - lf: pl.LazyFrame = entry['lazyframe'] - p = params or {} - - try: - # Collect cluster features - numeric only - available = lf.collect_schema() - available_names = available.names() - available_dtypes = available.dtypes() - numeric_dtypes = (pl.Int8, pl.Int16, pl.Int32, pl.Int64, - pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64, - pl.Float32, pl.Float64) - feature_cols = [ - c for c in cluster_columns - if c in available_names - and available_dtypes[available_names.index(c)] in numeric_dtypes - ] - if not feature_cols: - # Fall back: auto-select all numeric columns (exclude internal ones) - feature_cols = [ - available_names[i] for i, dt in enumerate(available_dtypes) - if dt in numeric_dtypes and not available_names[i].startswith('_') - ][:10] - if not feature_cols: - # Fallback: try coercing Utf8 columns to Float64 (common when data loaded from - # SQLite TEXT storage without proper dtype restoration). - utf8_cols = [ - available_names[i] for i, dt in enumerate(available_dtypes) - if dt == pl.Utf8 and not available_names[i].startswith('_') - ][:10] - if utf8_cols: - # Attempt per-column cast; filter to columns where coercion succeeded - for col in utf8_cols: - try: - casted = lf.select( - pl.col(col).cast(pl.Float64, strict=False).alias(col) - ).collect(streaming=True) - null_ratio = casted[col].is_null().sum() / max(len(casted), 1) - if null_ratio < 0.9: # at least 10% successfully parsed as floats - lf = lf.with_columns( - pl.col(col).cast(pl.Float64, strict=False) - ) - feature_cols.append(col) - except Exception: - pass - if not feature_cols: - all_numeric = [f"{available_names[i]}" for i, dt in enumerate(available_dtypes) - if dt in numeric_dtypes] - return { - 'error': f'No numeric cluster columns found in dataset. Available: {all_numeric}', - 'truncated': False - } - - # Downsample before collect: compute row count via fast lazy count - try: - row_count = lf.select(pl.len()).collect(streaming=True).item() - except Exception: - row_count = 0 - if row_count > 100000: - # Sample to 100K before collect to avoid OOM on 10M+ rows - lf = lf.collect(streaming=True).sample(n=100000, seed=42).lazy() - - df_features = lf.select(feature_cols).collect(streaming=True) - - # Drop all-NaN columns (keep rows, drop useless columns) - for col in df_features.columns: - if df_features[col].is_null().all(): - df_features = df_features.drop(col) - feature_cols = [c for c in feature_cols if c != col] - - if df_features.width == 0: - return {'error': 'No valid feature columns after dropping all-NaN columns', - 'truncated': False} - - data = df_features.to_numpy() - - # Fill remaining NaN with column mean - if data.dtype.kind == 'f': - col_mean = np.nanmean(data, axis=0) - col_mean = np.nan_to_num(col_mean, nan=0.0) - data = np.where(np.isnan(data), col_mean, data) - - # Zero-sample / too-few-samples guard - if len(data) < 3: - return { - 'cluster_result_id': _generate_id('clust'), - 'n_clusters': 0, - 'n_noise': 0, - 'quality_metrics': {'note': 'too few samples'}, - 'truncated': False, - } - - # ── D14: Low memory check (safe fallback if psutil not installed) ── - low_memory = False - try: - import psutil - mem = psutil.virtual_memory() - low_memory = mem.available < 2 * 1024**3 # 2 GB - except ImportError: - pass - - # ── D14: Auto-downsample for low memory (>100K rows → 50K) ──────── - if low_memory and len(data) > 100000: - rng = np.random.default_rng(42) - idx = rng.choice(len(data), 50000, replace=False) - data = data[idx] - - # ── D5: Downsampling → extract matching indices ────────────── - # Downsample to 50K max, store indices so _handle_extract_features - # can filter the same rows. - _ds_idx = None - if len(data) > 50000: - rng = np.random.default_rng(42) - _ds_idx = rng.choice(len(data), 50000, replace=False) - data = data[_ds_idx] - - # ── H17: Variance filtering ─────────────────────────────────────── - # Remove features with zero / near-zero variance (< 1e-10) - if data.shape[1] > 1: - variances = np.var(data, axis=0) - keep_var = np.where(variances >= 1e-10)[0] - if len(keep_var) == 0: - # All features have zero variance — fall back to unfiltered data - keep_var = np.arange(data.shape[1]) - if len(keep_var) < data.shape[1]: - data = data[:, keep_var] - feature_cols = [feature_cols[i] for i in keep_var] - - # Log clustering input - import logging - _clust_logger = logging.getLogger(__name__) - _clust_logger.info(f'[CLUSTER_INPUT] rows={len(data)} cols={data.shape[1] if len(data.shape) > 1 else 1} feature_cols={feature_cols}') - - # Standard scale - from sklearn.preprocessing import StandardScaler - _clust_logger.info('[CLUSTER_SCALE] Starting StandardScaler...') - data_scaled = StandardScaler().fit_transform(data) - _clust_logger.info(f'[CLUSTER_SCALE] Done. shape={data_scaled.shape}') - - # ── H17: Correlation filtering ──────────────────────────────────── - # Remove features with Pearson correlation > 0.95 (keep the first one) - if data_scaled.shape[1] > 1: - corr = np.corrcoef(data_scaled.T) - upper = np.triu(np.abs(corr), 1) - to_drop = [j for j in range(upper.shape[1]) if any(upper[:, j] > 0.95)] - if to_drop: - keep = [j for j in range(data_scaled.shape[1]) if j not in to_drop] - data_scaled = data_scaled[:, keep] - feature_cols = [feature_cols[j] for j in keep] - - # ── D14: SVD dimensionality reduction (if >50 features) ─────────── - n_features = data_scaled.shape[1] - if n_features > 50: - from sklearn.decomposition import TruncatedSVD - svd = TruncatedSVD(n_components=50, random_state=random_state) - data_scaled = svd.fit_transform(data_scaled) - - # ── H17: HDBSCAN hyperparameter auto-tuning based on data size ──── - n = len(data_scaled) - if algorithm == 'hdbscan': - min_cluster = p.get('min_cluster_size', max(3, min(50, n // 20))) - p['min_cluster_size'] = min_cluster - p['min_samples'] = p.get('min_samples', min_cluster) - p['cluster_selection_epsilon'] = p.get('cluster_selection_epsilon', 0.0) - else: - # KMeans safety cap - p['min_cluster_size'] = min( - p.get('min_cluster_size', 5), - max(n // 2, 2), - ) - - # Clustering - from sklearn.cluster import HDBSCAN, MiniBatchKMeans, AgglomerativeClustering - if algorithm == 'kmeans': - n_clusters = p.get('n_clusters', 3) - model = MiniBatchKMeans( - n_clusters=n_clusters, random_state=random_state, - batch_size=1024, n_init='auto', - ) - elif algorithm in ('agglomerative', 'ward'): - n_clusters = p.get('n_clusters', 5) - model = AgglomerativeClustering( - n_clusters=n_clusters, - metric=p.get('metric', 'euclidean'), - linkage=p.get('linkage', 'ward'), - distance_threshold=None, - ) - else: - model = HDBSCAN( - min_cluster_size=p['min_cluster_size'], - min_samples=p.get('min_samples', None), - metric=p.get('metric', 'euclidean'), - cluster_selection_epsilon=p.get('cluster_selection_epsilon', 0.0), - ) - - labels_array = model.fit_predict(data_scaled) - n_clusters_found = int(len(set(labels_array)) - (1 if -1 in labels_array else 0)) - noise_count = int((labels_array == -1).sum()) if algorithm == 'hdbscan' else 0 - noise_ratio = round(noise_count / len(labels_array), 4) if algorithm == 'hdbscan' else 0.0 - - # Map back to original rows - full_labels = labels_array - - # Quality metrics - quality = { - 'n_clusters': n_clusters_found, - 'n_noise': noise_count, - 'noise_ratio': noise_ratio, - 'algorithm': algorithm, - } - - if n_clusters_found > 1: - try: - from sklearn.metrics import silhouette_score - score = silhouette_score(data_scaled, labels_array, random_state=random_state) - quality['silhouette_score'] = round(float(score), 4) - except Exception: - quality['silhouette_score'] = None - try: - from sklearn.metrics import davies_bouldin_score - dbs = davies_bouldin_score(data_scaled, labels_array) - quality['davies_bouldin_score'] = round(float(dbs), 4) - except Exception: - quality['davies_bouldin_score'] = None - try: - from sklearn.metrics import calinski_harabasz_score - chs = calinski_harabasz_score(data_scaled, labels_array) - quality['calinski_harabasz_score'] = round(float(chs), 4) - except Exception: - quality['calinski_harabasz_score'] = None - - # Cluster sizes - cluster_sizes = {} - for label in np.unique(labels_array): - cluster_sizes[int(label)] = int((labels_array == label).sum()) - quality['cluster_sizes'] = cluster_sizes - - except Exception: - raise - - result_id = _generate_id('clust') - store.store_cluster_result( - result_id=result_id, - labels=full_labels.tolist(), - n_clusters=n_clusters_found, - params={ - 'algorithm': algorithm, - 'params': p, - 'random_state': random_state, - 'feature_columns': feature_cols, - '_ds_idx': _ds_idx.tolist() if _ds_idx is not None else None, - }, - parent_dataset_id=dataset_id, - ) - - return { - 'cluster_result_id': result_id, - 'n_clusters': n_clusters_found, - 'n_noise': noise_count, - 'labels_sample': full_labels[:50].tolist(), - 'quality_metrics': quality, - } - - -# ── 6. evaluate_clustering ────────────────────────────────────────────── - -async def _handle_evaluate_clustering( - cluster_result_id: str, - dataset_id: str, - metrics: Optional[list[str]] = None, -) -> dict: - """Evaluate clustering result against the dataset.""" - store = SessionStore() - cluster_entry = store.get_cluster_result(cluster_result_id) - if cluster_entry is None: - return {'error': f'Cluster result not found: {cluster_result_id}', - 'truncated': False} - - dataset_entry = store.get_dataset(dataset_id) - if dataset_entry is None: - return {'error': f'Dataset not found: {dataset_id}', 'truncated': False} - - labels = cluster_entry['labels'] - default_metrics = ['silhouette', 'davies_bouldin', 'calinski_harabasz', - 'noise_ratio', 'cluster_sizes'] - requested = metrics or default_metrics - - scores: dict = {'n_clusters': cluster_entry['n_clusters']} - - try: - lf: pl.LazyFrame = dataset_entry['lazyframe'] - params = cluster_entry.get('params', {}) - feature_cols = params.get('feature_columns', []) - if not feature_cols and 'feature_columns' in params: - pass - elif feature_cols: - df_features = lf.select(feature_cols).collect(streaming=True) - data = df_features.to_numpy() - - # Align labels with data (handle missing rows) - valid_mask = ~np.isnan(data).any(axis=1) if data.dtype.kind == 'f' else np.ones(len(data), dtype=bool) - data_clean = data[valid_mask] - labels_clean = np.array(labels)[:len(data_clean)] - - unique_labels = set(labels_clean) - n_clusters_actual = len(unique_labels - {-1}) - - for m in requested: - if m == 'silhouette' and n_clusters_actual >= 2: - from sklearn.metrics import silhouette_score - scores['silhouette'] = round(float( - silhouette_score(data_clean, labels_clean, random_state=42)), 4) - elif m == 'davies_bouldin' and n_clusters_actual >= 2: - from sklearn.metrics import davies_bouldin_score - scores['davies_bouldin'] = round(float( - davies_bouldin_score(data_clean, labels_clean)), 4) - elif m == 'calinski_harabasz' and n_clusters_actual >= 2: - from sklearn.metrics import calinski_harabasz_score - scores['calinski_harabasz'] = round(float( - calinski_harabasz_score(data_clean, labels_clean)), 4) - elif m == 'noise_ratio': - noise = int((labels_clean == -1).sum()) - scores['noise_ratio'] = round(noise / len(labels_clean), 4) - elif m == 'cluster_sizes': - sizes = {} - for label in unique_labels: - sizes[int(label)] = int((labels_clean == label).sum()) - scores['cluster_sizes'] = sizes - - scores['n_clusters'] = cluster_entry['n_clusters'] - - except Exception: - raise - - return {'scores': scores} - - -# ── 7. extract_features ───────────────────────────────────────────────── - -async def _handle_extract_features( - dataset_id: str, - cluster_result_id: str, - top_k: int = 10, - method: str = 'zscore', - save_to_db: bool = True, - run_id: int | None = None, -) -> dict: - """Compute per-cluster distinguishing features and optionally persist.""" - store = SessionStore() - dataset_entry = store.get_dataset(dataset_id) - if dataset_entry is None: - return {'error': f'Dataset not found: {dataset_id}', 'truncated': False} - - cluster_entry = store.get_cluster_result(cluster_result_id) - if cluster_entry is None: - return {'error': f'Cluster result not found: {cluster_result_id}', - 'truncated': False} - - try: - lf: pl.LazyFrame = dataset_entry['lazyframe'] - labels = cluster_entry['labels'] - - # D5: If clustering downsampled the data, filter matching rows - _ds_idx = cluster_entry.get('params', {}).get('_ds_idx') - if _ds_idx is not None: - # Filter in lazy plan before collect — avoids loading full 10M rows - df = (lf.with_row_index('__ds_idx') - .filter(pl.col('__ds_idx').is_in(_ds_idx)) - .drop('__ds_idx') - .collect(streaming=True)) - else: - df = lf.collect(streaming=True) - - # Truncate labels if df was filtered but labels weren't - if len(labels) != len(df): - labels = labels[:len(df)] - - # Only use numeric columns for feature extraction - numeric_cols = [c for c in df.columns - if df[c].dtype in (pl.Float32, pl.Float64, - pl.Int32, pl.Int64, - pl.UInt32, pl.UInt64)] - if len(labels) != len(df): - labels = labels[:len(df)] - - # Global statistics per column - global_stats = {} - for col in numeric_cols: - s = df[col].drop_nulls() - if len(s) > 0: - global_stats[col] = { - 'mean': float(s.mean()), - 'std': float(s.std()) if s.std() is not None else 1.0, - } - - unique_labels = sorted(set(labels)) - all_features = [] - - for label in unique_labels: - mask = np.array(labels) == label - if mask.sum() < 2: - continue - - cluster_df = df.filter(pl.Series('__mask__', mask)) - cluster_features = [] - - for col in numeric_cols: - c_series = cluster_df[col].drop_nulls() - gs = global_stats.get(col) - if gs is None or gs['std'] == 0 or len(c_series) == 0: - continue - - c_mean = float(c_series.mean()) - c_std = float(c_series.std()) if c_series.std() is not None else 0.0 - - if method == 'zscore': - score = (c_mean - gs['mean']) / max(gs['std'], 1e-10) - else: - # ANOVA-like: F = (between-group variance) / (within-group variance) - score = abs(c_mean - gs['mean']) / max(c_std + gs['std'], 1e-10) - - cluster_features.append({ - 'feature_name': col, - 'cluster_label': int(label), - 'mean': round(c_mean, 4), - 'std': round(c_std, 4), - 'global_mean': round(gs['mean'], 4), - 'global_std': round(gs['std'], 4), - 'distinguishing_score': round(float(score), 4), - 'distinguishing_method': method, - }) - - # Sort by distinguishing score descending, take top_k - cluster_features.sort(key=lambda x: abs(x['distinguishing_score']), reverse=True) - all_features.extend(cluster_features[:top_k]) - - # ── Compute UMAP-2D embedding ────────────────────────────────────── - if len(numeric_cols) >= 2 and len(df) > 2: - try: - from sklearn.preprocessing import StandardScaler - import umap - MAX_UMAP_TRAIN = 10000 - BATCH_SIZE = 1000 - data_matrix = df.select(numeric_cols).to_numpy() - if len(df) > MAX_UMAP_TRAIN: - # Train UMAP on 10K sample, batch-transform full dataset - idx_sample = np.random.RandomState(42).choice( - len(df), size=MAX_UMAP_TRAIN, replace=False) - mat_sample = data_matrix[idx_sample] - scaler = StandardScaler().fit(mat_sample) - mat_sample_scaled = np.nan_to_num(scaler.transform(mat_sample), - nan=0.0, posinf=0.0, neginf=0.0) - reducer = umap.UMAP(n_components=2, random_state=42, - n_neighbors=15, min_dist=0.1, - metric='euclidean') - reducer.fit(mat_sample_scaled) - # Batch transform in 1K-row batches - coords_list = [] - for start in range(0, len(df), BATCH_SIZE): - end = min(start + BATCH_SIZE, len(df)) - mat_batch = scaler.transform(data_matrix[start:end]) - mat_batch = np.nan_to_num(mat_batch, nan=0.0, posinf=0.0, neginf=0.0) - coords_list.append(reducer.transform(mat_batch)) - coords = np.vstack(coords_list) - else: - data_scaled = StandardScaler().fit_transform(data_matrix) - # Handle NaN/Inf values (np is imported at module level) - data_scaled = np.nan_to_num(data_scaled, nan=0.0, posinf=0.0, neginf=0.0) - reducer = umap.UMAP(n_components=2, random_state=42, - n_neighbors=15, min_dist=0.1, - metric='euclidean') - coords = reducer.fit_transform(data_scaled) - non_numeric = [c for c in df.columns if c not in numeric_cols and not c.startswith('_')] - entity_col_umap = non_numeric[0] if non_numeric else df.columns[0] - labels_list = cluster_entry.get('labels', []) - # Save: create EntityProfile records with coordinates - from analysis.models import EntityProfile as EP - from analysis.models import ClusterResult as CR - from analysis.models import AnalysisRun as AR - from asgiref.sync import sync_to_async - csv_glob = dataset_entry.get('metadata', {}).get('csv_glob', 'manual') - run_obj = await sync_to_async( - AR.objects.filter(csv_glob=csv_glob).order_by('-id').first, - thread_sensitive=True - )() - if run_obj: - run_id_for_ep = run_obj.id - profiles = [] - cr_cache = {} - for i, row in enumerate(df.iter_rows(named=True)): - if i >= len(coords): break - ev = str(row.get(entity_col_umap, '')) - if not ev: continue - lbl = labels_list[i] if i < len(labels_list) else -1 - cache_key = f'{run_id_for_ep}_{lbl}' - if cache_key not in cr_cache: - cr_cache[cache_key] = CR.objects.filter(run_id=run_id_for_ep, cluster_label=lbl).first() - profiles.append(EP( - entity_value=ev, run_id=run_id_for_ep, cluster_label=lbl, - cluster=cr_cache[cache_key], - embedding_x=float(coords[i, 0]), - embedding_y=float(coords[i, 1]), - )) - if profiles: - EP.objects.bulk_create(profiles, ignore_conflicts=True, batch_size=1000) - except Exception: - import traceback, sys; print(f'[UMAP_ERR] {traceback.format_exc()}', file=sys.stderr) - - # Persist to Django ORM - db_saved = False - if save_to_db: - try: - from asgiref.sync import sync_to_async - from analysis.models import AnalysisRun as AnalysisRunModel - ce = cluster_entry # local ref for closure - async def _do_save(ce_inner): - if run_id is not None: - run = await sync_to_async(AnalysisRunModel.objects.get)(id=run_id) - else: - csv_glob_fallback = dataset_entry.get('metadata', {}).get('csv_glob', 'manual') - run = await sync_to_async( - AnalysisRunModel.objects.filter(csv_glob=csv_glob_fallback).order_by('-id').first, - thread_sensitive=True - )() - if run is None: - run = await sync_to_async(AnalysisRunModel.objects.create)( - csv_glob=csv_glob_fallback, status='running', total_flows=0, - ) - run.status = 'completed' - run.cluster_count = ce_inner['n_clusters'] - await sync_to_async(run.save)() - return await sync_to_async(_save_features_to_db)(cluster_result_id, all_features, run_id=run.id) - db_saved = await _do_save(ce) - except Exception as e: - import traceback, sys; print(f'[DB_SAVE_ERROR] {e}\n{traceback.format_exc()}', file=sys.stderr) - db_saved = False - - except Exception: - raise - - result_id = _generate_id('feat') - store.store_feature_result( - result_id=result_id, - features=all_features, - parent_cluster_result_id=cluster_result_id, - ) - - return { - 'feature_result_id': result_id, - 'features': all_features, - 'n_features': len(all_features), - 'db_saved': db_saved, - } - - -@profile -def _save_features_to_db( - cluster_result_id: str, - features: list[dict], - run_id: int | None = None, -) -> bool: - """Save extracted features to the Django ORM ClusterFeature model. - - If run_id is provided, use it directly (skip csv_glob lookup). - """ - if run_id is not None: - from analysis.models import AnalysisRun as AnalysisRun2 - run = AnalysisRun2.objects.filter(id=run_id).first() - if run is None: - return False - else: - # Try to find the ClusterResult record by run association - store = SessionStore() - cluster_entry = store.get_cluster_result(cluster_result_id) - if cluster_entry is None: - return False - - parent_ds_id = cluster_entry.get('parent_dataset_id', '') - from analysis.models import AnalysisRun - ds_entry = store.get_dataset(parent_ds_id) - if ds_entry is None: - return False - - csv_glob = ds_entry.get('metadata', {}).get('csv_glob', '') - # Compute n_clusters from features before using it - unique_global = sorted(set(f['cluster_label'] for f in features)) - n_clusters_global = len([l for l in unique_global if l >= 0]) - try: - run = AnalysisRun.objects.filter(csv_glob=csv_glob).order_by('-created_at').first() - if run is None: - run = AnalysisRun.objects.create( - csv_glob=csv_glob, - status='completed', - cluster_count=n_clusters_global, - ) - except Exception: - return False - - from analysis.models import ClusterResult as ClusterResultModel - - # Compute actual cluster sizes from full label list (not from features count!) - # Use the cluster_entry which has the complete labels array - store = SessionStore() - cluster_entry = store.get_cluster_result(cluster_result_id) - label_counts = {} - if cluster_entry: - full_labels = cluster_entry.get('labels', []) - for lbl in full_labels: - label_counts[lbl] = label_counts.get(lbl, 0) + 1 - - unique_labels = sorted(set( - f['cluster_label'] for f in features - )) - # Compute n_clusters from features (excluding noise -1) - n_clusters = len([l for l in unique_labels if l >= 0]) - for label in unique_labels: - if label == -1: - continue # skip noise cluster - label_features = [f for f in features if f['cluster_label'] == label] - if not label_features: - continue - - cluster_db, _ = ClusterResultModel.objects.get_or_create( - run=run, - cluster_label=label, - defaults={'size': label_counts.get(label, 1)}, - ) - # Update size if already exists - if label_counts.get(label) is not None: - ClusterResultModel.objects.filter( - run=run, cluster_label=label - ).update(size=label_counts[label]) - - from analysis.models import ClusterFeature as ClusterFeatureModel - for feat in label_features: - try: - ClusterFeatureModel.objects.get_or_create( - cluster=cluster_db, - feature_name=feat['feature_name'], - defaults={ - 'mean': feat.get('mean'), - 'std': feat.get('std'), - 'median': None, - 'p25': None, - 'p75': None, - 'missing_rate': None, - 'distinguishing_score': feat.get('distinguishing_score'), - 'distinguishing_method': feat.get('distinguishing_method', 'zscore'), - } - ) - except Exception: - continue # skip problematic features - - # Update run counts - run.cluster_count = n_clusters - run.save(update_fields=['cluster_count']) - return True - - return True - - -# ── 8. export_results ─────────────────────────────────────────────────── - -async def _handle_export_results( - result_id: str, - output_path: str, - format: str = 'csv', - overwrite: bool = False, -) -> dict: - """Export a dataset (or result) to disk.""" - store = SessionStore() - - out = Path(output_path) - file_paths: list[str] = [] - - # Check if it's a dataset - dataset_entry = store.get_dataset(result_id) - if dataset_entry is not None: - lf: pl.LazyFrame = dataset_entry['lazyframe'] - try: - df = lf.collect(streaming=True) - if out.suffix == '': - out = out / f"export_{result_id}.{format}" - - if out.exists() and not overwrite: - return {'error': f'Output exists and overwrite=False: {out}', - 'truncated': False} - - out.parent.mkdir(parents=True, exist_ok=True) - - if format == 'csv': - df.write_csv(out) - elif format == 'parquet': - df.write_parquet(out) - elif format == 'json': - df.write_json(out) - else: - return {'error': f'Unsupported format: {format}', - 'truncated': False} - - file_paths.append(str(out.resolve())) - except Exception: - raise - - return {'file_paths': file_paths, 'format': format, - 'row_count': len(df)} - - # Check if it's a cluster result - cluster_entry = store.get_cluster_result(result_id) - if cluster_entry is not None: - if out.suffix == '': - out = out / f"cluster_{result_id}.json" - out.parent.mkdir(parents=True, exist_ok=True) - - if out.exists() and not overwrite: - return {'error': f'Output exists and overwrite=False: {out}', - 'truncated': False} - - export_data = { - 'result_id': result_id, - 'n_clusters': cluster_entry['n_clusters'], - 'params': cluster_entry['params'], - 'labels_sample': cluster_entry['labels'][:1000], - } - with open(out, 'w', encoding='utf-8') as f: - json.dump(export_data, f, default=str) - file_paths.append(str(out.resolve())) - - return {'file_paths': file_paths, 'format': 'json'} - - # Check feature result - feat_entry = store.get_feature_result(result_id) - if feat_entry is not None: - if out.suffix == '': - out = out / f"features_{result_id}.json" - out.parent.mkdir(parents=True, exist_ok=True) - - if out.exists() and not overwrite: - return {'error': f'Output exists and overwrite=False: {out}', - 'truncated': False} - - with open(out, 'w', encoding='utf-8') as f: - json.dump(feat_entry['features'], f, default=str) - file_paths.append(str(out.resolve())) - return {'file_paths': file_paths, 'format': 'json'} - - return {'error': f'No dataset or result found for ID: {result_id}', - 'truncated': False} - - -# ── 9. list_datasets ──────────────────────────────────────────────────── - -async def _handle_list_datasets( - _filter_type: Optional[str] = None, -) -> dict: - """List all active datasets and results.""" - store = SessionStore() - datasets = store.list_datasets() - return {'datasets': datasets, 'count': len(datasets)} - - -# ── 10. drop_dataset ──────────────────────────────────────────────────── - -async def _handle_drop_dataset(dataset_id: str) -> dict: - """Remove a dataset/result and release memory.""" - store = SessionStore() - - dropped = (store.drop_dataset(dataset_id) - or store.drop_cluster_result(dataset_id) - or store.drop_feature_result(dataset_id)) - - if dropped: - return {'dropped': True, 'dataset_id': dataset_id} - return {'error': f'No dataset or result found: {dataset_id}', - 'truncated': False} - - -# ── 11. clone_dataset ─────────────────────────────────────────────────── - -async def _handle_clone_dataset(dataset_id: str) -> dict: - """Shallow-clone a dataset.""" - store = SessionStore() - new_id = store.clone_dataset(dataset_id) - if new_id is None: - return {'error': f'Dataset not found: {dataset_id}', 'truncated': False} - return {'original_dataset_id': dataset_id, 'cloned_dataset_id': new_id} - - -# ── 12. build_entity_profiles ──────────────────────────────────────────── - -_ENTITY_KEYWORDS = [ - 'src_ip', 'dst_ip', 'source_ip', 'destination_ip', 'src_addr', 'dst_addr', - 'sip', 'dip', 'ip_src', 'ip_dst', 'saddr', 'daddr', - 'sni', 'host', 'domain', 'uri', 'url', 'fqdn', 'server_name', - 'mac', 'mac_addr', 'src_mac', 'dst_mac', - 'user_agent', 'ua', 'asn', 'organization', 'org', -] - -_NUMERIC_TYPES_FOR_AGG = (pl.Int8, pl.Int16, pl.Int32, pl.Int64, - pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64, - pl.Float32, pl.Float64) - - -async def _handle_build_entity_profiles( - dataset_id: str, - entity_column: Optional[str] = None, - entity_columns: Optional[list[str]] = None, - auto_detect: bool = True, -) -> dict: - """Group raw flow data by entity column(s) and compute per-entity aggregate features. - - Stores the result in the session store and returns the new dataset_id. - """ - store = SessionStore() - entry = store.get_dataset(dataset_id) - if entry is None: - return {'error': f'Dataset not found: {dataset_id}', 'truncated': False} - - lf: pl.LazyFrame = entry['lazyframe'] - schema = entry.get('schema', {}) - col_names = list(schema.keys()) - - # ── Resolve entity columns ────────────────────────────────────────── - if entity_column and entity_column in col_names: - group_cols = [entity_column] - elif entity_columns: - group_cols = [c for c in entity_columns if c in col_names] - if not group_cols: - return {'error': f'None of entity_columns {entity_columns} exist in dataset. Available: {col_names[:20]}', - 'truncated': False} - elif auto_detect: - # Auto-detect: find columns matching entity keywords - matched = [] - for kw in _ENTITY_KEYWORDS: - found = [c for c in col_names if kw.lower() in c.lower()] - matched.extend(found) - # Deduplicate while preserving order - seen = set() - group_cols = [c for c in matched if not (c in seen or seen.add(c))] - if not group_cols: - # Fallback: pick first non-numeric, non-internal column - for c in col_names: - if c.startswith('_'): - continue - group_cols = [c] - break - # Limit to at most 3 entity columns - group_cols = group_cols[:3] - else: - return {'error': 'No entity column specified and auto_detect is disabled', - 'truncated': False} - - if not group_cols: - return {'error': 'Could not determine entity column(s) for grouping', - 'truncated': False} - - # ── Build aggregation expressions ────────────────────────────────── - # Always: count of flows - agg_exprs: list[pl.Expr] = [ - pl.len().alias('flow_count'), - pl.col(group_cols[0]).n_unique().alias(f'unique_{group_cols[0]}'), - ] - - # Numeric aggregations (sum, mean) - numeric_cols = [c for c in col_names if c not in group_cols - and schema[c].startswith(('Int', 'UInt', 'Float'))] - for c in numeric_cols[:20]: - agg_exprs.append(pl.sum(c).alias(f'total_{c}')) - agg_exprs.append(pl.mean(c).alias(f'avg_{c}')) - - # Unique counts for string-like entity columns (dst_ip, dst_port, sni, etc.) - str_cols = [c for c in col_names if c not in group_cols - and schema[c].startswith(('Utf8', 'String', 'Cat'))] - for c in str_cols[:10]: - agg_exprs.append(pl.col(c).n_unique().alias(f'unique_{c}')) - - # Also count distinct values of specific keyword-matching columns - for kw in ['dst_ip', 'dip', 'dst_addr', 'daddr', 'ip_dst', 'dest_ip', - 'dst_port', 'dport', 'destination_port', 'dest_port', - 'port', 'protocol', 'proto', 'sni', 'server_name', - 'cipher_suite', 'cipher', 'tls_version', 'version', '0ver', - 'country', 'scnt', 'dcnt', 'asn', 'organization']: - found = [c for c in col_names if c not in group_cols and kw.lower() in c.lower()] - for c in found: - alias_name = f'unique_{c}' - if alias_name not in [str(a.meta.output_name()) if hasattr(a, 'meta') else '' for a in agg_exprs]: - agg_exprs.append(pl.col(c).n_unique().alias(alias_name)) - - # ── Execute grouping ─────────────────────────────────────────────── - try: - entity_lf = lf.group_by(group_cols).agg(agg_exprs) - # Estimate row count - n_entities = entity_lf.select(pl.len()).collect(streaming=True).item() - except Exception: - raise - - # ── Store result ─────────────────────────────────────────────────── - new_schema = {name: str(dtype) for name, dtype in - zip(entity_lf.collect_schema().names(), entity_lf.collect_schema().dtypes())} - new_id = f'entity_{_generate_id("ds")}' - store.store_dataset( - dataset_id=new_id, - lazyframe=entity_lf, - schema=new_schema, - metadata={ - 'row_count': n_entities, - 'entity_columns': group_cols, - 'parent_dataset_id': dataset_id, - 'n_agg_features': len(agg_exprs), - }, - parent_id=dataset_id, - ) - - return { - 'dataset_id': new_id, - 'entity_columns': group_cols, - 'n_entities': n_entities, - 'n_features': len(agg_exprs), - 'aggregated_columns': list(new_schema.keys())[:20], - } - - -# ── Adaptive anomaly scoring helpers (inlined from entity_aggregator) ───── - -_ANOMALY_FEATURE_COLS = [ - 'non_std_port_rate', 'modern_tls_rate', 'sni_missing_ratio', - 'recoverable_ratio', 'flow_count', 'total_bytes_sent', - 'total_packets', 'unique_dst_ips', 'unique_dst_ports', - 'unique_protocols', 'has_sni', 'avg_sni_length', - 'countries_visited', -] - - -@profile -def _add_adaptive_scores(lf: pl.LazyFrame, feature_names: list[str]) -> pl.LazyFrame: - """Add anomaly scores using data-driven weights (SVD + IQR + percentile). - - Steps: - 1. Find which anomaly feature columns exist in the aggregated data - 2. Collect a sample (max 10K rows) for SVD weight learning - 3. Learn feature weights from SVD first component loadings - 4. Compute weighted proxy_score across ALL rows (vectorized, no per-row ops) - 5. Add percentile rank calibration - 6. Infer risk thresholds from data distribution - """ - _agg_cols = lf.collect_schema().names() - avail = [c for c in _ANOMALY_FEATURE_COLS if c in _agg_cols] - if not avail: - return lf - - try: - # Step 1-2: Sample up to 10K rows for weight learning - n_rows = lf.select(pl.len()).collect(streaming=True).item() - sample_lf = lf if n_rows <= 10000 else lf.sample(n=10000, seed=42) - df_sample = sample_lf.select(avail).collect(streaming=True) - mat = df_sample.to_numpy() - mat = np.nan_to_num(mat, nan=0.0) - - # Step 3: SVD weight learning - from sklearn.decomposition import TruncatedSVD - n_comp = min(1, min(mat.shape) - 1) - if n_comp < 1 or mat.shape[1] < 2: - return lf - svd = TruncatedSVD(n_components=1, random_state=42).fit(mat) - raw_weights = np.abs(svd.components_[0]) - # Normalise to sum = 1 - weights = dict(zip(avail, raw_weights / raw_weights.sum())) - - # Step 4: Weighted sum expression (all vectorised, no loop) - expr_parts = [] - for col, w in weights.items(): - if col == 'modern_tls_rate': - # modern_tls_rate is good — invert for anomaly scoring - expr_parts.append((1.0 - pl.col(col).fill_null(0.5)) * w) - else: - expr_parts.append(pl.col(col).fill_null(0.0) * w) - if expr_parts: - proxy_expr = expr_parts[0] - for p in expr_parts[1:]: - proxy_expr = proxy_expr + p - else: - return lf - - lf = lf.with_columns(proxy_expr.clip(0, 1).alias('_raw_score')) - - # Step 5: Percentile rank calibration (0-1, relative position) - sample_scores = df_sample.select( - pl.sum_horizontal([ - (1.0 - pl.col('modern_tls_rate').fill_null(0.5)) * weights.get('modern_tls_rate', 0) - if c == 'modern_tls_rate' - else pl.col(c).fill_null(0.0) * weights.get(c, 0) - for c in avail - ]).clip(0, 1) - ).to_series().to_numpy() - - # Compute IQR thresholds from sample - q25, q50, q75 = np.percentile(sample_scores, [25, 50, 75]) - iqr = q75 - q25 - upper_fence = float(q75 + 1.5 * iqr) if iqr > 0 else 0.9 - extreme_fence = float(q75 + 3.0 * iqr) if iqr > 0 else 0.99 - - # Step 6: Risk level labels (0.0-1.0) - lf = lf.with_columns([ - pl.col('_raw_score').alias('proxy_score'), - pl.when(pl.col('_raw_score') >= extreme_fence) - .then(pl.lit('critical')) - .when(pl.col('_raw_score') >= upper_fence) - .then(pl.lit('suspicious')) - .when(pl.col('_raw_score') >= q50) - .then(pl.lit('watch')) - .otherwise(pl.lit('normal')) - .alias('risk_level'), - (pl.col('_raw_score') * 0.6 + 0.4 * (pl.col('_raw_score') > q75).cast(pl.Float64)) - .alias('threat_score'), - ]).drop('_raw_score') - - valid_counts = {c: int((mat[:, i] != 0).sum()) for i, c in enumerate(avail)} - import logging - logger = logging.getLogger(__name__) - logger.info( - '[SCORE] adaptive weights from SVD: %s | thresholds: p50=%.3f upper=%.3f extreme=%.3f | sample=%d/%d', - {c: f'{w:.3f}' for c, w in sorted(weights.items(), key=lambda x: -x[1])}, - float(q50), upper_fence, extreme_fence, - min(n_rows, 10000), n_rows, - ) - - except Exception as exc: - import logging - logging.getLogger(__name__).warning('[SCORE] adaptive scoring skipped: %s', exc) - - return lf - - -# ── 12. compute_scores ───────────────────────────────────────────────────── - -async def _handle_compute_scores(dataset_id: str) -> dict: - """Compute adaptive proxy/anomaly/threat scores on entity data.""" - store = SessionStore() - entry = store.get_dataset(dataset_id) - if entry is None: - return {'error': f'Dataset not found: {dataset_id}', 'truncated': False} - - lf = entry['lazyframe'] - lf = _add_adaptive_scores(lf, []) - # Update schema to include new score columns (proxy_score, risk_level, threat_score) - try: - updated_schema = lf.collect_schema() - new_schema = {name: str(dtype) for name, dtype in - zip(updated_schema.names(), updated_schema.dtypes())} - except Exception: - new_schema = entry.get('schema', {}) - store.store_dataset(dataset_id, lf, schema=new_schema, - metadata=entry.get('metadata', {})) - return {'status': 'scored', 'dataset_id': dataset_id} - - -# ── 13. filter_and_cluster ───────────────────────────────────────────────── - -async def _handle_filter_and_cluster( - dataset_id: str, - filters: list[dict], - logic: str = 'and', - algorithm: str = 'hdbscan', - params: dict | None = None, -) -> dict: - """Apply column-level filters then run clustering on the filtered result.""" - store = SessionStore() - entry = store.get_dataset(dataset_id) - if entry is None: - return {'error': f'Dataset not found: {dataset_id}', 'truncated': False} - - lf: pl.LazyFrame = entry['lazyframe'] - full_schema = entry.get('schema', {}) - - # ── Step 1: Apply filters ───────────────────────────────────────────── - try: - exprs = [_build_filter_expr( - f['column'], f.get('op', 'eq'), f.get('value'), - ) for f in filters] - - if exprs: - if logic == 'and': - combined = exprs[0] - for e in exprs[1:]: - combined = combined & e - else: - combined = exprs[0] - for e in exprs[1:]: - combined = combined | e - filtered_lf = lf.filter(combined) - else: - filtered_lf = lf - - # Estimate row count (fast streaming count) - try: - row_count = filtered_lf.select(pl.len()).collect(streaming=True).item() - except Exception: - row_count = None - - except Exception: - raise - - # ── Step 2: Auto-detect numeric feature columns ─────────────────────── - try: - available = filtered_lf.collect_schema() - available_names = available.names() - available_dtypes = available.dtypes() - numeric_dtypes = (pl.Int8, pl.Int16, pl.Int32, pl.Int64, - pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64, - pl.Float32, pl.Float64) - feature_cols = [ - available_names[i] for i, dt in enumerate(available_dtypes) - if dt in numeric_dtypes and not available_names[i].startswith('_') - ][:10] - - if not feature_cols: - return { - 'error': 'No numeric columns found for clustering after filtering', - 'filtered_dataset_id': None, - 'truncated': False, - } - except Exception: - raise - - # ── Step 3: Store filtered dataset ──────────────────────────────────── - filtered_dataset_id = _generate_id('flt') - store.store_dataset( - dataset_id=filtered_dataset_id, - lazyframe=filtered_lf, - schema=full_schema, - metadata={ - 'row_count': row_count, - 'parent_dataset_id': dataset_id, - 'filter_count': len(filters), - 'logic': logic, - }, - parent_id=dataset_id, - ) - - # ── Step 4: Run clustering (inlined from _handle_run_clustering) ─────── - p = params or {} - try: - # Downsample before collect - try: - total_count = filtered_lf.select(pl.len()).collect(streaming=True).item() - except Exception: - total_count = 0 - - clamp_lf = filtered_lf - if total_count > 100000: - clamp_lf = filtered_lf.collect(streaming=True).sample(n=100000, seed=42).lazy() - - df_features = clamp_lf.select(feature_cols).collect(streaming=True) - - # Drop all-NaN columns - for col in df_features.columns: - if df_features[col].is_null().all(): - df_features = df_features.drop(col) - feature_cols = [c for c in feature_cols if c != col] - - if df_features.width == 0: - return {'error': 'No valid feature columns after dropping all-NaN columns', - 'filtered_dataset_id': filtered_dataset_id, 'truncated': False} - - data = df_features.to_numpy() - - # Fill remaining NaN with column mean - if data.dtype.kind == 'f': - col_mean = np.nanmean(data, axis=0) - col_mean = np.nan_to_num(col_mean, nan=0.0) - data = np.where(np.isnan(data), col_mean, data) - - # Too-few-samples guard - if len(data) < 3: - return { - 'cluster_result_id': _generate_id('clust'), - 'n_clusters': 0, - 'n_noise': 0, - 'quality_metrics': {'note': 'too few samples'}, - 'filtered_dataset_id': filtered_dataset_id, - 'truncated': False, - } - - # Low memory check - low_memory = False - try: - import psutil - mem = psutil.virtual_memory() - low_memory = mem.available < 2 * 1024 ** 3 - except ImportError: - pass - - if low_memory and len(data) > 100000: - rng = np.random.default_rng(42) - idx = rng.choice(len(data), 50000, replace=False) - data = data[idx] - - # Downsample to 50K max - _ds_idx = None - if len(data) > 50000: - rng = np.random.default_rng(42) - _ds_idx = rng.choice(len(data), 50000, replace=False) - data = data[_ds_idx] - - # Variance filtering - if data.shape[1] > 1: - variances = np.var(data, axis=0) - keep_var = np.where(variances >= 1e-10)[0] - if len(keep_var) == 0: - keep_var = np.arange(data.shape[1]) - if len(keep_var) < data.shape[1]: - data = data[:, keep_var] - feature_cols = [feature_cols[i] for i in keep_var] - - # Standard scale - from sklearn.preprocessing import StandardScaler - data_scaled = StandardScaler().fit_transform(data) - - # Correlation filtering - if data_scaled.shape[1] > 1: - corr = np.corrcoef(data_scaled.T) - upper = np.triu(np.abs(corr), 1) - to_drop = [j for j in range(upper.shape[1]) if any(upper[:, j] > 0.95)] - if to_drop: - keep = [j for j in range(data_scaled.shape[1]) if j not in to_drop] - data_scaled = data_scaled[:, keep] - feature_cols = [feature_cols[j] for j in keep] - - # SVD dimensionality reduction (if >50 features) - n_features = data_scaled.shape[1] - if n_features > 50: - from sklearn.decomposition import TruncatedSVD - data_scaled = TruncatedSVD(n_components=50, random_state=42).fit_transform(data_scaled) - - # Hyperparameter auto-tuning - n = len(data_scaled) - if algorithm == 'hdbscan': - min_cluster = p.get('min_cluster_size', max(3, min(50, n // 20))) - p['min_cluster_size'] = min_cluster - p['min_samples'] = p.get('min_samples', min_cluster) - p['cluster_selection_epsilon'] = p.get('cluster_selection_epsilon', 0.0) - else: - p['min_cluster_size'] = min( - p.get('min_cluster_size', 5), - max(n // 2, 2), - ) - - # Fit model - from sklearn.cluster import HDBSCAN, MiniBatchKMeans - if algorithm == 'kmeans': - n_clusters_param = p.get('n_clusters', 3) - model = MiniBatchKMeans( - n_clusters=n_clusters_param, random_state=42, - batch_size=1024, n_init='auto', - ) - else: - model = HDBSCAN( - min_cluster_size=p['min_cluster_size'], - min_samples=p.get('min_samples', None), - metric=p.get('metric', 'euclidean'), - cluster_selection_epsilon=p.get('cluster_selection_epsilon', 0.0), - ) - - labels_array = model.fit_predict(data_scaled) - n_clusters_found = int(len(set(labels_array)) - (1 if -1 in labels_array else 0)) - noise_count = int((labels_array == -1).sum()) if algorithm == 'hdbscan' else 0 - noise_ratio = round(noise_count / len(labels_array), 4) if algorithm == 'hdbscan' else 0.0 - - # Quality metrics - quality = { - 'n_clusters': n_clusters_found, - 'n_noise': noise_count, - 'noise_ratio': noise_ratio, - 'algorithm': algorithm, - } - - if n_clusters_found > 1: - try: - from sklearn.metrics import silhouette_score - score = silhouette_score(data_scaled, labels_array, random_state=42) - quality['silhouette_score'] = round(float(score), 4) - except Exception: - quality['silhouette_score'] = None - try: - from sklearn.metrics import davies_bouldin_score - dbs = davies_bouldin_score(data_scaled, labels_array) - quality['davies_bouldin_score'] = round(float(dbs), 4) - except Exception: - quality['davies_bouldin_score'] = None - try: - from sklearn.metrics import calinski_harabasz_score - chs = calinski_harabasz_score(data_scaled, labels_array) - quality['calinski_harabasz_score'] = round(float(chs), 4) - except Exception: - quality['calinski_harabasz_score'] = None - - # Cluster sizes - cluster_sizes = {} - for label in np.unique(labels_array): - cluster_sizes[int(label)] = int((labels_array == label).sum()) - quality['cluster_sizes'] = cluster_sizes - - except Exception: - raise - - result_id = _generate_id('clust') - store.store_cluster_result( - result_id=result_id, - labels=labels_array.tolist(), - n_clusters=n_clusters_found, - params={ - 'algorithm': algorithm, - 'params': p, - 'random_state': 42, - 'feature_columns': feature_cols, - '_ds_idx': _ds_idx.tolist() if _ds_idx is not None else None, - }, - parent_dataset_id=filtered_dataset_id, - ) - - return { - 'cluster_result_id': result_id, - 'filtered_dataset_id': filtered_dataset_id, - 'n_clusters': n_clusters_found, - 'n_noise': noise_count, - 'labels_sample': labels_array[:50].tolist(), - 'quality_metrics': quality, - 'feature_columns': feature_cols, - } - - -# ── 14. detect_anomalies ────────────────────────────────────────────────── - -async def _handle_detect_anomalies(dataset_id: str, contamination: float = 0.05) -> dict: - """Run Isolation Forest on scored entity data.""" - from sklearn.ensemble import IsolationForest - store = SessionStore() - entry = store.get_dataset(dataset_id) - if entry is None: - return {'error': f'Dataset not found: {dataset_id}', 'truncated': False} - - lf: pl.LazyFrame = entry['lazyframe'] - schema = entry.get('schema', {}) - numeric_types = {'Int8', 'Int16', 'Int32', 'Int64', 'UInt8', 'UInt16', 'UInt32', 'UInt64', 'Float32', 'Float64'} - num_cols = [c for c in schema if schema[c].split('(')[0].strip() in numeric_types and not c.startswith('_')] - if not num_cols: - return {'error': 'No numeric columns for anomaly detection', 'truncated': False} - - # Collect scored data (already sampled by pipeline head param) - df = lf.select(num_cols).collect(streaming=True) - mat = df.to_numpy() - mat = np.nan_to_num(mat, nan=0.0) - - # Isolation Forest in chunks - n = len(mat) - chunk_size = min(50000, n) - labels = np.zeros(n, dtype=int) - for start in range(0, n, chunk_size): - end = min(start + chunk_size, n) - iso = IsolationForest(n_estimators=100, max_samples=min(256, chunk_size), - contamination=contamination, random_state=42, n_jobs=-1) - labels[start:end] = iso.fit_predict(mat[start:end]) - - anomaly_count = int((labels == -1).sum()) - return { - 'status': 'completed', - 'total_entities': n, - 'anomaly_count': anomaly_count, - 'anomaly_rate': round(anomaly_count / max(n, 1), 4), - 'dataset_id': dataset_id, - } - - -# ── 15. visualize_anomalies ─────────────────────────────────────────────── - -async def _handle_visualize_anomalies(dataset_id: str) -> dict: - """Generate UMAP-2D embedding for anomaly visualization.""" - from sklearn.preprocessing import StandardScaler - import umap - store = SessionStore() - entry = store.get_dataset(dataset_id) - if entry is None: - return {'error': f'Dataset not found: {dataset_id}', 'truncated': False} - - lf: pl.LazyFrame = entry['lazyframe'] - schema = entry.get('schema', {}) - numeric_types = {'Int8', 'Int16', 'Int32', 'Int64', 'UInt8', 'UInt16', 'UInt32', 'UInt64', 'Float32', 'Float64'} - num_cols = [c for c in schema if schema[c].split('(')[0].strip() in numeric_types and not c.startswith('_')] - if len(num_cols) < 2: - return {'error': 'Need at least 2 numeric columns for UMAP', 'truncated': False} - - # Sample for UMAP visualisation (max 10K) - try: - n_total = lf.select(pl.len()).collect(streaming=True).item() - except Exception: - n_total = 0 - sample_lf = lf if n_total <= 10000 else lf.collect(streaming=True).sample(n=10000, seed=42).lazy() - df = sample_lf.select(num_cols).collect(streaming=True) - mat = StandardScaler().fit_transform(df.to_numpy()) - mat = np.nan_to_num(mat, nan=0.0) - reducer = umap.UMAP(n_components=2, random_state=42, - n_neighbors=15, min_dist=0.1, metric='euclidean') - coords = reducer.fit_transform(mat) - - risk_col = next((c for c in schema if 'risk' in c.lower()), None) - score_col = next((c for c in schema if 'proxy_score' in c.lower() or 'threat' in c.lower()), None) - - non_num = [c for c in df.columns if c not in num_cols and not c.startswith('_')] - ent_col = non_num[0] if non_num else df.columns[0] - entities = df.select(ent_col).to_series().to_list() if ent_col in df.columns else [] - - scatter = [ - {'x': float(coords[i, 0]), 'y': float(coords[i, 1]), - 'entity': str(entities[i]) if i < len(entities) else '', - 'score': round(float(mat[i].mean()), 4)} - for i in range(len(coords)) - ] - return { - 'status': 'completed', - 'scatter_points': len(scatter), - 'scatter_data': scatter[:200], # cap for LLM context - 'dataset_id': dataset_id, - } - - -# ── 16. validate_data ────────────────────────────────────────────────── - -async def _handle_validate_data(dataset_id: str, fix_issues: bool = False) -> dict: - """Check data quality: schema consistency, missing values, type conflicts.""" - store = SessionStore() - entry = store.get_dataset(dataset_id) - if entry is None: - return {'error': f'Dataset not found: {dataset_id}', 'truncated': False} - lf = entry['lazyframe'] - schema = entry.get('schema', {}) - issues = [] - try: - df_sample = lf.head(1000).collect(streaming=True) - for col in schema: - null_count = df_sample[col].is_null().sum() - if null_count > 500: - issues.append(f'{col}: {null_count/10:.0f}% null') - try: - unique = df_sample[col].n_unique() - if unique <= 1 and null_count == 0: - issues.append(f'{col}: constant value (no variation)') - except Exception: - pass # n_unique not supported for this column type - except Exception as exc: - raise - total_rows = lf.select(pl.len()).collect(streaming=True).item() - return { - 'status': 'ok', - 'total_rows': total_rows, - 'columns': len(schema), - 'issues_found': len(issues), - 'issues': issues[:20], - 'recommendation': 'Try explore_distributions for detailed column analysis' if issues else 'No issues found', - } - - -# ── 17. explore_distributions ────────────────────────────────────────── - -async def _handle_explore_distributions(dataset_id: str, columns: list[str] = None, - max_sample: int = 10000) -> dict: - """Per-column distribution statistics.""" - store = SessionStore() - entry = store.get_dataset(dataset_id) - if entry is None: - return {'error': f'Dataset not found: {dataset_id}', 'truncated': False} - lf = entry['lazyframe'] - schema = entry.get('schema', {}) - numeric_types = {'Int8', 'Int16', 'Int32', 'Int64', 'UInt8', 'UInt16', 'UInt32', 'UInt64', 'Float32', 'Float64'} - - if not columns: - columns = [c for c in schema if schema[c].split('(')[0].strip() in numeric_types][:10] - # Limit columns to prevent excessive computation - if len(columns) > 20: - columns = columns[:20] - - try: - n = lf.select(pl.len()).collect(streaming=True).item() - if n > max_sample: - # Use LazyFrame.sample() on the limited columns to avoid full-materialize - sample_lf = lf.select(columns).sample(n=max_sample, seed=42) - else: - sample_lf = lf.select(columns) - df = sample_lf.collect(streaming=True) - except Exception as exc: - return {'error': f'explore_distributions failed: {exc}', 'truncated': False} - - stats = {} - for col in columns: - try: - s = df[col] - stats[col] = { - 'dtype': str(s.dtype), - 'null_pct': round(float(s.is_null().mean() * 100), 2), - 'unique': int(s.n_unique()), - 'top_values': [str(v) for v in s.head(5).to_list()], - } - if s.dtype in (pl.Float32, pl.Float64, pl.Int32, pl.Int64): - stats[col]['min'] = float(s.min()) if s.min() is not None else None - stats[col]['max'] = float(s.max()) if s.max() is not None else None - stats[col]['mean'] = round(float(s.mean()), 4) if s.mean() is not None else None - stats[col]['std'] = round(float(s.std()), 4) if s.std() is not None else None - except Exception: - stats[col] = {'error': 'failed to compute'} - return {'columns': stats, 'total_rows': n, 'sampled': min(n, max_sample)} - - -# ── 18. find_outliers ───────────────────────────────────────────────── - -async def _handle_find_outliers(dataset_id: str, columns: list[str] = None, - threshold: float = 3.0) -> dict: - """Find statistical outliers using IQR method.""" - store = SessionStore() - entry = store.get_dataset(dataset_id) - if entry is None: - return {'error': f'Dataset not found: {dataset_id}', 'truncated': False} - lf = entry['lazyframe'] - schema = entry.get('schema', {}) - numeric_types = {'Int8', 'Int16', 'Int32', 'Int64', 'UInt8', 'UInt16', 'UInt32', 'UInt64', 'Float32', 'Float64'} - if not columns: - columns = [c for c in schema if schema[c].split('(')[0].strip() in numeric_types] - try: - n = lf.select(pl.len()).collect(streaming=True).item() - sample_lf = lf if n <= 100000 else lf.collect(streaming=True).sample(n=100000, seed=42).lazy() - df = sample_lf.select(columns).collect(streaming=True) - except Exception as exc: - raise - - outlier_counts = {} - for col in columns: - try: - s = df[col] - q1, q3 = s.quantile(0.25), s.quantile(0.75) - iqr = q3 - q1 - lo = q1 - threshold * iqr - hi = q3 + threshold * iqr - n_out = ((s < lo) | (s > hi)).sum() - if n_out > 0: - outlier_counts[col] = int(n_out) - except Exception: - pass # quantile not supported for this column type - return {'outlier_counts': outlier_counts, 'total_sampled': len(df), 'threshold_iqr': threshold} - - -# ── 19. diagnose_clustering ─────────────────────────────────────────── - -async def _handle_diagnose_clustering(dataset_id: str, cluster_result_id: str) -> dict: - """Diagnose clustering quality issues.""" - from sklearn.decomposition import TruncatedSVD - store = SessionStore() - entry = store.get_dataset(dataset_id) - cluster_entry = store.get_cluster_result(cluster_result_id) - if entry is None: - return {'error': f'Dataset not found: {dataset_id}', 'truncated': False} - if cluster_entry is None: - return {'error': f'Cluster result not found: {cluster_result_id}', 'truncated': False} - lf = entry['lazyframe'] - schema = entry.get('schema', {}) - numeric_types = {'Int8', 'Int16', 'Int32', 'Int64', 'UInt8', 'UInt16', 'UInt32', 'UInt64', 'Float32', 'Float64'} - num_cols = [c for c in schema if schema[c].split('(')[0].strip() in numeric_types and not c.startswith('_')] - - n_clusters = cluster_entry.get('n_clusters', 0) - n_noise = cluster_entry.get('n_noise', 0) - quality = cluster_entry.get('quality_metrics', {}) - labels = cluster_entry.get('labels', []) - params = cluster_entry.get('params', {}) - algo = params.get('algorithm', 'unknown') - feature_cols = params.get('feature_columns', []) - - diagnosis = [] - if n_clusters <= 1: - diagnosis.append('Only 1 cluster — data may lack structure or need feature scaling') - if n_noise > 0.5 * len(labels) if labels else False: - diagnosis.append(f'High noise ratio ({n_noise}/{len(labels)}) — try HDBSCAN with larger min_cluster_size') - sil = quality.get('silhouette_score') - if sil and sil < 0.2: - diagnosis.append(f'Low silhouette ({sil:.3f}) — clusters overlap, try KMeans or feature selection') - if len(feature_cols) < 3: - diagnosis.append(f'Only {len(feature_cols)} features — add more columns for meaningful clusters') - - # SVD variance check - try: - df_sample = lf.select(num_cols).head(5000).collect(streaming=True) - mat = df_sample.to_numpy() - n_comp = min(mat.shape) - svd = TruncatedSVD(n_components=n_comp, random_state=42).fit(mat) - var_explained = svd.explained_variance_ratio_ - if var_explained[0] > 0.95: - diagnosis.append(f'First SVD component explains {var_explained[0]:.0%} variance — data nearly 1D') - except Exception: - pass # SVD fit failed — likely constant or all-null columns - - return { - 'diagnosis': diagnosis, - 'n_clusters': n_clusters, - 'n_noise': n_noise, - 'silhouette': sil, - 'algorithm': algo, - 'n_features': len(feature_cols), - 'suggestions': [ - f'Try algorithm={"kmeans" if algo == "hdbscan" else "hdbscan"}', - 'Increase min_cluster_size for fewer, larger clusters', - 'Use explore_distributions to check feature quality', - ], - } - - -# ── 20. compare_datasets ───────────────────────────────────────────── - -async def _handle_compare_datasets(dataset_id_a: str, dataset_id_b: str) -> dict: - """Compare two datasets.""" - store = SessionStore() - entry_a, entry_b = store.get_dataset(dataset_id_a), store.get_dataset(dataset_id_b) - if entry_a is None or entry_b is None: - return {'error': 'One or both datasets not found', 'truncated': False} - - def _get_info(entry): - schema = entry.get('schema', {}) - lf = entry['lazyframe'] - try: - n = lf.select(pl.len()).collect(streaming=True).item() - except Exception: - n = -1 - return {'schema': set(schema.keys()), 'row_count': n, 'columns': len(schema)} - - info_a, info_b = _get_info(entry_a), _get_info(entry_b) - common_cols = info_a['schema'] & info_b['schema'] - only_a = info_a['schema'] - info_b['schema'] - only_b = info_b['schema'] - info_a['schema'] - return { - 'dataset_a': {'id': dataset_id_a, **info_a}, - 'dataset_b': {'id': dataset_id_b, **info_b}, - 'common_columns': len(common_cols), - 'columns_only_in_a': list(only_a)[:10] if only_a else [], - 'columns_only_in_b': list(only_b)[:10] if only_b else [], - 'row_diff': info_a['row_count'] - info_b['row_count'], - 'identical_schema': len(only_a) == 0 and len(only_b) == 0, - } - - -# ── 21. export_debug_sample ─────────────────────────────────────────── - -async def _handle_export_debug_sample(dataset_id: str, max_rows: int = 50) -> dict: - """Export raw data sample as JSON for debugging.""" - store = SessionStore() - entry = store.get_dataset(dataset_id) - if entry is None: - return {'error': f'Dataset not found: {dataset_id}', 'truncated': False} - try: - df = entry['lazyframe'].head(max_rows).collect(streaming=True) - rows = df.to_dict(as_series=False) - # Convert to list-of-dicts, serialize safely - result = [] - for i in range(len(df)): - row = {} - for col in df.columns: - val = rows[col][i] - if isinstance(val, (float, int, str, bool)): - row[col] = val - elif val is None: - row[col] = None - else: - row[col] = str(val) - result.append(row) - return {'rows': result, 'count': len(result), 'columns': df.columns} - except Exception as exc: - raise - - -# ── 22. repair_schema ───────────────────────────────────────────────── - -async def _handle_repair_schema(dataset_id: str) -> dict: - """Attempt to repair schema mismatches by normalising column names.""" - store = SessionStore() - entry = store.get_dataset(dataset_id) - if entry is None: - return {'error': f'Dataset not found: {dataset_id}', 'truncated': False} - lf = entry['lazyframe'] - schema = entry.get('schema', {}) - # Normalise: replace hyphens/underscores/dots, lowercase - col_map = {} - for c in schema: - norm = c.lower().replace('-', '_').replace('.', '_') - col_map[c] = norm - if len(set(col_map.values())) < len(col_map): - # Conflicts exist — can't auto-repair - return {'status': 'conflict', 'message': 'Normalised names conflict', 'column_map': col_map} - # Rename columns - renames = {old: new for old, new in col_map.items() if old != new} - if renames: - lf = lf.rename(renames) - new_schema = {col_map.get(c, c): dtype for c, dtype in schema.items()} - store.store_dataset(dataset_id, lf, schema=new_schema, metadata=entry.get('metadata', {})) - return {'status': 'repaired', 'renamed': len(renames), 'column_map': col_map} - return {'status': 'no_change', 'message': 'Column names already normalised'} - - -# ── 23. analyze_patterns ────────────────────────────────────────────── - -async def _handle_analyze_patterns(dataset_id: str, top_n: int = 10) -> dict: - """Read-only: traffic pattern analysis.""" - store = SessionStore() - entry = store.get_dataset(dataset_id) - if entry is None: - return {'error': 'Dataset not found', 'truncated': False} - lf = entry['lazyframe'] - schema = entry.get('schema', {}) - n = lf.select(pl.len()).collect(streaming=True).item() - df = lf.head(100000).collect(streaming=True) - - result = {'total_rows': n, 'sampled': min(n, 100000)} - - # Top source IPs - for col_name, label in [(':ips', 'src_ips'), (':ipd', 'dst_ips'), - ('snam', 'snis'), ('cnam', 'cert_names')]: - if col_name in df.columns: - try: - top = df[col_name].value_counts(sort=True).head(top_n) - result[label] = { - 'top': [{'value': str(r[col_name]), 'count': int(r['count'])} - for r in top.iter_rows(named=True)], - 'unique': int(df[col_name].n_unique()), - } - except Exception: - pass # value_counts not supported for column, skip - - # Port distribution - for port_col in [':prs', ':prd']: - if port_col in df.columns: - try: - top = df[port_col].value_counts(sort=True).head(top_n) - result[f'{port_col}_dist'] = { - 'top': [{'port': int(r[port_col]), 'count': int(r['count'])} - for r in top.iter_rows(named=True)], - 'unique': int(df[port_col].n_unique()), - } - except Exception: - pass # port distribution extraction failed — skip - - # TLS version breakdown - for tls_col in ['0ver']: - if tls_col in df.columns: - try: - vc = df[tls_col].value_counts(sort=True) - result['tls_versions'] = [{'version': str(r[tls_col]), 'count': int(r['count'])} - for r in vc.iter_rows(named=True)] - except Exception: - pass # TLS version value_counts not supported — skip - - return result - - -# ── 24. analyze_temporal ───────────────────────────────────────────── - -async def _handle_analyze_temporal(dataset_id: str, timestamp_column: str = None) -> dict: - """Read-only: temporal traffic pattern analysis.""" - store = SessionStore() - entry = store.get_dataset(dataset_id) - if entry is None: - return {'error': 'Dataset not found', 'truncated': False} - lf = entry['lazyframe'] - schema = entry.get('schema', {}) - - if not timestamp_column: - for cand in ['timestamp', 'time', '8dbd', '8did']: - if cand in schema: - timestamp_column = cand - break - if not timestamp_column: - return {'error': 'No timestamp column found', 'truncated': False} - - n = lf.select(pl.len()).collect(streaming=True).item() - return {'total_rows': n, 'timestamp_column': timestamp_column, - 'note': 'Temporal analysis requires parsed timestamps; raw values shown in explore_distributions'} - - -# ── 24b. analyze_fft ───────────────────────────────────────────────── - -async def _handle_analyze_fft( - dataset_id: str, - timestamp_column: str, - top_n: int = 5, -) -> dict: - """FFT spectrum analysis on a timestamp column to detect periodic patterns. - - Computes the Fast Fourier Transform of sorted timestamp-derived values, - returning dominant frequencies, spectral centroid, and a periodicity score. - """ - store = SessionStore() - entry = store.get_dataset(dataset_id) - if entry is None: - return {'error': f'Dataset not found: {dataset_id}', 'truncated': False} - - lf: pl.LazyFrame = entry['lazyframe'] - schema = entry.get('schema', {}) - - # ── Resolve timestamp column ────────────────────────────────────── - col = None - # Try exact match first - if timestamp_column in schema: - col = timestamp_column - else: - # Try case-insensitive match - col_lower = timestamp_column.lower() - for c in schema: - if c.lower() == col_lower: - col = c - break - if col is None: - return {'error': f"Timestamp column '{timestamp_column}' not found in dataset. Available: {list(schema.keys())[:20]}", - 'truncated': False} - - # ── Extract and sort values ─────────────────────────────────────── - df = lf.select(pl.col(col).alias('__ts__')).collect(streaming=True) - - n_raw = len(df) - if n_raw < 4: - return {'error': f'Need at least 4 samples for FFT; got {n_raw}', 'truncated': False} - - # Determine dtype and convert to numeric values - ts_series = df['__ts__'] - dtype = ts_series.dtype - str_dtypes = (pl.Utf8, pl.String) - - if dtype in str_dtypes: - # Try parsing as datetime string, then convert to epoch seconds - try: - ts_parsed = ts_series.str.strptime(pl.Datetime, format=None, strict=False) - ts_numeric = ts_parsed.cast(pl.Int64) / 1_000_000_000.0 # ns → seconds - except Exception: - # Fallback: try to cast to float directly - try: - ts_numeric = ts_series.cast(pl.Float64) - except Exception as e: - return {'error': f'Cannot convert string column to numeric or datetime: {e}', - 'truncated': False} - elif dtype in (pl.Datetime, pl.Date): - ts_numeric = ts_series.cast(pl.Int64) / 1_000_000_000.0 - else: - # Float, Int types — cast to float - try: - ts_numeric = ts_series.cast(pl.Float64) - except Exception as e: - return {'error': f'Cannot cast column to float: {e}', 'truncated': False} - - # Build DataFrame with values, drop nulls, sort - df_sorted = pl.DataFrame({'__val__': ts_numeric}).drop_nulls().sort('__val__') - n = len(df_sorted) - if n < 4: - return {'error': f'After removing nulls, only {n} samples remain (need ≥4)', 'truncated': False} - - values = df_sorted['__val__'].to_numpy().astype(np.float64) - - # ── FFT computation ──────────────────────────────────────────────── - # Center by subtracting mean (DC offset removal) - values_centered = values - np.mean(values) - fft_complex = np.fft.fft(values_centered) - fft_magnitudes = np.abs(fft_complex) - freqs = np.fft.fftfreq(n) - - # Take positive half (excluding DC at index 0) - pos_mask = freqs > 0 - freqs_pos = freqs[pos_mask] - mags_pos = fft_magnitudes[pos_mask] - - if len(freqs_pos) == 0: - return {'error': 'Insufficient positive frequencies for FFT analysis', 'truncated': False} - - # ── Find top-N peaks ────────────────────────────────────────────── - peak_indices = np.argsort(mags_pos)[::-1][:min(top_n, len(mags_pos))] - top_periods = [] - for idx in peak_indices: - freq = float(freqs_pos[idx]) - mag = float(mags_pos[idx]) - # Convert frequency to period (in sample-index units) - period = float(1.0 / freq) if freq > 1e-10 else float('inf') - # Map to time-domain period if data is temporal - if n > 1: - time_span = float(values[-1] - values[0]) - sample_period = time_span / max(n - 1, 1) - time_period = period * sample_period - else: - time_period = float('inf') - top_periods.append({ - 'period': round(period, 2), - 'magnitude': round(mag, 4), - 'frequency': round(freq, 6), - 'time_domain_period_seconds': round(time_period, 2) if time_period != float('inf') else None, - }) - - # ── Spectral centroid (weighted mean frequency) ──────────────────── - mag_sum = float(np.sum(mags_pos)) - if mag_sum > 0: - spectral_centroid = float(np.sum(freqs_pos * mags_pos) / mag_sum) - else: - spectral_centroid = 0.0 - - # ── Periodicity score: ratio of top peak to mean magnitude ──────── - mean_mag = float(np.mean(mags_pos)) if len(mags_pos) > 0 else 1.0 - max_mag = float(mags_pos[peak_indices[0]]) if len(peak_indices) > 0 else 0.0 - periodicity_score = round(max_mag / max(mean_mag, 1e-10), 4) - - # ── Return decimated data for LLM context window ────────────────── - # Decimate to at most 200 frequency-magnitude pairs - decimate_step = max(1, len(freqs_pos) // 200) - freqs_decimated = freqs_pos[::decimate_step].tolist() - mags_decimated = mags_pos[::decimate_step].tolist() - - return { - 'frequencies': [round(float(f), 6) for f in freqs_decimated], - 'magnitudes': [round(float(m), 4) for m in mags_decimated], - 'top_periods': top_periods, - 'spectral_centroid': round(spectral_centroid, 6), - 'periodicity_score': periodicity_score, - 'n_samples': n, - 'n_raw': n_raw, - 'nan_dropped': n_raw - n, - 'timestamp_column': col, - } - - -# ── 25. analyze_tls_health ──────────────────────────────────────────── - -async def _handle_analyze_tls_health(dataset_id: str) -> dict: - """Read-only: TLS security assessment.""" - store = SessionStore() - entry = store.get_dataset(dataset_id) - if entry is None: - return {'error': 'Dataset not found', 'truncated': False} - lf = entry['lazyframe'] - schema = entry.get('schema', {}) - n = lf.select(pl.len()).collect(streaming=True).item() - df = lf.head(100000).collect(streaming=True) - - result = {'total_rows': n, 'sampled': min(n, 100000)} - - if '0ver' in df.columns: - vc = df['0ver'].value_counts(sort=True) - versions = {} - for r in vc.iter_rows(named=True): - v = str(r['0ver']).replace(' ', '') - if v in ('0303', '0304'): - versions['tls_1_2_plus'] = versions.get('tls_1_2_plus', 0) + int(r['count']) - else: - versions['tls_legacy'] = versions.get('tls_legacy', 0) + int(r['count']) - total = sum(versions.values()) - result.update({ - 'tls_1_2_plus_pct': round(versions.get('tls_1_2_plus', 0) / max(total, 1) * 100, 1), - 'tls_legacy_pct': round(versions.get('tls_legacy', 0) / max(total, 1) * 100, 1), - 'tls_legacy_versions': [{'version': str(r['0ver']), 'count': int(r['count'])} - for r in vc.iter_rows(named=True) - if str(r['0ver']).replace(' ', '') not in ('0303', '0304')], - }) - - if 'snam' in df.columns: - empty_sni = df['snam'].is_null().sum() + df['snam'].cast(pl.Utf8).is_in(['', 'null', 'None']).sum() - result['sni_missing_pct'] = round(float(empty_sni) / max(len(df), 1) * 100, 1) - - if 'cipher-suite' in df.columns: - weak_ciphers = ['rc4', 'cbc', 'null', 'export'] - total_ciphers = len(df) - weak_count = 0 - for w in weak_ciphers: - try: - weak_count += df['cipher-suite'].str.to_lowercase().str.contains(w).sum() - except Exception: - pass # str.contains not supported for cipher-suite column — skip - result['weak_cipher_pct'] = round(float(weak_count) / max(total_ciphers, 1) * 100, 1) if total_ciphers else 0 - - if result.get('tls_legacy_pct', 0) > 20: - result['warning'] = 'High proportion of legacy TLS — possible downgrade attacks' - if result.get('sni_missing_pct', 0) > 50: - result['warning'] = 'Most flows lack SNI — possible IP-direct connections (proxy/VPN)' - return result - - -# ── 26. analyze_geo_distribution ───────────────────────────────────── - -async def _handle_analyze_geo_distribution(dataset_id: str) -> dict: - """Read-only: geographic traffic analysis.""" - store = SessionStore() - entry = store.get_dataset(dataset_id) - if entry is None: - return {'error': 'Dataset not found', 'truncated': False} - lf = entry['lazyframe'] - schema = entry.get('schema', {}) - n = lf.select(pl.len()).collect(streaming=True).item() - df = lf.head(100000).collect(streaming=True) - - result = {'total_rows': n, 'sampled': min(n, 100000)} - - for geo_col, label in [('scnt', 'src_countries'), ('dcnt', 'dst_countries')]: - if geo_col in df.columns: - try: - vc = df[geo_col].value_counts(sort=True).head(15) - result[label] = [{'country': str(r[geo_col]), 'count': int(r['count'])} - for r in vc.iter_rows(named=True)] - result[f'{label}_unique'] = int(df[geo_col].n_unique()) - except Exception: - pass # country distribution not computable — skip - - if 'scnt' in df.columns and 'dcnt' in df.columns: - try: - same = (df['scnt'] == df['dcnt']).sum() - result['domestic_pct'] = round(float(same) / max(len(df), 1) * 100, 1) - result['international_pct'] = round(100.0 - result['domestic_pct'], 1) - except Exception: - pass # domestic/international comparison failed — skip - - return result - - -# ── 27. analyze_entity_detail ──────────────────────────────────────── - -async def _handle_analyze_entity_detail(dataset_id: str, entity_value: str) -> dict: - """Read-only: deep-dive into a single entity.""" - store = SessionStore() - entry = store.get_dataset(dataset_id) - if entry is None: - return {'error': 'Dataset not found', 'truncated': False} - lf = entry['lazyframe'] - schema = entry.get('schema', {}) - - # Find entity column (first non-numeric, non-internal column) - numeric_types = {'Int8', 'Int16', 'Int32', 'Int64', 'UInt8', 'UInt16', 'UInt32', 'UInt64', 'Float32', 'Float64'} - entity_col = None - for c in schema: - if c.startswith('_') or schema[c].split('(')[0].strip() in numeric_types or '.' in c: - continue - entity_col = c - break - if not entity_col: - entity_col = list(schema.keys())[0] - - try: - filtered = lf.filter(pl.col(entity_col).cast(pl.Utf8).str.contains(entity_value, literal=True)) - n = filtered.select(pl.len()).collect(streaming=True).item() - if n == 0: - return {'error': f"Entity '{entity_value}' not found in column '{entity_col}'", 'truncated': False} - df = filtered.collect(streaming=True) - except Exception as exc: - raise - - result = { - 'entity_value': entity_value, - 'entity_column': entity_col, - 'flow_count': n, - 'columns': list(df.columns[:15]), - } - - # Numeric summary - for c in df.columns: - if df[c].dtype in (pl.Float32, pl.Float64, pl.Int32, pl.Int64): - try: - result[f'{c}_stats'] = { - 'mean': round(float(df[c].mean()), 2), - 'min': float(df[c].min()) if df[c].min() is not None else None, - 'max': float(df[c].max()) if df[c].max() is not None else None, - } - except Exception: - pass # numeric stats not computable for this column — skip - - return result - - -# ── 28. compute_distance_matrix ────────────────────────────────────────── - -def _indent_function_body(body: str, indent: str = ' ') -> str: - """Indent each non-empty line of a code block for wrapping in a def.""" - return '\n'.join( - indent + line if line.strip() else line - for line in body.split('\n') - ) - - -async def _handle_compute_distance_matrix( - dataset_id: str, - python_function: str, - columns: Optional[list[str]] = None, - max_sample: int = 10000, -) -> dict: - """Evaluate a user-provided Python function on each row of a dataset. - - Wraps the function body as ``def distance_fn(row): ``, executes - it per-row in a restricted namespace (``math``, ``numpy`` only), and - returns a distance-score summary plus a sample of scored rows. - """ - store = SessionStore() - entry = store.get_dataset(dataset_id) - if entry is None: - return {'error': f'Dataset not found: {dataset_id}', 'truncated': False} - - lf: pl.LazyFrame = entry['lazyframe'] - - # Collect data (streaming for large datasets) - df = lf.collect(streaming=True) - - # Determine columns to pass to the function - if columns is None: - numeric_dtypes = (pl.Int8, pl.Int16, pl.Int32, pl.Int64, - pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64, - pl.Float32, pl.Float64) - columns = [c for c in df.columns - if df[c].dtype in numeric_dtypes] - - # Filter to columns that actually exist - available = set(df.columns) - columns = [c for c in columns if c in available] - if not columns: - return {'error': 'No valid columns found in dataset', - 'truncated': False} - - n_total = len(df) - - # Sample if larger than max_sample - if n_total > max_sample: - df_sampled = df.sample(n=max_sample, seed=42) - n_evaluated = max_sample - else: - df_sampled = df - n_evaluated = n_total - - # Build list-of-dicts for the selected columns - rows = df_sampled.select(columns).to_dicts() - - # ── Safe execution ──────────────────────────────────────────────────── - safe_builtins = { - 'abs': abs, 'len': len, 'range': range, - 'min': min, 'max': max, 'sum': sum, 'round': round, - 'int': int, 'float': float, 'str': str, 'bool': bool, - 'list': list, 'dict': dict, 'tuple': tuple, - 'isinstance': isinstance, 'type': type, - 'True': True, 'False': False, 'None': None, - 'enumerate': enumerate, 'zip': zip, 'map': map, - 'filter': filter, 'sorted': sorted, 'reversed': reversed, - } - - safe_ns = { - '__builtins__': safe_builtins, - 'math': __import__('math'), - 'np': __import__('numpy'), - } - - # Wrap the user's function body in a def statement - body = python_function.strip() - if body.startswith('def '): - # User provided the full function definition - full_code = body - else: - indented = _indent_function_body(body) - full_code = f'def distance_fn(row):\n{indented}' - - try: - exec(full_code, safe_ns) - # Resolve: the function name might be 'distance_fn' or whatever the user named it - distance_fn = safe_ns.get('distance_fn') - if distance_fn is None: - # Fallback: try to find any callable in safe_ns - for _name, _obj in safe_ns.items(): - if callable(_obj) and _name not in ('math', 'np', '__builtins__'): - distance_fn = _obj - break - if distance_fn is None: - return {'error': 'Could not locate the compiled function', - 'truncated': False} - except SyntaxError as e: - return {'error': f'Function syntax error: {e}', 'truncated': False} - except Exception as e: - return {'error': f'Function compilation failed: {e}', - 'truncated': False} - - # ── Apply function per row ──────────────────────────────────────────── - scores: list[float | None] = [] - errors = 0 - for row in rows: - try: - score = distance_fn(row) - if isinstance(score, (int, float)): - scores.append(float(score)) - else: - scores.append(None) - errors += 1 - except Exception: - scores.append(None) - errors += 1 - - valid_scores = [s for s in scores if s is not None] - if not valid_scores: - return {'error': 'No valid scores computed — all rows raised errors', - 'truncated': False} - - # ── Distance summary ────────────────────────────────────────────────── - arr = np.array(valid_scores) - summary = { - 'min': float(np.min(arr)), - 'max': float(np.max(arr)), - 'mean': float(np.mean(arr)), - 'std': float(np.std(arr)), - 'count': len(arr), - 'errors': errors, - 'total_rows_in_sample': len(rows), - 'matrix_shape': [len(rows), len(columns)], - } - - # Sample of scored rows (first 30, keep score + first 3 values) - scored_sample = [] - for i, row in enumerate(rows[:30]): - if i < len(scores): - scored_sample.append({ - 'index': i, - 'values': {c: row.get(c) for c in columns[:3]}, - 'score': scores[i], - }) - - return { - 'summary': summary, - 'scored_sample': scored_sample, - 'columns_used': columns, - 'total_rows_in_dataset': n_total, - 'rows_evaluated': n_evaluated, - } - - -# Auto-profile all public functions in this module -from analysis.profile_util import auto_profile_module # noqa: E402 -auto_profile_module(__name__) diff --git a/analysis/views/clustering.py b/analysis/views/clustering.py index 3dcd195..b4f6c47 100644 --- a/analysis/views/clustering.py +++ b/analysis/views/clustering.py @@ -1,4 +1,4 @@ -"""Cluster views: overview, detail, globe flows, and the clustering pipeline.""" +"""Cluster views: overview, detail, globe flows, and the clustering pipeline.""" import asyncio import json import traceback @@ -227,7 +227,6 @@ def _run_clustering_pipeline(run, store, ds_id, feature_columns, algorithm, Operates directly on raw data rows (no entity aggregation). Per-row results stored in EntityProfile with row index as identifier. -<<<<<<< HEAD Args: run: AnalysisRun ORM object. store: SessionStore instance. diff --git a/analysis/views_backup.py b/analysis/views_backup.py deleted file mode 100644 index e7bd0af..0000000 --- a/analysis/views_backup.py +++ /dev/null @@ -1,2308 +0,0 @@ -import json -import time -import urllib.request -import urllib.error -import threading -import os -from pathlib import Path - -from analysis.appdata import ensure_appdata_dir - -from django.shortcuts import render, get_object_or_404, redirect -from django.http import HttpResponse, JsonResponse -from django.views.decorators.csrf import csrf_exempt - -from config import get_config, save_config, Config -from .models import AnalysisRun, ClusterResult, EntityProfile -from . import nl_describe -from .profile_util import profile - - -def dashboard(request): - """Home page: recent runs overview.""" - runs = AnalysisRun.objects.all()[:20] - return render(request, 'analysis/dashboard.html', {'runs': runs}) - - -def run_list(request): - """List all analysis runs with pagination and optional ?type= filter.""" - run_type_filter = request.GET.get('type') - page = int(request.GET.get('page', 1)) - per_page = 20 - - qs = AnalysisRun.objects.all() - if run_type_filter in dict(AnalysisRun.RUN_TYPE_CHOICES): - qs = qs.filter(run_type=run_type_filter) - - total = qs.count() - runs = qs[(page - 1) * per_page: page * per_page] - return render(request, 'analysis/run_list.html', { - 'runs': runs, - 'page': page, - 'total_pages': (total + per_page - 1) // per_page if total else 1, - 'current_type': run_type_filter or '', - }) - - -def run_detail(request, display_id): - """Details for a single analysis run.""" - run = get_object_or_404(AnalysisRun, display_id=display_id) - clusters = run.clusters.all().order_by('-size') - return render(request, 'analysis/run_detail.html', { - 'run': run, - 'clusters': clusters, - }) - - -def _extract_lat(val): - """Try to extract a latitude from *val* (number or str).""" - if val is None: - return None - try: - v = float(val) - return None if v < -90 or v > 90 else v - except (ValueError, TypeError): - return None - - -def _extract_lon(val): - """Try to extract a longitude from *val* (number or str).""" - if val is None: - return None - try: - v = float(val) - return None if v < -180 or v > 180 else v - except (ValueError, TypeError): - return None - - -def cluster_overview(request, display_id): - """Overview of all clusters for a run, with UMAP scatter data and geo scatter data.""" - run = get_object_or_404(AnalysisRun, display_id=display_id) - clusters = run.clusters.exclude(cluster_label=-1).order_by('-size').prefetch_related('features') - noise = run.clusters.filter(cluster_label=-1).first() - - # Build UMAP scatter data for Canvas / Three.js 3D - entities = run.entities.exclude(embedding_x=None).exclude(embedding_y=None).only( - 'embedding_x', 'embedding_y', 'embedding_z', 'cluster_label', 'entity_value' - ) - has_z = any(e.embedding_z is not None for e in entities) - scatter_data = [ - {'x': e.embedding_x, 'y': e.embedding_y, 'z': e.embedding_z if has_z else 0, - 'label': e.cluster_label if e.cluster_label is not None else -1, 'entity': e.entity_value} - for e in entities - ] - - # Noise cluster: features + entity detail - noise_features = [] - noise_summary = '' - noise_entity_count = 0 - if noise: - noise_features_qs = noise.features.all().order_by('-distinguishing_score')[:15] - noise_features = [ - {'feature_name': f.feature_name, 'score': f.distinguishing_score, 'mean': f.mean, - 'std': f.std, 'p25': f.p25, 'p75': f.p75} - for f in noise_features_qs - ] - if noise_features: - noise_summary = nl_describe.describe_cluster(noise_features) - noise_entity_count = noise.size - - # LLM workflow timeline for auto runs - llm_timeline_json = '' - if run.run_type == 'auto': - llm_thinking = run.llm_thinking or '' - tool_calls = run.tool_calls_json or [] - if tool_calls or llm_thinking: - llm_timeline_json = json.dumps({ - 'thinking': llm_thinking, - 'tool_calls': tool_calls, - }) - - # Build geo scatter data from feature_json (lat/lon) - geo_data = [] - geo_skipped = 0 - for e in run.entities.all().only('feature_json', 'cluster_label', 'entity_value'): - fj = e.feature_json - if not fj: - continue - lat = _extract_lat(fj.get('avg_latitude')) - lon = _extract_lon(fj.get('avg_longitude')) - if lat is not None and lon is not None: - geo_data.append({ - 'x': lon, - 'y': lat, - 'label': e.cluster_label if e.cluster_label is not None else -1, - 'entity': e.entity_value, - }) - else: - geo_skipped += 1 - - # Cluster sizes for bar chart - cluster_sizes = [{'label': c.cluster_label, 'size': c.size, 'silhouette': c.silhouette_score} - for c in clusters] - - # Top features per cluster from ClusterFeature + NL summaries - top_features = {} - for c in clusters: - features_qs = c.features.all().order_by('-distinguishing_score')[:10] - feats = [ - {'feature_name': f.feature_name, 'score': f.distinguishing_score, 'mean': f.mean, - 'std': f.std} - for f in features_qs - ] - top_features[str(c.cluster_label)] = feats - # Attach features + NL summary directly to cluster object for template - c._top_features = feats - c.nl_summary = nl_describe.describe_cluster(feats) if feats else '' - - return render(request, 'analysis/cluster_overview.html', { - 'run': run, - 'clusters': clusters, - 'noise': noise, - 'scatter_data_json': json.dumps(scatter_data), - 'has_z': has_z, - 'geo_data_json': json.dumps(geo_data), - 'geo_skipped': geo_skipped, - 'geo_count': len(geo_data), - 'cluster_sizes_json': json.dumps(cluster_sizes), - 'top_features_json': json.dumps(top_features), - 'noise_features': noise_features, - 'noise_summary': noise_summary, - 'noise_entity_count': noise_entity_count, - 'llm_timeline_json': llm_timeline_json, - # Globe flow data for the sidebar - 'globe_flows_json': json.dumps(_get_globe_flows(run)), - }) - - -def _get_globe_flows(run, max_rows=500): - """Extract flow data for 3D globe visualization from a run.""" - import json - from analysis.data_loader import load_csv_directory, load_from_db - try: - lf = None - if run.sqlite_table: - lf = load_from_db(run.sqlite_table) - if lf is None and run.csv_glob: - lf, _, _, _, _ = load_csv_directory(run.csv_glob) - if lf is None: - return [] - df = lf.head(max_rows).collect() - from analysis.geoip import lookup as geo_lookup - flows = [] - TLS_HEX_MAP = {'0303': 'TLSv1.2', '0304': 'TLSv1.3', '0302': 'TLSv1.1', '0301': 'TLSv1.0'} - src_ip_col = next((c for c in df.columns if 'src_ip' in c.lower() or ':ips' in c.lower()), None) - dst_ip_col = next((c for c in df.columns if 'dst_ip' in c.lower() or ':ipd' in c.lower()), None) - tls_col = next((c for c in df.columns if c.lower() in ('0ver', 'tls_version', '_tlsver', 'version')), None) - bytes_col = next((c for c in df.columns if c.lower() in ('8ack', '8pak', '8byt', 'bytes_sent', 'bytes')), None) - for row in df.iter_rows(named=True): - sip = str(row.get(src_ip_col, '')) if src_ip_col else '' - dip = str(row.get(dst_ip_col, '')) if dst_ip_col else '' - if not sip or not dip: - continue - sg = geo_lookup(sip) - dg = geo_lookup(dip) - if not sg or not dg: - continue - tls_val = str(row.get(tls_col, '') or '') if tls_col else '' - tls_ver = TLS_HEX_MAP.get(tls_val.replace(' ', ''), tls_val) - flows.append({ - 'slat': sg['lat'], 'slon': sg['lon'], - 'dlat': dg['lat'], 'dlon': dg['lon'], - 'tls': tls_ver, - 'bytes': float(row.get(bytes_col, 0) or 0) if bytes_col else 0, - }) - return flows - except Exception: - return [] - - -def cluster_detail(request, display_id, cluster_label): - """Detailed view of a single cluster with entity list, SVD features, and NL summary.""" - run = get_object_or_404(AnalysisRun, display_id=display_id) - try: - cluster_label = int(cluster_label) - except (ValueError, TypeError): - raise Http404("Invalid cluster label") - cluster = get_object_or_404(ClusterResult, run=run, cluster_label=cluster_label) - features = cluster.features.all().order_by('-distinguishing_score')[:50] - entities = cluster.entities.all()[:100] - nl_summary = '' - feats_list = [{'feature_name': f.feature_name, 'score': f.distinguishing_score, - 'mean': f.mean, 'std': f.std} for f in features[:10]] - if feats_list: - nl_summary = nl_describe.describe_cluster(feats_list) - - # Build entity data with feature_json values - entity_data = [] - for e in entities: - entry = {'id': e.id, 'value': e.entity_value, - 'embedding_x': e.embedding_x, 'embedding_y': e.embedding_y} - if e.feature_json: - entry['features'] = {k: round(float(v), 4) if isinstance(v, (int, float)) else str(v) - for k, v in e.feature_json.items()} - entity_data.append(entry) - - return render(request, 'analysis/cluster_detail.html', { - 'run': run, - 'cluster': cluster, - 'features': features, - 'entities': entities, - 'entity_data_json': json.dumps(entity_data), - 'nl_summary': nl_summary, - }) - - -def entity_profile(request, entity_id): - """Detailed profile for a single entity.""" - entity = get_object_or_404(EntityProfile, id=entity_id) - run = entity.run - - # Compute deviation from cluster mean - deviations = {} - if entity.feature_json and entity.cluster: - cluster_features = { - f.feature_name: {'mean': f.mean, 'std': f.std} - for f in entity.cluster.features.all() - } - for col, val in entity.feature_json.items(): - if col in cluster_features and cluster_features[col].get('std') and cluster_features[col]['std'] > 1e-10: - deviations[col] = { - 'value': val, - 'cluster_mean': cluster_features[col]['mean'], - 'zscore': (val - cluster_features[col]['mean']) / cluster_features[col]['std'], - } - - return render(request, 'analysis/entity_profile.html', { - 'entity': entity, - 'run': run, - 'deviations': deviations, - }) - - -def start_analysis(request): - """Placeholder: API endpoint to trigger analysis via MCP.""" - return JsonResponse({'status': 'not_implemented', 'message': 'Use MCP tools to trigger analysis'}) - - -def config_view(request): - """Config page: GET renders current config, POST saves updates.""" - cfg = get_config() - - if request.method == 'POST': - # Build a new Config from POST data (fall back to current values) - server = Config.Server( - host=request.POST.get('server_host', cfg.server.host), - port=int(request.POST.get('server_port', cfg.server.port)), - debug=request.POST.get('server_debug') == 'true', - ) - data = Config.Data( - schema_strict=request.POST.get('data_schema_strict') == 'true', - recursive=request.POST.get('data_recursive') == 'true', - ) - clustering = Config.Clustering( - algorithm=request.POST.get('clustering_algorithm', cfg.clustering.algorithm), - min_cluster_size=int(request.POST.get('clustering_min_cluster_size', cfg.clustering.min_cluster_size)), - random_state=cfg.clustering.random_state, - ) - llm = Config.LLM( - enabled=request.POST.get('llm_enabled') == 'true', - base_url=request.POST.get('llm_base_url', cfg.llm.base_url), - api_key=request.POST.get('llm_api_key', cfg.llm.api_key), - model=request.POST.get('llm_model', cfg.llm.model), - ) - cfg = Config(server=server, data=data, clustering=clustering, llm=llm) - save_config(cfg) - # Re-render with saved flag - return render(request, 'tianxuan/config.html', {'config': cfg, 'saved': True}) - - return render(request, 'tianxuan/config.html', {'config': cfg}) - - -@csrf_exempt -def llm_test(request): - """LLM connectivity test: POST JSON {base_url, api_key, model} → chat/completions.""" - if request.method != 'POST': - return JsonResponse({'success': False, 'message': '仅支持 POST 请求', 'latency_ms': 0}, status=405) - - try: - body = json.loads(request.body) - except json.JSONDecodeError: - return JsonResponse({'success': False, 'message': '无效的 JSON 请求体', 'latency_ms': 0}) - - base_url = (body.get('base_url') or '').rstrip('/') - api_key = body.get('api_key') or '' - model = body.get('model') or '' - - if not base_url: - return JsonResponse({'success': False, 'message': 'Base URL 不能为空', 'latency_ms': 0}) - if not api_key: - return JsonResponse({'success': False, 'message': 'API Key 不能为空', 'latency_ms': 0}) - if not model: - return JsonResponse({'success': False, 'message': 'Model 不能为空', 'latency_ms': 0}) - - url = f'{base_url}/chat/completions' - payload = json.dumps({ - 'model': model, - 'messages': [{'role': 'user', 'content': 'respond with ok'}], - 'max_tokens': 5, - }).encode('utf-8') - - req = urllib.request.Request( - url, - data=payload, - headers={ - 'Content-Type': 'application/json', - 'Authorization': f'Bearer {api_key}', - }, - method='POST', - ) - - start = time.monotonic() - try: - with urllib.request.urlopen(req, timeout=5) as resp: - elapsed = int((time.monotonic() - start) * 1000) - data = json.loads(resp.read().decode('utf-8')) - if 'choices' in data and len(data['choices']) > 0: - logger.info(f'[LLM_TEST] base_url={base_url} model={model} success=True latency={elapsed}ms') - return JsonResponse({ - 'success': True, - 'message': f'连接成功，模型 {model} 返回正常', - 'latency_ms': elapsed, - }) - else: - logger.info(f'[LLM_TEST] base_url={base_url} model={model} success=True latency={elapsed}ms') - return JsonResponse({ - 'success': True, - 'message': f'连接成功，但响应中无 choices（原始响应已记录）', - 'latency_ms': elapsed, - }) - except urllib.error.HTTPError as e: - elapsed = int((time.monotonic() - start) * 1000) - try: - detail = json.loads(e.read().decode('utf-8', errors='replace')) - msg = detail.get('error', {}).get('message', str(e)) - except Exception: - msg = str(e) - logger.info(f'[LLM_TEST] base_url={base_url} model={model} success=False latency={elapsed}ms') - return JsonResponse({'success': False, 'message': f'HTTP {e.code}: {msg}', 'latency_ms': elapsed}) - except urllib.error.URLError as e: - elapsed = int((time.monotonic() - start) * 1000) - reason = str(e.reason) if e.reason else '连接失败' - logger.info(f'[LLM_TEST] base_url={base_url} model={model} success=False latency={elapsed}ms') - return JsonResponse({'success': False, 'message': f'连接失败: {reason}', 'latency_ms': elapsed}) - except TimeoutError: - elapsed = int((time.monotonic() - start) * 1000) - logger.info(f'[LLM_TEST] base_url={base_url} model={model} success=False latency={elapsed}ms') - return JsonResponse({'success': False, 'message': '请求超时（5秒）', 'latency_ms': elapsed}) - except Exception as e: - elapsed = int((time.monotonic() - start) * 1000) - logger.info(f'[LLM_TEST] base_url={base_url} model={model} success=False latency={elapsed}ms') - return JsonResponse({'success': False, 'message': f'未知错误: {str(e)}', 'latency_ms': elapsed}) - - -# ── Upload page ───────────────────────────────────────────────────────── - -def upload_page(request): - """CSV upload page.""" - runs = AnalysisRun.objects.all().order_by('-created_at')[:20] - return render(request, 'tianxuan/upload.html', {'runs': runs}) - - -@csrf_exempt -def upload_csv(request): - """Handle CSV upload AJAX POST.""" - if request.method != 'POST': - return JsonResponse({'error': 'POST required'}, status=405) - files = request.FILES.getlist('files') - if not files: - return JsonResponse({'error': 'No files uploaded'}) - logger.info('[UPLOAD] Received %d files', len(files)) - for f in files: - if f.size > 500 * 1024 * 1024: - return JsonResponse({'warning': f'文件 {f.name} 超过 500MB，建议分割后上传'}) - total_size = sum(f.size for f in files) - if total_size > 50 * 1024 * 1024 * 1024: - return JsonResponse({'error': '总大小超过 50GB 限制'}) - from django.utils import timezone - ts = timezone.now().strftime('%Y%m%d_%H%M%S') - from analysis.appdata import ensure_appdata_dir - base = ensure_appdata_dir() / 'data' / 'uploads' - upload_dir = base / ts - upload_dir.mkdir(parents=True, exist_ok=True) - saved = [] - for f in files: - path = upload_dir / f.name - with open(path, 'wb+') as dst: - for chunk in f.chunks(): - dst.write(chunk) - saved.append(str(path)) - # Create run with status='pending' — background processing deferred to finalize endpoint - run = AnalysisRun.objects.create( - csv_glob=str(upload_dir / '*.csv'), - status='pending', - run_type='upload', - ) - # Deferred: _background_process will be started by finalize_upload endpoint - return JsonResponse({'run_id': run.display_id, 'files': saved, 'status': 'pending'}) - - -@csrf_exempt -def upload_csv_batch(request, display_id): - """Append files to existing upload run. Does NOT re-trigger _background_process.""" - run = get_object_or_404(AnalysisRun, display_id=display_id) - files = request.FILES.getlist('files') - if not files: - return JsonResponse({'error': 'No files uploaded'}) - upload_dir = Path(run.csv_glob).parent if run.csv_glob else None - if upload_dir: - expected_base = ensure_appdata_dir() / 'data' / 'uploads' - if not str(upload_dir.resolve()).startswith(str(expected_base.resolve())): - return JsonResponse({'error': 'Invalid upload path'}, status=400) - if not upload_dir or not upload_dir.exists(): - return JsonResponse({'error': 'Upload directory not found'}) - saved = [] - for f in files: - path = upload_dir / f.name - with open(path, 'wb+') as dst: - for chunk in f.chunks(): - dst.write(chunk) - saved.append(str(path)) - # NO background process re-trigger — first batch handles it - return JsonResponse({'run_id': run.display_id, 'files': len(saved), 'status': 'ok'}) - - -@csrf_exempt -def finalize_upload(request, display_id): - """Finalize batch upload: validate upload_dir, set status='loading', start background processing.""" - if request.method != 'POST': - return JsonResponse({'error': 'POST required'}, status=405) - run = get_object_or_404(AnalysisRun, display_id=display_id) - if run.status != 'pending': - return JsonResponse({'error': f'Run status is {run.status}, expected pending'}, status=400) - upload_dir = Path(run.csv_glob).parent if run.csv_glob else None - if not upload_dir or not upload_dir.exists(): - return JsonResponse({'error': 'Upload directory not found'}, status=400) - run.status = 'loading' - run.save(update_fields=['status']) - t = threading.Thread(target=_background_process, args=(run.id, upload_dir), daemon=True) - t.start() - return JsonResponse({'run_id': run.display_id, 'status': 'loading'}) - - -@csrf_exempt -def delete_upload(request, display_id): - """Delete uploaded files, SQLite data table, and AnalysisRun. - - Handles all run statuses (loading, profiling, clustering, completed, failed). - Gracefully tolerates null csv_glob and missing/errored SQLite tables. - """ - run = get_object_or_404(AnalysisRun, display_id=display_id) - import logging - import shutil - - _logger = logging.getLogger(__name__) - - # 1. Drop the SQLite data table if one exists - if run.sqlite_table: - try: - from analysis.data_loader import drop_sqlite_table # fmt: skip - drop_sqlite_table(run.sqlite_table) - except Exception as exc: - _logger.warning('[DELETE] failed to drop SQLite table %s: %s', run.sqlite_table, exc) - - # 2. Remove the upload directory if csv_glob is set - if run.csv_glob: - p = Path(run.csv_glob).parent - expected_base = ensure_appdata_dir() / 'data' / 'uploads' - if not str(p.resolve()).startswith(str(expected_base.resolve())): - return JsonResponse({'error': 'Invalid upload path'}, status=400) - if p.exists(): - shutil.rmtree(p, ignore_errors=True) - - # 3. Delete the ORM record - run.delete() - return JsonResponse({'deleted': True}) - - -import traceback, logging -logger = logging.getLogger(__name__) - - -@profile -def _background_process(run_id, upload_dir): - """Background: stream-load CSVs → save to SQLite → ready for analysis.""" - import django - os.environ['DJANGO_SETTINGS_MODULE'] = 'tianxuan.settings' - django.setup() - from analysis.models import AnalysisRun - from analysis.session_store import SessionStore - - run = AnalysisRun.objects.get(id=run_id) - store = SessionStore() - try: - # Check upload directory still exists (may have been temp-cleaned) - if not upload_dir.is_dir(): - msg = '上传数据目录已不存在，请重新上传' - logger.error('[BACKGROUND] %s (run_id=%s, upload_dir=%s)', msg, run_id, upload_dir) - run.status = 'failed' - run.error_message = msg - run.save(update_fields=['status', 'error_message']) - return - - import glob - import polars as pl - csv_paths = sorted(glob.glob(str(upload_dir / '*.csv'))) - if not csv_paths: - msg = '没有找到 CSV 文件' - logger.error('[BACKGROUND] %s (run_id=%s)', msg, run_id) - run.status = 'failed' - run.error_message = msg - run.save(update_fields=['status', 'error_message']) - return - - # ── Detect lat/lon columns for schema override ── - import csv as _csv - _LAT_LON_KW = ('latd', 'lond', 'latitude', 'longitude') - lat_lon_overrides = {} - try: - with open(csv_paths[0], 'r', encoding='utf-8') as _f: - _reader = _csv.reader(_f) - _headers = next(_reader, []) - for _col in _headers: - _norm = _col.lower().replace('-', '_').replace('.', '_').replace(' ', '_') - if any(kw in _norm for kw in _LAT_LON_KW): - lat_lon_overrides[_col] = pl.Utf8 - logger.info('[BACKGROUND] Forcing lat/lon column %r to Utf8 for safe parsing', _col) - except Exception: - logger.warning('[BACKGROUND] Failed to detect lat/lon columns', exc_info=True) - - total_files = len(csv_paths) - logger.info('[BACKGROUND] Found %d CSV files in %s', total_files, upload_dir) - # Adaptive batch size: aim for ~20 batches total, cap at 500 files per batch - batch_size = max(1, min(500, total_files // 20 or 1)) - total_batches = (total_files + batch_size - 1) // batch_size - - run.status = 'loading'; run.progress_pct = 10 - run.progress_msg = f'正在流式加载 {total_files} 个文件（共 {total_batches} 批）...' - run.save(update_fields=['status', 'progress_pct', 'progress_msg']) - - from analysis.data_loader import save_to_db, drop_sqlite_table, load_from_db, _dtype_to_sqlite - - table_name = f'_data_{run_id}' - - # ── Phase 1: Pre-scan all files for full union column set ── - all_columns = [] - for _path in csv_paths: - with open(_path, 'r', encoding='utf-8') as _f: - _reader = _csv.reader(_f) - _headers = next(_reader, []) - for _h in _headers: - if _h not in all_columns: - all_columns.append(_h) - reference_columns = sorted(all_columns, key=str.lower) - schema = {col: 'Utf8' for col in all_columns} - - # ── Phase 2: Create SQLite table once with all columns as TEXT ── - import sqlite3 as _sqlite3 - from django.conf import settings as _dj_settings - _db_path = _dj_settings.DATABASES['default']['NAME'] - _conn = _sqlite3.connect(_db_path) - try: - _col_defs = [f'"{_col}" TEXT' for _col in reference_columns] - _conn.execute(f'DROP TABLE IF EXISTS "{table_name}"') - _conn.execute(f'CREATE TABLE "{table_name}" ({", ".join(_col_defs)})') - _conn.commit() - finally: - _conn.close() - - # ── Phase 3: Batched collection → SQLite append ── - schema_overrides = {col: pl.Utf8 for col in reference_columns} - completed_batches = 0 - total_rows = 0 - for i in range(0, total_files, batch_size): - batch = csv_paths[i:i+batch_size] - # Scan each file INDIVIDUALLY — batch scan fails when files have different schemas - dfs = [] - for fp in batch: - lf = pl.scan_csv(fp, infer_schema_length=0, - schema_overrides=schema_overrides) - df = lf.collect(streaming=True) - # Fill missing columns with NULL - for col in reference_columns: - if col not in df.columns: - df = df.with_columns(pl.lit(None).alias(col)) - # Select all reference_columns in canonical order - df = df.select(reference_columns) - dfs.append(df) - df_batch = pl.concat(dfs, how='diagonal_relaxed') - - # Append to pre-created SQLite table - save_to_db(run.id, df_batch.lazy(), mode='append') - - # Delete temp files after successful SQLite write - for f in batch: - try: - os.unlink(f) - except OSError: - pass - - completed_batches += 1 - run.progress_pct = int((completed_batches / total_batches) * 80 + 10) - run.progress_msg = f'正在加载数据... ({completed_batches}/{total_batches} 批)' - run.save(update_fields=['progress_pct', 'progress_msg']) - - # ── Phase 3: Reload from SQLite with correct types ── - lf = load_from_db(table_name, schema_overrides=schema) - if lf is None: - raise RuntimeError(f'SQLite 表 {table_name} 加载失败') - - # Apply _coerce_to_float on lat/lon columns (restore Float64 from Utf8 storage) - from analysis.data_loader import _coerce_to_float as _coerce_to_float_fn - for lat_lon_col in lat_lon_overrides: - if lat_lon_col in lf.columns: - lf = lf.with_columns( - pl.col(lat_lon_col).map_batches(_coerce_to_float_fn, return_dtype=pl.Float64).alias(lat_lon_col) - ) - - # ── Restore numeric types for Utf8 columns coerced during CSV load ── - # All columns were forced to Utf8 for safe SQLite storage. Now try to - # recover the original numeric types (Int64 / Float64) from the data. - numeric_types = (pl.Int8, pl.Int16, pl.Int32, pl.Int64, - pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64, - pl.Float32, pl.Float64) - # Get current schema to know which columns are still Utf8 - live_schema = lf.collect_schema() - live_names = live_schema.names() - live_dtypes = list(live_schema.dtypes()) - utf8_cols = [ - live_names[i] for i, dt in enumerate(live_dtypes) - if dt == pl.Utf8 and not live_names[i].startswith('_') - ] - if utf8_cols: - # Infer types from a small sample to decide Float64 vs Int64 - sample_df = lf.select(utf8_cols).head(1000).collect(streaming=True) - type_map: dict[str, pl.DataType] = {} - for col in utf8_cols: - try: - # Try Float64 first (most general numeric type) - casted = sample_df[col].cast(pl.Float64, strict=False) - non_null = casted.is_not_null().sum() - if non_null > len(casted) * 0.5: - # Check if all non-null values are integers → use Int64 - all_int = True - for v in casted.head(100).to_list(): - if v is not None and v != int(v): - all_int = False - break - type_map[col] = pl.Int64 if all_int else pl.Float64 - except Exception: - pass - - if type_map: - # Update schema dict to reflect restored types - for col, dtype in type_map.items(): - schema[col] = str(dtype) - # Apply casts to the LazyFrame - casts = [pl.col(col).cast(dtype, strict=False) for col, dtype in type_map.items()] - lf = lf.with_columns(casts) - logger.info('[BACKGROUND] Restored numeric types for %d columns: %s', - len(type_map), list(type_map.keys())[:10]) - - # ── SVD dimensionality reduction ── - svd_components = 0 - # Only operate on columns with known numeric types in the schema - numeric_dtypes = {'Int64', 'Float64', 'Int32', 'Float32', - 'Int8', 'Int16', 'UInt8', 'UInt16', 'UInt32', 'UInt64'} - numeric_cols = [c for c, dt in schema.items() if dt in numeric_dtypes] - n_numeric = len(numeric_cols) - - if n_numeric >= 2: - n_components = min(50, n_numeric - 1) - try: - from sklearn.decomposition import TruncatedSVD # fmt: skip - import numpy as np # fmt: skip - - # Collect numeric data for SVD (full materialise → transform → add back) - df_full = lf.select(numeric_cols).collect(streaming=True) - X = df_full.to_numpy() - X = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0) - - svd = TruncatedSVD(n_components=n_components, random_state=42) - X_svd = svd.fit_transform(X) - svd_components = n_components - - # Build a new LazyFrame that includes _svd_* columns - # Strategy: collect full LazyFrame once, add SVD columns, re-wrap as lazy - df_all = lf.collect(streaming=True) - for i in range(n_components): - col_name = f'_svd_{i}' - schema[col_name] = 'Float64' - df_all = df_all.with_columns( - pl.Series(col_name, X_svd[:, i]) - ) - lf = df_all.lazy() - - logger.info('[BACKGROUND] SVD: %d components from %d numeric columns ' - '(explained variance ratio sum=%.4f)', - svd_components, n_numeric, - float(svd.explained_variance_ratio_.sum())) - except Exception as e: - logger.warning('[BACKGROUND] SVD failed (non-fatal): %s', e) - elif n_numeric == 1: - logger.info('[BACKGROUND] SVD skipped: only 1 numeric column (%s)', - numeric_cols[0]) - - # Get row count - df_count = lf.select(pl.len()).collect(streaming=True) - total_rows = df_count[0, 0] if df_count.height > 0 else 0 - - run.sqlite_table = table_name - run.total_flows = total_rows - run.save(update_fields=['sqlite_table', 'total_flows']) - - ds_id = f'upload_{run.display_id}' - store.store_dataset(ds_id, lf, schema=schema, metadata={ - 'row_count': total_rows, 'file_count': total_files, 'upload_dir': str(upload_dir), - 'csv_glob': str(upload_dir / '*.csv'), - 'sqlite_table': table_name, - 'svd_components': svd_components, - }) - - # ── Phase 3: Data ready for filtering/clustering via MCP tools ── - run.status = 'ready'; run.progress_pct = 60; run.progress_msg = '数据准备就绪' - run.save(update_fields=['status', 'progress_pct', 'progress_msg']) - except Exception: - tb = traceback.format_exc() - logger.error(tb) - # Drop partial SQLite table on failure (idempotent) - try: - from analysis.data_loader import drop_sqlite_table # fmt: skip - drop_sqlite_table(f'_data_{run_id}') - except Exception: - pass # Expected: drop_sqlite_table may fail if table already gone - run.status = 'failed' - run.error_message = tb - run.run_log += f'\n[ERROR] {tb}' - run.save(update_fields=['status', 'error_message', 'run_log']) - - -# ── Manual analysis page ─────────────────────────────────────────────── - -def manual_page(request): - """Workflow builder: select dataset, add MCP tool steps, execute.""" - from analysis.session_store import SessionStore - from analysis.tool_registry import get_tools_meta - import json as _json - - # Get all tools metadata - tools = get_tools_meta() - core_names = {'profile_data', 'build_entity_profiles', 'compute_scores', - 'run_clustering', 'extract_features', 'detect_anomalies', - 'visualize_anomalies'} - diag_names = {'validate_data', 'explore_distributions', 'find_outliers', - 'diagnose_clustering', 'compare_datasets', 'export_debug_sample', - 'repair_schema'} - analysis_names = {'analyze_patterns', 'analyze_temporal', 'analyze_tls_health', - 'analyze_geo_distribution', 'analyze_entity_detail'} - core_tools, diag_tools, analysis_tools, other_tools = [], [], [], [] - for t in tools: - if t.name in core_names: core_tools.append(t) - elif t.name in diag_names: diag_tools.append(t) - elif t.name in analysis_names: analysis_tools.append(t) - else: other_tools.append(t) - - meta_list = [{'name': t.name, 'description': t.description, 'inputSchema': t.inputSchema} for t in tools] - - # Get datasets from SessionStore - store = SessionStore() - session_datasets = store.list_datasets() if hasattr(store, 'list_datasets') else [] - session_ds_map = {ds['dataset_id']: ds for ds in session_datasets} - - # Build dataset entries from ALL AnalysisRun records (upload/manual/auto, any status) - from analysis.models import AnalysisRun - all_runs = AnalysisRun.objects.all().order_by('-created_at')[:100] - datasets_with_status = [] - - for run in all_runs: - # Determine how the dataset was keyed in SessionStore - if run.run_type == 'upload': - ds_id = f'upload_{run.display_id}' - else: - ds_id = f'run_{run.display_id}' - - session_entry = session_ds_map.get(ds_id) - - if session_entry: - # Full data available in SessionStore — merge with run status - datasets_with_status.append({ - **session_entry, - 'dataset_id': ds_id, - 'run_status': run.status, - 'total_flows': run.total_flows, - 'display_id': run.display_id, - 'needs_reload': False, - }) - elif run.csv_glob or run.sqlite_table: - # Data not in SessionStore but recoverable from disk/DB - datasets_with_status.append({ - 'dataset_id': ds_id, - 'run_status': run.status, - 'display_id': run.display_id, - 'total_flows': run.total_flows, - 'row_count': run.total_flows, - 'file_count': None, - 'column_count': 0, - 'columns': [], - 'svd_components': 0, - 'needs_reload': True, - 'csv_glob': run.csv_glob, - 'sqlite_table': run.sqlite_table or '', - }) - - # Map statuses to user-facing labels for the filter tabs - # ready → 待分析, completed → 已分析, failed → 错误 - # Everything else (pending/loading/profiling/aggregating/clustering/extracting) → analyzing → 分析中 - for ds_info in datasets_with_status: - s = ds_info['run_status'] - if s in ('completed',): - ds_info['status_label'] = '已分析' - ds_info['badge_class'] = 'completed' - elif s in ('failed',): - ds_info['status_label'] = '错误' - ds_info['badge_class'] = 'failed' - elif s in ('pending', 'loading', 'profiling', 'aggregating', 'clustering', 'extracting'): - ds_info['status_label'] = '分析中' - ds_info['badge_class'] = 'analyzing' - else: - ds_info['status_label'] = '待分析' - ds_info['badge_class'] = 'ready' - - return render(request, 'tianxuan/manual.html', { - 'core_tools': core_tools, - 'diag_tools': diag_tools, - 'analysis_tools': analysis_tools, - 'other_tools': other_tools, - 'tools_meta': _json.dumps(meta_list, ensure_ascii=False), - 'datasets': datasets_with_status, - 'datasets_with_status_json': _json.dumps(datasets_with_status, ensure_ascii=False), - }) - - -# ── Pipeline worker ───────────────────────────────────────────────────────── - - -def _run_pipeline_worker(run_id, analysis_fn, **ctx): - """Unified background worker: setup / ORM / progress / error handling.""" - import django - os.environ['DJANGO_SETTINGS_MODULE'] = 'tianxuan.settings' - os.environ['DJANGO_ALLOW_ASYNC_UNSAFE'] = 'true' - django.setup() - - from analysis.db_utils import retry_on_lock - from django.db import close_old_connections - - close_old_connections() - - @retry_on_lock() - def _get_run(): - return AnalysisRun.objects.get(id=run_id) - - run = _get_run() - try: - analysis_fn(run, ctx) - except Exception: - import traceback - tb = traceback.format_exc() - logger.error(tb) - run.status = 'failed' - run.error_message = tb - run.run_log += f'\n[FATAL] {tb}' - retry_on_lock()(lambda: run.save(update_fields=['status', 'error_message', 'run_log']))() - - -# ── Shared clustering pipeline ───────────────────────────────────────── - -@profile -def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorithm, - min_cluster_size, run_umap=True, head=None): - """Unified clustering pipeline: clustering → feature extraction → UMAP → ORM save. - - Args: - run: AnalysisRun ORM object. - store: SessionStore instance. - entity_ds_id: dataset ID in SessionStore pointing to entity-aggregated data. - feature_columns: list of feature column names (None = auto-detect). - algorithm: 'hdbscan' or 'kmeans'. - min_cluster_size: int for HDBSCAN. - run_umap: bool, whether to compute and save UMAP-2D embeddings. - head: optional int, downsample to at most this many rows (min 100). - """ - import warnings - warnings.filterwarnings('ignore', category=RuntimeWarning, module='sklearn') - import asyncio, traceback, logging - logger = logging.getLogger(__name__) - - try: - from analysis.tool_registry import _handle_run_clustering, _handle_extract_features - import polars as pl - - entry = store.get_dataset(entity_ds_id) - if entry is None: - run.status = 'failed' - run.error_message = 'Entity dataset not found in session store' - run.save(update_fields=['status', 'error_message']) - return - - schema = entry.get('schema', {}) - - # ── Downsampling ── - if head is not None and head > 0: - lf = entry['lazyframe'] - try: - row_count = lf.select(pl.len()).collect(streaming=True).item() - except Exception: - row_count = 0 - - if row_count > head: - head = max(head, 100) # ensure minimum rows for clustering - run.progress_msg = f'正在采样 {head}/{row_count} 行...' - run.save(update_fields=['progress_msg']) - lf = lf.head(head) - store.store_dataset( - entity_ds_id, lf, - schema=schema, - metadata=entry.get('metadata', {}), - ) - entry = store.get_dataset(entity_ds_id) # refresh - row_count = head - - # Resolve feature columns — use actual LazyFrame dtypes (not stored schema dict - # which may be stale after Utf8 coercion in _background_process). - numeric_types = (pl.Int8, pl.Int16, pl.Int32, pl.Int64, - pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64, - pl.Float32, pl.Float64) - - lf = entry['lazyframe'] - live_schema = lf.collect_schema() - live_names = live_schema.names() - live_dtypes = list(live_schema.dtypes()) - - if feature_columns: - feature_cols = [ - c for c in feature_columns - if c in live_names and live_dtypes[live_names.index(c)] in numeric_types - ] - else: - feature_cols = [ - live_names[i] for i, dt in enumerate(live_dtypes) - if dt in numeric_types and not live_names[i].startswith('_') - ][:10] - - # ── Step 1: Clustering ──────────────────────────────────────────────── - # If no numeric columns detected, pass empty list to let - # _handle_run_clustering apply its Utf8→Float64 coercion fallback. - run.status = 'clustering' - run.progress_pct = 70 - run.progress_msg = '正在进行聚类分析...' - run.save(update_fields=['status', 'progress_pct', 'progress_msg']) - - result = asyncio.run(_handle_run_clustering( - dataset_id=entity_ds_id, - cluster_columns=feature_cols, - algorithm=algorithm, - params={'min_cluster_size': min_cluster_size}, - random_state=42, - )) - - if 'error' in result: - err = result['error'] - if 'numeric' in err.lower(): - available = [ - f"{k}({v})" for k, v in schema.items() - if v.split('(')[0].strip() in numeric_types and not k.startswith('_') - ] - err += f"。可用数值列: {available}" - raise Exception(err) - - cluster_id = result.get('cluster_result_id', '') - run.cluster_count = result.get('n_clusters', 0) - run.save(update_fields=['cluster_count']) - - # ── Step 2: Feature extraction ────────────────────────────────────── - run.status = 'extracting' - run.progress_pct = 90 - run.progress_msg = '正在提取聚类特征...' - run.save(update_fields=['status', 'progress_pct', 'progress_msg']) - - feat_result = asyncio.run(_handle_extract_features( - dataset_id=entity_ds_id, - cluster_result_id=cluster_id, - top_k=10, method='zscore', save_to_db=True, - run_id=run.id, - )) - - if 'error' in feat_result: - raise Exception(feat_result['error']) - - # ── Step 2b: Cluster SVD feature extraction ────────────────────── - run.progress_msg = '正在进行聚类SVD特征提取...' - run.save(update_fields=['progress_msg']) - try: - from analysis.distance import cluster_svd_extract - lf = entry['lazyframe'] - cluster_entry = store.get_cluster_result(cluster_id) - labels_list = cluster_entry.get('labels', []) if cluster_entry else [] - - if labels_list and feature_cols: - svd_results = cluster_svd_extract(lf, labels_list, feature_cols) - - # Persist SVD features to ClusterFeature (method='cluster_svd') - from analysis.models import ClusterFeature as CF - from analysis.models import ClusterResult as CR - for label, svd_info in svd_results.items(): - cr = CR.objects.filter(run=run, cluster_label=label).first() - if cr is None: - continue - # Delete old zscore entries for this cluster, replace with SVD - CF.objects.filter(cluster=cr, distinguishing_method='zscore').delete() - for feat_name in svd_info.get('features', [])[:10]: - idx = svd_info['features'].index(feat_name) if feat_name in svd_info['features'] else -1 - strength = (svd_info.get('feature_strength', [])[idx] - if idx >= 0 and idx < len(svd_info.get('feature_strength', [])) - else None) - CF.objects.get_or_create( - cluster=cr, - feature_name=feat_name, - defaults={ - 'mean': None, 'std': None, 'median': None, - 'p25': None, 'p75': None, 'missing_rate': None, - 'distinguishing_score': float(strength) if strength is not None else None, - 'distinguishing_method': 'cluster_svd', - }, - ) - except Exception as svd_err: - logger.warning(f'Cluster SVD extraction skipped: {svd_err}') - - # ── Step 3: UMAP-3D embedding (2D fallback) ─────────────────────── - if run_umap: - try: - lf = entry['lazyframe'] - # UMAP: sample max 10K for training, batch-transform rest in 1K batches - MAX_UMAP_TRAIN = 10000 - BATCH_SIZE = 1000 - try: - n_total = lf.select(pl.len()).collect(streaming=True).item() - except Exception: - n_total = 0 - from sklearn.preprocessing import StandardScaler - import umap - import numpy as np - - live_schema = lf.collect_schema() - num_cols = [name for name, dt in zip(live_schema.names(), live_schema.dtypes()) - if dt in (pl.Int8, pl.Int16, pl.Int32, pl.Int64, - pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64, - pl.Float32, pl.Float64) and not name.startswith('_')] - - if len(num_cols) >= 2: - if n_total > MAX_UMAP_TRAIN: - run.progress_msg = f'UMAP 采样 {MAX_UMAP_TRAIN}/{n_total} 实体训练...' - run.save(update_fields=['progress_msg']) - df_sample = lf.sample(n=MAX_UMAP_TRAIN, seed=42).collect(streaming=True) - df_umap = lf.collect(streaming=True) - else: - df_umap = lf.collect(streaming=True) - - umap_components = 3 # try 3D first - coords = None - reducer = None - - if len(df_umap) > 2: - for attempt_n in (3, 2): - try: - if n_total > MAX_UMAP_TRAIN: - # Train UMAP on 10K sample, batch-transform full dataset - mat_sample = df_sample.select(num_cols).to_numpy() - scaler = StandardScaler().fit(mat_sample) - mat_sample_scaled = np.nan_to_num( - scaler.transform(mat_sample), nan=0.0) - reducer = umap.UMAP(n_components=attempt_n, - random_state=42, - n_neighbors=15, min_dist=0.1, - metric='euclidean') - reducer.fit(mat_sample_scaled) - # Batch transform in 1K-row batches - coords_list = [] - for start in range(0, len(df_umap), BATCH_SIZE): - end = min(start + BATCH_SIZE, len(df_umap)) - mat_batch = scaler.transform( - df_umap[start:end].select(num_cols).to_numpy()) - mat_batch = np.nan_to_num(mat_batch, nan=0.0) - coords_list.append(reducer.transform(mat_batch)) - coords = np.vstack(coords_list) - else: - mat = np.nan_to_num(StandardScaler().fit_transform( - df_umap.select(num_cols).to_numpy()), nan=0.0) - reducer = umap.UMAP(n_components=attempt_n, - random_state=42, - n_neighbors=15, min_dist=0.1, - metric='euclidean') - coords = reducer.fit_transform(mat) - umap_components = attempt_n - break - except Exception as e_umap: - if attempt_n == 2: - raise - logger.warning(f'UMAP 3D failed ({e_umap}), falling back to 2D') - - if coords is None: - raise Exception('UMAP produced no output') - - non_num = [c for c in df_umap.columns - if c not in num_cols and not c.startswith('_')] - ent_col = non_num[0] if non_num else df_umap.columns[0] - - cluster_entry = store.get_cluster_result(cluster_id) - labels_list = cluster_entry.get('labels', []) if cluster_entry else [] - - from analysis.models import EntityProfile as EP - from analysis.models import ClusterResult as CR - - profiles = [] - cr_cache = {} - for i, row in enumerate(df_umap.iter_rows(named=True)): - ev = str(row.get(ent_col, '')) or 'entity' - ev = f'{ev}_{i}' - lbl = labels_list[i] if i < len(labels_list) else -1 - cache_key = f'{run.id}_{lbl}' - if cache_key not in cr_cache: - cr_cache[cache_key] = CR.objects.filter( - run=run, cluster_label=lbl).first() - has_z = umap_components >= 3 and coords.shape[1] >= 3 - profiles.append(EP( - entity_value=ev, run=run, cluster_label=lbl, - cluster=cr_cache[cache_key], - embedding_x=float(coords[i, 0]) if i < len(coords) else None, - embedding_y=float(coords[i, 1]) if i < len(coords) else None, - embedding_z=float(coords[i, 2]) if has_z - and i < len(coords) else None, - )) - if profiles: - EP.objects.bulk_create(profiles, ignore_conflicts=True, batch_size=1000) - run.entity_count = len(profiles) - run.save(update_fields=['entity_count']) - except Exception as umap_err: - logger.warning(f'UMAP embedding skipped: {umap_err}') - - # Fallback: if entity_count not set (UMAP skipped), derive from cluster sizes - if run.entity_count is None: - from django.db.models import Sum - total = run.clusters.aggregate(total=Sum('size'))['total'] or 0 - run.entity_count = total - run.save(update_fields=['entity_count']) - - # ── Done ── - run.status = 'completed' - run.progress_pct = 100 - run.progress_msg = '分析完成' - run.save(update_fields=['status', 'progress_pct', 'progress_msg', 'entity_count']) - - except Exception: - tb = traceback.format_exc() - logger.error(tb) - try: - run.status = 'failed' - run.error_message = tb - run.progress_msg = f'失败: {tb}' - run.run_log += f'\n[ERROR] {tb}' - run.save(update_fields=['status', 'error_message', 'progress_msg', 'run_log']) - except Exception as e: - logger.warning('save failed after pipeline error: %s', e) - - -@csrf_exempt -def manual_run_analysis(request): - """AJAX POST: run clustering pipeline on the upload dataset.""" - if request.method != 'POST': - return JsonResponse({'error': 'POST required'}, status=405) - import json - body = json.loads(request.body) - display_id = body.get('run_id') - run = get_object_or_404(AnalysisRun, display_id=display_id) - pk = run.id - run.run_type = 'manual' - run.save(update_fields=['run_type']) - feature_columns = body.get('feature_columns') - algorithm = body.get('algorithm', 'agglomerative') - min_cluster_size = int(body.get('min_cluster_size', 5)) - head = body.get('head') - cluster_mode = body.get('cluster_mode', 'raw') - - def _analysis_fn(run, ctx): - from analysis.session_store import SessionStore - - feature_columns = ctx['feature_columns'] - algorithm = ctx['algorithm'] - min_cluster_size = ctx['min_cluster_size'] - head = ctx.get('head') - pk = ctx['pk'] - cluster_mode = ctx.get('cluster_mode', 'raw') - - store = SessionStore() - try: - # Use display_id for dataset key (consistent with _background_process) - display_id = ctx.get('display_id', pk) - upload_ds_id = f'upload_{display_id}' - entry = store.get_dataset(upload_ds_id) - if entry is None: - # Fallback: try PK-based key (old pipeline) - entry = store.get_dataset(f'upload_{pk}') - if entry is None: - # Fallback: try entity dataset (from older pipeline) - entry = store.get_dataset(f'entity_{pk}') - if entry is None: - run.error_message = '请先上传并等待预处理完成' - run.status = 'failed' - run.save(update_fields=['status', 'error_message']) - return - - ds_id = upload_ds_id if store.get_dataset(upload_ds_id) else f'upload_{pk}' - - # Use the unified clustering pipeline (clustering → extraction → UMAP) - from analysis.views import _run_clustering_pipeline - _run_clustering_pipeline( - run=run, store=store, entity_ds_id=ds_id, - feature_columns=feature_columns, - algorithm=algorithm, min_cluster_size=min_cluster_size, - run_umap=True, head=head, - ) - except Exception: - tb = traceback.format_exc() - logger.error(tb) - run.status = 'failed' - run.error_message = tb - run.progress_msg = f'失败: {tb}' - run.run_log += f'\n[ERROR] {tb}' - run.save(update_fields=['status', 'error_message', 'progress_msg', 'run_log']) - - t = threading.Thread( - target=_run_pipeline_worker, - args=(pk, _analysis_fn), - kwargs=dict(feature_columns=feature_columns, algorithm=algorithm, - min_cluster_size=min_cluster_size, head=head, pk=pk, - display_id=run.display_id, cluster_mode=cluster_mode), - daemon=True, - ) - t.start() - return JsonResponse({'status': 'started', 'redirect': f'/runs/{run.display_id}/'}) - - - - - - - -# ── Filter endpoint for manual analysis page ────────────────────────── - -@csrf_exempt -def apply_filter(request): - """POST {dataset_id, filters, logic} → apply filter_data tool → return filtered_X id.""" - if request.method != 'POST': - return JsonResponse({'error': 'POST required'}, status=405) - import json as _json - import asyncio - body = _json.loads(request.body) - dataset_id = body.get('dataset_id') - filters = body.get('filters', []) - logic = body.get('logic', 'and') - - if not dataset_id: - return JsonResponse({'error': 'dataset_id is required'}) - if not filters: - return JsonResponse({'error': 'filters is required'}) - - try: - from analysis.tool_registry import _handle_filter_data - result = asyncio.run(_handle_filter_data( - dataset_id=dataset_id, - filters=filters, - logic=logic, - )) - if 'error' in result: - return JsonResponse({'error': result['error']}) - - filtered_id = result.get('filtered_dataset_id', '') - row_count = result.get('row_count', 0) - - # Use the unique flt_xxx ID from _handle_filter_data — never rename - # to filtered_{num} which would collide on repeated filters of the same source. - return JsonResponse({ - 'dataset_id': filtered_id, - 'row_count': row_count, - }) - except Exception as e: - import traceback - logger.error(f'[apply_filter] {traceback.format_exc()}') - return JsonResponse({'error': str(e)}) - - -# ── LLM auto analysis page ───────────────────────────────────────────── - - -@csrf_exempt -def run_llm_analysis_view(request): - """AJAX POST: run LLM-driven analysis pipeline.""" - if request.method != 'POST': - return JsonResponse({'error': 'POST required'}, status=405) - import json - body = json.loads(request.body) - display_id = body.get('run_id') - run = get_object_or_404(AnalysisRun, display_id=display_id) - pk = run.id - run.run_type = 'auto' - run.save(update_fields=['run_type']) - head = body.get('head') - - from analysis.session_store import SessionStore - store = SessionStore() - - # Determine dataset key format (consistent with manual_page) - upload_ds_id = f'upload_{display_id}' - run_ds_id = f'run_{display_id}' - entry = store.get_dataset(upload_ds_id) - if entry is None: - entry = store.get_dataset(run_ds_id) - - if entry is None: - # Try primary key fallback - upload_pk_id = f'upload_{pk}' - entity_pk_id = f'entity_{pk}' - entry = store.get_dataset(upload_pk_id) - if entry is None: - entry = store.get_dataset(entity_pk_id) - - if entry is None: - # Auto-reload from sqlite_table or csv_glob - from analysis.data_loader import load_csv_directory, load_from_db - lf = None - schema = {} - if run.sqlite_table: - try: - lf = load_from_db(run.sqlite_table) - except Exception: - pass - if lf is None and run.csv_glob: - try: - lf, schema, _, _, _ = load_csv_directory(run.csv_glob) - except Exception: - pass - if lf is None: - return JsonResponse({'error': '数据集不存在，请先上传并等待预处理完成'}) - entry_key = upload_ds_id if run.run_type == 'upload' else run_ds_id - store.store_dataset(entry_key, lf, schema=schema, metadata={ - 'row_count': run.total_flows, - 'csv_glob': run.csv_glob or '', - 'sqlite_table': run.sqlite_table or '', - }) - # Re-fetch entry - entry = store.get_dataset(entry_key) - - # Determine the active dataset_id - for candidate in [upload_ds_id, run_ds_id, f'upload_{pk}', f'entity_{pk}']: - if store.get_dataset(candidate): - dataset_id = candidate - break - else: - return JsonResponse({'error': '数据集不存在，请先上传并等待预处理完成'}) - - # ── Optional: apply filter before LLM pipeline ── - filters = body.get('filters', []) - if filters and len(filters) > 0: - logic = body.get('logic', 'and') - try: - from analysis.tool_registry import _handle_filter_data - filter_result = asyncio.run(_handle_filter_data( - dataset_id=dataset_id, - filters=filters, - logic=logic, - )) - if 'error' not in filter_result: - filtered_id = filter_result.get('filtered_dataset_id', '') - if filtered_id and store.get_dataset(filtered_id): - dataset_id = filtered_id - except Exception: - import traceback - logger.warning(f'[LLM auto] Filter failed (non-fatal): {traceback.format_exc()}') - - def _analysis_fn(run, ctx): - from tianxuan.llm_orchestrator import run_llm_pipeline, LLMConfig - dataset_id = ctx['dataset_id'] - pk = ctx['pk'] - - def progress_callback(step, tool_name, result=None, meta=None): - nonlocal run - try: - run = AnalysisRun.objects.get(id=pk) - pct = min(10 + step * 6, 95) - run.progress_pct = pct - run.progress_msg = f'步骤 {step}: {tool_name}' - - save_fields = ['progress_pct', 'progress_msg'] - if tool_name == 'thinking': - # Accumulate LLM thinking text (appended to llm_thinking) - if result: - run.llm_thinking = (run.llm_thinking or '') + f'\n[{step}] {result}' - save_fields = ['progress_pct', 'progress_msg', 'llm_thinking'] - elif tool_name == 'complete': - pass # No extra tracking needed - else: - # Single tool call — always has result - if isinstance(result, dict): - import json as _json - summary = _json.dumps(result, default=str, ensure_ascii=False)[:500] - else: - summary = str(result)[:500] - run.run_log += f'[{step}] {tool_name}: {summary}...\n' - - # One entry per tool call - if meta and isinstance(meta, dict) and 'args' in meta: - tool_calls = list(run.tool_calls_json or []) - tool_calls.append({ - 'step': step, - 'name': tool_name, - 'input': meta['args'], - 'output': result, - }) - run.tool_calls_json = tool_calls - save_fields = ['progress_pct', 'progress_msg', 'run_log', 'tool_calls_json'] - - run.save(update_fields=save_fields) - except Exception as e: - logger.warning('progress_callback error: %s', e) - - try: - from config import get_config - cfg = get_config() - llm_cfg = LLMConfig(cfg.llm.base_url, cfg.llm.api_key, cfg.llm.model) - run.run_log = '启动 LLM 分析...\n' - run.status = 'loading' - run.progress_pct = 5 - run.progress_msg = 'LLM 初始化中...' - run.save(update_fields=['status', 'progress_pct', 'progress_msg', 'run_log']) - - result = run_llm_pipeline(dataset_id, config=llm_cfg, callback=progress_callback) - - if 'error' in result: - run.status = 'failed' - run.progress_msg = f'失败: {result["error"]}' - run.run_log += f'\n[ERROR] {result["error"]}' - run.save(update_fields=['status', 'progress_pct', 'progress_msg', 'run_log']) - else: - run.progress_pct = 55 - run.progress_msg = f'LLM分析完成: {result.get("result", "")[:200]}' - run.run_log += f'\n[完成] {result.get("result", "")}\n[INFO] 开始自动聚类流程...' - run.save(update_fields=['progress_pct', 'progress_msg', 'run_log']) - - # ── Auto-run clustering pipeline after LLM analysis ── - entity_ds_id = None - tool_calls = run.tool_calls_json or [] - for tc in tool_calls: - tool_name = tc.get('name', '') - tool_input = tc.get('input', {}) - if tool_name in ('build_entity_profiles', 'run_clustering'): - entity_ds_id = tool_input.get('dataset_id', '') - if entity_ds_id and store.get_dataset(entity_ds_id): - break - entity_ds_id = None - - # Fallback: try common dataset ID patterns - if not entity_ds_id: - candidates = [ - ctx.get('dataset_id', ''), - f'entity_{pk}', - f'entity_{run.display_id}', - f'upload_{run.display_id}', - ] - for cid in candidates: - if cid and store.get_dataset(cid): - entity_ds_id = cid - break - - if entity_ds_id: - _run_clustering_pipeline( - run=run, - store=store, - entity_ds_id=entity_ds_id, - feature_columns=None, - algorithm='agglomerative', - min_cluster_size=5, - run_umap=True, - head=ctx.get('head'), - ) - else: - logger.warning( - f'[LLM auto] No entity dataset found for clustering. ' - f'Tool calls: {[tc.get("name") for tc in tool_calls]}. ' - f'Marking run as completed without clustering.' - ) - run.status = 'completed' - run.progress_pct = 100 - run.progress_msg = 'LLM分析完成（无实体数据，跳过聚类）' - run.save(update_fields=['status', 'progress_pct', 'progress_msg']) - - # ── Auto-save workflow as loadable plan ── - try: - tool_calls_data = run.tool_calls_json or [] - steps = [] - for tc in tool_calls_data: - tc_name = tc.get('name', '') - if tc_name == 'thinking': - continue - steps.append({ - 'tool_name': tc_name, - 'params': tc.get('input', {}), - }) - if steps: - import json as _json - plan_data = { - 'name': f'_auto_{run.display_id}', - 'steps': steps, - 'auto': True, - 'display_id': run.display_id, - 'created_at': str(run.created_at) if run.created_at else '', - } - plan_path = _PLANS_DIR / f'_auto_{run.display_id}.json' - plan_path.write_text( - _json.dumps(plan_data, ensure_ascii=False, indent=2), - encoding='utf-8', - ) - _add_to_auto_index(f'_auto_{run.display_id}.json') - logger.info( - '[LLM auto] Saved auto plan %s (%d steps)', - plan_path.name, len(steps), - ) - except Exception as save_err: - logger.warning('[LLM auto] Failed to save auto plan: %s', save_err) - - except Exception: - tb = traceback.format_exc() - logger.error(tb) - try: - run = AnalysisRun.objects.get(id=pk) - run.status = 'failed' - run.progress_msg = f'失败: {tb}' - run.run_log += f'\n[ERROR] {tb}' - run.error_message = tb - run.save(update_fields=['status', 'progress_msg', 'run_log', 'error_message']) - except Exception as e: - logger.warning('save failed after pipeline error: %s', e) - - t = threading.Thread( - target=_run_pipeline_worker, - args=(pk, _analysis_fn), - kwargs=dict(dataset_id=dataset_id, head=head, pk=pk), - daemon=True, - ) - t.start() - return JsonResponse({'status': 'started', 'run_id': run.display_id}) - - -def auto_page(request): - """LLM-driven auto analysis page.""" - from analysis.session_store import SessionStore - import json as _json - - cfg = get_config() - - # Get datasets from SessionStore + ALL runs from DB - store = SessionStore() - session_datasets = store.list_datasets() if hasattr(store, 'list_datasets') else [] - session_ds_map = {ds['dataset_id']: ds for ds in session_datasets} - - # Build dataset entries from ALL AnalysisRun records - all_runs = AnalysisRun.objects.all().order_by('-created_at')[:100] - datasets_with_status = [] - - for run in all_runs: - if run.run_type == 'upload': - ds_id = f'upload_{run.display_id}' - else: - ds_id = f'run_{run.display_id}' - - session_entry = session_ds_map.get(ds_id) - - if session_entry: - datasets_with_status.append({ - **session_entry, - 'dataset_id': ds_id, - 'run_status': run.status, - 'total_flows': run.total_flows, - 'display_id': run.display_id, - 'needs_reload': False, - 'csv_glob': run.csv_glob or '', - 'sqlite_table': run.sqlite_table or '', - }) - elif run.csv_glob or run.sqlite_table: - datasets_with_status.append({ - 'dataset_id': ds_id, - 'run_status': run.status, - 'display_id': run.display_id, - 'total_flows': run.total_flows, - 'row_count': run.total_flows, - 'file_count': None, - 'columns': [], - 'column_count': 0, - 'svd_components': 0, - 'needs_reload': True, - 'csv_glob': run.csv_glob or '', - 'sqlite_table': run.sqlite_table or '', - }) - - for ds_info in datasets_with_status: - s = ds_info['run_status'] - if s in ('completed',): - ds_info['status_label'] = '已分析' - ds_info['badge_class'] = 'completed' - elif s in ('failed',): - ds_info['status_label'] = '错误' - ds_info['badge_class'] = 'failed' - elif s in ('pending', 'loading', 'profiling', 'aggregating', 'clustering', 'extracting'): - ds_info['status_label'] = '分析中' - ds_info['badge_class'] = 'analyzing' - else: - ds_info['status_label'] = '待分析' - ds_info['badge_class'] = 'ready' - - return render(request, 'tianxuan/auto.html', { - 'llm_configured': bool(cfg.llm.base_url and cfg.llm.api_key), - 'datasets_with_status_json': _json.dumps(datasets_with_status, ensure_ascii=False), - }) - - -# ── Progress API ─────────────────────────────────────────────────────── - -def run_status_api(request, display_id): - """Return current run status as JSON.""" - run = get_object_or_404(AnalysisRun, display_id=display_id) - return JsonResponse({ - 'id': run.display_id, - 'status': run.status, - 'entity_count': run.entity_count, - 'cluster_count': run.cluster_count, - 'error_message': run.error_message, - 'progress_pct': run.progress_pct, - 'progress_msg': run.progress_msg, - 'run_log': run.run_log[-2000:], # last 2000 chars - 'llm_thinking': run.llm_thinking[-5000:], # last 5K chars - 'tool_calls': run.tool_calls_json, - }) - - -def log_viewer(request): - """Display last 200 lines of the app log (DEBUG only). - With ?pos=N, returns text: line_count\\nnew_content for AJAX polling. - """ - from django.conf import settings - if not settings.DEBUG: - return HttpResponse('Log viewer only available in DEBUG mode', status=403) - log_path = settings.BASE_DIR / 'logs' / 'tianxuan.log' - pos = request.GET.get('pos') - if pos is not None: - # AJAX polling mode: return new content since byte position - try: - pos = int(pos) - except ValueError: - pos = 0 - if log_path.exists(): - with open(log_path, 'r', encoding='utf-8') as f: - total = f.seek(0, 2) # seek to end - if pos >= total: - return HttpResponse(f'{total}\n', content_type='text/plain') - f.seek(pos) - new_content = f.read() - return HttpResponse(f'{total}\n{new_content}', content_type='text/plain') - return HttpResponse('0\n', content_type='text/plain') - - # HTML page mode - lines = [] - if log_path.exists(): - with open(log_path, 'r', encoding='utf-8') as f: - lines = f.readlines()[-200:] - return render(request, 'tianxuan/log_viewer.html', {'lines': lines}) - -# ── 3D Globe view ───────────────────────────────────────────────────────── - -def _extract_flows_from_df(df, MAX_ROWS): - """Extract flow data (lat/lon/TLS/bytes) from a Polars DataFrame.""" - from analysis.geoip import lookup as geo_lookup - TLS_HEX_MAP = {'0303': 'TLSv1.2', '0304': 'TLSv1.3', '0302': 'TLSv1.1', '0301': 'TLSv1.0'} - # NOTE: Duplicated in type_classifier.py — keep both in sync. - - src_lat_col = next((c for c in df.columns if c.lower() in ( - ':ips.latd', ':ips_latd', ':ips_lat', 'src_latitude', 'src_lat', 'source_latitude', 'source_lat')), None) - src_lon_col = next((c for c in df.columns if c.lower() in ( - ':ips.lond', ':ips_lond', ':ips_lon', 'src_longitude', 'src_lon', 'source_longitude', 'source_lon')), None) - dst_lat_col = next((c for c in df.columns if c.lower() in ( - ':ipd.latd', ':ipd_latd', ':ipd_lat', 'dst_latitude', 'dst_lat', 'dest_latitude', 'dest_lat')), None) - dst_lon_col = next((c for c in df.columns if c.lower() in ( - ':ipd.lond', ':ipd_lond', ':ipd_lon', 'dst_longitude', 'dst_lon', 'dest_longitude', 'dest_lon')), None) - src_ip_col = next((c for c in df.columns if 'src_ip' in c.lower() or ':ips' in c.lower()), None) - dst_ip_col = next((c for c in df.columns if 'dst_ip' in c.lower() or ':ipd' in c.lower()), None) - tls_col = next((c for c in df.columns if c.lower() in ('0ver', 'tls_version', '_tlsver', 'version')), None) - bytes_col = next((c for c in df.columns if c.lower() in ('8ack', '8pak', '8byt', 'bytes_sent', 'bytes')), None) - time_col = next((c for c in df.columns if c.lower() in ( - 'timestamp', 'time', '_time', 'ts', 'datetime', 'epoch', - 'created_at', 'created', 'event_time')), None) - - flows = [] - for row in df.iter_rows(named=True): - slat = slon = dlat = dlon = None - - if src_lat_col and src_lon_col: - slat = _extract_lat(row.get(src_lat_col)) - slon = _extract_lon(row.get(src_lon_col)) - if dst_lat_col and dst_lon_col: - dlat = _extract_lat(row.get(dst_lat_col)) - dlon = _extract_lon(row.get(dst_lon_col)) - - if (slat is None or slon is None) and src_ip_col: - sg = geo_lookup(str(row.get(src_ip_col, ''))) - if sg: - slat, slon = sg['lat'], sg['lon'] - if (dlat is None or dlon is None) and dst_ip_col: - dg = geo_lookup(str(row.get(dst_ip_col, ''))) - if dg: - dlat, dlon = dg['lat'], dg['lon'] - - if slat is None or slon is None or dlat is None or dlon is None: - continue - - tls_val = str(row.get(tls_col or '', '')) - # Check hex format first (e.g. "03 03" or "0303") - hex_key = tls_val.replace(' ', '').strip() - if hex_key in TLS_HEX_MAP: - tls_label = TLS_HEX_MAP[hex_key] - elif '1.3' in tls_val: - tls_label = 'TLSv1.3' - elif '1.2' in tls_val: - tls_label = 'TLSv1.2' - else: - tls_label = 'other' - - bs = float(row.get(bytes_col or '', 0) or 0) - - # Extract timestamp for animation ordering - t_raw = row.get(time_col) if time_col else None - t_val = None - if t_raw is not None: - if isinstance(t_raw, (int, float)): - t_val = float(t_raw) - elif hasattr(t_raw, 'timestamp'): - t_val = t_raw.timestamp() - else: - try: - t_val = float(t_raw) - except (ValueError, TypeError): - pass - - flows.append({ - 'slat': slat, 'slon': slon, - 'dlat': dlat, 'dlon': dlon, - 'tls': tls_label, - 'bytes': bs, - 'time': t_val, - }) - - # Normalize timestamps to 0-1 and sort flows by time - timed = [f for f in flows if f['time'] is not None] - if len(timed) > 1: - timed.sort(key=lambda f: f['time']) - t_min, t_max = timed[0]['time'], timed[-1]['time'] - t_range = max(t_max - t_min, 1) - for f in timed: - f['time'] = (f['time'] - t_min) / t_range - # Untimed flows get evenly spaced after timed ones - untimed = [f for f in flows if f['time'] is None] - for i, f in enumerate(untimed): - f['time'] = min(1.0, (len(timed) + i) / max(len(flows) - 1, 1)) - flows = timed + untimed - else: - for i, f in enumerate(flows): - f['time'] = i / max(len(flows) - 1, 1) - - return flows - - -def globe_view(request): - import json - from analysis.geoip import lookup as geo_lookup - from analysis.data_loader import load_csv_directory, load_from_db - from analysis.models import AnalysisRun - from analysis.session_store import SessionStore - - MAX_ROWS_PER_RUN = 300 # Limit per run to prevent browser overload - - # Check for ?data=dataset_id parameter (direct data loading from session store) - data_param = request.GET.get('data') - if data_param: - store = SessionStore() - flows = [] - all_runs = list(AnalysisRun.objects.filter(status='completed').order_by('-id').values('display_id', 'csv_glob')) - try: - entry = store.get_dataset(data_param) - if entry is not None: - lf = entry['lazyframe'] - df = lf.head(MAX_ROWS_PER_RUN).collect() - flows = _extract_flows_from_df(df, MAX_ROWS_PER_RUN) - except Exception as exc: - import logging - logging.getLogger(__name__).warning(f'globe ?data= param failed: {exc}') - return render(request, 'tianxuan/globe.html', { - 'geo_flows': json.dumps(flows), - 'all_runs': all_runs, - 'selected_ids': [], - }) - - # Get all completed runs for the selector - all_runs = list(AnalysisRun.objects.filter(status='completed').order_by('-id').values('display_id', 'csv_glob')) - - # Get selected run IDs from query params (multiple checkboxes) - selected_ids_list = request.GET.getlist('runs') - selected_runs = [] - if selected_ids_list: - for sid in selected_ids_list: - sid = sid.strip() - if sid.isdigit(): - try: - r = AnalysisRun.objects.get(display_id=int(sid), status='completed') - selected_runs.append(r) - except AnalysisRun.DoesNotExist: - pass - - # Check if user-selected runs have valid files; if not, fall through - all_selected_failed = bool(selected_ids_list) - fallback_run = None - - if not selected_runs: - all_selected_failed = False - - flows = [] - - for run in selected_runs: - try: - lf = None - if run.sqlite_table: - lf = load_from_db(run.sqlite_table) - if lf is None: - lf, schema, rc, fc, mem = load_csv_directory(run.csv_glob) - df = lf.head(MAX_ROWS_PER_RUN).collect() - flows.extend(_extract_flows_from_df(df, MAX_ROWS_PER_RUN)) - all_selected_failed = False - break # stop after first successful load - except Exception as exc: - logger.warning('[GLOBE] Failed to load run %s (%s): %s', run.display_id, run.csv_glob, exc) - continue - - if all_selected_failed or not flows: - # Fallback: latest completed run with valid data (SQLite or CSV) - for latest in AnalysisRun.objects.filter(status='completed').order_by('-id'): - if not latest.csv_glob and not latest.sqlite_table: - continue - try: - lf = None - if latest.sqlite_table: - lf = load_from_db(latest.sqlite_table) - if lf is None: - if not latest.csv_glob: - continue - lf, schema, rc, fc, mem = load_csv_directory(latest.csv_glob) - df = lf.head(MAX_ROWS_PER_RUN).collect() - flows.extend(_extract_flows_from_df(df, MAX_ROWS_PER_RUN)) - selected_runs = [latest] - break - except Exception: - continue - - resp = render(request, 'tianxuan/globe.html', { - 'geo_flows': json.dumps(flows), - 'all_runs': all_runs, - 'selected_ids': [r.display_id for r in selected_runs], - }) - resp['Cache-Control'] = 'no-cache, no-store, must-revalidate' - resp['Pragma'] = 'no-cache' - resp['Expires'] = '0' - return resp - - -# ── MCP Tool Lab — manual tool execution ───────────────────────────── - - -def tool_lab(request): - """List all available MCP tools with descriptions and input schemas.""" - from analysis.tool_registry import get_tools_meta - tools = get_tools_meta() - # Group tools by category - core_names = {'profile_data', 'build_entity_profiles', 'compute_scores', - 'run_clustering', 'extract_features', 'detect_anomalies', - 'visualize_anomalies'} - diag_names = {'validate_data', 'explore_distributions', 'find_outliers', - 'diagnose_clustering', 'compare_datasets', 'export_debug_sample', - 'repair_schema'} - analysis_names = {'analyze_patterns', 'analyze_temporal', 'analyze_tls_health', - 'analyze_geo_distribution', 'analyze_entity_detail'} - - core_tools, diag_tools, analysis_tools, other_tools = [], [], [], [] - for t in tools: - if t.name in core_names: - core_tools.append(t) - elif t.name in diag_names: - diag_tools.append(t) - elif t.name in analysis_names: - analysis_tools.append(t) - else: - other_tools.append(t) - - # Get list of datasets for parameter autofill - from analysis.session_store import SessionStore - store = SessionStore() - ds_list = store.list_datasets() if hasattr(store, 'list_datasets') else [] - - import json as _json - # Serialize tools metadata for JS - meta_list = [] - for t in tools: - meta_list.append({ - 'name': t.name, - 'description': t.description, - 'inputSchema': t.inputSchema, - }) - return render(request, 'analysis/tool_lab.html', { - 'core_tools': core_tools, - 'diag_tools': diag_tools, - 'analysis_tools': analysis_tools, - 'other_tools': other_tools, - 'datasets': ds_list, - 'tools_meta': _json.dumps(meta_list, ensure_ascii=False), - }) - - -@csrf_exempt -def tool_lab_run(request): - """Execute a single MCP tool via AJAX POST.""" - if request.method != 'POST': - return JsonResponse({'error': 'POST required'}, status=405) - import json as _json - try: - body = _json.loads(request.body) - except Exception: - return JsonResponse({'error': 'Invalid JSON body'}, status=400) - - tool_name = body.get('tool') - params = body.get('params', {}) - if not tool_name: - return JsonResponse({'error': 'Missing tool name'}, status=400) - - # ── Backend validation: check required params + dataset existence ── - from analysis.tool_registry import handle_call, get_tools_meta - from analysis.session_store import SessionStore - tools_meta = get_tools_meta() - tool_schema = None - for t in tools_meta: - if t.name == tool_name: - tool_schema = t.inputSchema - break - if tool_schema: - required_fields = tool_schema.get('required', []) - for field in required_fields: - val = params.get(field) - if val is None or (isinstance(val, str) and val.strip() == ''): - return JsonResponse({'error': f'Missing required parameter: {field}'}, status=400) - # If field is a dataset_id, verify it exists in the store - if 'dataset' in field.lower(): - store = SessionStore() - if not store.get_dataset(str(val)): - return JsonResponse({'error': f'Dataset not found: {val}'}, status=400) - - import asyncio - result = asyncio.run(handle_call(tool_name, params)) - return JsonResponse({'status': 'ok', 'result': result}) - - -@csrf_exempt -def reload_run_data(request): - """Reload a run's data from csv_glob or sqlite_table into SessionStore.""" - if request.method != 'POST': - return JsonResponse({'error': 'POST required'}, status=405) - import json as _json - try: - body = _json.loads(request.body) - except Exception: - return JsonResponse({'error': 'Invalid JSON body'}, status=400) - - display_id = body.get('display_id') - if not display_id: - return JsonResponse({'error': 'Missing display_id'}, status=400) - - from analysis.models import AnalysisRun - from analysis.data_loader import load_csv_directory, load_from_db - - run = get_object_or_404(AnalysisRun, display_id=display_id) - - # Determine dataset_id (same keying as manual_page) - ds_id = f'upload_{display_id}' if run.run_type == 'upload' else f'run_{display_id}' - - store = SessionStore() - # Check if already loaded - existing = store.get_dataset(ds_id) - if existing is not None: - return JsonResponse({'status': 'ok', 'dataset_id': ds_id, 'reloaded': False}) - - # Try loading from sqlite_table first, then csv_glob - lf = None - schema = {} - row_count = 0 - file_count = 0 - - if run.sqlite_table: - try: - lf = load_from_db(run.sqlite_table) - if lf is not None: - row_count = lf.select(pl.len()).collect(streaming=True)[0, 0] - except Exception as exc: - import logging - logging.getLogger(__name__).warning(f'Failed to load from sqlite_table: {exc}') - - if lf is None and run.csv_glob: - try: - from pathlib import Path - csv_path = Path(run.csv_glob) - if csv_path.is_file() or csv_path.is_dir() or csv_path.parent.is_dir(): - lf, schema, row_count, file_count, _ = load_csv_directory(run.csv_glob) - except Exception as exc: - import logging - logging.getLogger(__name__).warning(f'Failed to load from csv_glob: {exc}') - - if lf is None: - return JsonResponse({'error': 'No data source available (upload dir or sqlite table gone)'}, status=404) - - store.store_dataset(ds_id, lf, schema=schema, metadata={ - 'row_count': row_count, - 'file_count': file_count, - 'csv_glob': run.csv_glob or '', - 'sqlite_table': run.sqlite_table or '', - }) - - return JsonResponse({ - 'status': 'ok', - 'dataset_id': ds_id, - 'reloaded': True, - 'row_count': row_count, - 'columns': list(schema.keys()), - }) - - -# ── Save / Load / Delete analysis plans ───────────────────────────── - -from pathlib import Path -_PLANS_DIR = Path(__file__).resolve().parent.parent / '.omo' / 'plans' -_PLANS_DIR.mkdir(parents=True, exist_ok=True) - -_AUTO_INDEX_PATH = _PLANS_DIR / '_auto_index.json' -_MAX_AUTO_PLANS = 10 - - -def _read_auto_index(): - """Read the auto index file, returning list of {file, pinned}.""" - import json as _j - if not _AUTO_INDEX_PATH.exists(): - return [] - try: - data = _j.loads(_AUTO_INDEX_PATH.read_text(encoding='utf-8')) - if not isinstance(data, list): - return [] - return data - except Exception: - return [] - - -def _write_auto_index(index): - """Write the auto index list.""" - import json as _j - _AUTO_INDEX_PATH.write_text( - _j.dumps(index, ensure_ascii=False, indent=2), - encoding='utf-8', - ) - - -def _add_to_auto_index(filename): - """Prepend filename to auto index, evict unpinned entries when >10.""" - index = _read_auto_index() - # Remove if already present - index = [entry for entry in index if entry.get('file') != filename] - # Prepend - index.insert(0, {'file': filename, 'pinned': False}) - # Evict unpinned entries beyond max - pinned_entries = [e for e in index if e.get('pinned')] - unpinned_entries = [e for e in index if not e.get('pinned')] - # Keep all pinned + fill remaining slots with unpinned - result = pinned_entries + unpinned_entries[:_MAX_AUTO_PLANS - len(pinned_entries)] - _write_auto_index(result) - - -@csrf_exempt -def tool_plan(request): - """Save, list, load, delete, or pin/unpin an analysis plan.""" - import json as _j - if request.method == 'GET': - name = request.GET.get('name') - if name: - # Auto plans stored with _auto_ prefix - if name.startswith('_auto_'): - path = _PLANS_DIR / f'{name}.json' - else: - path = _PLANS_DIR / f'{name}.json' - if not path.exists(): - return JsonResponse({'error': f'Plan "{name}" not found'}, status=404) - try: - data = _j.loads(path.read_text(encoding='utf-8')) - return JsonResponse(data) - except Exception as e: - return JsonResponse({'error': str(e)}, status=500) - - # List all plans (regular + auto) - auto_index = _read_auto_index() - auto_index_map = {e.get('file', ''): e.get('pinned', False) for e in auto_index} - auto_names = set() - plans = [] - for p in sorted(_PLANS_DIR.glob('*.json')): - # Skip index file and auto plans (handled separately) - if p.name == '_auto_index.json': - continue - if p.name.startswith('_auto_'): - auto_names.add(p.name) - continue - try: - content = _j.loads(p.read_text(encoding='utf-8')) - steps = len(content.get('steps', [])) - plans.append({'name': p.stem, 'steps': steps}) - except Exception: - plans.append({'name': p.stem, 'steps': 0}) - - # Add auto plans with markers - auto_plans = [] - for auto_name in sorted(auto_names): - try: - path = _PLANS_DIR / auto_name - content = _j.loads(path.read_text(encoding='utf-8')) - steps = len(content.get('steps', [])) - auto_plans.append({ - 'name': auto_name.replace('.json', ''), - 'steps': steps, - 'auto': True, - 'pinned': auto_index_map.get(auto_name, False), - }) - except Exception: - auto_plans.append({ - 'name': auto_name.replace('.json', ''), - 'steps': 0, - 'auto': True, - 'pinned': auto_index_map.get(auto_name, False), - }) - # Interleave: auto plans first, then regular - return JsonResponse({'plans': auto_plans + plans}) - - elif request.method == 'POST': - try: - body = _j.loads(request.body) - # Pin/unpin toggle for auto plans - if 'pin' in body and body.get('name', '').startswith('_auto_'): - filename = body['name'].replace('.json', '') + '.json' \ - if not body['name'].endswith('.json') else body['name'] - index = _read_auto_index() - new_pinned = bool(body['pin']) - found = False - for entry in index: - if entry.get('file') == filename: - entry['pinned'] = new_pinned - found = True - break - if not found and new_pinned: - index.append({'file': filename, 'pinned': True}) - _write_auto_index(index) - return JsonResponse({ - 'status': 'pinned' if new_pinned else 'unpinned', - 'name': body['name'], - 'pinned': new_pinned, - }) - - # Regular save - name = body.get('name', '').strip() - steps = body.get('steps', []) - if not name: - return JsonResponse({'error': 'Missing plan name'}, status=400) - path = _PLANS_DIR / f'{name}.json' - path.write_text(_j.dumps({'name': name, 'steps': steps}, ensure_ascii=False, indent=2), encoding='utf-8') - return JsonResponse({'status': 'saved', 'name': name, 'steps': len(steps)}) - except Exception as e: - return JsonResponse({'error': str(e)}, status=500) - - elif request.method == 'DELETE': - name = request.GET.get('name', '').strip() - if not name: - return JsonResponse({'error': 'Missing plan name'}, status=400) - path = _PLANS_DIR / f'{name}.json' - if path.exists(): - path.unlink() - # Also remove from auto index if applicable - if name.startswith('_auto_'): - filename = name + '.json' if not name.endswith('.json') else name - index = _read_auto_index() - index = [e for e in index if e.get('file') != filename] - _write_auto_index(index) - return JsonResponse({'status': 'deleted', 'name': name}) - - return JsonResponse({'error': 'Method not allowed'}, status=405) - - -@csrf_exempt -def retry_run(request, display_id): - """Reset a failed run and re-trigger based on run_type.""" - if request.method != 'POST': - return JsonResponse({'error': 'POST required'}, status=405) - - run = get_object_or_404(AnalysisRun, display_id=display_id) - if run.status != 'failed': - return JsonResponse({'error': '只能重试失败的分析运行'}, status=400) - - run.status = 'pending' - run.error_message = '' - run.progress_pct = 0 - run.progress_msg = '' - run.save(update_fields=['status', 'error_message', 'progress_pct', 'progress_msg']) - - if run.run_type == 'upload' and run.csv_glob: - upload_dir = Path(run.csv_glob).parent - expected_base = ensure_appdata_dir() / 'data' / 'uploads' - if not str(upload_dir.resolve()).startswith(str(expected_base.resolve())): - return JsonResponse({'error': 'Invalid upload path'}, status=400) - if upload_dir.is_dir(): - t = threading.Thread( - target=_background_process, - args=(run.id, upload_dir), - daemon=True, - ) - t.start() - else: - run.error_message = '上传数据目录已不存在，请重新上传' - run.status = 'failed' - run.save(update_fields=['status', 'error_message']) - - return redirect('analysis:run_detail', display_id=run.display_id) - - -# Auto-profile all public functions in this module -from analysis.profile_util import auto_profile_module # noqa: E402 -auto_profile_module(__name__) - diff --git a/scripts/fix_merge_conflict.py b/scripts/fix_merge_conflict.py new file mode 100644 index 0000000..3dfa9e7 --- /dev/null +++ b/scripts/fix_merge_conflict.py @@ -0,0 +1,11 @@ +"""Fix the unresolved merge conflict in views/clustering.py.""" +import re +with open('analysis/views/clustering.py', 'r', encoding='utf-8-sig') as f: + lines = f.readlines() + +# Find and remove lines containing conflict markers +new_lines = [l for l in lines if '<<<<<<<' not in l and '>>>>>>>' not in l and '=======' not in l] +with open('analysis/views/clustering.py', 'w', encoding='utf-8') as f: + f.writelines(new_lines) +print(f'Removed {len(lines) - len(new_lines)} conflict marker lines') + diff --git a/static/tianxuan/cluster.css b/static/tianxuan/cluster.css deleted file mode 100644 index 51060b1..0000000 --- a/static/tianxuan/cluster.css +++ /dev/null @@ -1,133 +0,0 @@ -/** - * Shared cluster page styles. - * Extracted from templates/analysis/cluster_overview.html and cluster_detail.html - */ - -/* ── Legend pills ── */ -.legend-pills { - display: flex; - flex-wrap: wrap; - gap: 4px 8px; - padding: 6px 0; -} -.pill { - display: inline-flex; - align-items: center; - gap: 4px; - padding: 2px 10px; - border-radius: 12px; - font-size: 0.75rem; - cursor: pointer; - border: 2px solid transparent; - transition: all 0.15s; - white-space: nowrap; -} -.pill:hover { - opacity: 0.8; -} -.pill.active { - border-color: #333; - font-weight: 600; -} - -/* ── Cluster card ── */ -.cluster-card { - border: 1px solid #e0e0e0; - border-radius: 8px; - padding: 1rem; - background: #fff; - cursor: pointer; - transition: box-shadow 0.15s; -} -.cluster-card:hover { - box-shadow: 0 2px 8px rgba(0,0,0,0.1); -} -.cluster-card.selected { - border-color: #4361ee; - box-shadow: 0 0 0 2px #4361ee33; -} - -/* ── NL summary ── */ -.nl-summary { - background: #f8f9fa; - border-left: 3px solid #4361ee; - padding: 0.6rem; - margin: 0.5rem 0; - border-radius: 0 4px 4px 0; - font-size: 0.85rem; - line-height: 1.6; - color: #333; -} - -/* ── Feature table ── */ -.feat-table { - width: 100%; - border-collapse: collapse; - font-size: 0.8rem; -} -.feat-table th { - background: #f5f5f5; - padding: 0.3rem 0.5rem; - text-align: left; - font-weight: 600; - border-bottom: 2px solid #ddd; - white-space: nowrap; -} -.feat-table td { - padding: 0.25rem 0.5rem; - border-bottom: 1px solid #eee; -} -.feat-table td code { - font-size: 0.72rem; -} - -/* ── Feature scores ── */ -.feat-positive { - color: #2e7d32; -} -.feat-negative { - color: #c62828; -} - -/* ── Detail panel (overlay) ── */ -.detail-overlay { - position: fixed; - bottom: 0; - left: 0; - right: 0; - z-index: 100; - transform: translateY(100%); - transition: transform 0.35s cubic-bezier(.4,0,.2,1); -} -.detail-overlay.open { - transform: translateY(0); -} -.detail-content { - background: #fff; - border-radius: 16px 16px 0 0; - box-shadow: 0 -4px 24px rgba(0,0,0,0.15); - max-height: 45vh; - overflow-y: auto; - padding: 1rem 2rem 2rem; -} -.detail-handle { - width: 40px; - height: 4px; - background: #ccc; - border-radius: 2px; - margin: 0.5rem auto; - cursor: pointer; -} -.detail-handle:hover { - background: #999; -} -.detail-close { - float: right; - cursor: pointer; - font-size: 1.5rem; - color: #999; - line-height: 1; -} -.detail-close:hover { - color: #333; -} diff --git a/static/tianxuan/globe_embed.js b/static/tianxuan/globe_embed.js deleted file mode 100644 index 800637c..0000000 --- a/static/tianxuan/globe_embed.js +++ /dev/null @@ -1,129 +0,0 @@ -/** - * Three.js 3D globe rendering for TLS flow analysis. - * Extracted from templates/tianxuan/globe_embed.html - * - * Usage: initGlobe('globeContainer', geoFlows, options) - */ -function initGlobe(containerId, geoFlows, options) { - options = options || {}; - var W = window.innerWidth, H = window.innerHeight; - - var scene = new THREE.Scene(); - var camera = new THREE.PerspectiveCamera(45, W / H, 0.1, 1000); - camera.position.set(0, 2, 12); - - var renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true }); - renderer.setSize(W, H); - renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2)); - document.getElementById(containerId).appendChild(renderer.domElement); - - // Earth - var earthGeo = new THREE.SphereGeometry(5, 64, 64); - var earthMat = new THREE.MeshPhongMaterial({ color: 0x1a3a5c, emissive: 0x0a1a2a, specular: new THREE.Color(0x333333), shininess: 5 }); - var earth = new THREE.Mesh(earthGeo, earthMat); - scene.add(earth); - - // Glow - var glow = new THREE.Mesh(new THREE.SphereGeometry(5.1, 64, 64), new THREE.MeshBasicMaterial({ color: 0x224488, transparent: true, opacity: 0.1 })); - scene.add(glow); - - // Lights - scene.add(new THREE.AmbientLight(0x222244)); - var dl = new THREE.DirectionalLight(0xffffff, 1); - dl.position.set(5, 10, 7); - scene.add(dl); - scene.add(new THREE.DirectionalLight(0x4488ff, 0.3)); - - // Stars - var starGeo = new THREE.BufferGeometry(); - var starPos = new Float32Array(1000 * 3); - for (var i = 0; i < 1000 * 3; i++) starPos[i] = (Math.random() - 0.5) * 200; - starGeo.setAttribute('position', new THREE.BufferAttribute(starPos, 3)); - scene.add(new THREE.Points(starGeo, new THREE.PointsMaterial({ color: 0xffffff, size: 0.15 }))); - - // Lat/lon grid - var gridMat = new THREE.LineBasicMaterial({ color: 0x6699cc, transparent: true, opacity: 0.1 }); - for (var lat = -80; lat <= 80; lat += 20) { - var pts = []; - for (var lon = 0; lon <= 360; lon += 5) { - var phi = (90 - lat) * Math.PI / 180; - var theta = (lon + 180) * Math.PI / 180; - pts.push(new THREE.Vector3(-5.02 * Math.sin(phi) * Math.cos(theta), 5.02 * Math.cos(phi), 5.02 * Math.sin(phi) * Math.sin(theta))); - } - scene.add(new THREE.Line(new THREE.BufferGeometry().setFromPoints(pts), gridMat)); - } - for (var lon = 0; lon < 360; lon += 20) { - var pts = []; - for (var lat = -90; lat <= 90; lat += 5) { - var phi = (90 - lat) * Math.PI / 180; - var theta = (lon + 180) * Math.PI / 180; - pts.push(new THREE.Vector3(-5.02 * Math.sin(phi) * Math.cos(theta), 5.02 * Math.cos(phi), 5.02 * Math.sin(phi) * Math.sin(theta))); - } - scene.add(new THREE.Line(new THREE.BufferGeometry().setFromPoints(pts), gridMat)); - } - - // Flow arcs - var tlsColors = { 'TLSv1.3': 0x4cc9f0, 'TLSv1.2': 0x43aa8b, 'TLSv1.1': 0xf8961e, 'TLSv1.0': 0xf94144 }; - var flowGroup = new THREE.Group(); - scene.add(flowGroup); - - function latLonToVec3(lat, lon, r) { - if (lat == null || lon == null) return null; - var phi = (90 - lat) * Math.PI / 180; - var theta = (lon + 180) * Math.PI / 180; - return new THREE.Vector3(-r * Math.sin(phi) * Math.cos(theta), r * Math.cos(phi), r * Math.sin(phi) * Math.sin(theta)); - } - - var arcMeshes = []; - var flowData = Array.isArray(geoFlows) ? geoFlows : (typeof geoFlows === 'string' ? JSON.parse(geoFlows) : []); - flowData.forEach(function (flow) { - var from = latLonToVec3(flow.slat, flow.slon, 5); - var to = latLonToVec3(flow.dlat, flow.dlon, 5); - if (!from || !to) return; - var mid = new THREE.Vector3().addVectors(from, to).multiplyScalar(0.5).normalize().multiplyScalar(7); - var curve = new THREE.QuadraticBezierCurve3(from, mid, to); - var color = tlsColors[flow.tls] || 0x888888; - var mat = new THREE.MeshBasicMaterial({ color: color, transparent: true, opacity: 0.6 }); - var mesh = new THREE.Mesh(new THREE.TubeGeometry(curve, 20, 0.02, 4, false), mat); - mesh.userData = flow; - flowGroup.add(mesh); - arcMeshes.push(mesh); - }); - - // Mouse rotation - var isDragging = false, prevMouse = { x: 0, y: 0 }; - renderer.domElement.addEventListener('mousedown', function (e) { isDragging = true; prevMouse = { x: e.clientX, y: e.clientY }; }); - window.addEventListener('mouseup', function () { isDragging = false; }); - window.addEventListener('mousemove', function (e) { - if (!isDragging) return; - var dx = e.clientX - prevMouse.x, dy = e.clientY - prevMouse.y; - earth.rotation.y += dx * 0.005; - earth.rotation.z += dy * 0.005; - flowGroup.rotation.y = earth.rotation.y; - flowGroup.rotation.z = earth.rotation.z; - prevMouse = { x: e.clientX, y: e.clientY }; - }); - - // Listen for cluster filter from parent - window.addEventListener('message', function (e) { - if (e.data && e.data.type === 'globe_filter') { - var label = e.data.cluster_label; - arcMeshes.forEach(function (mesh) { - if (label === null) { - mesh.material.opacity = 0.6; - mesh.material.color.setHex(tlsColors[mesh.userData.tls] || 0x888888); - } else { - mesh.material.opacity = 0.06; - mesh.material.color.setHex(0x666666); - } - }); - } - }); - - // Auto-rotate - (function animate() { - requestAnimationFrame(animate); - if (!isDragging) { earth.rotation.y += 0.003; flowGroup.rotation.y = earth.rotation.y; } - renderer.render(scene, camera); - })(); -} diff --git a/static/tianxuan/markdown.js b/static/tianxuan/markdown.js deleted file mode 100644 index 0e290eb..0000000 --- a/static/tianxuan/markdown.js +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Simple Markdown-to-HTML converter. - * Extracted from templates/tianxuan/auto.html - * - * Usage: renderMarkdown(text) → HTML string - */ -function renderMarkdown(text) { - if (!text) return ''; - // Escape HTML first - var html = escapeHtml(text); - // Code blocks (```...```) — must be before inline code - html = html.replace(/```(\w*)\n?([\s\S]*?)```/g, '
$2
'); - // Inline code - html = html.replace(/`([^`]+)`/g, '$1'); - // Bold (**text** or __text__) - html = html.replace(/\*\*([^*]+)\*\*/g, '$1'); - html = html.replace(/__([^_]+)__/g, '$1'); - // Italic (*text* or _text_) - html = html.replace(/\*([^*]+)\*/g, '$1'); - html = html.replace(/(?$1'); - // Links [text](url) - html = html.replace(/\[([^\]]+)\]\(([^)]+)\)/g, '$1'); - // Headers (# ## ###) - html = html.replace(/^### (.+)$/gm, '

$1

'); - html = html.replace(/^## (.+)$/gm, '

$1

'); - html = html.replace(/^# (.+)$/gm, '

$1

'); - // Blockquotes - html = html.replace(/^>\s?(.+)$/gm, '
$1
'); - // Unordered lists - html = html.replace(/^[\s]*[-*+]\s+(.+)$/gm, '
  • $1
    • '); - html = html.replace(/()\n(?!' + m + ''; }); - // Ordered lists - html = html.replace(/^\d+\.\s+(.+)$/gm, '
  • $1
    • '); - // Horizontal rules - html = html.replace(/^---$/gm, '
    '); - // Paragraphs (double newlines → paragraphs) - html = html.replace(/\n\n/g, '

    '); - // Line breaks (single newlines → br) - html = html.replace(/\n/g, '
    '); - // Wrap in paragraph if not already wrapped - if (!html.startsWith('<')) html = '

    ' + html + '

    '; - return html; -} diff --git a/static/tianxuan/timeline.js b/static/tianxuan/timeline.js deleted file mode 100644 index 627b11b..0000000 --- a/static/tianxuan/timeline.js +++ /dev/null @@ -1,162 +0,0 @@ -/** - * LLM workflow timeline rendering. - * Extracted from templates/tianxuan/auto.html and templates/analysis/run_detail.html - * - * Provides: - * buildTimeline(llmThinking, toolCalls) → entries[] - * renderTimeline(entries, containerId) - * escapeHtml(str) - * prettyJson(obj) - * toggleStep(el) - */ - -/** - * Escape HTML special characters. - */ -function escapeHtml(str) { - if (str == null) return ''; - return String(str).replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"'); -} - -/** - * Pretty-print object as JSON string. - */ -function prettyJson(obj) { - try { return JSON.stringify(obj, null, 2); } - catch (e) { return String(obj); } -} - -/** - * Toggle a collapsible step card. - */ -function toggleStep(el) { - el.classList.toggle('open'); - var body = el.nextElementSibling; - if (body) body.classList.toggle('open'); -} - -/** - * Build timeline entries from LLM thinking text and tool calls. - * - * @param {string} llmThinking - Raw LLM thinking text with [step] markers - * @param {Array} toolCalls - Array of { step, name, input, output } - * @returns {Array} entries - Interleaved array of { step, type, text/name/input/output } - */ -function buildTimeline(llmThinking, toolCalls) { - var thinkingByStep = {}; - var thinkOrder = []; - if (llmThinking) { - var parts = llmThinking.split(/\n(?=\[\d+\])/); - for (var pi = 0; pi < parts.length; pi++) { - var m = parts[pi].match(/^\[(\d+)\]\s*(.*)/s); - if (m) { - var step = parseInt(m[1]); - if (!(step in thinkingByStep)) { - thinkOrder.push(step); - } - thinkingByStep[step] = (thinkingByStep[step] || '') + (thinkingByStep[step] ? '\n' : '') + m[2].trim(); - } - } - } - - var toolMap = {}; - (toolCalls || []).forEach(function (tc) { - var s = tc.step; - if (!toolMap[s]) toolMap[s] = []; - toolMap[s].push({ - type: 'tool', - name: tc.name, - input: tc.input, - output: tc.output, - }); - }); - - var entries = []; - var allSteps = new Set(thinkOrder.concat(Object.keys(toolMap).map(Number))); - var sortedSteps = Array.from(allSteps).sort(function (a, b) { return a - b; }); - - for (var si = 0; si < sortedSteps.length; si++) { - var step = sortedSteps[si]; - if (thinkingByStep[step] !== undefined) { - var isLastStep = step === sortedSteps[sortedSteps.length - 1]; - var hasTool = toolMap[step] && toolMap[step].length > 0; - if (isLastStep && !hasTool) { - entries.push({ step: step, type: 'answer', text: thinkingByStep[step] }); - } else { - entries.push({ step: step, type: 'thinking', text: thinkingByStep[step] }); - } - } - - if (toolMap[step]) { - for (var tj = 0; tj < toolMap[step].length; tj++) { - var te = toolMap[step][tj]; - entries.push({ step: step, type: 'tool', name: te.name, input: te.input, output: te.output, thinking: '' }); - } - } - } - - return entries; -} - -/** - * Render timeline entries into a DOM container. - * - * @param {Array} entries - From buildTimeline() - * @param {string} containerId - DOM element ID to render into - */ -function renderTimeline(entries, containerId) { - var container = document.getElementById(containerId); - if (!container || !entries || entries.length === 0) return; - - container.innerHTML = ''; - var lastStep = -1; - - for (var ei = 0; ei < entries.length; ei++) { - var entry = entries[ei]; - - if (entry.step !== lastStep) { - var header = document.createElement('div'); - header.style.cssText = 'font-weight:700;font-size:0.85rem;color:#4361ee;padding:0.6rem 0 0.3rem 0;border-top:1px solid #eee;margin-top:0.4rem;'; - header.textContent = '\u6b65\u9aa4 ' + entry.step; - container.appendChild(header); - lastStep = entry.step; - } - - if (entry.type === 'thinking') { - var div = document.createElement('div'); - div.style.cssText = 'background:#eef2ff;padding:0.6rem;border-radius:6px;font-size:0.78rem;line-height:1.6;color:#333;max-height:300px;overflow:auto;margin-bottom:0.3rem;border:1px solid #d0d8f0;'; - div.innerHTML = - '\ud83d\udcad \u601d\u8003' + - (typeof renderMarkdown === 'function' ? renderMarkdown(entry.text) : escapeHtml(entry.text).replace(/\n/g, '
    ')); - container.appendChild(div); - } else if (entry.type === 'answer') { - var div = document.createElement('div'); - div.style.cssText = 'background:#e8f5e9;padding:0.6rem 0.8rem;border-radius:6px;font-size:0.82rem;line-height:1.7;color:#333;max-height:400px;overflow:auto;margin-bottom:0.3rem;border:1px solid #c8e6c9;'; - div.innerHTML = - '\u2705 \u56de\u7b54' + - (typeof renderMarkdown === 'function' ? renderMarkdown(entry.text) : escapeHtml(entry.text).replace(/\n/g, '
    ')); - container.appendChild(div); - } else if (entry.type === 'tool') { - var card = document.createElement('div'); - card.style.cssText = 'border:1px solid #e0e0e0;border-radius:6px;margin-bottom:0.3rem;overflow:hidden;background:#fff;'; - var inputStr = prettyJson(entry.input); - var outputStr = prettyJson(entry.output); - var bodyInner = ''; - bodyInner += - '
    \u8f93\u5165 (INPUT):
    ' + - '
    ' + escapeHtml(inputStr) + '
    ' + - '
    \u8f93\u51fa (OUTPUT):
    ' + - '
    ' + escapeHtml(outputStr) + '
    '; - card.innerHTML = - '
    ' + - '\ud83d\udd27' + - '' + escapeHtml(entry.name || '?') + '' + - '\u25b6' + - '
    ' + - '
    ' + - bodyInner + - '
    '; - container.appendChild(card); - } - } -}