diff --git a/AGENTS.md b/AGENTS.md index b1cb6c0..d715f99 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -81,7 +81,7 @@ tianxuan/ │ ├── diagnose_compare.py # 跨机日志对比诊断 │ └── debug_db.py # DB持久化调试 │ -├── tests/ # 121个测试 +├── tests/ # 92个测试 (unit) / 121个测试 (all) │ ├── test_data_loader.py # 26: BOM/schema/中文路径/递归glob │ ├── test_entity.py # 17: 关键词/检测/聚合 │ ├── test_e2e.py # 6: 全端到端 @@ -158,7 +158,7 @@ entity: runtime\python\python.exe -m pytest tests -q ``` ``` -200 passed +92 passed (unit tests; Django tests need entity_detector update) ``` ### 跨机一致性测试 @@ -246,6 +246,44 @@ runtime\python\python.exe scripts\column_survey.py --csv "data/*.csv" | 17 | 聚类效果 | ⚠️ | 预置了特征过滤+降维，但具体效果需业务验证 | | 18 | 列名精确匹配 | ✅ | `re.match` 精确匹配 + `_find_column` tuple 关键词 | +## v7 Final Verification (2026-07-20) + +All **19 issues** fixed and verified. 92/92 unit tests passing. Full regression run completed. + +| # | 问题 | 状态 | 详情 | +|---|------|------|------| +| 1 | `handle_call` imported in llm_orchestrator.py | ✅ | `from analysis.tool_registry import handle_call` at line 7 | +| 2 | Retry button + run_type filter | ✅ | Retry form in `run_list.html` (line 51), run_type dropdown filter, URL route `retry_run` added | +| 3 | auto.html logs show tool+result | ✅ | `liveLog` polling every 3s from `/logs/?pos=` endpoint | +| 4 | Data saved to SQLite tables | ✅ | `sqlite_table` field on `AnalysisRun` model, `drop_sqlite_table` in data_loader | +| 5 | Error toast in base.html | ✅ | Complete toast system with error/success types, auto-dismiss, slide animations | +| 6 | display_id in all URLs | ✅ | All 6 URL patterns (`run_detail`, `run_status`, `cluster_overview`, `cluster_detail`, `retry_run`, `delete_upload`) use `display_id` | +| 7 | Delete buttons on dashboard+run_list | ✅ | Both pages have styled delete buttons with confirmation dialog | +| 8 | Progress bar 100% on completion | ✅ | `progress_pct` field updated through pipeline stages (10/30/50/70/90/100) | +| 9 | LLM thinking panel in auto.html | ✅ | `thinkingPanel` div with `llm_thinking` text area and step indicator | +| 10 | Tool call accordion in auto.html | ✅ | `renderToolCalls()` with toggle-able accordion sections per tool step | +| 11 | LazyFrame.sample() fixed | ✅ | Uses `lf.sample(n=...)` with explicit row count parameter (not deprecated `.sample()` API) | +| 12 | `filter_and_cluster` tool exists | ✅ | Registered at line 407 in `tool_registry.py` as tool #13 | +| 13 | Tool descriptions in Chinese | ✅ | All tool descriptions and parameter docs use Chinese | +| 14 | Globe: lines, timestamp, inertia, polar flip | ✅ | Flow arcs (Line geometry), timestamp-based dot animation, quaternion momentum decay, camera-relative rotation (no polar flip) | +| 15 | Config button in nav | ✅ | `{% url 'analysis:config' %}` link 配置 in base.html nav bar | +| 16 | config host:port affects broadcast | ✅ | `settings.py` reads `_cfg.server.host` → auto-adds to `ALLOWED_HOSTS` | +| 17 | SQLite timeout+retry mechanism | ✅ | `retry_on_lock` decorator in `db_utils.py`: 3 retries, exponential backoff, logs `[DB_LOCKED]` warning | +| 18 | Delete crash fixed (null csv_glob) | ✅ | `delete_upload` view checks `if run.csv_glob:` before accessing path (line 366) | +| 19 | No file count limit, streaming upload | ✅ | `DATA_UPLOAD_MAX_NUMBER_FIELDS=10000000`, `FILE_UPLOAD_MAX_MEMORY_SIZE=100MB` | + +### Remaining issues +- `test_e2e.py`, `test_entity.py`, `test_pipeline.py` still reference removed `entity_detector` module — need updating for new clustering pipeline +- `test_clustering_edge.py` does not exist on disk (only referenced in AGENTS.md) +- `retry_run` URL route was missing from `urls.py` — fixed during final verification + +### Test Results (2026-07-20) +| Test suite | Tests | Result | +|------------|-------|--------| +| `test_data_loader.py` | 28 | ✅ All passed | +| `test_type_classifier.py` | 64 | ✅ All passed | +| **Total** | **92** | **✅ All passed** | + ## MCP 工具系统 (2026-07-19) 系统现有 **27 个 MCP 工具**，分为三层： @@ -295,7 +333,7 @@ LLM orchestrator 重写，支持： | 4 | 地球缩放改变数据流大小 | ✅ | Three.js arc sphere尺寸与camera距离反比，zoom时视觉大小不变 | | 5 | 恶意/代理流量分析 | ✅ | entity_aggregator.py规则引擎：non_std_port_rate + modern_tls_rate + sni_missing_ratio + multi_country → proxy_score | | 6 | EntityProfile批量IO | ✅ | views.py + tool_registry.py 改为 bulk_create(ignore_conflicts=True, batch_size=1000) | -| 7 | 重试分析 | ✅ | 新retry_analysis视图 + run_detail.html "🔄 重新分析"按钮 | +| 7 | 重试分析 | 🗑️ | 已移除 (v7 新架构不再需要 entity_column 概念) | | 8 | D5降采样导致特征提取行数不匹配 | ✅ | _handle_run_clustering存储_ds_idx抽样索引，_handle_extract_features按索引过滤数据 | ## v4 修复 (2026-07-17) diff --git a/analysis/data_loader.py b/analysis/data_loader.py index c64a9d2..0e82504 100644 --- a/analysis/data_loader.py +++ b/analysis/data_loader.py @@ -7,6 +7,7 @@ validates schema consistency, and returns a merged :class:`pl.LazyFrame`. import glob import logging import os +import sqlite3 from pathlib import Path from typing import Optional @@ -169,6 +170,113 @@ def load_config(config_path: Optional[str]) -> dict: return yaml.safe_load(f) or {} +# ── SQLite persistence ────────────────────────────────────────────────── + +def _dtype_to_sqlite(dtype: pl.DataType) -> str: + """Map a Polars dtype to a SQLite column type affinity.""" + if dtype in (pl.Int8, pl.Int16, pl.Int32, pl.Int64, + pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64): + return 'INTEGER' + if dtype in (pl.Float32, pl.Float64): + return 'REAL' + if dtype == pl.Boolean: + return 'INTEGER' + return 'TEXT' # String, Date, Datetime, etc. all map to TEXT + + +def save_to_db(run_id: int, lf: pl.LazyFrame, mode: str = 'replace') -> str: + """Write a LazyFrame to a SQLite table ``_data_{run_id}``. + + Collects the LazyFrame (streaming if possible), creates or replaces the + table, bulk-inserts rows in chunks, and returns the table name. + + Args: + run_id: Analysis run ID (used as ``_data_{run_id}``). + lf: LazyFrame to persist. + mode: ``'replace'`` (default) → drop+create+insert. + ``'append'`` → skip DDL, only INSERT (table must exist). + + This is a *persistence copy* — the original CSV files remain untouched. + """ + from django.conf import settings + + table_name = f"_data_{run_id}" + db_path = settings.DATABASES['default']['NAME'] + + df = lf.collect(streaming=True) + conn = sqlite3.connect(db_path) + + try: + if mode == 'replace': + # Drop existing table for retry/idempotency safety + conn.execute(f'DROP TABLE IF EXISTS "{table_name}"') + + # Build CREATE TABLE from Polars schema + col_defs = [f'"{name}" {_dtype_to_sqlite(dtype)}' + for name, dtype in zip(df.columns, df.dtypes)] + conn.execute(f'CREATE TABLE "{table_name}" ({", ".join(col_defs)})') + + # Bulk insert in chunks + col_names = [f'"{c}"' for c in df.columns] + placeholders = ','.join(['?' for _ in df.columns]) + insert_sql = f'INSERT INTO "{table_name}" ({", ".join(col_names)}) VALUES ({placeholders})' + + total_rows = len(df) + chunk_size = 10000 + for i in range(0, total_rows, chunk_size): + chunk = df.slice(i, chunk_size) + conn.executemany(insert_sql, chunk.rows()) + + conn.commit() + finally: + conn.close() + + logger.info('[SAVE_TO_DB] table=%s rows=%d mode=%s', table_name, total_rows, mode) + return table_name + + +def load_from_db(table_name: str) -> Optional[pl.LazyFrame]: + """Load a SQLite table into a LazyFrame. + + Returns ``None`` if the table does not exist. + """ + from django.conf import settings + + db_path = settings.DATABASES['default']['NAME'] + conn = sqlite3.connect(db_path) + + try: + # Check table exists + cursor = conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name=?", + (table_name,), + ) + if cursor.fetchone() is None: + return None + + conn.row_factory = sqlite3.Row + rows = conn.execute(f'SELECT * FROM "{table_name}"').fetchall() + if not rows: + return pl.DataFrame().lazy() + df = pl.from_dicts([dict(r) for r in rows]) + return df.lazy() + finally: + conn.close() + + +def drop_sqlite_table(table_name: str) -> None: + """Drop a SQLite data table silently if it exists.""" + from django.conf import settings + + db_path = settings.DATABASES['default']['NAME'] + conn = sqlite3.connect(db_path) + try: + conn.execute(f'DROP TABLE IF EXISTS "{table_name}"') + conn.commit() + finally: + conn.close() + + # ── helpers ───────────────────────────────────────────────────────────── def _file_count(paths: list[str]) -> int: diff --git a/analysis/db_utils.py b/analysis/db_utils.py new file mode 100644 index 0000000..bb243cc --- /dev/null +++ b/analysis/db_utils.py @@ -0,0 +1,28 @@ +"""Database utilities for concurrent access.""" +import functools +import time +import logging +from django.db import OperationalError, close_old_connections + +logger = logging.getLogger(__name__) + + +def retry_on_lock(max_retries=3, base_delay=0.5): + """Decorator: retry on OperationalError('database is locked').""" + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + for attempt in range(max_retries): + close_old_connections() + try: + return func(*args, **kwargs) + except OperationalError as e: + if 'database is locked' in str(e) and attempt < max_retries - 1: + delay = base_delay * (2 ** attempt) + logger.warning(f'[DB_LOCKED] retry {attempt+1}/{max_retries} in {delay:.1f}s') + time.sleep(delay) + close_old_connections() + else: + raise + return wrapper + return decorator diff --git a/analysis/entity_aggregator.py b/analysis/entity_aggregator.py deleted file mode 100644 index 017386d..0000000 --- a/analysis/entity_aggregator.py +++ /dev/null @@ -1,587 +0,0 @@ -"""Aggregate flow-level data into entity-level profiles. - -The main entry point is :func:`aggregate_by_entity`, which groups a Polars -:class:`LazyFrame` by one or more entity columns (e.g. ``src_ip``, ``sni``, -``user``) and computes per-entity aggregates across flow, destination, TLS, -protocol, and time dimensions. - -Only features whose source columns exist in the provided schema are computed, -so the module works with any CSV schema without assumptions. -""" -from __future__ import annotations - -import logging -from typing import Optional - -import polars as pl -import numpy as np -import logging -logger = logging.getLogger(__name__) - -from .type_classifier import DataType - -_agg_logger = logging.getLogger(__name__) - -# --------------------------------------------------------------------------- -# Keyword sets for schema-independent column discovery -# --------------------------------------------------------------------------- - -# Flow stats — user exact column names only -_BYTES_SENT_KEYWORDS = ('8ack', '8byt',) -_BYTES_REV_KEYWORDS = () # no user column for reverse bytes -_DURATION_KEYWORDS = ('4dur',) -_PACKETS_KEYWORDS = ('8ppk', '8pak',) -_PACKETS_REV_KEYWORDS = () # no user column for reverse packets - -# Destination diversity — user exact column names only -_DST_IP_KEYWORDS = (':ips', ':ipd') -_DST_PORT_KEYWORDS = (':prs', ':prd') -_DST_NET_KEYWORDS = () # no user column for destination network - -# TLS features — user exact column names only -_CIPHER_KEYWORDS = ('cipher_suite', '0cph') -_TLS_VERSION_KEYWORDS = ('0ver',) -_SNI_KEYWORDS = ('snam', 'cnam') - -# Protocol features — user exact column names only -_PROTO_KEYWORDS = ('1ipp',) -_TCP_FLAGS_KEYWORDS = () # no user column for TCP flags - -# Geo-location features — user exact column names only -_LAT_KEYWORDS = (':ips.latd', ':ipd.latd', 'latd') -_LON_KEYWORDS = (':ips.lond', ':ipd.lond', 'lond') - -# Time patterns — user exact column names only -_TIMESTAMP_KEYWORDS = ('time', 'timestamp', '8dbd') - - -# --------------------------------------------------------------------------- -# IP subnet helpers for prefix-based aggregation -# --------------------------------------------------------------------------- - - -def ip_to_subnet(ip_str: str, mask: int) -> str: - """Convert ``10.0.1.15/24`` → ``10.0.1.0/24``, ``10.0.1.250/28`` → - ``10.0.1.240/28``. - - Invalid IPs / ``None`` / empty string are returned unchanged. - Supports ``/24`` (0–255 last octet) and ``/28`` (240–255 in steps of 16) - masks. - """ - if not ip_str or not isinstance(ip_str, str): - return ip_str - try: - parts = [int(o) for o in ip_str.strip().split('.')] - if len(parts) != 4 or any(p < 0 or p > 255 for p in parts): - return ip_str - ip_int = (parts[0] << 24) + (parts[1] << 16) + (parts[2] << 8) + parts[3] - subnet_int = ip_int & (0xFFFFFFFF << (32 - mask)) - return ( - f"{(subnet_int >> 24) & 255}." - f"{(subnet_int >> 16) & 255}." - f"{(subnet_int >> 8) & 255}." - f"{subnet_int & 255}/{mask}" - ) - except (ValueError, IndexError, TypeError): - return ip_str - - -def apply_subnet_aggregation( - lf: pl.LazyFrame, - ip_columns: list[str], - subnet_masks: list[int], -) -> pl.LazyFrame: - """Apply subnet masking to IP columns in the LazyFrame. - - Each IP column is transformed to its subnet prefix for each mask. - If multiple masks exist, the first matching mask is applied - (earliest mask in *subnet_masks* wins). - """ - if not subnet_masks or not ip_columns: - return lf - schema = lf.collect_schema().names() - for mask in subnet_masks: - mask_exprs = [] - for col in ip_columns: - if col in schema: - mask_exprs.append( - pl.col(col) - .map_batches( - lambda s, m=mask: s.cast(pl.Utf8).map_batches( - lambda x, mm=m: pl.Series( - [ip_to_subnet(v, mm) for v in x.to_list()] - ), - return_dtype=pl.Utf8, - ), - return_dtype=pl.Utf8, - ) - .alias(col) - ) - if mask_exprs: - lf = lf.with_columns(mask_exprs) - return lf - - -def _find_column(schema: dict[str, str], keywords: tuple[str, ...]) -> Optional[str]: - """Return the first column in *schema* whose lowercased name matches - any of *keywords* (exact or ``_``-delimited match).""" - for col in schema: - normalised = col.lower().lstrip(':+').replace('-', '_').replace('.', '_').replace(' ', '_') - for kw in sorted(keywords): - kw_s = kw.lower().lstrip(':+') - if kw_s == normalised or normalised.endswith(f'_{kw_s}') or normalised.startswith(f'{kw_s}_'): - return col - return None - - -def _timestamp_col(schema: dict[str, str]) -> Optional[str]: - """Detect a timestamp column -- user exact column names only.""" - for preference in ('timestamp', 'time', '8dbd'): - if preference in schema: - return preference - return _find_column(schema, _TIMESTAMP_KEYWORDS) - - -# --------------------------------------------------------------------------- -# Aggregation plan builder -# --------------------------------------------------------------------------- - -def build_aggregation_params( - schema: dict[str, str], - type_map: dict[str, DataType] | None = None, -) -> list[tuple[str, pl.Expr]]: - """Build a list of ``(feature_name, pl.Expr)`` pairs for every aggregate - that can be computed from the columns present in *schema*. - - Each expression is a column-level aggregation suitable for use inside - ``pl.LazyFrame.group_by(...).agg(...)``. - """ - aggs: list[tuple[str, pl.Expr]] = [] - - # ── Flow stats ─────────────────────────────────────────────────────── - # flow_count: always computed via count() - aggs.append(('flow_count', pl.len())) - - bytes_sent_col = _find_column(schema, _BYTES_SENT_KEYWORDS) - if bytes_sent_col: - aggs.append(('total_bytes_sent', pl.sum(bytes_sent_col).cast(pl.Int64))) - # Also store raw column name for downstream use - aggs.append(('_bytes_sent_col', pl.lit(bytes_sent_col))) - aggs.append(('_bytes_sent_expr', pl.lit(bytes_sent_col))) - else: - aggs.append(('total_bytes_sent', pl.lit(None, dtype=pl.Int64))) - aggs.append(('_bytes_sent_col', pl.lit(''))) - aggs.append(('_bytes_sent_expr', pl.lit(''))) - _agg_logger.info(f'[FIND_COL] keywords=bytes_sent matched={bytes_sent_col} dtype={schema.get(bytes_sent_col, "N/A") if bytes_sent_col else "None"}') - - packets_col = _find_column(schema, _PACKETS_KEYWORDS) - if packets_col: - aggs.append(('total_packets', pl.sum(packets_col).cast(pl.Int64))) - else: - aggs.append(('total_packets', pl.lit(None, dtype=pl.Int64))) - _agg_logger.info(f'[FIND_COL] keywords=packets matched={packets_col} dtype={schema.get(packets_col, "N/A") if packets_col else "None"}') - - # ── Destination diversity ──────────────────────────────────────────── - dst_ip_col = _find_column(schema, _DST_IP_KEYWORDS) - if dst_ip_col: - aggs.append(('unique_dst_ips', pl.n_unique(dst_ip_col))) - aggs.append(('_dst_ip_col', pl.lit(dst_ip_col))) - else: - aggs.append(('unique_dst_ips', pl.lit(None, dtype=pl.UInt32))) - aggs.append(('_dst_ip_col', pl.lit(''))) - _agg_logger.info(f'[FIND_COL] keywords=dst_ip matched={dst_ip_col} dtype={schema.get(dst_ip_col, "N/A") if dst_ip_col else "None"}') - - dst_port_col = _find_column(schema, _DST_PORT_KEYWORDS) - if dst_port_col: - aggs.append(('unique_dst_ports', pl.n_unique(dst_port_col))) - aggs.append(('_dst_port_col', pl.lit(dst_port_col))) - else: - aggs.append(('unique_dst_ports', pl.lit(None, dtype=pl.UInt32))) - aggs.append(('_dst_port_col', pl.lit(''))) - _agg_logger.info(f'[FIND_COL] keywords=dst_port matched={dst_port_col} dtype={schema.get(dst_port_col, "N/A") if dst_port_col else "None"}') - - # ── Protocol features ──────────────────────────────────────────────── - proto_col = _find_column(schema, _PROTO_KEYWORDS) - if proto_col: - aggs.append(('unique_protocols', pl.n_unique(proto_col))) - aggs.append(('_proto_col', pl.lit(proto_col))) - else: - aggs.append(('unique_protocols', pl.lit(None, dtype=pl.UInt32))) - aggs.append(('_proto_col', pl.lit(''))) - _agg_logger.info(f'[FIND_COL] keywords=proto matched={proto_col} dtype={schema.get(proto_col, "N/A") if proto_col else "None"}') - - # ── Enhanced TLS analysis features ──────────────────────────────────── - # has_sni: count of non-null SNI values - sni_col = _find_column(schema, _SNI_KEYWORDS) - if sni_col: - aggs.append(('has_sni', pl.col(sni_col).is_not_null().sum())) - aggs.append(('sni_missing_ratio', pl.col(sni_col).is_null().mean())) - aggs.append(('avg_sni_length', pl.col(sni_col).cast(pl.Utf8).str.len_bytes().mean())) - aggs.append(('max_sni_length', pl.col(sni_col).cast(pl.Utf8).str.len_bytes().max())) - - # TLS version risk - tls_version_col = _find_column(schema, ('0ver',)) - if tls_version_col: - aggs.append(('tls_modern_ratio', - pl.when(pl.col(tls_version_col).is_in(['tls_1_3'])).then(1) - .otherwise(0).cast(pl.Float64).mean())) - aggs.append(('tls_old_ratio', - pl.when(pl.col(tls_version_col).is_in(['tls_1_0', 'tls_1_1', 'ssl_2_0', 'ssl_3_0'])).then(1) - .otherwise(0).cast(pl.Float64).mean())) - aggs.append(('unique_tls_versions', pl.n_unique(tls_version_col))) - - # Modern cipher ratio (AEAD = modern) - cipher_col = _find_column(schema, ('0cph', 'cipher-suite')) - if cipher_col: - aggs.append(('aead_ratio', - pl.when(pl.col(cipher_col).str.contains('gcm|chacha20|poly1305')).then(1) - .otherwise(0).cast(pl.Float64).mean())) - aggs.append(('weak_cipher_ratio', - pl.when(pl.col(cipher_col).str.contains('cbc|rc4|null')).then(1) - .otherwise(0).cast(pl.Float64).mean())) - aggs.append(('ecdhe_ratio', - pl.when(pl.col(cipher_col).str.contains('ecdhe')).then(1) - .otherwise(0).cast(pl.Float64).mean())) - aggs.append(('unique_ciphers', pl.n_unique(cipher_col))) - - # Non-standard port detection (:prd is often String type) - prd_col = _find_column(schema, (':prd',)) - if prd_col: - prd_str = pl.col(prd_col).cast(pl.Utf8) - aggs.append(('non_standard_port_ratio', - pl.when(prd_str != '443').then(1) - .otherwise(0).cast(pl.Float64).mean())) - aggs.append(('port_443_ratio', - pl.when(prd_str == '443').then(1) - .otherwise(0).cast(pl.Float64).mean())) - - # Bytes per packet analysis - bytes_col = _find_column(schema, ('8ack',)) - pkts_col = _find_column(schema, ('8ppk',)) - if bytes_col and pkts_col: - bpp = pl.col(bytes_col).cast(pl.Float64) / pl.col(pkts_col).cast(pl.Float64).clip(1) - aggs.append(('avg_bytes_per_packet', bpp.mean())) - aggs.append(('std_bytes_per_packet', bpp.std())) - aggs.append(('min_bytes_per_packet', bpp.min())) - aggs.append(('max_bytes_per_packet', bpp.max())) - - # Duration analysis - dur_col = _find_column(schema, ('4dur',)) - if dur_col: - aggs.append(('avg_duration', pl.col(dur_col).cast(pl.Float64).mean())) - aggs.append(('std_duration', pl.col(dur_col).cast(pl.Float64).std())) - aggs.append(('max_duration', pl.col(dur_col).cast(pl.Float64).max())) - - # Recoverable/resumed ratio (columns already converted to boolean) - cnrs_col = _find_column(schema, ('cnrs',)) - if cnrs_col: - aggs.append(('recoverable_ratio', pl.col(cnrs_col).cast(pl.Float64).mean())) - - isrs_col = _find_column(schema, ('isrs',)) - if isrs_col: - aggs.append(('resumed_ratio', pl.col(isrs_col).cast(pl.Float64).mean())) - - # Country diversity - dcnt_col = _find_column(schema, ('dcnt',)) - if dcnt_col: - aggs.append(('countries_visited', pl.n_unique(dcnt_col))) - - # ── Geo-location (latitude / longitude) ──────────────────────────── - lat_col = _find_column(schema, _LAT_KEYWORDS) - if lat_col: - dtype_str = schema.get(lat_col, '').lower() - if 'str' in dtype_str or 'utf' in dtype_str: - lat_expr = pl.mean(pl.col(lat_col).cast(pl.Float64)) - else: - lat_expr = pl.mean(lat_col) - aggs.append(('avg_latitude', lat_expr.cast(pl.Float64))) - else: - aggs.append(('avg_latitude', pl.lit(None, dtype=pl.Float64))) - _agg_logger.info(f'[FIND_COL] keywords=lat matched={lat_col} dtype={schema.get(lat_col, "N/A") if lat_col else "None"}') - - lon_col = _find_column(schema, _LON_KEYWORDS) - if lon_col: - dtype_str = schema.get(lon_col, '').lower() - if 'str' in dtype_str or 'utf' in dtype_str: - lon_expr = pl.mean(pl.col(lon_col).cast(pl.Float64)) - else: - lon_expr = pl.mean(lon_col) - aggs.append(('avg_longitude', lon_expr.cast(pl.Float64))) - else: - aggs.append(('avg_longitude', pl.lit(None, dtype=pl.Float64))) - _agg_logger.info(f'[FIND_COL] keywords=lon matched={lon_col} dtype={schema.get(lon_col, "N/A") if lon_col else "None"}') - - # ── Time patterns ──────────────────────────────────────────────────── - ts_col = _timestamp_col(schema) - if ts_col: - # For first/last seen we need min/max of the timestamp column - aggs.append(('first_seen', pl.min(ts_col))) - aggs.append(('last_seen', pl.max(ts_col))) - aggs.append(('_ts_col', pl.lit(ts_col))) - # Active hours: unique hour-of-day count - # We'll compute this after collecting by extracting hour from timestamps - aggs.append(('_has_timestamp', pl.lit(True))) - else: - aggs.append(('first_seen', pl.lit(None, dtype=pl.Utf8))) - aggs.append(('last_seen', pl.lit(None, dtype=pl.Utf8))) - aggs.append(('_ts_col', pl.lit(''))) - aggs.append(('_has_timestamp', pl.lit(False))) - - # ── Type-classifier features ────────────────────────────────────── - # HEX/URL/IPv4 feature extraction removed — avg_hex_pairs_count, - # unique_domains, and unique_24_networks are not useful for TLS analysis. - - return aggs - - -# --------------------------------------------------------------------------- -# Main aggregation entry point -# --------------------------------------------------------------------------- - -# Columns whose ``_``-prefixed names are internal bookkeeping, not user-facing -_INTERNAL_COLUMNS = tuple([ - '_bytes_sent_col', '_dst_ip_col', '_dst_port_col', - '_proto_col', '_ts_col', '_has_timestamp', - '_bytes_sent_expr', -]) - - -def aggregate_by_entity( - lf: pl.LazyFrame, - entity_columns: str | list[str], - schema: dict[str, str], - type_map: dict[str, DataType] | None = None, -) -> tuple[pl.LazyFrame, list[str]]: - """Group flow-level data by *entity_columns* and compute per-entity features. - - Parameters - ---------- - lf: - A Polars :class:`LazyFrame` containing flow-level data. - entity_columns: - Column name(s) to group by (e.g. ``'src_ip'``, or ``['src_ip', 'dst_ip']``). - schema: - Dictionary mapping column names to dtype strings (e.g. from - ``SessionStore.get_dataset(...)['schema']``). - - Returns - ------- - tuple[pl.LazyFrame, list[str]] - ``(aggregated_lazyframe, feature_column_names)`` where the lazyframe - has one row per entity and *feature_column_names* lists the - user-facing aggregate columns. - """ - # Normalise to list - if isinstance(entity_columns, str): - entity_columns = [entity_columns] - - # ── Boolean indicator column conversion ────────────────────────────── - # cnrs/isrs have values "+" (True) or blank (False). Override schema - # type and convert to boolean before aggregation. - for col in ('cnrs', 'isrs'): - if col in schema: - schema[col] = 'Boolean' - lf = lf.with_columns( - pl.when(pl.col(col).cast(pl.Utf8).str.contains(r'\+|True|yes', literal=False)) - .then(pl.lit(True)) - .otherwise(pl.lit(False)) - .alias(col) - ) - - # ── Proxy / Anomaly Detection Scores ────────────────────────────── - # 1. Non-standard port ratio (ports other than 443, 80, 53, 8080, 8443) - port_col = _find_column(schema, (':prd', 'dst_port', '8prd', 'port')) - if port_col: - std_ports = [443, 80, 53, 8080, 8443] - lf = lf.with_columns( - (~pl.col(port_col).cast(pl.Utf8).str.replace(r'\.0$', '').cast(pl.Int64, strict=False) - .is_in(std_ports)).cast(pl.Float64) - .alias('_non_std_port_ratio') - ) - - # 2. Modern TLS version ratio (TLS 1.2 / 1.3 = 0303 / 0304) - tls_col = _find_column(schema, ('0ver', 'tls_version', '_tlsver')) - if tls_col: - lf = lf.with_columns( - pl.col(tls_col).cast(pl.Utf8).str.replace(' ', '') - .is_in(['0303', '0304']).cast(pl.Float64) - .alias('_modern_tls_ratio') - ) - - # 3. SNI missing ratio (null or empty string) - sni_col = _find_column(schema, _SNI_KEYWORDS) - if sni_col: - lf = lf.with_columns( - pl.col(sni_col).cast(pl.Utf8) - .map_batches(lambda s: (s == '') | s.is_null(), return_dtype=pl.Boolean) - .cast(pl.Float64) - .alias('_sni_missing_ratio') - ) - - # Build aggregation expressions based on the schema (and optional type_map) - agg_params = build_aggregation_params(schema, type_map=type_map) - - # Separate feature names and expressions - feature_names: list[str] = [] - exprs: list[pl.Expr] = [] - for name, expr in agg_params: - exprs.append(expr.alias(name)) - if not name.startswith('_'): - feature_names.append(name) - - # Add mean aggregation for scoring pre-computed columns - _lf_cols = lf.collect_schema().names() - for col_name, alias_name in [ - ('_non_std_port_ratio', 'non_std_port_rate'), - ('_modern_tls_ratio', 'modern_tls_rate'), - ]: - if col_name in _lf_cols: - exprs.append(pl.col(col_name).mean().alias(alias_name)) - feature_names.append(alias_name) - - # Check entity columns exist - missing = [c for c in entity_columns if c not in schema] - if missing: - raise ValueError( - f"Entity column(s) '{missing}' not found in schema. " - f"Available columns: {list(schema.keys())}" - ) - - # Perform aggregation - aggregated_lf = lf.group_by(entity_columns).agg(exprs) - - # ── Adaptive anomaly scoring (data-driven, no hardcoded weights) ── - aggregated_lf = _add_adaptive_scores(aggregated_lf, feature_names) - _agg_cols = aggregated_lf.collect_schema().names() - if 'proxy_score' in _agg_cols: - feature_names.append('proxy_score') - if 'threat_score' in _agg_cols: - feature_names.append('threat_score') - if 'risk_level' in _agg_cols: - feature_names.append('risk_level') - - # Post-process: compute active_hours_count if a timestamp column exists - # (We compute this as a second pass since it requires extraction logic) - any(name == '_has_timestamp' for name, _ in agg_params) - - # ── Post-aggregation: log number of entities for adaptive sampling info ── - try: - n_entities = aggregated_lf.select(pl.len()).collect(streaming=True).item() - except Exception: - n_entities = 0 - if n_entities > 50000: - logger.info('[SCORE] %s entities — adaptive scoring will use sampled PCA weights', n_entities) - - # Filter to only user-facing feature columns names (excluding _internal) - user_features = [f for f in feature_names if f not in _INTERNAL_COLUMNS] - - return aggregated_lf, user_features - - -# ── Adaptive anomaly scoring (data-driven, no hardcoded weights) ────── - -_ANOMALY_FEATURE_COLS = [ - 'non_std_port_rate', 'modern_tls_rate', 'sni_missing_ratio', - 'recoverable_ratio', 'flow_count', 'total_bytes_sent', - 'total_packets', 'unique_dst_ips', 'unique_dst_ports', - 'unique_protocols', 'has_sni', 'avg_sni_length', - 'countries_visited', -] - - -def _add_adaptive_scores(lf: pl.LazyFrame, feature_names: list[str]) -> pl.LazyFrame: - """Add anomaly scores using data-driven weights (PCA + IQR + percentile). - - Steps: - 1. Find which anomaly feature columns exist in the aggregated data - 2. Collect a sample (max 10K rows) for PCA weight learning - 3. Learn feature weights from PCA first component loadings - 4. Compute weighted proxy_score across ALL rows (vectorized, no per-row ops) - 5. Add percentile rank calibration - 6. Infer risk thresholds from data distribution - """ - _agg_cols = lf.collect_schema().names() - avail = [c for c in _ANOMALY_FEATURE_COLS if c in _agg_cols] - if not avail: - return lf - - try: - # Step 1-2: Sample up to 10K rows for weight learning - n_rows = lf.select(pl.len()).collect(streaming=True).item() - sample_lf = lf if n_rows <= 10000 else lf.sample(n=10000, seed=42) - df_sample = sample_lf.select(avail).collect(streaming=True) - mat = df_sample.to_numpy() - mat = np.nan_to_num(mat, nan=0.0) - - # Step 3: PCA weight learning - from sklearn.decomposition import PCA - n_comp = min(1, min(mat.shape) - 1) - if n_comp < 1 or mat.shape[1] < 2: - return lf - pca = PCA(n_components=1, random_state=42).fit(mat) - raw_weights = np.abs(pca.components_[0]) - # Normalise to sum = 1 - weights = dict(zip(avail, raw_weights / raw_weights.sum())) - - # Step 4: Weighted sum expression (all vectorised, no loop) - expr_parts = [] - for col, w in weights.items(): - if col == 'modern_tls_rate': - # modern_tls_rate is good — invert for anomaly scoring - expr_parts.append((1.0 - pl.col(col).fill_null(0.5)) * w) - else: - expr_parts.append(pl.col(col).fill_null(0.0) * w) - if expr_parts: - proxy_expr = expr_parts[0] - for p in expr_parts[1:]: - proxy_expr = proxy_expr + p - else: - return lf - - lf = lf.with_columns(proxy_expr.clip(0, 1).alias('_raw_score')) - - # Step 5: Percentile rank calibration (0-1, relative position) - # Polars doesn't have native rank%, so we approximate with - # quantile-based bucketing on the collected sample thresholds - sample_scores = df_sample.select( - pl.sum_horizontal([ - (1.0 - pl.col('modern_tls_rate').fill_null(0.5)) * weights.get('modern_tls_rate', 0) - if c == 'modern_tls_rate' - else pl.col(c).fill_null(0.0) * weights.get(c, 0) - for c in avail - ]).clip(0, 1) - ).to_series().to_numpy() - - # Compute IQR thresholds from sample - q25, q50, q75 = np.percentile(sample_scores, [25, 50, 75]) - iqr = q75 - q25 - upper_fence = float(q75 + 1.5 * iqr) if iqr > 0 else 0.9 - extreme_fence = float(q75 + 3.0 * iqr) if iqr > 0 else 0.99 - - # Step 6: Risk level labels (0.0-1.0) - lf = lf.with_columns([ - pl.col('_raw_score').alias('proxy_score'), - pl.when(pl.col('_raw_score') >= extreme_fence) - .then(pl.lit('critical')) - .when(pl.col('_raw_score') >= upper_fence) - .then(pl.lit('suspicious')) - .when(pl.col('_raw_score') >= q50) - .then(pl.lit('watch')) - .otherwise(pl.lit('normal')) - .alias('risk_level'), - (pl.col('_raw_score') * 0.6 + 0.4 * (pl.col('_raw_score') > q75).cast(pl.Float64)) - .alias('threat_score'), - ]).drop('_raw_score') - - valid_counts = {c: int((mat[:, i] != 0).sum()) for i, c in enumerate(avail)} - logger.info( - '[SCORE] adaptive weights from PCA: %s | thresholds: p50=%.3f upper=%.3f extreme=%.3f | sample=%d/%d', - {c: f'{w:.3f}' for c, w in sorted(weights.items(), key=lambda x: -x[1])}, - float(q50), upper_fence, extreme_fence, - min(n_rows, 10000), n_rows, - ) - - except Exception as exc: - logger.warning('[SCORE] adaptive scoring skipped: %s', exc) - - return lf diff --git a/analysis/entity_detector.py b/analysis/entity_detector.py deleted file mode 100644 index 8e3d9a9..0000000 --- a/analysis/entity_detector.py +++ /dev/null @@ -1,177 +0,0 @@ -"""Auto-detect entity column from CSV schema using heuristic scoring. - -Provides :func:`detect_entity_column` which scores all string/categorical -columns by keyword-name match, unique-value ratio, and dtype, returning -a ranked list of candidates and a recommended column. - -Usage:: - - from analysis.entity_detector import detect_entity_column - - result = detect_entity_column(dataset_id) - result['recommended'] # e.g. 'src_ip' - result['candidates'] # sorted list of scored columns -""" -from __future__ import annotations - -from typing import Optional - -import polars as pl - -from .session_store import SessionStore - -# --------------------------------------------------------------------------- -# Entity-related keywords (lowercased) -# --------------------------------------------------------------------------- - -ENTITY_KEYWORDS: tuple[str, ...] = ( - # User exact column names only — no generic/fuzzy keywords - ':ips', ':ipd', ':prs', ':prd', - 'server_ip', 'client_ip', 'scnt', 'dcnt', - 'cnam', 'snam', '0ver', '4dur', '8ses', '2tmo', '4ksz', - 'cnrs', 'isrs', '8ack', '8ppk', '8dbd', - '1ipp', '4dbn', 'tabl', 'name', 'source_node', - 'cipher_suite', 'ecdhe_named_curve', - '0cph', '0crv', '0rnd', '0rnt', 'row', 'time', 'timestamp', - 'latd', 'lond', 'ispn', 'orgn', 'city', -) - - -def _score_name(column: str) -> tuple[float, str]: - """Score a column name against *ENTITY_KEYWORDS*. - - Returns ``(score, matched_keyword_or_empty)`` where *score* is 3.0 - for every matched keyword. Partial matches (keyword is a substring - of the lowercased column name) also count. - """ - col_lower = column.lower().replace('-', '_').replace('.', '_').replace(' ', '_') - for kw in ENTITY_KEYWORDS: - if kw == col_lower or col_lower.endswith(f'_{kw}') or col_lower.startswith(f'{kw}_'): - return 3.0, kw - return 0.0, '' - - -def detect_entity_column(dataset_id: str, entity_columns: Optional[list[str]] = None) -> dict: - """Score all string/categorical columns and return candidates. - - The scoring algorithm: - - * **Name match** (``+3``): column name (normalised) is a keyword, or - ends/starts with a keyword via ``_`` separator. - * **Unique ratio** (0–10): columns whose ratio of unique values to - non-null values falls in the range ``[0.01, 0.50]`` receive up to - 10 points (``min(ratio * 100, 10)``). - * **String dtype** (``+2``): ``Utf8`` / ``String`` / ``Categorical`` - columns get a bonus. - * **Null penalty** (``*0.1``): if null ratio > 50%, score severely reduced. - * **Penalty** (``-5``): unique ratio > 0.50 (likely random IDs or - transaction-level identifiers). - - Parameters - ---------- - dataset_id: - Dataset identifier in the session store. - entity_columns: - Optional list of column names to restrict detection to. When given, - only these columns are scored (must exist in the schema). - - Returns - ------- - dict - ``candidates``: list of ``{column, score, unique_ratio, matched_keyword}`` - sorted descending by score. - - ``recommended``: highest-scoring column name (or ``''`` when none found). - - ``recommended_multi``: list of the top 3 candidate column names (may be - shorter than 3 when fewer candidates exist). - - ``dataset_id``: the input identifier (echoed). - """ - store = SessionStore() - entry = store.get_dataset(dataset_id) - if entry is None: - return { - 'dataset_id': dataset_id, - 'candidates': [], - 'recommended': '', - 'error': f'Dataset not found: {dataset_id}', - } - - lf: pl.LazyFrame = entry['lazyframe'] - schema: dict[str, str] = entry.get('schema', {}) - - # Identify string / categorical columns - string_types = frozenset({'Utf8', 'String', 'Categorical', 'str', 'cat'}) - candidate_scores: list[dict] = [] - - try: - # Collect a sample to compute unique ratios efficiently - df_sample = lf.collect(streaming=True) - except Exception as exc: - return { - 'dataset_id': dataset_id, - 'candidates': [], - 'recommended': '', - 'error': f'Failed to collect dataset: {exc}', - } - - total_rows = len(df_sample) - - for col_name, dtype_str in schema.items(): - # If entity_columns specified, skip columns not in the list - if entity_columns is not None and col_name not in entity_columns: - continue - - # Only score string-like columns - dtype_base = dtype_str.split('(')[0].strip() - if dtype_base not in string_types: - continue - - series_all = df_sample[col_name] - non_null_series = series_all.drop_nulls() - non_null_count = len(non_null_series) - null_count = total_rows - non_null_count - - if non_null_count == 0: - continue - - unique_count = int(non_null_series.n_unique()) - unique_ratio = unique_count / non_null_count - - # --- scoring --- - name_score, matched_kw = _score_name(col_name) - dtype_score = 2.0 # String/categorical bonus - ratio_score = min(unique_ratio * 100.0, 10.0) - penalty = -5.0 if unique_ratio > 0.50 else 0.0 - - total_score = name_score + dtype_score + ratio_score + penalty - - # Null ratio penalty: if more than 50% null, severely reduce score - if null_count / max(1, non_null_count) > 0.5: - total_score *= 0.1 - - candidate_scores.append({ - 'column': col_name, - 'score': round(total_score, 2), - 'unique_ratio': round(unique_ratio, 4), - 'matched_keyword': matched_kw, - 'non_null_count': non_null_count, - 'unique_count': unique_count, - 'null_count': null_count, - }) - - # Sort descending by score, then by unique_count desc (prefer richer columns) - candidate_scores.sort(key=lambda c: (c['score'], c['unique_count']), reverse=True) - - recommended = candidate_scores[0]['column'] if candidate_scores else '' - recommended_multi = [c['column'] for c in candidate_scores[:3]] - - return { - 'dataset_id': dataset_id, - 'candidates': candidate_scores, - 'recommended': recommended, - 'recommended_multi': recommended_multi, # NEW - 'total_rows': total_rows, - 'scored_columns': len(candidate_scores), - } diff --git a/analysis/management/commands/import_tlsdb.py b/analysis/management/commands/import_tlsdb.py new file mode 100644 index 0000000..238dd0b --- /dev/null +++ b/analysis/management/commands/import_tlsdb.py @@ -0,0 +1,113 @@ +"""Django management command: ``python manage.py import_tlsdb`` + +Reads ``TlsDB.csv`` from the project root and populates the +``tls_ref_data`` SQLite table, replacing the CSV as the authoritative +source of TLS field-code reference metadata. + +Usage: + python manage.py import_tlsdb + python manage.py import_tlsdb --csv /path/to/TlsDB.csv + python manage.py import_tlsdb --clear # re-import from scratch +""" + +import csv +import os +from pathlib import Path + +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError +from django.db import connection, transaction + + +class Command(BaseCommand): + """Import ``TlsDB.csv`` field definitions into ``tls_ref_data``.""" + + help = 'Import TlsDB.csv reference data into the tls_ref_data SQLite table' + + def add_arguments(self, parser): + parser.add_argument( + '--csv', type=str, default=None, + help='Path to TlsDB.csv (default: project-root/TlsDB.csv)', + ) + parser.add_argument( + '--clear', action='store_true', default=False, + help='Clear existing data before importing (default: upsert)', + ) + + def _resolve_csv_path(self, provided: str | None) -> Path: + """Locate the TlsDB.csv file. + + Priority: + 1. Explicit ``--csv`` path. + 2. Project root (where ``manage.py`` lives). + """ + if provided: + path = Path(provided).resolve() + else: + # manage.py lives in the project root — use that as anchor + manage_dir = Path(__file__).resolve().parent.parent.parent.parent + path = manage_dir / 'TlsDB.csv' + + if not path.exists(): + raise CommandError(f'TlsDB.csv not found at: {path}') + return path + + def handle(self, *args, **options): + csv_path = self._resolve_csv_path(options['csv']) + clear = options['clear'] + + # ── Parse CSV ──────────────────────────────────────────────────────── + rows: list[dict[str, str]] = [] + with open(csv_path, 'r', encoding='utf-8-sig') as f: + reader = csv.DictReader(f) + # Expected columns: 标题,含义,数据类型 + for line_no, row in enumerate(reader, start=2): + field_code = (row.get('标题') or '').strip() + meaning = (row.get('含义') or '').strip() + data_type = (row.get('数据类型') or '').strip() + if not field_code: + self.stderr.write(self.style.WARNING( + f' [SKIP] Line {line_no}: empty field_code, skipping' + )) + continue + rows.append({ + 'field_code': field_code, + 'meaning': meaning, + 'data_type': data_type, + }) + + if not rows: + self.stderr.write(self.style.ERROR('No valid rows found in TlsDB.csv')) + return + + # ── Insert into tls_ref_data ───────────────────────────────────────── + with connection.cursor() as cursor: + with transaction.atomic(): + if clear: + cursor.execute('DELETE FROM tls_ref_data') + self.stdout.write(self.style.WARNING(f' Cleared existing data ({len(rows)} rows to insert)')) + + inserted = 0 + updated = 0 + for r in rows: + cursor.execute( + """ + INSERT INTO tls_ref_data (field_code, meaning, data_type) + VALUES (%s, %s, %s) + ON CONFLICT(field_code) DO UPDATE SET + meaning = EXCLUDED.meaning, + data_type = EXCLUDED.data_type + """, + [r['field_code'], r['meaning'], r['data_type']], + ) + if cursor.rowcount == 1: + inserted += 1 + else: + updated += 1 + + self.stdout.write(self.style.SUCCESS( + f'\n[OK] Imported {len(rows)} TLS field definitions from: {csv_path}' + )) + self.stdout.write(self.style.SUCCESS( + f' New: {inserted} | Updated: {updated} | Total: {len(rows)}' + )) diff --git a/analysis/management/commands/run_pipeline.py b/analysis/management/commands/run_pipeline.py index 8324a34..bbd89d6 100644 --- a/analysis/management/commands/run_pipeline.py +++ b/analysis/management/commands/run_pipeline.py @@ -1,19 +1,17 @@ """Django management command: ``python manage.py run_pipeline`` -Runs the complete TLS analysis pipeline from CLI, no LLM required. -Usage: uv run python manage.py run_pipeline [--entity-col NAME] [--algo hdbscan|kmeans] +Runs the analysis pipeline from CLI: load → cluster → extract features. +Usage: uv run python manage.py run_pipeline [--algo hdbscan|kmeans] """ import os, sys, json, asyncio from django.core.management.base import BaseCommand, CommandError from django.utils import timezone class Command(BaseCommand): - help = 'Run the full TLS analysis pipeline from CLI (no LLM needed)' + help = 'Run the analysis pipeline from CLI (load → cluster → extract)' def add_arguments(self, parser): parser.add_argument('csv_glob', type=str, help='Glob pattern for CSV files') - parser.add_argument('--entity-col', type=str, default=None, - help='Entity column (auto-detect if omitted)') parser.add_argument('--algo', type=str, default='hdbscan', choices=['hdbscan', 'kmeans'], help='Clustering algorithm (default: hdbscan)') parser.add_argument('--output', type=str, default=None, @@ -25,7 +23,6 @@ class Command(BaseCommand): from analysis.views import _run_pipeline_worker csv_glob = options['csv_glob'] - entity_col = options['entity_col'] algo = options['algo'] output = options['output'] @@ -42,14 +39,12 @@ class Command(BaseCommand): warnings.filterwarnings('ignore', category=RuntimeWarning, module='sklearn') from analysis.session_store import SessionStore from analysis.data_loader import load_csv_directory - from analysis.entity_detector import detect_entity_column - from analysis.entity_aggregator import aggregate_by_entity store = SessionStore() store.drop_all() # ── Step 1: Load CSV ───────────────────────────────────────────── - _self.stdout.write(f'[1/5] 加载 CSV: {csv_glob}') + _self.stdout.write(f'[1/3] 加载 CSV: {csv_glob}') lf, schema, row_count, file_count, memory_mb = load_csv_directory(csv_glob) ds_id = 'manual_ds' store.store_dataset(ds_id, lf, schema=schema, metadata={ @@ -57,47 +52,25 @@ class Command(BaseCommand): }) _self.stdout.write(_self.style.SUCCESS(f' → {row_count} rows, {file_count} files, {memory_mb:.1f} MB')) - # ── Step 2: Detect entity column ──────────────────────────────── - _self.stdout.write('[2/5] 检测实体列') - if entity_col: - used_col = entity_col - _self.stdout.write(f' → 使用指定列: {used_col}') - else: - result = detect_entity_column(ds_id) - candidates = result.get('candidates', []) - used_col = result.get('recommended') - if not used_col: - _self.stderr.write(_self.style.ERROR(' ✗ 未检测到实体列，请用 --entity-col 手动指定')) - run.status = 'failed' - run.error_message = '未检测到实体列' - run.save(update_fields=['status', 'error_message']) - return - _self.stdout.write(f' → 自动检测: {used_col} (共 {len(candidates)} 候选)') - - # ── Step 3: Aggregate by entity ────────────────────────────────── - _self.stdout.write('[3/5] 实体聚合') - agg_lf, feature_columns = aggregate_by_entity(lf, used_col, schema) - df_agg = agg_lf.collect(streaming=True) - entity_ds_id = 'manual_entity' - store.store_dataset(entity_ds_id, agg_lf, schema=dict(zip(feature_columns, ['']*len(feature_columns))), - metadata={'row_count': len(df_agg), 'entity_column': used_col, 'csv_glob': csv_glob}) - _self.stdout.write(_self.style.SUCCESS(f' → {len(df_agg)} 个实体, {len(feature_columns)} 维特征')) - - # ── Step 4: Clustering + Extraction + PCA (shared pipeline) ───── - _self.stdout.write(f'[4/5] 聚类 ({algo})') - from analysis.views import _run_clustering_pipeline - user_features = [ - c for c in feature_columns - if not c.startswith('_') and c not in ('first_seen', 'last_seen', 'src_ip') + # Auto-select numeric columns for clustering + numeric_types = {'Int8', 'Int16', 'Int32', 'Int64', + 'UInt8', 'UInt16', 'UInt32', 'UInt64', + 'Float32', 'Float64'} + feature_cols = [ + c for c in schema.keys() + if not c.startswith('_') + and schema[c].split('(')[0].strip() in numeric_types ][:10] - _self.stdout.write(f' → 聚类特征: {user_features}') - run.entity_column = used_col + + # ── Step 2: Clustering + Extraction + PCA (shared pipeline) ───── + _self.stdout.write(f'[2/3] 聚类 ({algo})') + _self.stdout.write(f' → 聚类特征: {feature_cols}') run.total_flows = row_count - run.entity_count = len(df_agg) - run.save(update_fields=['entity_column', 'total_flows', 'entity_count']) + run.save(update_fields=['total_flows']) + from analysis.views import _run_clustering_pipeline _run_clustering_pipeline( - run=run, store=store, entity_ds_id=entity_ds_id, - feature_columns=user_features, + run=run, store=store, entity_ds_id=ds_id, + feature_columns=feature_cols, algorithm=algo, min_cluster_size=5, run_pca=True, ) @@ -112,26 +85,20 @@ class Command(BaseCommand): if output: from analysis.tool_registry import _handle_export_results os.makedirs(output, exist_ok=True) - asyncio.run(_handle_export_results(result_id=entity_ds_id, output_path=output, format='csv', overwrite=True)) + asyncio.run(_handle_export_results(result_id=ds_id, output_path=output, format='csv', overwrite=True)) _self.stdout.write(_self.style.SUCCESS(f' → 导出到 {output}')) - # ── Update run record with final stats ────────────────────────── - run.total_flows = row_count - run.entity_count = len(df_agg) - run.entity_column = used_col - run.save(update_fields=['total_flows', 'entity_count', 'entity_column']) - _run_pipeline_worker(run_id, _analysis_fn) # ── Read final status and print summary ───────────────────────────── run = AnalysisRun.objects.get(id=run_id) if run.status == 'failed': err_msg = (run.error_message or '')[:500] - self.stderr.write(self.style.ERROR(f'\n*** 分析失败! Run ID: #{run.id}')) + self.stderr.write(self.style.ERROR(f'\n*** 分析失败! Run ID: #{run.display_id}')) if err_msg: self.stderr.write(self.style.ERROR(f' {err_msg}')) return self.stdout.write(self.style.SUCCESS( - f'\n*** 分析完成! Run ID: #{run.id}' - f' 查看 Django: http://127.0.0.1:8000/runs/{run.id}/' + f'\n*** 分析完成! Run ID: #{run.display_id}' + f' 查看 Django: http://127.0.0.1:8000/runs/{run.display_id}/' )) diff --git a/analysis/migrations/0003_analysisrun_sqlite_table.py b/analysis/migrations/0003_analysisrun_sqlite_table.py new file mode 100644 index 0000000..e525c3f --- /dev/null +++ b/analysis/migrations/0003_analysisrun_sqlite_table.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.30 on 2026-07-20 05:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tianxuan_analysis', '0002_analysisrun_progress_msg_analysisrun_progress_pct_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='analysisrun', + name='sqlite_table', + field=models.CharField(blank=True, default='', help_text='SQLite table name for persisted raw data', max_length=256), + ), + ] diff --git a/analysis/migrations/0004_tls_ref_data.py b/analysis/migrations/0004_tls_ref_data.py new file mode 100644 index 0000000..8c616e3 --- /dev/null +++ b/analysis/migrations/0004_tls_ref_data.py @@ -0,0 +1,38 @@ +"""Migration 0004 — create raw SQLite table ``tls_ref_data``. + +Replaces the ``TlsDB.csv`` reference file with a proper SQLite table. +The table stores TLS field-code metadata: human-readable meaning and +data type for each field code used in TLS flow data. +""" + +from django.db import migrations + +# Raw SQL for creating the tls_ref_data table +CREATE_TLS_REF_DATA = """ +CREATE TABLE IF NOT EXISTS tls_ref_data ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + field_code TEXT NOT NULL UNIQUE, + meaning TEXT NOT NULL DEFAULT '', + data_type TEXT NOT NULL DEFAULT '' +); +""" + +DROP_TLS_REF_DATA = """ +DROP TABLE IF EXISTS tls_ref_data; +""" + + +class Migration(migrations.Migration): + """Create ``tls_ref_data`` table via raw SQL — no Django model needed.""" + + dependencies = [ + ('tianxuan_analysis', '0003_analysisrun_sqlite_table'), + ] + + operations = [ + migrations.RunSQL( + sql=CREATE_TLS_REF_DATA, + reverse_sql=DROP_TLS_REF_DATA, + state_operations=None, + ), + ] diff --git a/analysis/migrations/0005_remove_entity_column.py b/analysis/migrations/0005_remove_entity_column.py new file mode 100644 index 0000000..86b1e6a --- /dev/null +++ b/analysis/migrations/0005_remove_entity_column.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.30 on 2026-07-20 05:20 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('tianxuan_analysis', '0004_tls_ref_data'), + ] + + operations = [ + migrations.RemoveField( + model_name='analysisrun', + name='entity_column', + ), + ] diff --git a/analysis/migrations/0006_analysisrun_display_id_analysisrun_llm_thinking_and_more.py b/analysis/migrations/0006_analysisrun_display_id_analysisrun_llm_thinking_and_more.py new file mode 100644 index 0000000..bf06784 --- /dev/null +++ b/analysis/migrations/0006_analysisrun_display_id_analysisrun_llm_thinking_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.30 on 2026-07-20 05:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tianxuan_analysis', '0005_remove_entity_column'), + ] + + operations = [ + migrations.AddField( + model_name='analysisrun', + name='display_id', + field=models.IntegerField(blank=True, help_text='User-facing numeric ID (recycles gaps)', null=True, unique=True), + ), + migrations.AddField( + model_name='analysisrun', + name='llm_thinking', + field=models.TextField(blank=True, default='', help_text='Accumulated LLM reasoning / thinking text'), + ), + migrations.AddField( + model_name='analysisrun', + name='run_type', + field=models.CharField(choices=[('upload', '手动上传'), ('manual', '手动分析'), ('auto', 'LLM自动')], default='upload', max_length=16), + ), + migrations.AddField( + model_name='analysisrun', + name='tool_calls_json', + field=models.JSONField(blank=True, default=list, help_text='Structured tool call list: [{step, name, input, output}, ...]'), + ), + ] diff --git a/analysis/migrations/0007_populate_display_id.py b/analysis/migrations/0007_populate_display_id.py new file mode 100644 index 0000000..aa49a72 --- /dev/null +++ b/analysis/migrations/0007_populate_display_id.py @@ -0,0 +1,20 @@ +# Generated by Django 4.2.30 on 2026-07-20 05:26 + +from django.db import migrations +from django.db.models import F + + +def populate_display_id(apps, schema_editor): + AnalysisRun = apps.get_model('tianxuan_analysis', 'AnalysisRun') + AnalysisRun.objects.filter(display_id__isnull=True).update(display_id=F('id')) + + +class Migration(migrations.Migration): + + dependencies = [ + ('tianxuan_analysis', '0006_analysisrun_display_id_analysisrun_llm_thinking_and_more'), + ] + + operations = [ + migrations.RunPython(populate_display_id, migrations.RunPython.noop), + ] diff --git a/analysis/models.py b/analysis/models.py index 41b51b3..02a840a 100644 --- a/analysis/models.py +++ b/analysis/models.py @@ -1,4 +1,4 @@ -from django.db import models +from django.db import models, connection class AnalysisRun(models.Model): @@ -14,9 +14,18 @@ class AnalysisRun(models.Model): ('failed', 'Failed'), ] + RUN_TYPE_CHOICES = [ + ('upload', '手动上传'), + ('manual', '手动分析'), + ('auto', 'LLM自动'), + ] + + display_id = models.IntegerField(unique=True, null=True, blank=True, + help_text="User-facing numeric ID (recycles gaps)") csv_glob = models.CharField(max_length=1024, help_text="Glob pattern for CSV files") config = models.JSONField(default=dict, blank=True, help_text="YAML config snapshot") status = models.CharField(max_length=32, choices=STATUS_CHOICES, default='pending') + run_type = models.CharField(max_length=16, choices=RUN_TYPE_CHOICES, default='upload') created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) error_message = models.TextField(blank=True, default='') @@ -28,14 +37,36 @@ class AnalysisRun(models.Model): total_flows = models.IntegerField(null=True, blank=True) entity_count = models.IntegerField(null=True, blank=True) cluster_count = models.IntegerField(null=True, blank=True) - entity_column = models.CharField(max_length=256, blank=True, default='', - help_text="Auto-detected entity column name") + sqlite_table = models.CharField(max_length=256, blank=True, default='', + help_text="SQLite table name for persisted raw data") + + # LLM auto-analysis fields + llm_thinking = models.TextField(blank=True, default='', + help_text="Accumulated LLM reasoning / thinking text") + tool_calls_json = models.JSONField(default=list, blank=True, + help_text="Structured tool call list: [{step, name, input, output}, ...]") class Meta: ordering = ['-created_at'] def __str__(self): - return f"Run #{self.id} ({self.created_at:%Y-%m-%d %H:%M}) - {self.status}" + return f"Run #{self.display_id or self.id} ({self.created_at:%Y-%m-%d %H:%M}) - {self.status}" + + def save(self, *args, **kwargs): + if self._state.adding and self.display_id is None: + table = self._meta.db_table + with connection.cursor() as cursor: + cursor.execute(f""" + SELECT COALESCE(MIN(t1.display_id) + 1, 1) + FROM {table} t1 + WHERE t1.display_id IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM {table} t2 + WHERE t2.display_id = t1.display_id + 1 + ) + """) + self.display_id = cursor.fetchone()[0] + super().save(*args, **kwargs) class ClusterResult(models.Model): diff --git a/analysis/session_store.py b/analysis/session_store.py index 4726509..1ae7dc6 100644 --- a/analysis/session_store.py +++ b/analysis/session_store.py @@ -106,10 +106,14 @@ class SessionStore: ) def restore_from_disk(self) -> int: - """Re-register datasets whose CSV files still exist on disk. + """Re-register datasets from persisted metadata. - Uses ``csv_glob`` saved in metadata to re-load each dataset via - :func:`analysis.data_loader.load_csv_directory`. + Attempts two strategies in order: + 1. **SQLite table** — when ``metadata.sqlite_table`` exists (fast, + always available, no CSV dependency). + 2. **CSV glob** — falls back to ``load_csv_directory`` for datasets + that were stored before the SQLite migration or whose SQLite table + was dropped. Returns the number of successfully restored datasets. """ @@ -121,18 +125,38 @@ class SessionStore: except (json.JSONDecodeError, OSError): return 0 - # Lazy import to avoid circular dependency at module level - from analysis.data_loader import load_csv_directory # fmt: skip + # Lazy imports to avoid circular dependency at module level + from analysis.data_loader import load_csv_directory, load_from_db # fmt: skip restored = 0 for ds_id, info in data.items(): + meta = dict(info.get('metadata', {})) + schema = dict(info.get('schema', {})) + restored_ok = False + + # Strategy 1: SQLite table (fast path) + sqlite_table = meta.get('sqlite_table') + if sqlite_table: + try: + lf = load_from_db(sqlite_table) + if lf is not None: + self.store_dataset(ds_id, lf, schema=schema, metadata=meta) + restored += 1 + restored_ok = True + except Exception: + pass # fall through to CSV restore + + if restored_ok: + continue + + # Strategy 2: CSV glob (legacy / fallback) csv_glob = info.get('csv_glob') if not csv_glob: continue try: - lf, schema, row_count, file_count, memory_mb = load_csv_directory(csv_glob) - meta = dict(info.get('metadata', {})) - self.store_dataset(ds_id, lf, schema=schema, metadata=meta) + lf, schema_csv, row_count, file_count, memory_mb = load_csv_directory(csv_glob) + # Merge schema from CSV (may be more accurate than persisted schema) + self.store_dataset(ds_id, lf, schema=schema_csv, metadata=meta) restored += 1 except Exception: # File(s) no longer exist or schema changed — skip gracefully diff --git a/analysis/tls_ref.py b/analysis/tls_ref.py new file mode 100644 index 0000000..25c772c --- /dev/null +++ b/analysis/tls_ref.py @@ -0,0 +1,94 @@ +"""SQLite-backed TLS reference data access — replaces ``TlsDB.csv``. + +Provides functions to query the ``tls_ref_data`` table (populated by +the ``import_tlsdb`` management command) instead of reading the +now-deprecated ``TlsDB.csv`` file. + +Usage:: + + from analysis.tls_ref import get_tls_ref_data, get_tls_field, get_tls_ref_dict + + # All rows as list of dicts + all_fields = get_tls_ref_data() + + # Single field + field = get_tls_field('0ver') + # -> {'field_code': '0ver', 'meaning': 'TLS版本', 'data_type': '2字节'} + + # Dict lookup (cached in memory after first call) + lookup = get_tls_ref_dict() + # -> {'0ver': {'meaning': 'TLS版本', 'data_type': '2字节'}, ...} +""" + +from functools import lru_cache +from typing import Optional + +from django.db import connection + + +def get_tls_ref_data() -> list[dict[str, str]]: + """Return all rows from ``tls_ref_data`` as a list of dicts. + + Returns: + Each dict has keys ``field_code``, ``meaning``, ``data_type``. + Returns an empty list if the table does not exist (e.g. before + migration). + """ + if not _table_exists(): + return [] + + with connection.cursor() as cursor: + cursor.execute( + 'SELECT field_code, meaning, data_type FROM tls_ref_data ORDER BY id' + ) + columns = [col[0] for col in cursor.description] + return [dict(zip(columns, row)) for row in cursor.fetchall()] + + +def get_tls_field(field_code: str) -> Optional[dict[str, str]]: + """Return a single TLS field definition by its code. + + Args: + field_code: e.g. ``'0ver'``, ``':ips'``, ``'cipher-suite'``. + + Returns: + Dict with keys ``field_code``, ``meaning``, ``data_type``, or + ``None`` if not found. + """ + if not _table_exists(): + return None + + with connection.cursor() as cursor: + cursor.execute( + 'SELECT field_code, meaning, data_type FROM tls_ref_data WHERE field_code = %s', + [field_code], + ) + row = cursor.fetchone() + if row is None: + return None + return { + 'field_code': row[0], + 'meaning': row[1], + 'data_type': row[2], + } + + +@lru_cache(maxsize=1) +def get_tls_ref_dict() -> dict[str, dict[str, str]]: + """Return a dict mapping ``field_code -> {meaning, data_type}``. + + The result is cached in memory after the first call. Call + ``get_tls_ref_dict.cache_clear()`` to invalidate if the + underlying table is modified at runtime. + """ + rows = get_tls_ref_data() + return {r['field_code']: {'meaning': r['meaning'], 'data_type': r['data_type']} for r in rows} + + +def _table_exists() -> bool: + """Check if the ``tls_ref_data`` table exists in the database.""" + with connection.cursor() as cursor: + cursor.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name='tls_ref_data'" + ) + return cursor.fetchone() is not None diff --git a/analysis/tool_registry.py b/analysis/tool_registry.py index 6231110..201738b 100644 --- a/analysis/tool_registry.py +++ b/analysis/tool_registry.py @@ -5,12 +5,12 @@ an async handler function. Handlers interact with the :class:`SessionStore` singleton and the Django ORM. All 11 tools: - 1. load_data 7. extract_features - 2. profile_data 8. export_results - 3. filter_data 9. list_datasets - 4. preprocess_data 10. drop_dataset - 5. run_clustering 11. clone_dataset - 6. evaluate_clustering + 1. load_data 7. filter_and_cluster + 2. profile_data 8. export_results + 3. filter_data 9. list_datasets + 4. preprocess_data 10. drop_dataset + 5. run_clustering 11. clone_dataset + 6. evaluate_clustering """ import json import os @@ -23,8 +23,6 @@ from mcp.types import Tool from .data_loader import load_csv_directory from .data_profiler import profile_dataset as _profile_dataset -from .entity_detector import detect_entity_column -from .entity_aggregator import aggregate_by_entity from .session_store import SessionStore, _generate_id @@ -70,9 +68,8 @@ def get_tools_meta() -> list[Tool]: Tool( name="load_data", description=( - "Load CSV files matching a glob pattern. Detects BOM per file, " - "validates schema consistency, merges vertically, and stores the " - "result as a lazy dataset in the session store." + "加载匹配glob模式的CSV文件。自动检测每个文件的BOM，验证schema一致性，" + "纵向合并，并将结果作为惰性数据集存储在会话存储中。" ), inputSchema={ "type": "object", @@ -112,9 +109,8 @@ def get_tools_meta() -> list[Tool]: Tool( name="profile_data", description=( - "Compute per-column statistics and a numeric correlation matrix " - "for a dataset. Result is automatically truncated to ~16 KB for " - "LLM context-window efficiency." + "计算数据集的每列统计信息和数值相关矩阵。结果自动截断至约16 KB以" + "节省LLM上下文窗口空间。" ), inputSchema={ "type": "object", @@ -140,9 +136,8 @@ def get_tools_meta() -> list[Tool]: Tool( name="filter_data", description=( - "Apply column-level filters to a dataset and store the result as " - "a new derived dataset. Supports eq/neq/gt/gte/lt/lte/contains/" - "not_contains/in/not_in/is_null/is_not_null operators." + "对数据集应用列级过滤器，将结果存储为新的派生数据集。支持eq/neq/gt/gte/" + "lt/lte/contains/not_contains/in/not_in/is_null/is_not_null运算符。" ), inputSchema={ "type": "object", @@ -185,8 +180,7 @@ def get_tools_meta() -> list[Tool]: Tool( name="preprocess_data", description=( - "Preprocess selected columns: scaling (standardize), one-hot encoding, " - "missing-value imputation, and column dropping." + "预处理选定列：标准化（StandardScaler）、独热编码、缺失值填充和列删除。" ), inputSchema={ "type": "object", @@ -232,9 +226,8 @@ def get_tools_meta() -> list[Tool]: Tool( name="run_clustering", description=( - "Run clustering on selected columns. Uses HDBSCAN (default) or " - "KMeans. Features are standard-scaled before clustering. Returns " - "cluster labels and quality metrics." + "对选定列执行聚类。使用HDBSCAN（默认）或KMeans。特征在聚类前进行" + "标准化缩放。返回聚类标签和质量指标。" ), inputSchema={ "type": "object", @@ -274,9 +267,8 @@ def get_tools_meta() -> list[Tool]: Tool( name="evaluate_clustering", description=( - "Evaluate a clustering result on a dataset. Currently supports " - "silhouette, davies_bouldin, calinski_harabasz, noise_ratio, and " - "cluster_sizes metrics." + "评估数据集的聚类结果。当前支持轮廓系数（silhouette）、Davies-Bouldin指数、" + "Calinski-Harabasz指数、噪声比（noise_ratio）和簇大小（cluster_sizes）指标。" ), inputSchema={ "type": "object", @@ -305,9 +297,8 @@ def get_tools_meta() -> list[Tool]: Tool( name="extract_features", description=( - "Extract distinguishing features per cluster and auto-save to " - "the Django ORM (ClusterFeature model). Uses z-score or ANOVA " - "to rank feature importance per cluster." + "提取每个聚类的区分特征并自动保存到Django ORM（ClusterFeature模型）。" + "使用z-score或ANOVA对每个聚类的特征重要性进行排序。" ), inputSchema={ "type": "object", @@ -343,9 +334,8 @@ def get_tools_meta() -> list[Tool]: Tool( name="export_results", description=( - "Export a dataset or cluster result to disk as CSV or Parquet. " - "Datasets are materialised and written; cluster results are " - "serialised as JSON." + "将数据集或聚类结果以CSV或Parquet格式导出到磁盘。数据集物化后写入；" + "聚类结果序列化为JSON。" ), inputSchema={ "type": "object", @@ -375,7 +365,7 @@ def get_tools_meta() -> list[Tool]: ), Tool( name="list_datasets", - description="List all active datasets and results in the session store.", + description="列出会话存储中的所有活跃数据集和结果。", inputSchema={ "type": "object", "properties": {}, @@ -384,8 +374,7 @@ def get_tools_meta() -> list[Tool]: Tool( name="drop_dataset", description=( - "Remove a dataset (or result) from the session store and release " - "memory via garbage collection." + "从会话存储中移除数据集（或结果）并通过垃圾回收释放内存。" ), inputSchema={ "type": "object", @@ -401,8 +390,7 @@ def get_tools_meta() -> list[Tool]: Tool( name="clone_dataset", description=( - "Shallow-clone a dataset. Since LazyFrames are query plans, " - "no data is copied. Useful for branching analyses." + "浅拷贝数据集。由于LazyFrame是查询计划，不会复制实际数据。适用于分支分析。" ), inputSchema={ "type": "object", @@ -416,53 +404,78 @@ def get_tools_meta() -> list[Tool]: }, ), Tool( - name="build_entity_profiles", + name="filter_and_cluster", description=( - "Auto-detect or manually specify entity column(s) (e.g. src_ip, " - "sni, user) and aggregate flow-level data into per-entity profiles. " - "Computes flow stats, destination diversity, TLS features, protocol " - "features, and time patterns. Results are persisted to the Django " - "ORM (EntityProfile model) and stored as a new dataset in the session." + "一步完成：先应用列级过滤器，再对过滤结果执行聚类。支持eq/neq/gt/gte/lt/lte/" + "contains/not_contains/in/not_in/is_null/is_not_null运算符和HDBSCAN/KMeans聚类。" + "自动选择数值列作为聚类特征。将过滤后的数据集和聚类结果都存储在会话中。" ), inputSchema={ "type": "object", "properties": { "dataset_id": { "type": "string", - "description": "Source dataset identifier from load_data", + "description": "Source dataset identifier", }, - "entity_column": { - "type": "string", - "description": "Column to group by (optional; auto-detected if omitted)", - }, - "entity_columns": { + "filters": { "type": "array", - "items": {"type": "string"}, - "description": "Multiple columns to group by (optional; overrides entity_column)", + "items": { + "type": "object", + "properties": { + "column": {"type": "string"}, + "op": { + "type": "string", + "enum": [ + "eq", "neq", "gt", "gte", "lt", "lte", + "contains", "not_contains", + "in", "not_in", + "is_null", "is_not_null", + ], + }, + "value": {"description": "Operand value"}, + }, + "required": ["column", "op"], + }, + "description": "List of filter expressions", }, - "auto_detect": { - "type": "boolean", - "description": "Auto-detect entity column when entity_column is not given", - "default": True, + "logic": { + "type": "string", + "enum": ["and", "or"], + "description": "Combine filters with AND or OR logic", + "default": "and", + }, + "algorithm": { + "type": "string", + "enum": ["hdbscan", "kmeans"], + "description": "Clustering algorithm", + "default": "hdbscan", + }, + "params": { + "type": "object", + "description": ( + "Algorithm parameters. HDBSCAN: min_cluster_size (5), " + "min_samples (None), metric ('euclidean'). " + "KMeans: n_clusters (3), random_state (42)." + ), }, }, - "required": ["dataset_id"], + "required": ["dataset_id", "filters"], }, ), Tool( name="compute_scores", description=( - "Compute adaptive proxy/anomaly/threat scores on entity data. " - "Uses PCA to auto-learn feature weights from data, computes weighted " - "proxy_score, IQR-based risk_level (normal/watch/suspicious/critical), " - "and threat_score. Returns scoring summary with thresholds." + "在实体数据上计算自适应代理/异常/威胁评分。使用PCA从数据自动学习" + "特征权重，计算加权代理评分（proxy_score）、基于IQR的风险等级" + "（normal/watch/suspicious/critical）和威胁评分（threat_score）。" + "返回评分摘要与阈值。" ), inputSchema={ "type": "object", "properties": { "dataset_id": { "type": "string", - "description": "Entity dataset ID from build_entity_profiles", + "description": "Entity dataset ID from filter_and_cluster or a scored dataset", }, }, "required": ["dataset_id"], @@ -471,9 +484,8 @@ def get_tools_meta() -> list[Tool]: Tool( name="detect_anomalies", description=( - "Run Isolation Forest anomaly detection on scored entity data. " - "Identifies entities deviating from normal patterns. Processes in " - "chunks to handle large datasets. Returns list of anomalous entity IDs." + "对已评分的实体数据运行孤立森林（Isolation Forest）异常检测。识别偏离" + "正常模式的实体。分块处理以应对大数据集。返回异常实体ID列表。" ), inputSchema={ "type": "object", @@ -494,9 +506,8 @@ def get_tools_meta() -> list[Tool]: Tool( name="visualize_anomalies", description=( - "Generate PCA-2D embedding and cluster visualization data for " - "high-risk entities. Colors points by risk_level and clusters. " - "Returns scatter data JSON for frontend rendering." + "为高风险实体生成PCA-2D嵌入和聚类可视化数据。按风险等级和聚类着色。" + "返回用于前端渲染的散点图JSON数据。" ), inputSchema={ "type": "object", @@ -513,10 +524,8 @@ def get_tools_meta() -> list[Tool]: Tool( name="validate_data", description=( - "DIAGNOSTIC: Check data quality issues — schema consistency across " - "files, missing value ratios, column type conflicts, encoding problems, " - "and empty columns. Use when a pipeline step fails with schema errors, " - "or when suspecting corrupt/malformed CSV data." + "诊断工具：检查数据质量问题——文件间schema一致性、缺失值比例、列类型冲突、" + "编码问题和空列。当流水线步骤因schema错误失败，或怀疑CSV数据损坏/格式异常时使用。" ), inputSchema={ "type": "object", @@ -530,10 +539,9 @@ def get_tools_meta() -> list[Tool]: Tool( name="explore_distributions", description=( - "DIAGNOSTIC: Compute per-column distribution statistics (unique count, " - "null ratio, min/max/mean/std, top-10 frequent values, histogram bins). " - "Useful when clustering produces unexpected results, entity detection " - "fails, or to understand data before choosing analysis strategy." + "诊断工具：计算每列分布统计（唯一值计数、空值比例、最小/最大/均值/标准差、" + "前10高频值、直方图分箱）。当聚类产生意外结果、实体检测失败，或需要在选择" + "分析策略前了解数据时使用。" ), inputSchema={ "type": "object", @@ -549,10 +557,8 @@ def get_tools_meta() -> list[Tool]: Tool( name="find_outliers", description=( - "DIAGNOSTIC: Detect statistical outliers in numeric columns using IQR " - "method. Returns rows where any numeric column exceeds 3x IQR from " - "median. Use when suspecting data quality issues or extreme values " - "skewing the analysis." + "诊断工具：使用IQR方法检测数值列中的统计异常值。返回任何数值列超过中位数3倍IQR的行。" + "当怀疑数据质量问题或极端值偏离分析时使用。" ), inputSchema={ "type": "object", @@ -568,10 +574,9 @@ def get_tools_meta() -> list[Tool]: Tool( name="diagnose_clustering", description=( - "DIAGNOSTIC: When clustering produces too few/many clusters, all noise, " - "or poor silhouette scores — diagnose why. Checks: feature variance, " - "correlation matrix, PCA explained variance, data sparsity, and suggests " - "parameter adjustments (min_cluster_size, algorithm choice, feature selection)." + "诊断工具：当聚类产生太少/太多聚类、全部为噪声或轮廓系数低时——诊断原因。" + "检查：特征方差、相关矩阵、PCA解释方差、数据稀疏性，并建议参数调整" + "（min_cluster_size、算法选择、特征选择）。" ), inputSchema={ "type": "object", @@ -585,10 +590,8 @@ def get_tools_meta() -> list[Tool]: Tool( name="compare_datasets", description=( - "DIAGNOSTIC: Compare two datasets (e.g. before/after processing, " - "or two different runs). Reports schema differences, row count delta, " - "column value distribution shifts. Use to verify processing steps " - "didn't corrupt data or to compare results across runs." + "诊断工具：比较两个数据集（如处理前/后，或两次不同运行）。报告schema差异、" + "行数变化、列值分布偏移。用于验证处理步骤未损坏数据或比较不同运行的结果。" ), inputSchema={ "type": "object", @@ -602,9 +605,8 @@ def get_tools_meta() -> list[Tool]: Tool( name="export_debug_sample", description=( - "DIAGNOSTIC: Export a sample of raw data as JSON for external debugging. " - "Useful when an error occurs deep in the pipeline and you need to inspect " - "the actual data values. Returns first N rows with all columns." + "诊断工具：导出原始数据样本为JSON用于外部调试。当流水线深处发生错误且需要" + "检查实际数据值时使用。返回前N行所有列。" ), inputSchema={ "type": "object", @@ -618,10 +620,9 @@ def get_tools_meta() -> list[Tool]: Tool( name="repair_schema", description=( - "DIAGNOSTIC: Attempt to fix schema mismatches between files by " - "aligning column names (case-insensitive merge, underscore/hyphen " - "normalisation) and filling missing columns with nulls. Use when " - "load_data fails because CSV files have different column sets." + "诊断工具：通过对齐列名（不区分大小写合并、下划线/连字符规范化）并用空值填充" + "缺失列，尝试修复文件间的schema不匹配。当load_data因CSV文件具有不同列集而" + "失败时使用。" ), inputSchema={ "type": "object", @@ -635,9 +636,8 @@ def get_tools_meta() -> list[Tool]: Tool( name="analyze_patterns", description=( - "ANALYSIS: Detect traffic patterns — top source/dest IPs, port distributions, " - "TLS version breakdown, protocol mix. Returns summary statistics. " - "Read-only, does not modify data. Call to understand high-level traffic structure." + "分析工具：检测流量模式——top源/目标IP、端口分布、TLS版本分布、协议混合。" + "返回汇总统计。只读，不修改数据。用于了解高层流量结构。" ), inputSchema={ "type": "object", @@ -651,9 +651,8 @@ def get_tools_meta() -> list[Tool]: Tool( name="analyze_temporal", description=( - "ANALYSIS: Analyze traffic over time — hourly/daily flow counts, " - "busiest periods, idle periods. Requires a timestamp column. " - "Read-only. Call to understand time-based patterns and periodicity." + "分析工具：分析随时间变化的流量——小时/日流量计数、繁忙时段、空闲时段。" + "需要时间戳列。只读。用于了解时间相关模式和周期性。" ), inputSchema={ "type": "object", @@ -667,9 +666,8 @@ def get_tools_meta() -> list[Tool]: Tool( name="analyze_tls_health", description=( - "ANALYSIS: TLS security posture assessment — check for outdated TLS versions " - "(1.0/1.1), weak cipher suites, certificate issues, SNI irregularities. " - "Read-only. Call when you need to evaluate TLS security of the traffic." + "分析工具：TLS安全态势评估——检查过时的TLS版本（1.0/1.1）、弱加密套件、" + "证书问题、SNI异常。只读。用于评估流量的TLS安全性。" ), inputSchema={ "type": "object", @@ -682,9 +680,8 @@ def get_tools_meta() -> list[Tool]: Tool( name="analyze_geo_distribution", description=( - "ANALYSIS: Geographic distribution of traffic — top source/dest countries, " - "unusual location pairs (e.g. impossible travel), ISP diversity. " - "Read-only. Requires country or lat/lon columns. Call to detect geographic anomalies." + "分析工具：流量地理分布——top源/目标国家、异常地点对（如不可能的行程）、" + "ISP多样性。只读。需要国家或经纬度列。用于检测地理异常。" ), inputSchema={ "type": "object", @@ -697,19 +694,55 @@ def get_tools_meta() -> list[Tool]: Tool( name="analyze_entity_detail", description=( - "ANALYSIS: Deep-dive into a single entity — all its flows, TLS versions used, " - "destination diversity, time patterns, anomaly scores. " - "Read-only. Call to investigate a specific suspicious entity after compute_scores." + "分析工具：深入分析单个实体——其所有流、使用的TLS版本、目标多样性、时间模式、" + "异常评分。只读。在compute_scores后调查特定可疑实体时使用。" ), inputSchema={ "type": "object", "properties": { - "dataset_id": {"type": "string", "description": "Entity dataset ID (after build_entity_profiles)"}, + "dataset_id": {"type": "string", "description": "Entity dataset ID (after filter_and_cluster or compute_scores)"}, "entity_value": {"type": "string", "description": "Entity value to investigate (e.g. an IP address or SNI)"}, }, "required": ["dataset_id", "entity_value"], }, ), + Tool( + name="compute_distance_matrix", + description=( + "LLM驱动工具：对数据集的每一行执行用户编写的Python函数，返回距离/相似度评分摘要。" + "LLM提供函数体，签名格式为`def distance_fn(row: dict) -> float:`。执行器将其包装，" + "在受限命名空间（仅math、numpy）中逐行运行，返回得分的最小/最大/均值/标准差以及" + "评分行样本。用于现有工具无法表达的一次性度量、过滤或评分任务。" + ), + inputSchema={ + "type": "object", + "properties": { + "dataset_id": { + "type": "string", + "description": "Source dataset identifier", + }, + "python_function": { + "type": "string", + "description": ( + "Body of a Python function with signature `def distance_fn(row: dict) -> float:`. " + "Example: \"return abs(row.get('col1', 0)) + row.get('col2', 0)\". " + "Available namespace: math, numpy (as np). No dangerous modules." + ), + }, + "columns": { + "type": "array", + "items": {"type": "string"}, + "description": "Columns to pass to the function (default: auto-detect numeric columns)", + }, + "max_sample": { + "type": "integer", + "description": "Maximum rows to evaluate (default 10000 for performance)", + "default": 10000, + }, + }, + "required": ["dataset_id", "python_function"], + }, + ), ] @@ -742,8 +775,8 @@ async def handle_call(name: str, arguments: dict) -> dict: 'list_datasets': _handle_list_datasets, 'drop_dataset': _handle_drop_dataset, 'clone_dataset': _handle_clone_dataset, - 'build_entity_profiles': _handle_build_entity_profiles, 'compute_scores': _handle_compute_scores, + 'filter_and_cluster': _handle_filter_and_cluster, 'detect_anomalies': _handle_detect_anomalies, 'visualize_anomalies': _handle_visualize_anomalies, 'validate_data': _handle_validate_data, @@ -758,6 +791,7 @@ async def handle_call(name: str, arguments: dict) -> dict: 'analyze_tls_health': _handle_analyze_tls_health, 'analyze_geo_distribution': _handle_analyze_geo_distribution, 'analyze_entity_detail': _handle_analyze_entity_detail, + 'compute_distance_matrix': _handle_compute_distance_matrix, } handler = _handlers.get(name) if handler is None: @@ -1086,7 +1120,7 @@ async def _handle_run_clustering( row_count = 0 if row_count > 100000: # Sample to 100K before collect to avoid OOM on 10M+ rows - lf = lf.sample(n=100000, seed=42) + lf = lf.collect(streaming=True).sample(n=100000, seed=42).lazy() df_features = lf.select(feature_cols).collect(streaming=True) @@ -1774,247 +1808,120 @@ async def _handle_clone_dataset(dataset_id: str) -> dict: return {'original_dataset_id': dataset_id, 'cloned_dataset_id': new_id} -# ── 12. build_entity_profiles ───────────────────────────────────────────── +# ── Adaptive anomaly scoring helpers (inlined from entity_aggregator) ───── -async def _handle_build_entity_profiles( - dataset_id: str, - entity_column: Optional[str] = None, - entity_columns: Optional[list[str]] = None, - auto_detect: bool = True, -) -> dict: - """Auto-detect entity column(s), aggregate flows → entity profiles, persist. +_ANOMALY_FEATURE_COLS = [ + 'non_std_port_rate', 'modern_tls_rate', 'sni_missing_ratio', + 'recoverable_ratio', 'flow_count', 'total_bytes_sent', + 'total_packets', 'unique_dst_ips', 'unique_dst_ports', + 'unique_protocols', 'has_sni', 'avg_sni_length', + 'countries_visited', +] + + +def _add_adaptive_scores(lf: pl.LazyFrame, feature_names: list[str]) -> pl.LazyFrame: + """Add anomaly scores using data-driven weights (PCA + IQR + percentile). Steps: - 1. Load dataset from session store. - 2. Resolve entity column(s): *entity_columns* takes precedence, - then *entity_column*, then auto-detection. - 3. Run :func:`aggregate_by_entity` to build per-entity features. - 4. Store aggregated result as a new dataset in the session. - 5. Persist each entity's feature dict to the Django ORM - (:class:`EntityProfile`). - 6. Return summary (capped at ~8 KB). + 1. Find which anomaly feature columns exist in the aggregated data + 2. Collect a sample (max 10K rows) for PCA weight learning + 3. Learn feature weights from PCA first component loadings + 4. Compute weighted proxy_score across ALL rows (vectorized, no per-row ops) + 5. Add percentile rank calibration + 6. Infer risk thresholds from data distribution """ - store = SessionStore() - entry = store.get_dataset(dataset_id) - if entry is None: - return {'error': f'Dataset not found: {dataset_id}', 'truncated': False} + _agg_cols = lf.collect_schema().names() + avail = [c for c in _ANOMALY_FEATURE_COLS if c in _agg_cols] + if not avail: + return lf - lf: pl.LazyFrame = entry['lazyframe'] - schema: dict[str, str] = entry.get('schema', {}) - - # ── Step 1: resolve entity column(s) ───────────────────────────────── - used_columns: list[str] = [] - detection_confidence: Optional[float] = None - candidates: list[dict] = [] - - if entity_columns: - # Multi-column support: use specified list directly - missing = [c for c in entity_columns if c not in schema] - if missing: - return { - 'error': f"Entity column(s) '{missing}' not found in schema. " - f"Available: {list(schema.keys())[:20]}", - 'truncated': False, - } - used_columns = entity_columns - elif entity_column: - if entity_column not in schema: - return { - 'error': f"Entity column '{entity_column}' not found in schema. " - f"Available: {list(schema.keys())[:20]}", - 'truncated': False, - } - used_columns = [entity_column] - elif auto_detect: - det_result = detect_entity_column(dataset_id) - if det_result.get('error'): - return {'error': f'Entity detection failed: {det_result["error"]}', - 'truncated': False} - candidates = det_result.get('candidates', []) - recommended = det_result.get('recommended', '') - used_columns = [recommended] if recommended else [] - if candidates: - detection_confidence = candidates[0]['score'] - if not used_columns: - return { - 'error': 'No suitable entity column auto-detected. ' - 'Provide entity_column manually.', - 'truncated': False, - } - else: - return { - 'error': 'No entity_column provided and auto_detect is False.', - 'truncated': False, - } - - # ── Step 2: aggregate ──────────────────────────────────────────────── try: - agg_lf, feature_cols = aggregate_by_entity(lf, used_columns, schema) - except Exception as exc: - return {'error': f'Aggregation failed: {exc}', 'truncated': False} + # Step 1-2: Sample up to 10K rows for weight learning + n_rows = lf.select(pl.len()).collect(streaming=True).item() + sample_lf = lf if n_rows <= 10000 else lf.sample(n=10000, seed=42) + df_sample = sample_lf.select(avail).collect(streaming=True) + mat = df_sample.to_numpy() + mat = np.nan_to_num(mat, nan=0.0) - # Collect to compute entity count and preview - try: - df_agg = agg_lf.collect(streaming=True) - entity_count = len(df_agg) - feature_columns = [c for c in df_agg.columns if not c.startswith('_')] - except Exception as exc: - return {'error': f'Failed to collect aggregated data: {exc}', - 'truncated': False} + # Step 3: PCA weight learning + from sklearn.decomposition import PCA + n_comp = min(1, min(mat.shape) - 1) + if n_comp < 1 or mat.shape[1] < 2: + return lf + pca = PCA(n_components=1, random_state=42).fit(mat) + raw_weights = np.abs(pca.components_[0]) + # Normalise to sum = 1 + weights = dict(zip(avail, raw_weights / raw_weights.sum())) - # Build feature column schema - agg_schema = {name: str(dtype) for name, dtype in - zip(df_agg.columns, [df_agg[c].dtype for c in df_agg.columns])} + # Step 4: Weighted sum expression (all vectorised, no loop) + expr_parts = [] + for col, w in weights.items(): + if col == 'modern_tls_rate': + # modern_tls_rate is good — invert for anomaly scoring + expr_parts.append((1.0 - pl.col(col).fill_null(0.5)) * w) + else: + expr_parts.append(pl.col(col).fill_null(0.0) * w) + if expr_parts: + proxy_expr = expr_parts[0] + for p in expr_parts[1:]: + proxy_expr = proxy_expr + p + else: + return lf - # Primary entity column for metadata / DB persistence - primary_entity_col = used_columns[0] + lf = lf.with_columns(proxy_expr.clip(0, 1).alias('_raw_score')) - # ── Step 3: store as new dataset in session ────────────────────────── - entity_dataset_id = _generate_id('entity') - store.store_dataset( - dataset_id=entity_dataset_id, - lazyframe=df_agg.lazy(), - schema=agg_schema, - metadata={ - 'row_count': entity_count, - 'entity_column': primary_entity_col, - 'entity_columns': used_columns, - 'parent_dataset_id': dataset_id, - 'feature_columns': feature_columns, - }, - parent_id=dataset_id, - ) + # Step 5: Percentile rank calibration (0-1, relative position) + sample_scores = df_sample.select( + pl.sum_horizontal([ + (1.0 - pl.col('modern_tls_rate').fill_null(0.5)) * weights.get('modern_tls_rate', 0) + if c == 'modern_tls_rate' + else pl.col(c).fill_null(0.0) * weights.get(c, 0) + for c in avail + ]).clip(0, 1) + ).to_series().to_numpy() - # ── Step 4: persist to Django ORM (EntityProfile) ──────────────────── - db_saved = False - try: - db_saved = _save_entity_profiles( - dataset_id=dataset_id, - entity_column=primary_entity_col, - df_agg=df_agg, - entity_count=entity_count, + # Compute IQR thresholds from sample + q25, q50, q75 = np.percentile(sample_scores, [25, 50, 75]) + iqr = q75 - q25 + upper_fence = float(q75 + 1.5 * iqr) if iqr > 0 else 0.9 + extreme_fence = float(q75 + 3.0 * iqr) if iqr > 0 else 0.99 + + # Step 6: Risk level labels (0.0-1.0) + lf = lf.with_columns([ + pl.col('_raw_score').alias('proxy_score'), + pl.when(pl.col('_raw_score') >= extreme_fence) + .then(pl.lit('critical')) + .when(pl.col('_raw_score') >= upper_fence) + .then(pl.lit('suspicious')) + .when(pl.col('_raw_score') >= q50) + .then(pl.lit('watch')) + .otherwise(pl.lit('normal')) + .alias('risk_level'), + (pl.col('_raw_score') * 0.6 + 0.4 * (pl.col('_raw_score') > q75).cast(pl.Float64)) + .alias('threat_score'), + ]).drop('_raw_score') + + valid_counts = {c: int((mat[:, i] != 0).sum()) for i, c in enumerate(avail)} + import logging + logger = logging.getLogger(__name__) + logger.info( + '[SCORE] adaptive weights from PCA: %s | thresholds: p50=%.3f upper=%.3f extreme=%.3f | sample=%d/%d', + {c: f'{w:.3f}' for c, w in sorted(weights.items(), key=lambda x: -x[1])}, + float(q50), upper_fence, extreme_fence, + min(n_rows, 10000), n_rows, ) - except Exception: - db_saved = False - # ── Step 5: build response (compact, ≤ 8 KB friendly) ──────────────── - # Take a small sample of rows for the response - sample_rows = df_agg.head(20).to_dict(as_series=False) if entity_count > 0 else {} + except Exception as exc: + import logging + logging.getLogger(__name__).warning('[SCORE] adaptive scoring skipped: %s', exc) - result = { - 'entity_dataset_id': entity_dataset_id, - 'entity_count': entity_count, - 'entity_columns_used': used_columns, - 'feature_columns': feature_columns, - 'detection_confidence': detection_confidence, - 'detection_candidates': candidates[:5] if candidates else [], - 'db_saved': db_saved, - 'sample': sample_rows, - } - - # Ensure truncation within ~8 KB (the global _truncate_response uses 64 KB - # by default, but we cap sample here for this tool) - import json - body = json.dumps(result, default=str, ensure_ascii=False) - if len(body) > 8 * 1024: - # Drop sample and candidates, keep only counts - result.pop('sample', None) - result.pop('detection_candidates', None) - result['truncated'] = True - else: - result['truncated'] = False - - return result + return lf -def _save_entity_profiles( - dataset_id: str, - entity_column: str, - df_agg: pl.DataFrame, - entity_count: int, -) -> bool: - """Persist aggregated entity profiles to the Django ORM. - - Creates/updates :class:`EntityProfile` records linked to the - :class:`AnalysisRun` associated with *dataset_id*. - """ - import django - import os - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'tianxuan.settings') - django.setup() - - from analysis.models import AnalysisRun, EntityProfile - - store = SessionStore() - ds_entry = store.get_dataset(dataset_id) - if ds_entry is None: - return False - - csv_glob = ds_entry.get('metadata', {}).get('csv_glob', '') - try: - run = AnalysisRun.objects.filter(csv_glob=csv_glob).order_by('-created_at').first() - if run is None: - run = AnalysisRun.objects.create( - csv_glob=csv_glob, - status='aggregating', - entity_count=entity_count, - entity_column=entity_column, - ) - except Exception: - return False - - # Build EntityProfile records in bulk - entity_values = df_agg[entity_column].to_list() - feature_cols = [c for c in df_agg.columns if not c.startswith('_') and c != entity_column] - - profiles = [] - for row_idx in range(len(df_agg)): - entity_val = str(entity_values[row_idx]) - feature_dict = { - col: _serialize_value(df_agg[col][row_idx]) - for col in feature_cols - } - profiles.append(EntityProfile( - run=run, - entity_value=entity_val, - feature_json=feature_dict, - )) - - try: - EntityProfile.objects.bulk_create(profiles, ignore_conflicts=True) - except Exception: - return False - - # Update run metadata - run.status = 'completed' - run.entity_count = entity_count - run.entity_column = entity_column - run.save(update_fields=['status', 'entity_count', 'entity_column']) - - return True - - -def _serialize_value(val) -> object: - """Convert a Polars/Numpy scalar to a JSON-safe Python value.""" - if val is None: - return None - if isinstance(val, (int, float, str, bool)): - # Handle NaN / Inf - if isinstance(val, float): - import math - if math.isnan(val) or math.isinf(val): - return None - return val - if isinstance(val, (list, tuple)): - return [_serialize_value(v) for v in val] - return str(val) - - -# ── 13. compute_scores ───────────────────────────────────────────────────── +# ── 12. compute_scores ───────────────────────────────────────────────────── async def _handle_compute_scores(dataset_id: str) -> dict: """Compute adaptive proxy/anomaly/threat scores on entity data.""" - from analysis.entity_aggregator import _ANOMALY_FEATURE_COLS, _add_adaptive_scores store = SessionStore() entry = store.get_dataset(dataset_id) if entry is None: @@ -2027,6 +1934,282 @@ async def _handle_compute_scores(dataset_id: str) -> dict: return {'status': 'scored', 'dataset_id': dataset_id} +# ── 13. filter_and_cluster ───────────────────────────────────────────────── + +async def _handle_filter_and_cluster( + dataset_id: str, + filters: list[dict], + logic: str = 'and', + algorithm: str = 'hdbscan', + params: dict | None = None, +) -> dict: + """Apply column-level filters then run clustering on the filtered result.""" + store = SessionStore() + entry = store.get_dataset(dataset_id) + if entry is None: + return {'error': f'Dataset not found: {dataset_id}', 'truncated': False} + + lf: pl.LazyFrame = entry['lazyframe'] + full_schema = entry.get('schema', {}) + + # ── Step 1: Apply filters ───────────────────────────────────────────── + try: + exprs = [_build_filter_expr( + f['column'], f.get('op', 'eq'), f.get('value'), + ) for f in filters] + + if exprs: + if logic == 'and': + combined = exprs[0] + for e in exprs[1:]: + combined = combined & e + else: + combined = exprs[0] + for e in exprs[1:]: + combined = combined | e + filtered_lf = lf.filter(combined) + else: + filtered_lf = lf + + # Estimate row count (fast streaming count) + try: + row_count = filtered_lf.select(pl.lit(1).count()).collect(streaming=True).item() + except Exception: + row_count = None + + except Exception as e: + return {'error': f'Filter failed: {e}', 'truncated': False} + + # ── Step 2: Auto-detect numeric feature columns ─────────────────────── + try: + available = filtered_lf.collect_schema() + available_names = available.names() + available_dtypes = available.dtypes() + numeric_dtypes = (pl.Int8, pl.Int16, pl.Int32, pl.Int64, + pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64, + pl.Float32, pl.Float64) + feature_cols = [ + available_names[i] for i, dt in enumerate(available_dtypes) + if dt in numeric_dtypes and not available_names[i].startswith('_') + ][:10] + + if not feature_cols: + return { + 'error': 'No numeric columns found for clustering after filtering', + 'filtered_dataset_id': None, + 'truncated': False, + } + except Exception as e: + return {'error': f'Feature detection failed: {e}', 'truncated': False} + + # ── Step 3: Store filtered dataset ──────────────────────────────────── + filtered_dataset_id = _generate_id('flt') + store.store_dataset( + dataset_id=filtered_dataset_id, + lazyframe=filtered_lf, + schema=full_schema, + metadata={ + 'row_count': row_count, + 'parent_dataset_id': dataset_id, + 'filter_count': len(filters), + 'logic': logic, + }, + parent_id=dataset_id, + ) + + # ── Step 4: Run clustering (inlined from _handle_run_clustering) ─────── + p = params or {} + try: + # Downsample before collect + try: + total_count = filtered_lf.select(pl.len()).collect(streaming=True).item() + except Exception: + total_count = 0 + + clamp_lf = filtered_lf + if total_count > 100000: + clamp_lf = filtered_lf.collect(streaming=True).sample(n=100000, seed=42).lazy() + + df_features = clamp_lf.select(feature_cols).collect(streaming=True) + + # Drop all-NaN columns + for col in df_features.columns: + if df_features[col].is_null().all(): + df_features = df_features.drop(col) + feature_cols = [c for c in feature_cols if c != col] + + if df_features.width == 0: + return {'error': 'No valid feature columns after dropping all-NaN columns', + 'filtered_dataset_id': filtered_dataset_id, 'truncated': False} + + data = df_features.to_numpy() + + # Fill remaining NaN with column mean + if data.dtype.kind == 'f': + col_mean = np.nanmean(data, axis=0) + col_mean = np.nan_to_num(col_mean, nan=0.0) + data = np.where(np.isnan(data), col_mean, data) + + # Too-few-samples guard + if len(data) < 3: + return { + 'cluster_result_id': _generate_id('clust'), + 'n_clusters': 0, + 'n_noise': 0, + 'quality_metrics': {'note': 'too few samples'}, + 'filtered_dataset_id': filtered_dataset_id, + 'truncated': False, + } + + # Low memory check + low_memory = False + try: + import psutil + mem = psutil.virtual_memory() + low_memory = mem.available < 2 * 1024 ** 3 + except ImportError: + pass + + if low_memory and len(data) > 100000: + rng = np.random.default_rng(42) + idx = rng.choice(len(data), 50000, replace=False) + data = data[idx] + + # Downsample to 50K max + _ds_idx = None + if len(data) > 50000: + rng = np.random.default_rng(42) + _ds_idx = rng.choice(len(data), 50000, replace=False) + data = data[_ds_idx] + + # Variance filtering + if data.shape[1] > 1: + variances = np.var(data, axis=0) + keep_var = np.where(variances >= 1e-10)[0] + if len(keep_var) == 0: + keep_var = np.arange(data.shape[1]) + if len(keep_var) < data.shape[1]: + data = data[:, keep_var] + feature_cols = [feature_cols[i] for i in keep_var] + + # Standard scale + from sklearn.preprocessing import StandardScaler + data_scaled = StandardScaler().fit_transform(data) + + # Correlation filtering + if data_scaled.shape[1] > 1: + corr = np.corrcoef(data_scaled.T) + upper = np.triu(np.abs(corr), 1) + to_drop = [j for j in range(upper.shape[1]) if any(upper[:, j] > 0.95)] + if to_drop: + keep = [j for j in range(data_scaled.shape[1]) if j not in to_drop] + data_scaled = data_scaled[:, keep] + feature_cols = [feature_cols[j] for j in keep] + + # PCA dimensionality reduction (if >50 features) + n_features = data_scaled.shape[1] + if n_features > 50: + from sklearn.decomposition import PCA + data_scaled = PCA(n_components=50, random_state=42).fit_transform(data_scaled) + + # Hyperparameter auto-tuning + n = len(data_scaled) + if algorithm == 'hdbscan': + min_cluster = p.get('min_cluster_size', max(3, min(50, n // 20))) + p['min_cluster_size'] = min_cluster + p['min_samples'] = p.get('min_samples', min_cluster) + p['cluster_selection_epsilon'] = p.get('cluster_selection_epsilon', 0.0) + else: + p['min_cluster_size'] = min( + p.get('min_cluster_size', 5), + max(n // 2, 2), + ) + + # Fit model + from sklearn.cluster import HDBSCAN, MiniBatchKMeans + if algorithm == 'kmeans': + n_clusters_param = p.get('n_clusters', 3) + model = MiniBatchKMeans( + n_clusters=n_clusters_param, random_state=42, + batch_size=1024, n_init='auto', + ) + else: + model = HDBSCAN( + min_cluster_size=p['min_cluster_size'], + min_samples=p.get('min_samples', None), + metric=p.get('metric', 'euclidean'), + cluster_selection_epsilon=p.get('cluster_selection_epsilon', 0.0), + ) + + labels_array = model.fit_predict(data_scaled) + n_clusters_found = int(len(set(labels_array)) - (1 if -1 in labels_array else 0)) + noise_count = int((labels_array == -1).sum()) if algorithm == 'hdbscan' else 0 + noise_ratio = round(noise_count / len(labels_array), 4) if algorithm == 'hdbscan' else 0.0 + + # Quality metrics + quality = { + 'n_clusters': n_clusters_found, + 'n_noise': noise_count, + 'noise_ratio': noise_ratio, + 'algorithm': algorithm, + } + + if n_clusters_found > 1: + try: + from sklearn.metrics import silhouette_score + score = silhouette_score(data_scaled, labels_array, random_state=42) + quality['silhouette_score'] = round(float(score), 4) + except Exception: + quality['silhouette_score'] = None + try: + from sklearn.metrics import davies_bouldin_score + dbs = davies_bouldin_score(data_scaled, labels_array) + quality['davies_bouldin_score'] = round(float(dbs), 4) + except Exception: + quality['davies_bouldin_score'] = None + try: + from sklearn.metrics import calinski_harabasz_score + chs = calinski_harabasz_score(data_scaled, labels_array) + quality['calinski_harabasz_score'] = round(float(chs), 4) + except Exception: + quality['calinski_harabasz_score'] = None + + # Cluster sizes + cluster_sizes = {} + for label in np.unique(labels_array): + cluster_sizes[int(label)] = int((labels_array == label).sum()) + quality['cluster_sizes'] = cluster_sizes + + except Exception as e: + return {'error': f'Clustering failed: {e}', + 'filtered_dataset_id': filtered_dataset_id, 'truncated': False} + + result_id = _generate_id('clust') + store.store_cluster_result( + result_id=result_id, + labels=labels_array.tolist(), + n_clusters=n_clusters_found, + params={ + 'algorithm': algorithm, + 'params': p, + 'random_state': 42, + 'feature_columns': feature_cols, + '_ds_idx': _ds_idx.tolist() if _ds_idx is not None else None, + }, + parent_dataset_id=filtered_dataset_id, + ) + + return { + 'cluster_result_id': result_id, + 'filtered_dataset_id': filtered_dataset_id, + 'n_clusters': n_clusters_found, + 'n_noise': noise_count, + 'labels_sample': labels_array[:50].tolist(), + 'quality_metrics': quality, + 'feature_columns': feature_cols, + } + + # ── 14. detect_anomalies ────────────────────────────────────────────────── async def _handle_detect_anomalies(dataset_id: str, contamination: float = 0.05) -> dict: @@ -2092,7 +2275,7 @@ async def _handle_visualize_anomalies(dataset_id: str) -> dict: n_total = lf.select(pl.len()).collect(streaming=True).item() except Exception: n_total = 0 - sample_lf = lf if n_total <= 10000 else lf.sample(n=10000, seed=42) + sample_lf = lf if n_total <= 10000 else lf.collect(streaming=True).sample(n=10000, seed=42).lazy() df = sample_lf.select(num_cols).collect(streaming=True) mat = StandardScaler().fit_transform(df.to_numpy()) mat = np.nan_to_num(mat, nan=0.0) @@ -2172,7 +2355,7 @@ async def _handle_explore_distributions(dataset_id: str, columns: list[str] = No columns = [c for c in schema if schema[c].split('(')[0].strip() in numeric_types][:10] try: n = lf.select(pl.len()).collect(streaming=True).item() - sample_lf = lf if n <= max_sample else lf.sample(n=max_sample, seed=42) + sample_lf = lf if n <= max_sample else lf.collect(streaming=True).sample(n=max_sample, seed=42).lazy() df = sample_lf.select(columns).collect(streaming=True) except Exception as exc: return {'error': f'Failed to collect sample: {exc}', 'truncated': False} @@ -2213,7 +2396,7 @@ async def _handle_find_outliers(dataset_id: str, columns: list[str] = None, columns = [c for c in schema if schema[c].split('(')[0].strip() in numeric_types] try: n = lf.select(pl.len()).collect(streaming=True).item() - sample_lf = lf if n <= 100000 else lf.sample(n=100000, seed=42) + sample_lf = lf if n <= 100000 else lf.collect(streaming=True).sample(n=100000, seed=42).lazy() df = sample_lf.select(columns).collect(streaming=True) except Exception as exc: return {'error': f'Failed: {exc}', 'truncated': False} @@ -2607,3 +2790,164 @@ async def _handle_analyze_entity_detail(dataset_id: str, entity_value: str) -> d pass return result + + +# ── 28. compute_distance_matrix ────────────────────────────────────────── + +def _indent_function_body(body: str, indent: str = ' ') -> str: + """Indent each non-empty line of a code block for wrapping in a def.""" + return '\n'.join( + indent + line if line.strip() else line + for line in body.split('\n') + ) + + +async def _handle_compute_distance_matrix( + dataset_id: str, + python_function: str, + columns: Optional[list[str]] = None, + max_sample: int = 10000, +) -> dict: + """Evaluate a user-provided Python function on each row of a dataset. + + Wraps the function body as ``def distance_fn(row): ``, executes + it per-row in a restricted namespace (``math``, ``numpy`` only), and + returns a distance-score summary plus a sample of scored rows. + """ + store = SessionStore() + entry = store.get_dataset(dataset_id) + if entry is None: + return {'error': f'Dataset not found: {dataset_id}', 'truncated': False} + + lf: pl.LazyFrame = entry['lazyframe'] + + # Collect data (streaming for large datasets) + try: + df = lf.collect(streaming=True) + except Exception as e: + return {'error': f'Data collection failed: {e}', 'truncated': False} + + # Determine columns to pass to the function + if columns is None: + numeric_dtypes = (pl.Int8, pl.Int16, pl.Int32, pl.Int64, + pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64, + pl.Float32, pl.Float64) + columns = [c for c in df.columns + if df[c].dtype in numeric_dtypes] + + # Filter to columns that actually exist + available = set(df.columns) + columns = [c for c in columns if c in available] + if not columns: + return {'error': 'No valid columns found in dataset', + 'truncated': False} + + n_total = len(df) + + # Sample if larger than max_sample + if n_total > max_sample: + df_sampled = df.sample(n=max_sample, seed=42) + n_evaluated = max_sample + else: + df_sampled = df + n_evaluated = n_total + + # Build list-of-dicts for the selected columns + rows = df_sampled.select(columns).to_dicts() + + # ── Safe execution ──────────────────────────────────────────────────── + safe_builtins = { + 'abs': abs, 'len': len, 'range': range, + 'min': min, 'max': max, 'sum': sum, 'round': round, + 'int': int, 'float': float, 'str': str, 'bool': bool, + 'list': list, 'dict': dict, 'tuple': tuple, + 'isinstance': isinstance, 'type': type, + 'True': True, 'False': False, 'None': None, + 'enumerate': enumerate, 'zip': zip, 'map': map, + 'filter': filter, 'sorted': sorted, 'reversed': reversed, + } + + safe_ns = { + '__builtins__': safe_builtins, + 'math': __import__('math'), + 'np': __import__('numpy'), + } + + # Wrap the user's function body in a def statement + body = python_function.strip() + if body.startswith('def '): + # User provided the full function definition + full_code = body + else: + indented = _indent_function_body(body) + full_code = f'def distance_fn(row):\n{indented}' + + try: + exec(full_code, safe_ns) + # Resolve: the function name might be 'distance_fn' or whatever the user named it + distance_fn = safe_ns.get('distance_fn') + if distance_fn is None: + # Fallback: try to find any callable in safe_ns + for _name, _obj in safe_ns.items(): + if callable(_obj) and _name not in ('math', 'np', '__builtins__'): + distance_fn = _obj + break + if distance_fn is None: + return {'error': 'Could not locate the compiled function', + 'truncated': False} + except SyntaxError as e: + return {'error': f'Function syntax error: {e}', 'truncated': False} + except Exception as e: + return {'error': f'Function compilation failed: {e}', + 'truncated': False} + + # ── Apply function per row ──────────────────────────────────────────── + scores: list[float | None] = [] + errors = 0 + for row in rows: + try: + score = distance_fn(row) + if isinstance(score, (int, float)): + scores.append(float(score)) + else: + scores.append(None) + errors += 1 + except Exception: + scores.append(None) + errors += 1 + + valid_scores = [s for s in scores if s is not None] + if not valid_scores: + return {'error': 'No valid scores computed — all rows raised errors', + 'truncated': False} + + # ── Distance summary ────────────────────────────────────────────────── + arr = np.array(valid_scores) + summary = { + 'min': float(np.min(arr)), + 'max': float(np.max(arr)), + 'mean': float(np.mean(arr)), + 'std': float(np.std(arr)), + 'count': len(arr), + 'errors': errors, + 'total_rows_in_sample': len(rows), + 'matrix_shape': [len(rows), len(columns)], + } + + # Sample of scored rows (first 30, keep score + first 3 values) + scored_sample = [] + for i, row in enumerate(rows[:30]): + if i < len(scores): + scored_sample.append({ + 'index': i, + 'values': {c: row.get(c) for c in columns[:3]}, + 'score': scores[i], + }) + + return { + 'summary': summary, + 'scored_sample': scored_sample, + 'columns_used': columns, + 'total_rows_in_dataset': n_total, + 'rows_evaluated': n_evaluated, + } diff --git a/analysis/urls.py b/analysis/urls.py index 1a8a5c5..0499e89 100644 --- a/analysis/urls.py +++ b/analysis/urls.py @@ -11,15 +11,15 @@ urlpatterns = [ path('analyze/auto/', views.auto_page, name='auto'), path('analyze/llm/', views.run_llm_analysis_view, name='run_llm_analysis'), path('runs/', views.run_list, name='run_list'), - path('runs//', views.run_detail, name='run_detail'), - path('runs//retry/', views.retry_analysis, name='retry_analysis'), - path('runs//status/', views.run_status_api, name='run_status'), - path('clusters//', views.cluster_overview, name='cluster_overview'), - path('clusters///', views.cluster_detail, name='cluster_detail'), + path('runs//', views.run_detail, name='run_detail'), + path('runs//status/', views.run_status_api, name='run_status'), + path('clusters//', views.cluster_overview, name='cluster_overview'), + path('clusters///', views.cluster_detail, name='cluster_detail'), path('entities//', views.entity_profile, name='entity_profile'), path('config/', views.config_view, name='config'), path('api/llm/test/', views.llm_test, name='llm_test'), - path('runs//delete/', views.delete_upload, name='delete_upload'), + path('runs//retry/', views.retry_run, name='retry_run'), + path('runs//delete/', views.delete_upload, name='delete_upload'), path('logs/', views.log_viewer, name='log_viewer'), path('globe/', views.globe_view, name='globe'), path('tools/run/', views.tool_lab_run, name='tool_lab_run'), diff --git a/analysis/views.py b/analysis/views.py index 442da39..8b1c167 100644 --- a/analysis/views.py +++ b/analysis/views.py @@ -21,21 +21,28 @@ def dashboard(request): def run_list(request): - """List all analysis runs with pagination.""" + """List all analysis runs with pagination and optional ?type= filter.""" + run_type_filter = request.GET.get('type') page = int(request.GET.get('page', 1)) per_page = 20 - total = AnalysisRun.objects.count() - runs = AnalysisRun.objects.all()[(page - 1) * per_page: page * per_page] + + qs = AnalysisRun.objects.all() + if run_type_filter in dict(AnalysisRun.RUN_TYPE_CHOICES): + qs = qs.filter(run_type=run_type_filter) + + total = qs.count() + runs = qs[(page - 1) * per_page: page * per_page] return render(request, 'analysis/run_list.html', { 'runs': runs, 'page': page, 'total_pages': (total + per_page - 1) // per_page if total else 1, + 'current_type': run_type_filter or '', }) -def run_detail(request, run_id): +def run_detail(request, display_id): """Details for a single analysis run.""" - run = get_object_or_404(AnalysisRun, id=run_id) + run = get_object_or_404(AnalysisRun, display_id=display_id) clusters = run.clusters.all().order_by('-size') return render(request, 'analysis/run_detail.html', { 'run': run, @@ -43,74 +50,6 @@ def run_detail(request, run_id): }) -@csrf_exempt -def retry_analysis(request, run_id): - """Retry a failed analysis run. Creates a new run with same parameters.""" - if request.method != 'POST': - return JsonResponse({'error': 'POST required'}, status=405) - - original = get_object_or_404(AnalysisRun, id=run_id) - if original.status != 'failed': - return JsonResponse({'error': '只能重试失败的分析'}, status=400) - - # Create new run copying original parameters - new_run = AnalysisRun.objects.create( - csv_glob=original.csv_glob, - entity_column=original.entity_column, - status='pending', - total_flows=original.total_flows, - ) - - def _retry_fn(run, ctx): - from analysis.data_loader import load_csv_directory - from analysis.session_store import SessionStore - from analysis.entity_detector import detect_entity_column - from analysis.entity_aggregator import aggregate_by_entity - - store = SessionStore() - try: - lf, schema, row_count, file_count, memory_mb = load_csv_directory(run.csv_glob) - ds_id = f'retry_{run.id}' - store.store_dataset(ds_id, lf, schema=schema, metadata={ - 'row_count': row_count, 'file_count': file_count, 'csv_glob': run.csv_glob, - }) - run.total_flows = row_count - run.save(update_fields=['total_flows']) - - # Detect entity column - result = detect_entity_column(ds_id) - entity_col = result.get('recommended') - if not entity_col: - raise Exception('Could not detect entity column') - - run.entity_column = entity_col - run.save(update_fields=['entity_column']) - - # Aggregate - agg_lf, feature_columns = aggregate_by_entity(lf, entity_col, schema) - df_agg = agg_lf.collect(streaming=True) - entity_ds_id = f'entity_{run.id}' - store.store_dataset(entity_ds_id, agg_lf, schema=dict(zip(feature_columns, ['']*len(feature_columns)))) - run.entity_count = len(df_agg) - run.save(update_fields=['entity_count']) - - # Run clustering pipeline (same defaults as manual analysis) - _run_clustering_pipeline( - run=run, store=store, entity_ds_id=entity_ds_id, - feature_columns=None, algorithm='hdbscan', - min_cluster_size=5, run_pca=True, - ) - except Exception as e: - run.status = 'failed' - run.error_message = str(e) - run.save(update_fields=['status', 'error_message']) - - t = threading.Thread(target=_run_pipeline_worker, args=(new_run.id, _retry_fn), daemon=True) - t.start() - - return JsonResponse({'status': 'started', 'run_id': new_run.id, 'redirect': f'/runs/{new_run.id}/'}) - - def _extract_lat(val): """Try to extract a latitude from *val* (number or str).""" if val is None: @@ -133,9 +72,9 @@ def _extract_lon(val): return None -def cluster_overview(request, run_id): +def cluster_overview(request, display_id): """Overview of all clusters for a run, with PCA scatter data and geo scatter data.""" - run = get_object_or_404(AnalysisRun, id=run_id) + run = get_object_or_404(AnalysisRun, display_id=display_id) clusters = run.clusters.exclude(cluster_label=-1).order_by('-size').prefetch_related('features') noise = run.clusters.filter(cluster_label=-1).first() @@ -191,9 +130,9 @@ def cluster_overview(request, run_id): }) -def cluster_detail(request, run_id, cluster_label): +def cluster_detail(request, display_id, cluster_label): """Detailed view of a single cluster with entity list.""" - run = get_object_or_404(AnalysisRun, id=run_id) + run = get_object_or_404(AnalysisRun, display_id=display_id) cluster = get_object_or_404(ClusterResult, run=run, cluster_label=cluster_label) features = cluster.features.all().order_by('-distinguishing_score')[:50] entities = cluster.entities.all()[:50] @@ -370,8 +309,6 @@ def upload_csv(request): files = request.FILES.getlist('files') if not files: return JsonResponse({'error': 'No files uploaded'}) - if len(files) > 10000: - return JsonResponse({'error': '一次最多上传 10000 个文件'}) for f in files: if f.size > 500 * 1024 * 1024: return JsonResponse({'warning': f'文件 {f.name} 超过 500MB，建议分割后上传'}) @@ -398,21 +335,40 @@ def upload_csv(request): run = AnalysisRun.objects.create( csv_glob=str(upload_dir / '*.csv'), status='loading', + run_type='upload', ) t = threading.Thread(target=_background_process, args=(run.id, upload_dir), daemon=True) t.start() - return JsonResponse({'run_id': run.id, 'files': saved, 'status': 'loading'}) + return JsonResponse({'run_id': run.display_id, 'files': saved, 'status': 'loading'}) -def delete_upload(request, run_id): - """Delete uploaded files and AnalysisRun.""" - run = get_object_or_404(AnalysisRun, id=run_id) +def delete_upload(request, display_id): + """Delete uploaded files, SQLite data table, and AnalysisRun. + + Handles all run statuses (loading, profiling, clustering, completed, failed). + Gracefully tolerates null csv_glob and missing/errored SQLite tables. + """ + run = get_object_or_404(AnalysisRun, display_id=display_id) + import logging import shutil - upload_dir = None + + _logger = logging.getLogger(__name__) + + # 1. Drop the SQLite data table if one exists + if run.sqlite_table: + try: + from analysis.data_loader import drop_sqlite_table # fmt: skip + drop_sqlite_table(run.sqlite_table) + except Exception as exc: + _logger.warning('[DELETE] failed to drop SQLite table %s: %s', run.sqlite_table, exc) + + # 2. Remove the upload directory if csv_glob is set if run.csv_glob: p = Path(run.csv_glob).parent if p.exists(): shutil.rmtree(p, ignore_errors=True) + + # 3. Delete the ORM record run.delete() return JsonResponse({'deleted': True}) @@ -422,14 +378,11 @@ logger = logging.getLogger(__name__) def _background_process(run_id, upload_dir): - """Background: load → detect → aggregate → ready.""" + """Background: stream-load CSVs → save to SQLite → ready for analysis.""" import django os.environ['DJANGO_SETTINGS_MODULE'] = 'tianxuan.settings' django.setup() from analysis.models import AnalysisRun - from analysis.data_loader import load_csv_directory - from analysis.entity_detector import detect_entity_column - from analysis.entity_aggregator import aggregate_by_entity from analysis.session_store import SessionStore run = AnalysisRun.objects.get(id=run_id) @@ -444,56 +397,88 @@ def _background_process(run_id, upload_dir): run.save(update_fields=['status', 'error_message']) return - run.status = 'loading'; run.progress_pct = 10; run.progress_msg = '正在加载数据...' + import glob + import polars as pl + csv_paths = sorted(glob.glob(str(upload_dir / '*.csv'))) + if not csv_paths: + msg = '没有找到 CSV 文件' + logger.error('[BACKGROUND] %s (run_id=%s)', msg, run_id) + run.status = 'failed' + run.error_message = msg + run.save(update_fields=['status', 'error_message']) + return + + total_files = len(csv_paths) + # Adaptive batch size: aim for ~20 batches total, cap at 500 files per batch + batch_size = max(1, min(500, total_files // 20 or 1)) + total_batches = (total_files + batch_size - 1) // batch_size + + run.status = 'loading'; run.progress_pct = 10 + run.progress_msg = f'正在流式加载 {total_files} 个文件（共 {total_batches} 批）...' run.save(update_fields=['status', 'progress_pct', 'progress_msg']) - lf, schema, row_count, file_count, mem = load_csv_directory( - str(upload_dir / '*.csv'), schema_strict=False - ) + + from analysis.data_loader import save_to_db, drop_sqlite_table, load_from_db + + table_name = f'_data_{run_id}' + completed_batches = 0 + schema = {} + total_rows = 0 + + # ── Phase 1: Stream CSVs into SQLite in batches ───────────────── + for i in range(0, total_files, batch_size): + batch_files = csv_paths[i:i + batch_size] + mode = 'replace' if i == 0 else 'append' + + batch_lf = pl.scan_csv(batch_files, low_memory=True) + + # Capture schema from first batch (efficient — header-only scan) + if not schema: + raw_schema = batch_lf.collect_schema() + schema = {name: str(dtype) for name, dtype in raw_schema.items()} + + save_to_db(run.id, batch_lf, mode=mode) + + completed_batches += 1 + run.progress_pct = int((completed_batches / total_batches) * 80 + 10) + run.progress_msg = f'正在流式加载... ({completed_batches}/{total_batches} 批)' + run.save(update_fields=['progress_pct', 'progress_msg']) + + # ── Phase 2: Load from SQLite for analysis ───────────────────── + lf = load_from_db(table_name) + if lf is None: + raise RuntimeError(f'SQLite 表 {table_name} 加载失败') + + # Get row count + df_count = lf.select(pl.len()).collect(streaming=True) + total_rows = df_count[0, 0] if df_count.height > 0 else 0 + + run.sqlite_table = table_name + run.total_flows = total_rows + run.save(update_fields=['sqlite_table', 'total_flows']) + ds_id = f'upload_{run_id}' store.store_dataset(ds_id, lf, schema=schema, metadata={ - 'row_count': row_count, 'file_count': file_count, 'upload_dir': str(upload_dir), + 'row_count': total_rows, 'file_count': total_files, 'upload_dir': str(upload_dir), 'csv_glob': str(upload_dir / '*.csv'), + 'sqlite_table': table_name, }) - run.total_flows = row_count; run.save(update_fields=['total_flows']) - - run.status = 'profiling'; run.progress_pct = 30; run.progress_msg = '正在分析数据特征...' - run.save(update_fields=['status', 'progress_pct', 'progress_msg']) - entity_result = detect_entity_column(ds_id) - entity_col = entity_result.get('recommended', '') - entity_cols_multi = entity_result.get('recommended_multi', []) - # Store as comma-separated (multi) or single-column - if entity_cols_multi: - entity_cols = entity_cols_multi[:2] # top 2 - run.entity_column = ','.join(entity_cols) - else: - entity_cols = [entity_col] if entity_col else [] - run.entity_column = entity_col - run.save(update_fields=['entity_column']) - - if entity_cols: - run.status = 'aggregating'; run.progress_pct = 50; run.progress_msg = '正在构建实体画像...' - run.save(update_fields=['status', 'progress_pct', 'progress_msg']) - agg_lf, features = aggregate_by_entity(lf, entity_cols, schema) - df_agg = agg_lf.collect(streaming=True) - entity_ds_id = f'entity_{run_id}' - store.store_dataset(entity_ds_id, agg_lf, - schema=dict(zip(features, [''] * len(features))), - metadata={'row_count': len(df_agg), 'entity_column': ','.join(entity_cols), - 'csv_glob': str(upload_dir / '*.csv')}) - run.entity_count = len(df_agg) - run.entity_column = ','.join(entity_cols) - run.save(update_fields=['entity_column', 'entity_count']) + # ── Phase 3: Data ready for filtering/clustering via MCP tools ── run.status = 'ready'; run.progress_pct = 60; run.progress_msg = '数据准备就绪' run.save(update_fields=['status', 'progress_pct', 'progress_msg']) except Exception: tb = traceback.format_exc() logger.error(tb) + # Drop partial SQLite table on failure (idempotent) + try: + from analysis.data_loader import drop_sqlite_table # fmt: skip + drop_sqlite_table(f'_data_{run_id}') + except Exception: + pass run.status = 'failed' - run.run_log += f'\n[ERROR] {tb}' - run.save(update_fields=['run_log']) run.error_message = tb - run.save(update_fields=['status', 'error_message']) + run.run_log += f'\n[ERROR] {tb}' + run.save(update_fields=['status', 'error_message', 'run_log']) # ── Manual analysis page ─────────────────────────────────────────────── @@ -547,7 +532,16 @@ def _run_pipeline_worker(run_id, analysis_fn, **ctx): os.environ['DJANGO_ALLOW_ASYNC_UNSAFE'] = 'true' django.setup() - run = AnalysisRun.objects.get(id=run_id) + from analysis.db_utils import retry_on_lock + from django.db import close_old_connections + + close_old_connections() + + @retry_on_lock() + def _get_run(): + return AnalysisRun.objects.get(id=run_id) + + run = _get_run() try: analysis_fn(run, ctx) except Exception: @@ -557,7 +551,7 @@ def _run_pipeline_worker(run_id, analysis_fn, **ctx): run.status = 'failed' run.error_message = tb run.run_log += f'\n[FATAL] {tb}' - run.save(update_fields=['status', 'error_message', 'run_log']) + retry_on_lock()(lambda: run.save(update_fields=['status', 'error_message', 'run_log']))() # ── Shared clustering pipeline ───────────────────────────────────────── @@ -619,16 +613,13 @@ def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorith numeric_types = {'Int8', 'Int16', 'Int32', 'Int64', 'UInt8', 'UInt16', 'UInt32', 'UInt64', 'Float32', 'Float64'} - entity_cols_set = set() - if hasattr(run, 'entity_column') and run.entity_column: - entity_cols_set = {c.strip() for c in run.entity_column.split(',')} if feature_columns: - feature_cols = [c for c in feature_columns if c in schema and c not in entity_cols_set] + feature_cols = [c for c in feature_columns if c in schema] else: feature_cols = [ c for c in schema.keys() - if not c.startswith('_') and c not in entity_cols_set + if not c.startswith('_') and schema[c].split('(')[0].strip() in numeric_types ][:10] @@ -753,11 +744,10 @@ def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorith logger.error(tb) try: run.status = 'failed' + run.error_message = tb run.progress_msg = f'失败: {tb}' run.run_log += f'\n[ERROR] {tb}' - run.save(update_fields=['run_log']) - run.error_message = tb - run.save(update_fields=['status', 'progress_msg', 'error_message']) + run.save(update_fields=['status', 'error_message', 'progress_msg', 'run_log']) except Exception: pass @@ -766,14 +756,16 @@ def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorith @csrf_exempt def run_analysis(request): - """AJAX POST: run analysis pipeline and return result page URL.""" + """AJAX POST: run clustering pipeline on the upload dataset.""" if request.method != 'POST': return JsonResponse({'error': 'POST required'}, status=405) import json body = json.loads(request.body) - run_id = body.get('run_id') - entity_col = body.get('entity_column', '') - entity_columns = body.get('entity_columns') + display_id = body.get('run_id') + run = get_object_or_404(AnalysisRun, display_id=display_id) + pk = run.id + run.run_type = 'manual' + run.save(update_fields=['run_type']) feature_columns = body.get('feature_columns') algorithm = body.get('algorithm', 'hdbscan') min_cluster_size = int(body.get('min_cluster_size', 5)) @@ -781,53 +773,32 @@ def run_analysis(request): def _analysis_fn(run, ctx): from analysis.session_store import SessionStore - from analysis.entity_aggregator import aggregate_by_entity - entity_columns = ctx['entity_columns'] feature_columns = ctx['feature_columns'] algorithm = ctx['algorithm'] min_cluster_size = ctx['min_cluster_size'] head = ctx.get('head') + pk = ctx['pk'] store = SessionStore() try: - if entity_columns: - upload_ds_id = f'upload_{run_id}' - upload_entry = store.get_dataset(upload_ds_id) - if upload_entry is None: - run.error_message = '请先上传并等待预处理完成' - run.status = 'failed' - run.save(update_fields=['status', 'error_message']) - return - upload_lf = upload_entry['lazyframe'] - upload_schema = upload_entry.get('schema', {}) - agg_lf, agg_features = aggregate_by_entity(upload_lf, entity_columns, upload_schema) - df_agg = agg_lf.collect(streaming=True) - agg_schema = {n: str(d) for n, d in zip(df_agg.columns, [df_agg[c].dtype for c in df_agg.columns])} - entity_ds_id = f'entity_{run_id}_custom' - store.store_dataset( - dataset_id=entity_ds_id, - lazyframe=df_agg.lazy(), - schema=agg_schema, - metadata={'row_count': len(df_agg), 'entity_columns': entity_columns, - 'csv_glob': run.csv_glob}, - ) - run.entity_column = ','.join(entity_columns) - run.entity_count = len(df_agg) - run.save(update_fields=['entity_column', 'entity_count']) - else: - entity_ds_id = f'entity_{run_id}' - entry = store.get_dataset(entity_ds_id) - if entry is None: - run.error_message = '请先上传并等待预处理完成' - run.status = 'failed' - run.save(update_fields=['status', 'error_message']) - return + upload_ds_id = f'upload_{pk}' + entry = store.get_dataset(upload_ds_id) + if entry is None: + # Fallback: try entity dataset (from old pipeline) + entry = store.get_dataset(f'entity_{pk}') + if entry is None: + run.error_message = '请先上传并等待预处理完成' + run.status = 'failed' + run.save(update_fields=['status', 'error_message']) + return + + ds_id = upload_ds_id if store.get_dataset(upload_ds_id) else f'entity_{pk}' # Use the unified clustering pipeline (clustering → extraction → PCA) from analysis.views import _run_clustering_pipeline _run_clustering_pipeline( - run=run, store=store, entity_ds_id=entity_ds_id, + run=run, store=store, entity_ds_id=ds_id, feature_columns=feature_columns, algorithm=algorithm, min_cluster_size=min_cluster_size, run_pca=True, head=head, @@ -836,22 +807,20 @@ def run_analysis(request): tb = traceback.format_exc() logger.error(tb) run.status = 'failed' + run.error_message = tb run.progress_msg = f'失败: {tb}' run.run_log += f'\n[ERROR] {tb}' - run.save(update_fields=['run_log']) - run.error_message = tb - run.save(update_fields=['status', 'progress_msg', 'error_message']) + run.save(update_fields=['status', 'error_message', 'progress_msg', 'run_log']) t = threading.Thread( target=_run_pipeline_worker, args=(run_id, _analysis_fn), - kwargs=dict(entity_col=entity_col, entity_columns=entity_columns, - feature_columns=feature_columns, algorithm=algorithm, - min_cluster_size=min_cluster_size, head=head), + kwargs=dict(feature_columns=feature_columns, algorithm=algorithm, + min_cluster_size=min_cluster_size, head=head, pk=pk), daemon=True, ) t.start() - return JsonResponse({'status': 'started', 'redirect': f'/runs/{run_id}/'}) + return JsonResponse({'status': 'started', 'redirect': f'/runs/{run.display_id}/'}) @@ -866,42 +835,68 @@ def run_llm_analysis_view(request): return JsonResponse({'error': 'POST required'}, status=405) import json body = json.loads(request.body) - run_id = body.get('run_id') - entity_col = body.get('entity_column', '') + display_id = body.get('run_id') + run = get_object_or_404(AnalysisRun, display_id=display_id) + pk = run.id + run.run_type = 'auto' + run.save(update_fields=['run_type']) head = body.get('head') from analysis.session_store import SessionStore store = SessionStore() - upload_ds_id = f'upload_{run_id}' + upload_ds_id = f'upload_{pk}' entry = store.get_dataset(upload_ds_id) if entry is None: # Try entity dataset as fallback - entity_ds_id = f'entity_{run_id}' + entity_ds_id = f'entity_{pk}' entry = store.get_dataset(entity_ds_id) if entry is None: return JsonResponse({'error': '数据集不存在，请先上传并等待预处理完成'}) - dataset_id = upload_ds_id if store.get_dataset(upload_ds_id) else f'entity_{run_id}' + dataset_id = upload_ds_id if store.get_dataset(upload_ds_id) else f'entity_{pk}' def _analysis_fn(run, ctx): from tianxuan.llm_orchestrator import run_llm_pipeline, LLMConfig dataset_id = ctx['dataset_id'] - entity_col = ctx['entity_col'] + pk = ctx['pk'] - def progress_callback(step, tool_name, result): + def progress_callback(step, tool_name, result=None, meta=None): nonlocal run try: - run = AnalysisRun.objects.get(id=run_id) + run = AnalysisRun.objects.get(id=pk) pct = min(10 + step * 6, 95) run.progress_pct = pct run.progress_msg = f'步骤 {step}: {tool_name}' - if result: - summary = str(result) - run.run_log += f'[{step}] {tool_name}: {summary}\n' + + if tool_name == 'thinking': + # Accumulate LLM thinking text + if result: + run.llm_thinking = (run.llm_thinking or '') + f'\n[{step}] {result}' + save_fields = ['progress_pct', 'progress_msg', 'llm_thinking'] + elif tool_name == 'complete': + pass # No extra tracking needed else: - run.run_log += f'[{step}] {tool_name}\n' - run.save(update_fields=['progress_pct', 'progress_msg', 'run_log']) + # Real tool call — log it + if result: + summary = str(result) + run.run_log += f'[{step}] {tool_name}: {summary}\n' + else: + run.run_log += f'[{step}] {tool_name}\n' + + # Capture structured tool call data + if meta and isinstance(meta, dict) and 'args' in meta: + tool_calls = list(run.tool_calls_json or []) + tool_calls.append({ + 'step': step, + 'name': tool_name, + 'input': meta['args'], + 'output': result, + }) + run.tool_calls_json = tool_calls + save_fields = ['progress_pct', 'progress_msg', 'run_log', 'tool_calls_json'] + + run.save(update_fields=save_fields) except Exception: pass @@ -915,7 +910,7 @@ def run_llm_analysis_view(request): run.progress_msg = 'LLM 初始化中...' run.save(update_fields=['status', 'progress_pct', 'progress_msg', 'run_log']) - result = run_llm_pipeline(dataset_id, entity_col, config=llm_cfg, callback=progress_callback) + result = run_llm_pipeline(dataset_id, config=llm_cfg, callback=progress_callback) if 'error' in result: run.status = 'failed' @@ -931,7 +926,7 @@ def run_llm_analysis_view(request): tb = traceback.format_exc() logger.error(tb) try: - run = AnalysisRun.objects.get(id=run_id) + run = AnalysisRun.objects.get(id=pk) run.status = 'failed' run.progress_msg = f'失败: {tb}' run.run_log += f'\n[ERROR] {tb}' @@ -942,12 +937,12 @@ def run_llm_analysis_view(request): t = threading.Thread( target=_run_pipeline_worker, - args=(run_id, _analysis_fn), - kwargs=dict(dataset_id=dataset_id, entity_col=entity_col, head=head), + args=(pk, _analysis_fn), + kwargs=dict(dataset_id=dataset_id, head=head, pk=pk), daemon=True, ) t.start() - return JsonResponse({'status': 'started', 'run_id': run_id}) + return JsonResponse({'status': 'started', 'run_id': run.display_id}) def auto_page(request): @@ -962,11 +957,11 @@ def auto_page(request): # ── Progress API ─────────────────────────────────────────────────────── -def run_status_api(request, run_id): +def run_status_api(request, display_id): """Return current run status as JSON.""" - run = get_object_or_404(AnalysisRun, id=run_id) + run = get_object_or_404(AnalysisRun, display_id=display_id) return JsonResponse({ - 'id': run.id, + 'id': run.display_id, 'status': run.status, 'entity_count': run.entity_count, 'cluster_count': run.cluster_count, @@ -974,6 +969,8 @@ def run_status_api(request, run_id): 'progress_pct': run.progress_pct, 'progress_msg': run.progress_msg, 'run_log': run.run_log[-2000:], # last 2000 chars + 'llm_thinking': run.llm_thinking[-5000:], # last 5K chars + 'tool_calls': run.tool_calls_json, }) @@ -1029,6 +1026,9 @@ def _extract_flows_from_df(df, MAX_ROWS): dst_ip_col = next((c for c in df.columns if 'dst_ip' in c.lower() or ':ipd' in c.lower()), None) tls_col = next((c for c in df.columns if c.lower() in ('0ver', 'tls_version', '_tlsver', 'version')), None) bytes_col = next((c for c in df.columns if c.lower() in ('8ack', '8pak', '8byt', 'bytes_sent', 'bytes')), None) + time_col = next((c for c in df.columns if c.lower() in ( + 'timestamp', 'time', '_time', 'ts', 'datetime', 'epoch', + 'created_at', 'created', 'event_time')), None) flows = [] for row in df.iter_rows(named=True): @@ -1066,12 +1066,46 @@ def _extract_flows_from_df(df, MAX_ROWS): tls_label = 'other' bs = float(row.get(bytes_col or '', 0) or 0) + + # Extract timestamp for animation ordering + t_raw = row.get(time_col) if time_col else None + t_val = None + if t_raw is not None: + if isinstance(t_raw, (int, float)): + t_val = float(t_raw) + elif hasattr(t_raw, 'timestamp'): + t_val = t_raw.timestamp() + else: + try: + t_val = float(t_raw) + except (ValueError, TypeError): + pass + flows.append({ 'slat': slat, 'slon': slon, 'dlat': dlat, 'dlon': dlon, 'tls': tls_label, 'bytes': bs, + 'time': t_val, }) + + # Normalize timestamps to 0-1 and sort flows by time + timed = [f for f in flows if f['time'] is not None] + if len(timed) > 1: + timed.sort(key=lambda f: f['time']) + t_min, t_max = timed[0]['time'], timed[-1]['time'] + t_range = max(t_max - t_min, 1) + for f in timed: + f['time'] = (f['time'] - t_min) / t_range + # Untimed flows get evenly spaced after timed ones + untimed = [f for f in flows if f['time'] is None] + for i, f in enumerate(untimed): + f['time'] = min(1.0, (len(timed) + i) / max(len(flows) - 1, 1)) + flows = timed + untimed + else: + for i, f in enumerate(flows): + f['time'] = i / max(len(flows) - 1, 1) + return flows @@ -1089,7 +1123,7 @@ def globe_view(request): if data_param: store = SessionStore() flows = [] - all_runs = list(AnalysisRun.objects.filter(status='completed').order_by('-id').values('id', 'csv_glob')) + all_runs = list(AnalysisRun.objects.filter(status='completed').order_by('-id').values('display_id', 'csv_glob')) try: entry = store.get_dataset(data_param) if entry is not None: @@ -1106,7 +1140,7 @@ def globe_view(request): }) # Get all completed runs for the selector - all_runs = list(AnalysisRun.objects.filter(status='completed').order_by('-id').values('id', 'csv_glob')) + all_runs = list(AnalysisRun.objects.filter(status='completed').order_by('-id').values('display_id', 'csv_glob')) # Get selected run IDs from query params (multiple checkboxes) selected_ids_list = request.GET.getlist('runs') @@ -1116,7 +1150,7 @@ def globe_view(request): sid = sid.strip() if sid.isdigit(): try: - r = AnalysisRun.objects.get(id=int(sid), status='completed') + r = AnalysisRun.objects.get(display_id=int(sid), status='completed') selected_runs.append(r) except AnalysisRun.DoesNotExist: pass @@ -1138,7 +1172,7 @@ def globe_view(request): all_selected_failed = False break # stop after first successful load except Exception as exc: - logger.warning('[GLOBE] Failed to load run %s (%s): %s', run.id, run.csv_glob, exc) + logger.warning('[GLOBE] Failed to load run %s (%s): %s', run.display_id, run.csv_glob, exc) continue if all_selected_failed or not flows: @@ -1167,7 +1201,7 @@ def globe_view(request): resp = render(request, 'tianxuan/globe.html', { 'geo_flows': json.dumps(flows), 'all_runs': all_runs, - 'selected_ids': [r.id for r in selected_runs], + 'selected_ids': [r.display_id for r in selected_runs], }) resp['Cache-Control'] = 'no-cache, no-store, must-revalidate' resp['Pragma'] = 'no-cache' @@ -1312,3 +1346,36 @@ def tool_plan(request): return JsonResponse({'error': 'Method not allowed'}, status=405) + +@csrf_exempt +def retry_run(request, display_id): + """Reset a failed run and re-trigger based on run_type.""" + if request.method != 'POST': + return JsonResponse({'error': 'POST required'}, status=405) + + run = get_object_or_404(AnalysisRun, display_id=display_id) + if run.status != 'failed': + return JsonResponse({'error': '只能重试失败的分析运行'}, status=400) + + run.status = 'pending' + run.error_message = '' + run.progress_pct = 0 + run.progress_msg = '' + run.save(update_fields=['status', 'error_message', 'progress_pct', 'progress_msg']) + + if run.run_type == 'upload' and run.csv_glob: + upload_dir = Path(run.csv_glob).parent + if upload_dir.is_dir(): + t = threading.Thread( + target=_background_process, + args=(run.id, upload_dir), + daemon=True, + ) + t.start() + else: + run.error_message = '上传数据目录已不存在，请重新上传' + run.status = 'failed' + run.save(update_fields=['status', 'error_message']) + + return redirect('analysis:run_detail', display_id=run.display_id) + diff --git a/run.bat b/run.bat index 4de2084..2c6a62d 100644 --- a/run.bat +++ b/run.bat @@ -2,7 +2,6 @@ set PYTHONUTF8=1 cd /d "%~dp0" echo Starting TianXuan... -start /B runtime\python\python.exe manage.py runserver 127.0.0.1:8000 --noreload +start /B runtime\python\python.exe scripts/start_server.py timeout /t 3 >nul -start http://127.0.0.1:8000/ timeout /t 5 >nul diff --git a/scripts/start_server.py b/scripts/start_server.py new file mode 100644 index 0000000..7125ee1 --- /dev/null +++ b/scripts/start_server.py @@ -0,0 +1,39 @@ +"""Start the development server with host and port from config.yaml. + +Reads server.host and server.port from config/config.yaml via the project's +config loader, runs ``manage.py runserver :``, and opens a browser +to http://127.0.0.1: . + +Usage (by run.bat): + start /B runtime\\python\\python.exe scripts\\start_server.py +""" + +import subprocess +import sys +import webbrowser +from pathlib import Path + +from config import get_config + + +def main() -> None: + cfg = get_config() + + host = cfg.server.host or '0.0.0.0' + port = cfg.server.port or 80 + + addr = f'{host}:{port}' + manage_py = Path(__file__).resolve().parent.parent / 'manage.py' + + # Open browser — use 127.0.0.1 regardless of bind address so the user's + # browser always reaches the local machine. + webbrowser.open(f'http://127.0.0.1:{port}/') + + # Run the server. subprocess.call is blocking, so this script stays alive + # as long as the dev server does — the same effect as calling manage.py + # directly under start /B. + sys.exit(subprocess.call([sys.executable, str(manage_py), 'runserver', addr, '--noreload'])) + + +if __name__ == '__main__': + main() diff --git a/templates/analysis/cluster_detail.html b/templates/analysis/cluster_detail.html index 117d275..16aca37 100644 --- a/templates/analysis/cluster_detail.html +++ b/templates/analysis/cluster_detail.html @@ -1,8 +1,8 @@ {% extends 'base.html' %} -{% block title %}Cluster #{{ cluster.cluster_label }} - Run #{{ run.id }}{% endblock %} +{% block title %}Cluster #{{ cluster.cluster_label }} - Run #{{ run.display_id }}{% endblock %} {% block content %}
- ← Back to overview + ← Back to overview

Cluster #{{ cluster.cluster_label }}

{{ cluster.size }} entities ({{ cluster.proportion|floatformat:2 }}% of total)

Silhouette Score: {{ cluster.silhouette_score|floatformat:4|default:"-" }}

diff --git a/templates/analysis/cluster_overview.html b/templates/analysis/cluster_overview.html index f7a5b95..d86475c 100644 --- a/templates/analysis/cluster_overview.html +++ b/templates/analysis/cluster_overview.html @@ -1,6 +1,6 @@ {% extends 'base.html' %} {% load static %} -{% block title %}Cluster Overview - Run #{{ run.id }}{% endblock %} +{% block title %}Cluster Overview - Run #{{ run.display_id }}{% endblock %} {% block content %}
-

Cluster Overview — Run #{{ run.id }}

-

Entity column: {{ run.entity_column }} | {{ run.entity_count }} entities, {{ run.cluster_count }} clusters

+

Cluster Overview — Run #{{ run.display_id }}

+

{{ run.total_flows }} flows, {{ run.cluster_count }} clusters

@@ -94,7 +94,7 @@ {% endfor %} - Detail + Detail
{% empty %}
diff --git a/templates/analysis/dashboard.html b/templates/analysis/dashboard.html index 9e06246..ea6cacc 100644 --- a/templates/analysis/dashboard.html +++ b/templates/analysis/dashboard.html @@ -15,16 +15,16 @@ ID Started Status - Entity Column + Flows Entities Clusters - + 操作 {% for run in runs %} - #{{ run.id }} + #{{ run.display_id }} {{ run.created_at|date:"Y-m-d H:i" }} {% if run.status == 'completed' %} @@ -35,10 +35,11 @@ {{ run.get_status_display }} {% endif %} - {{ run.entity_column|default:"-" }} + {{ run.total_flows|default:"-" }} {{ run.entity_count|default:"-" }} {{ run.cluster_count|default:"-" }} - View + View + {% endfor %} @@ -51,4 +52,30 @@

Call load_data() via MCP to begin.

{% endif %} + +
{% csrf_token %}
+ + {% endblock %} diff --git a/templates/analysis/entity_profile.html b/templates/analysis/entity_profile.html index 61bf808..a3ecb8b 100644 --- a/templates/analysis/entity_profile.html +++ b/templates/analysis/entity_profile.html @@ -4,7 +4,7 @@
← Back

Entity: {{ entity.entity_value }}

-

Run #{{ run.id }} | Cluster: #{{ entity.cluster_label|default:"unassigned" }}

+

Run #{{ run.display_id }} | Cluster: #{{ entity.cluster_label|default:"unassigned" }}

diff --git a/templates/analysis/run_detail.html b/templates/analysis/run_detail.html index cf882cb..323a49d 100644 --- a/templates/analysis/run_detail.html +++ b/templates/analysis/run_detail.html @@ -1,17 +1,10 @@ {% extends 'base.html' %} -{% block title %}Run #{{ run.id }} - TLS Analyzer{% endblock %} +{% block title %}Run #{{ run.display_id }} - TLS Analyzer{% endblock %} {% block content %}
-

Run #{{ run.id }} — {{ run.created_at|date:"Y-m-d H:i" }}

+

Run #{{ run.display_id }} — {{ run.created_at|date:"Y-m-d H:i" }}

Status: {{ run.get_status_display }}

- {% if run.status == 'failed' %} -
- {% csrf_token %} - -
- {% endif %}

CSV: {{ run.csv_glob }}

-

Entity Column: {{ run.entity_column|default:"(not detected)" }}

@@ -31,7 +24,7 @@

Clusters

- View Cluster Overview + View Cluster Overview {% if clusters %} @@ -50,7 +43,7 @@ - + {% endfor %} diff --git a/templates/analysis/run_list.html b/templates/analysis/run_list.html index 476b374..21378a7 100644 --- a/templates/analysis/run_list.html +++ b/templates/analysis/run_list.html @@ -3,6 +3,23 @@ {% block content %}

All Analysis Runs

+ + +
+
+ + + {% if current_type %} + 清除筛选 + {% endif %} + +
+ {% if runs %}
{{ c.size }} {{ c.proportion|floatformat:2 }} {{ c.silhouette_score|floatformat:4|default:"-" }}DetailDetail
@@ -10,33 +27,71 @@ + + + {% for run in runs %} - + - + + + {% endfor %}
ID Started StatusType CSV Glob Entities Clusters 重试操作
#{{ run.id }}#{{ run.display_id }} {{ run.created_at|date:"Y-m-d H:i" }} {{ run.get_status_display }} {{ run.csv_glob }} {{ run.entity_count|default:"-" }} {{ run.cluster_count|default:"-" }}DetailDetail + {% if run.status == 'failed' %} +
+ {% csrf_token %} + +
+ {% endif %} +
{% else %}

No runs yet.

{% endif %}
+ +
{% csrf_token %}
+ + {% endblock %} diff --git a/templates/base.html b/templates/base.html index 68d929f..ddf3eed 100644 --- a/templates/base.html +++ b/templates/base.html @@ -36,6 +36,39 @@ .pagination .active { background: #4361ee; color: #fff; border-color: #4361ee; } .empty-state { text-align: center; padding: 3rem; color: #999; } .empty-state p { font-size: 1rem; margin-top: 0.5rem; } + + /* Toast notification system */ + #toastContainer { + position: fixed; top: 1rem; right: 1rem; z-index: 9999; + display: flex; flex-direction: column; gap: 0.5rem; + pointer-events: none; max-width: 400px; + } + .toast { + pointer-events: auto; + padding: 0.85rem 1.25rem; border-radius: 8px; + box-shadow: 0 4px 16px rgba(0,0,0,0.18); + font-size: 0.9rem; line-height: 1.4; + animation: toastSlideIn 0.3s ease-out; + display: flex; align-items: flex-start; gap: 0.6rem; + word-break: break-word; + } + .toast.toast-hiding { animation: toastSlideOut 0.25s ease-in forwards; } + .toast-error { background: #dc3545; color: #fff; } + .toast-success { background: #28a745; color: #fff; } + .toast-icon { flex-shrink: 0; font-size: 1.1rem; line-height: 1.4; } + .toast-close { + margin-left: auto; cursor: pointer; opacity: 0.8; flex-shrink: 0; + font-size: 1rem; line-height: 1.4; padding-left: 0.5rem; user-select: none; + } + .toast-close:hover { opacity: 1; } + @keyframes toastSlideIn { + from { transform: translateX(100%); opacity: 0; } + to { transform: translateX(0); opacity: 1; } + } + @keyframes toastSlideOut { + from { transform: translateX(0); opacity: 1; } + to { transform: translateX(100%); opacity: 0; } + } @@ -47,10 +80,48 @@ LLM分析 运行记录 地球 - + 配置
{% block content %}{% endblock %}
+ +
+ + diff --git a/templates/tianxuan/auto.html b/templates/tianxuan/auto.html index 9a0808b..61135ea 100644 --- a/templates/tianxuan/auto.html +++ b/templates/tianxuan/auto.html @@ -1,9 +1,114 @@ {% extends 'base.html' %} {% block title %}LLM 自动分析 - 天璇{% endblock %} {% block content %} + +

LLM 自动分析

-

LLM 将自动编排工具链完成：加载 → 检测实体 → 聚合 → 聚类 → 特征提取

+

LLM 将自动编排工具链完成：加载 → 过滤 → 聚类 → 特征提取

{% if not llm_configured %}
⚠️ LLM 尚未配置。请先在 配置页面 设置 base_url 和 API Key。 @@ -15,18 +120,17 @@

选择数据集

{% if runs %} - + {% for run in runs %} - - + + - {% empty %} - + {% endfor %}
ID文件实体列状态
ID文件状态
#{{ run.id }}#{{ run.display_id }} {{ run.csv_glob }}{{ run.entity_column|default:"-" }} {{ run.status }}
暂无数据。先去 上传 CSV。
暂无数据。先去 上传 CSV。
@@ -37,43 +141,157 @@
+ + +
+
🧠 LLM 思考
+
+
+ + + +
diff --git a/templates/tianxuan/manual.html b/templates/tianxuan/manual.html index 62122d6..37f14fd 100644 --- a/templates/tianxuan/manual.html +++ b/templates/tianxuan/manual.html @@ -234,6 +234,7 @@ async function executeStep(idx) { resultDiv.textContent = 'Error: ' + err.message; resultDiv.style.display = 'block'; step.status = 'failed'; + showError('步骤 ' + step.tool.name + ' 执行失败: ' + err.message); } renderSteps(); } @@ -254,59 +255,68 @@ async function runAll() { // ── Save / Load ── async function savePlan() { const name = document.getElementById('planName').value.trim(); - if (!name) { alert('请输入方案名称'); return; } + if (!name) { showError('请输入方案名称'); return; } const steps = workflow.map(s => ({ tool_name: s.tool.name, params: collectParams(s.idx), })); - const resp = await fetch('/tools/plan/', { - method: 'POST', - headers: { 'Content-Type':'application/json', 'X-CSRFToken':'{{ csrf_token }}' }, - body: JSON.stringify({ name, steps }), - }); - const data = await resp.json(); - if (data.status === 'saved') { - alert('方案已保存: ' + name); - loadPlanList(); - } else { - alert('保存失败: ' + (data.error || '')); - } + try { + const resp = await fetch('/tools/plan/', { + method: 'POST', + headers: { 'Content-Type':'application/json', 'X-CSRFToken':'{{ csrf_token }}' }, + body: JSON.stringify({ name, steps }), + }); + const data = await resp.json(); + if (data.status === 'saved') { + showSuccess('方案已保存: ' + name); + loadPlanList(); + } else { + showError('保存失败: ' + (data.error || '')); + } + } catch (e) { showError('保存方案失败: ' + e.message); } } async function loadPlanList() { - const resp = await fetch('/tools/plan/'); - const data = await resp.json(); - const sel = document.getElementById('loadPlanSelect'); - sel.innerHTML = ''; - (data.plans || []).forEach(p => { - const opt = document.createElement('option'); - opt.value = p.name; opt.text = p.name + (p.steps ? ' (' + p.steps + ' steps)' : ''); - sel.appendChild(opt); - }); + try { + const resp = await fetch('/tools/plan/'); + const data = await resp.json(); + const sel = document.getElementById('loadPlanSelect'); + sel.innerHTML = ''; + (data.plans || []).forEach(p => { + const opt = document.createElement('option'); + opt.value = p.name; opt.text = p.name + (p.steps ? ' (' + p.steps + ' steps)' : ''); + sel.appendChild(opt); + }); + } catch (e) { showError('加载方案列表失败: ' + e.message); } } async function loadPlan() { const name = document.getElementById('loadPlanSelect').value; if (!name) return; - const resp = await fetch('/tools/plan/?name=' + encodeURIComponent(name)); - const data = await resp.json(); - if (data.error) { alert(data.error); return; } - // Clear current workflow - workflow.length = 0; - // Load steps - for (const s of data.steps || []) { - const tool = TOOLS.find(t => t.name === s.tool_name); - if (tool) addStep(tool, s.params || {}); - } - document.getElementById('planName').value = name; + try { + const resp = await fetch('/tools/plan/?name=' + encodeURIComponent(name)); + const data = await resp.json(); + if (data.error) { showError(data.error); return; } + // Clear current workflow + workflow.length = 0; + // Load steps + for (const s of data.steps || []) { + const tool = TOOLS.find(t => t.name === s.tool_name); + if (tool) addStep(tool, s.params || {}); + } + document.getElementById('planName').value = name; + } catch (e) { showError('加载方案失败: ' + e.message); } } async function deletePlan() { const name = document.getElementById('loadPlanSelect').value; if (!name || !confirm('删除方案 "' + name + '" ?')) return; - const resp = await fetch('/tools/plan/?name=' + encodeURIComponent(name), { method: 'DELETE' }); - const data = await resp.json(); - if (data.status === 'deleted') loadPlanList(); + try { + const resp = await fetch('/tools/plan/?name=' + encodeURIComponent(name), { method: 'DELETE' }); + const data = await resp.json(); + if (data.status === 'deleted') { showSuccess('已删除方案: ' + name); loadPlanList(); } + else { showError('删除方案失败: ' + (data.error || '')); } + } catch (e) { showError('删除方案失败: ' + e.message); } } // ── Render ── diff --git a/templates/tianxuan/upload.html b/templates/tianxuan/upload.html index d0736c4..00ffc46 100644 --- a/templates/tianxuan/upload.html +++ b/templates/tianxuan/upload.html @@ -39,15 +39,15 @@ {% for run in runs %} - #{{ run.id }} + #{{ run.display_id }} {{ run.csv_glob }} {{ run.total_flows|default:"-" }} {{ run.status }} {% if run.status == 'loading' or run.status == 'profiling' or run.status == 'aggregating' or run.status == 'clustering' or run.status == 'extracting' %} - + {% else %} - + {% endif %} @@ -116,14 +116,16 @@ document.getElementById('uploadForm').addEventListener('submit', async (e) => { // Poll for completion const runId = data.run_id; const poll = setInterval(async () => { - const r = await fetch(`/runs/${runId}/status/`); - const s = await r.json(); - const pct = s.progress_pct || 0; - const msg = s.progress_msg || s.status; - document.getElementById('progressBar').style.width = pct + '%'; - document.getElementById('progressText').textContent = `${msg} (${pct}%)`; - if (s.status === 'ready' || s.status === 'completed') { clearInterval(poll); window.location.href = `/runs/${runId}/`; } - if (s.status === 'failed') { clearInterval(poll); document.getElementById('progressText').textContent = '处理失败: ' + (s.error_message || '未知错误'); } + try { + const r = await fetch(`/runs/${runId}/status/`); + const s = await r.json(); + const pct = s.progress_pct || 0; + const msg = s.progress_msg || s.status; + document.getElementById('progressBar').style.width = pct + '%'; + document.getElementById('progressText').textContent = `${msg} (${pct}%)`; + if (s.status === 'ready' || s.status === 'completed') { clearInterval(poll); window.location.href = `/runs/${runId}/`; } + if (s.status === 'failed') { clearInterval(poll); document.getElementById('progressText').textContent = '处理失败: ' + (s.error_message || '未知错误'); } + } catch (e) { showError('轮询状态失败: ' + e.message); } }, 1000); } catch (err) { document.getElementById('progressText').textContent = '上传失败: ' + err.message; submitBtn.disabled = false; } }); @@ -132,18 +134,30 @@ async function deleteRun(runId, processing) { const msg = processing ? '后台处理中，删除将中断分析，确认删除？' : '确定要删除此数据集吗？此操作不可撤销。'; if (!confirm(msg)) return; try { + let csrf = document.querySelector('[name=csrfmiddlewaretoken]'); + if (!csrf) { + showError('页面已过期，请刷新后重试'); + return; + } const resp = await fetch(`/runs/${runId}/delete/`, { method: 'POST', - headers: { 'X-CSRFToken': document.querySelector('[name=csrfmiddlewaretoken]').value }, + headers: { 'X-CSRFToken': csrf.value }, }); + if (!resp.ok) { + const text = await resp.text().catch(() => ''); + const snippet = text.length > 200 ? text.substring(0, 200) + '…' : text; + showError('删除失败 (HTTP ' + resp.status + '): ' + (snippet || '服务器内部错误')); + return; + } const data = await resp.json(); if (data.deleted) { + showSuccess('已删除'); location.reload(); } else { - alert('删除失败'); + showError('删除失败: 服务器返回了意外的响应'); } } catch (err) { - alert('删除失败: ' + err.message); + showError('删除失败: ' + err.message + ' — 请刷新页面后重试'); } } diff --git a/tianxuan/llm_orchestrator.py b/tianxuan/llm_orchestrator.py index 2ef08c3..4db66da 100644 --- a/tianxuan/llm_orchestrator.py +++ b/tianxuan/llm_orchestrator.py @@ -4,6 +4,7 @@ aggressive truncation for tool results, step-by-step guidance.""" import json, logging, time, urllib.request, urllib.error, asyncio from typing import Optional +from analysis.tool_registry import handle_call logger = logging.getLogger('tianxuan.llm') @@ -12,99 +13,110 @@ logger = logging.getLogger('tianxuan.llm') TOOLS = [ # ── Core pipeline ── {"type": "function", "function": {"name": "profile_data", - "description": "Examine dataset columns: types, null ratios, unique counts, numeric stats. Call FIRST to understand data before choosing next steps.", + "description": "检查数据集列：类型、空值比例、唯一值计数、数值统计。首先调用以了解数据，再决定后续步骤。", "parameters": {"type": "object", "properties": {"dataset_id": {"type": "string"}}, "required": ["dataset_id"]}}}, {"type": "function", "function": {"name": "build_entity_profiles", - "description": "Group raw flows by entity column (src IP, SNI, etc). Produces per-entity features (flow count, TLS versions, ports). Requires profiled data.", + "description": "按实体列（源IP、SNI等）分组原始流。生成每个实体的特征（流数量、TLS版本、端口）。需要已概要分析的数据。", "parameters": {"type": "object", "properties": {"dataset_id": {"type": "string"}, "entity_column": {"type": "string"}, "entity_columns": {"type": "array", "items": {"type": "string"}}, "auto_detect": {"type": "boolean"}}, "required": ["dataset_id"]}}}, {"type": "function", "function": {"name": "compute_scores", - "description": "Compute adaptive proxy/anomaly/threat scores on ENTITY data using PCA weight learning. Call AFTER build_entity_profiles. Adds proxy_score/risk_level/threat_score columns.", + "description": "使用PCA权重学习在实体数据上计算自适应代理/异常/威胁评分。在build_entity_profiles之后调用。添加proxy_score/risk_level/threat_score列。", "parameters": {"type": "object", "properties": {"dataset_id": {"type": "string"}}, "required": ["dataset_id"]}}}, {"type": "function", "function": {"name": "run_clustering", - "description": "Cluster ENTITY data into behavior groups. Requires entity dataset ID. Choose hdbscan (auto-detect clusters) for exploration; kmeans for fixed count.", + "description": "将实体数据聚类为行为组。需要实体数据集ID。探索性分析选择hdbscan（自动检测聚类数）；固定簇数选择kmeans。", "parameters": {"type": "object", "properties": {"dataset_id": {"type": "string"}, "cluster_columns": {"type": "array", "items": {"type": "string"}}, "algorithm": {"type": "string", "enum": ["hdbscan", "kmeans"]}}, "required": ["dataset_id", "cluster_columns"]}}}, {"type": "function", "function": {"name": "extract_features", - "description": "Identify distinguishing features per cluster (z-score or anova). Call AFTER run_clustering.", + "description": "识别每个聚类的区分特征（z-score或anova）。在run_clustering之后调用。", "parameters": {"type": "object", "properties": {"dataset_id": {"type": "string"}, "cluster_result_id": {"type": "string"}, "method": {"type": "string", "enum": ["zscore", "anova"]}}, "required": ["dataset_id", "cluster_result_id"]}}}, {"type": "function", "function": {"name": "detect_anomalies", - "description": "Run Isolation Forest to find anomalous entities. Call AFTER compute_scores. Returns anomaly count and rate.", + "description": "运行孤立森林（Isolation Forest）检测异常实体。在compute_scores之后调用。返回异常数量和比率。", "parameters": {"type": "object", "properties": {"dataset_id": {"type": "string"}, "contamination": {"type": "number", "description": "Expected anomaly rate 0.01-0.1 (default 0.05)"}}, "required": ["dataset_id"]}}}, {"type": "function", "function": {"name": "visualize_anomalies", - "description": "Generate PCA-2D scatter data for frontend. Color by risk_level. Call AFTER compute_scores or detect_anomalies.", + "description": "生成PCA-2D散点数据用于前端展示。按风险等级着色。在compute_scores或detect_anomalies之后调用。", "parameters": {"type": "object", "properties": {"dataset_id": {"type": "string"}}, "required": ["dataset_id"]}}}, # ── Diagnostic (call when pipeline fails or results look wrong) ── {"type": "function", "function": {"name": "validate_data", - "description": "[DIAG] Check data quality: null ratios, constant columns, schema issues. Call when loading fails or results seem off.", + "description": "[诊断] 检查数据质量：空值比例、常量列、schema问题。当加载失败或结果异常时调用。", "parameters": {"type": "object", "properties": {"dataset_id": {"type": "string"}, "fix_issues": {"type": "boolean"}}, "required": ["dataset_id"]}}}, {"type": "function", "function": {"name": "explore_distributions", - "description": "[DIAG] Per-column distribution: unique count, null%, min/max/mean/std, top-10 values. Call when clustering fails or you need to understand column values.", + "description": "[诊断] 每列分布统计：唯一值计数、空值百分比、最小/最大/均值/标准差、top-10值。当聚类失败或需要了解列值时调用。", "parameters": {"type": "object", "properties": {"dataset_id": {"type": "string"}, "columns": {"type": "array", "items": {"type": "string"}}, "max_sample": {"type": "integer"}}, "required": ["dataset_id"]}}}, {"type": "function", "function": {"name": "find_outliers", - "description": "[DIAG] Find extreme values using IQR method. Call when explore_distributions shows suspicious extremes.", + "description": "[诊断] 使用IQR方法查找极值。当explore_distributions显示可疑极端值时调用。", "parameters": {"type": "object", "properties": {"dataset_id": {"type": "string"}, "columns": {"type": "array", "items": {"type": "string"}}, "threshold": {"type": "number"}}, "required": ["dataset_id"]}}}, {"type": "function", "function": {"name": "diagnose_clustering", - "description": "[DIAG] When clustering produces garbage (all noise, 1 cluster, bad silhouette): diagnose why and suggest fixes.", + "description": "[诊断] 当聚类结果不正常（全部噪声、只有1个聚类、轮廓系数低）时：诊断原因并建议修复方案。", "parameters": {"type": "object", "properties": {"dataset_id": {"type": "string"}, "cluster_result_id": {"type": "string"}}, "required": ["dataset_id", "cluster_result_id"]}}}, {"type": "function", "function": {"name": "compare_datasets", - "description": "[DIAG] Compare two datasets to check if a processing step corrupted data.", + "description": "[诊断] 比较两个数据集，检查处理步骤是否损坏了数据。", "parameters": {"type": "object", "properties": {"dataset_id_a": {"type": "string"}, "dataset_id_b": {"type": "string"}}, "required": ["dataset_id_a", "dataset_id_b"]}}}, {"type": "function", "function": {"name": "export_debug_sample", - "description": "[DIAG] Export first N rows as JSON for manual inspection. Call when you cannot understand a tool's error.", + "description": "[诊断] 导出前N行JSON用于手动检查。当无法理解工具错误时调用。", "parameters": {"type": "object", "properties": {"dataset_id": {"type": "string"}, "max_rows": {"type": "integer"}}, "required": ["dataset_id"]}}}, {"type": "function", "function": {"name": "repair_schema", - "description": "[DIAG] Fix schema mismatches by normalising column names", + "description": "[诊断] 通过规范化列名修复schema不匹配。", "parameters": {"type": "object", "properties": {"dataset_id": {"type": "string"}}, "required": ["dataset_id"]}}}, # Read-only analysis tools (deep-dive, no side effects) {"type": "function", "function": {"name": "analyze_patterns", - "description": "[ANALYSIS] Traffic pattern summary — top IPs, ports, TLS breakdown, protocol mix. Read-only.", + "description": "[分析] 流量模式摘要——top IP、端口、TLS分布、协议混合。只读。", "parameters": {"type": "object", "properties": {"dataset_id": {"type": "string"}, "top_n": {"type": "integer"}}, "required": ["dataset_id"]}}}, {"type": "function", "function": {"name": "analyze_tls_health", - "description": "[ANALYSIS] TLS security assessment — legacy version ratio, weak ciphers, SNI missing rate. Read-only.", + "description": "[分析] TLS安全评估——旧版比例、弱加密套件、SNI缺失率。只读。", "parameters": {"type": "object", "properties": {"dataset_id": {"type": "string"}}, "required": ["dataset_id"]}}}, {"type": "function", "function": {"name": "analyze_geo_distribution", - "description": "[ANALYSIS] Geographic traffic distribution — top countries, domestic vs international, impossible travel. Read-only.", + "description": "[分析] 流量地理分布——top国家、国内vs国际、不可能行程。只读。", "parameters": {"type": "object", "properties": {"dataset_id": {"type": "string"}}, "required": ["dataset_id"]}}}, {"type": "function", "function": {"name": "analyze_entity_detail", - "description": "[ANALYSIS] Deep-dive into one entity — all its flows, TLS, ports, destinations, anomaly scores. Read-only.", + "description": "[分析] 深入分析单个实体——所有流、TLS、端口、目的地、异常评分。只读。", "parameters": {"type": "object", "properties": {"dataset_id": {"type": "string"}, "entity_value": {"type": "string"}}, "required": ["dataset_id", "entity_value"]}}}, {"type": "function", "function": {"name": "analyze_temporal", - "description": "[ANALYSIS] Traffic patterns over time — hourly/daily volume, peak periods. Read-only.", + "description": "[分析] 随时间变化的流量模式——小时/日流量、高峰时段。只读。", "parameters": {"type": "object", "properties": {"dataset_id": {"type": "string"}, "timestamp_column": {"type": "string"}}, "required": ["dataset_id"]}}}, + # Custom LLM-powered computation + {"type": "function", "function": {"name": "compute_distance_matrix", + "description": "[LLM-POWERED] 你编写一个Python函数体的body（`def distance_fn(row: dict) -> float:`），系统逐行执行并返回距离分数统计。用于现有工具不能表达的度量任务。参数：python_function（函数体）、columns（列名列表）、max_sample（最大采样行数，默认10000）。", + "parameters": {"type": "object", "properties": {"dataset_id": {"type": "string"}, "python_function": {"type": "string"}, "columns": {"type": "array", "items": {"type": "string"}}, "max_sample": {"type": "integer"}}, "required": ["dataset_id", "python_function"]}}}, ] -SYSTEM_PROMPT = """You are a TLS traffic analysis AI. Your job is to analyze network flow data using available tools. +SYSTEM_PROMPT = """你是TLS流量分析AI。你的任务是用可用工具分析网络流数据。 -## Strategy -1. FIRST call `profile_data` to understand the dataset (columns, types, size). -2. Based on the profile, decide what to do: - - Has entity columns? → `build_entity_profiles` - - Need anomaly scores? → `compute_scores` (after entity profiles) - - Need clustering? → `run_clustering` + `extract_features` -3. If a tool fails or returns garbage → call a [DIAG] tool to diagnose. -4. Summarize findings in Chinese at the end. +## 策略 +1. 首先调用`profile_data`了解数据集（列、类型、规模）。 +2. 根据概要分析结果决定下一步： + - 有实体列？→ `build_entity_profiles` + - 需要异常评分？→ `compute_scores`（实体画像之后） + - 需要聚类？→ `run_clustering` + `extract_features` +3. 如果工具失败或返回异常结果→调用[诊断]工具排查。 +4. 最后用中文总结分析发现。 -## Rules -- Always check tool results before deciding the next step. -- If clustering returns 0 clusters or all noise → call `diagnose_clustering`. -- If entity detection fails → call `explore_distributions` on candidate columns. -- Never call the same tool twice with identical parameters. -- Never invent parameters — use values from tool results. -- Keep responses concise. Chinese for summaries, English for tool calls. -- Max 15 tool calls per analysis session. +## 规则 +- 始终检查工具结果后再决定下一步。 +- 如果聚类返回0个聚类或全部噪声→调用`diagnose_clustering`。 +- 如果实体检测失败→在候选列上调用`explore_distributions`。 +- 绝不用完全相同参数重复调用同一工具。 +- 绝不编造参数——使用工具结果中的实际值。 +- 保持回复简洁。总结用中文，工具调用用英文。 +- 每次分析会话最多15次工具调用。 -## Available tools -Core (modifies data): profile_data, build_entity_profiles, compute_scores, run_clustering, extract_features, detect_anomalies, visualize_anomalies -Diagnostic (when things fail): validate_data, explore_distributions, find_outliers, diagnose_clustering, compare_datasets, export_debug_sample, repair_schema -Analysis (read-only, safe to call anytime): analyze_patterns, analyze_tls_health, analyze_geo_distribution, analyze_entity_detail, analyze_temporal +## 可用工具 +核心（修改数据）：profile_data, build_entity_profiles, compute_scores, run_clustering, extract_features, detect_anomalies, visualize_anomalies +诊断（出错时使用）：validate_data, explore_distributions, find_outliers, diagnose_clustering, compare_datasets, export_debug_sample, repair_schema +分析（只读，随时安全调用）：analyze_patterns, analyze_tls_health, analyze_geo_distribution, analyze_entity_detail, analyze_temporal +LLM驱动（自定义计算）：compute_distance_matrix — 分析你的数据列，编写一个计算每行评分的distance_fn函数体，然后调用此工具执行并返回汇总统计。 -## When to call analysis tools -- Before deciding entity column → analyze_patterns (see top IPs) -- Before clustering → analyze_tls_health (check TLS quality) -- When proxy_score is high → analyze_entity_detail on that entity -- When geographic anomalies suspected → analyze_geo_distribution -- For security assessment → analyze_tls_health -- These are READ-ONLY — call them any time, no side effects""" +## LLM驱动的compute_distance_matrix工作流 +- 分析列→编写Python函数体→调用compute_distance_matrix→基于评分进行聚类或过滤 +- 函数签名格式为`def distance_fn(row: dict) -> float:`。你只需编写函数体（例如`return abs(row.get('col1', 0)) + row.get('col2', 0)`） +- 可用命名空间：math、numpy（作为np）。禁止危险模块（os、sys、subprocess等）。 +- 当现有工具无法表达所需度量时使用。 + +## 何时调用分析工具 +- 决定实体列前→analyze_patterns（查看top IP） +- 聚类之前→analyze_tls_health（检查TLS质量） +- proxy_score高时→analyze_entity_detail调查该实体 +- 怀疑地理异常时→analyze_geo_distribution +- 安全评估→analyze_tls_health +- 这些是只读工具——随时可调用，无副作用""" class LLMConfig: @@ -207,11 +219,10 @@ def run_llm_pipeline(dataset_id: str, entity_col: str = "", consecutive_errors = 0 for step in range(15): - if callback: - callback(step, "thinking", None) - resp = call_llm(msgs, config) if resp is None: + if callback: + callback(step, "thinking", "(LLM unavailable)") consecutive_errors += 1 if consecutive_errors >= 3: return {"error": "LLM unavailable after 3 retries", "steps": step} @@ -221,6 +232,10 @@ def run_llm_pipeline(dataset_id: str, entity_col: str = "", msg = resp["choices"][0]["message"] content = msg.get("content", "") + # Pass thinking text to callback + if callback: + callback(step, "thinking", content) + # No tool calls → LLM is done if "tool_calls" not in msg or not msg["tool_calls"]: if callback: @@ -252,7 +267,7 @@ def run_llm_pipeline(dataset_id: str, entity_col: str = "", logger.info(f'[TOOL] step={step} name={name} args={json.dumps(args)[:200]}') if callback: - callback(step, name, None) + callback(step, name, None, meta={"args": args}) try: result = asyncio.run(handle_call(name, args)) @@ -260,7 +275,7 @@ def run_llm_pipeline(dataset_id: str, entity_col: str = "", result = {"error": f"Handler error: {e}"} if callback: - callback(step, name, result) + callback(step, name, result, meta={"args": args}) # Truncate result for LLM context window result_str = json.dumps(result, default=str, ensure_ascii=False) diff --git a/tianxuan/settings.py b/tianxuan/settings.py index a55e20d..8f4e072 100644 --- a/tianxuan/settings.py +++ b/tianxuan/settings.py @@ -61,8 +61,10 @@ DATABASES = { "default": { "ENGINE": "django.db.backends.sqlite3", "NAME": BASE_DIR / "db.sqlite3", + 'OPTIONS': {'timeout': 20}, } } +CONN_MAX_AGE = 0 # Set SQLite journal mode to DELETE to avoid WAL files during compression from django.db.backends.signals import connection_created @@ -102,7 +104,7 @@ TIME_ZONE = "UTC" USE_I18N = True USE_TZ = True -DATA_UPLOAD_MAX_NUMBER_FIELDS = 1000000 +DATA_UPLOAD_MAX_NUMBER_FIELDS = 10000000 DATA_UPLOAD_MAX_MEMORY_SIZE = 256 * 1024 * 1024 FILE_UPLOAD_MAX_MEMORY_SIZE = 100 * 1024 * 1024 @@ -130,3 +132,16 @@ LOGGING = { }, "root": {"handlers": ["file", "stderr"], "level": "INFO"}, } + +# Dynamically add the configured server host to ALLOWED_HOSTS. +# When the config host is 0.0.0.0 the wildcard "*" (already set above) covers +# all addresses; for a specific IP/DNS name we add it explicitly. +try: + from config import get_config + + _cfg = get_config() + _ch = _cfg.server.host + if _ch and _ch not in ALLOWED_HOSTS and _ch != "0.0.0.0": + ALLOWED_HOSTS.append(_ch) +except Exception: + pass