chore: bump VERSION to v2.0.0beta, reimport TlsDB, remove stale test files

TlsDB.csv

@@ -11,8 +11,8 @@
 :ipd.orgn,目标IP组织名称,字符串,97.61%
 :ips.city,源IP所在城市,字符串,99.26%
 :ipd.city,目标IP所在城市,字符串,98.74%
-:ips.anon,源IP匿名状态,"anon, hosting"或无,12.66%
-:ipd.anon,目标IP匿名状态,"anon, hosting"或无,15.54%
+:ips.anon,源IP匿名状态,"anon, hosting"或"anon, vpn"或"anon, server"或无,12.66%
+:ipd.anon,目标IP匿名状态,"anon, hosting"或"anon, vpn"或"anon, server"或无,15.54%
 :ips.doma,源IP域名,字符串,28.91%
 :ipd.doma,目标IP域名,字符串,13.48%
 :prs,源端口,整数,100%

VERSION

@@ -1 +1 @@
-v1.1.6
+v2.0.0beta

analysis/management/commands/run_pipeline.py

@@ -62,7 +62,7 @@ class Command(BaseCommand):
                 and schema[c].split('(')[0].strip() in numeric_types
             ][:10]
 
-            # ── Step 2: Clustering + Extraction + PCA (shared pipeline) ─────
+            # ── Step 2: Clustering + Extraction + UMAP (shared pipeline) ─────
             _self.stdout.write(f'[2/3] 聚类 ({algo})')
             _self.stdout.write(f'  → 聚类特征: {feature_cols}')
             run.total_flows = row_count
@@ -72,7 +72,7 @@ class Command(BaseCommand):
                 run=run, store=store, entity_ds_id=ds_id,
                 feature_columns=feature_cols,
                 algorithm=algo, min_cluster_size=5,
-                run_pca=True,
+                run_umap=True,
             )
             if run.status == 'failed':
                 _self.stderr.write(_self.style.ERROR(f'  ✗ 分析失败: {run.error_message}'))

analysis/models.py

@@ -94,8 +94,8 @@ class EntityProfile(models.Model):
                                  related_name='entities')
     feature_json = models.JSONField(default=dict, blank=True,
                                      help_text="Aggregated entity features as dict")
-    embedding_x = models.FloatField(null=True, blank=True, help_text="PCA-2D X coordinate")
-    embedding_y = models.FloatField(null=True, blank=True, help_text="PCA-2D Y coordinate")
+    embedding_x = models.FloatField(null=True, blank=True, help_text="UMAP-2D X coordinate")
+    embedding_y = models.FloatField(null=True, blank=True, help_text="UMAP-2D Y coordinate")
 
     class Meta:
         unique_together = ['run', 'entity_value']

analysis/session_store.py

@@ -252,6 +252,7 @@ class SessionStore:
                     'parent_id': entry.get('parent_id'),
                     'row_count': meta.get('row_count'),
                     'file_count': meta.get('file_count'),
+                    'svd_components': meta.get('svd_components', 0),
                     'columns': list(entry.get('schema', {}).keys()),
                     'column_count': len(entry.get('schema', {})),
                 })

analysis/tool_registry.py

@@ -498,7 +498,7 @@ def get_tools_meta() -> list[Tool]:
         Tool(
             name="compute_scores",
             description=(
-                "在实体数据上计算自适应代理/异常/威胁评分。使用PCA从数据自动学习"
+                "在实体数据上计算自适应代理/异常/威胁评分。从数据自动学习"
                 "特征权重，计算加权代理评分（proxy_score）、基于IQR的风险等级"
                 "（normal/watch/suspicious/critical）和威胁评分（threat_score）。"
                 "返回评分摘要与阈值。"
@@ -539,7 +539,7 @@ def get_tools_meta() -> list[Tool]:
         Tool(
             name="visualize_anomalies",
             description=(
-                "为高风险实体生成PCA-2D嵌入和聚类可视化数据。按风险等级和聚类着色。"
+                "为高风险实体生成UMAP-2D嵌入和聚类可视化数据。按风险等级和聚类着色。"
                 "返回用于前端渲染的散点图JSON数据。"
             ),
             inputSchema={
@@ -608,7 +608,7 @@ def get_tools_meta() -> list[Tool]:
             name="diagnose_clustering",
             description=(
                 "诊断工具：当聚类产生太少/太多聚类、全部为噪声或轮廓系数低时——诊断原因。"
-                "检查：特征方差、相关矩阵、PCA解释方差、数据稀疏性，并建议参数调整"
+                "检查：特征方差、相关矩阵、特征降维分析、数据稀疏性，并建议参数调整"
                 "（min_cluster_size、算法选择、特征选择）。"
             ),
             inputSchema={
@@ -1598,7 +1598,7 @@ async def _handle_extract_features(
                                         metric='euclidean')
                     coords = reducer.fit_transform(data_scaled)
                 non_numeric = [c for c in df.columns if c not in numeric_cols and not c.startswith('_')]
-                entity_col_pca = non_numeric[0] if non_numeric else df.columns[0]
+                entity_col_umap = non_numeric[0] if non_numeric else df.columns[0]
                 labels_list = cluster_entry.get('labels', [])
                 # Save: create EntityProfile records with coordinates
                 from analysis.models import EntityProfile as EP
@@ -1616,7 +1616,7 @@ async def _handle_extract_features(
                     cr_cache = {}
                     for i, row in enumerate(df.iter_rows(named=True)):
                         if i >= len(coords): break
-                        ev = str(row.get(entity_col_pca, ''))
+                        ev = str(row.get(entity_col_umap, ''))
                         if not ev: continue
                         lbl = labels_list[i] if i < len(labels_list) else -1
                         cache_key = f'{run_id_for_ep}_{lbl}'

analysis/views.py

@@ -580,6 +580,50 @@ def _background_process(run_id, upload_dir):
                 logger.info('[BACKGROUND] Restored numeric types for %d columns: %s',
                             len(type_map), list(type_map.keys())[:10])
 
+        # ── SVD dimensionality reduction ──
+        svd_components = 0
+        # Only operate on columns with known numeric types in the schema
+        numeric_dtypes = {'Int64', 'Float64', 'Int32', 'Float32',
+                          'Int8', 'Int16', 'UInt8', 'UInt16', 'UInt32', 'UInt64'}
+        numeric_cols = [c for c, dt in schema.items() if dt in numeric_dtypes]
+        n_numeric = len(numeric_cols)
+
+        if n_numeric >= 2:
+            n_components = min(50, n_numeric - 1)
+            try:
+                from sklearn.decomposition import TruncatedSVD  # fmt: skip
+                import numpy as np  # fmt: skip
+
+                # Collect numeric data for SVD (full materialise → transform → add back)
+                df_full = lf.select(numeric_cols).collect(streaming=True)
+                X = df_full.to_numpy()
+                X = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)
+
+                svd = TruncatedSVD(n_components=n_components, random_state=42)
+                X_svd = svd.fit_transform(X)
+                svd_components = n_components
+
+                # Build a new LazyFrame that includes _svd_* columns
+                # Strategy: collect full LazyFrame once, add SVD columns, re-wrap as lazy
+                df_all = lf.collect(streaming=True)
+                for i in range(n_components):
+                    col_name = f'_svd_{i}'
+                    schema[col_name] = 'Float64'
+                    df_all = df_all.with_columns(
+                        pl.Series(col_name, X_svd[:, i])
+                    )
+                lf = df_all.lazy()
+
+                logger.info('[BACKGROUND] SVD: %d components from %d numeric columns '
+                            '(explained variance ratio sum=%.4f)',
+                            svd_components, n_numeric,
+                            float(svd.explained_variance_ratio_.sum()))
+            except Exception as e:
+                logger.warning('[BACKGROUND] SVD failed (non-fatal): %s', e)
+        elif n_numeric == 1:
+            logger.info('[BACKGROUND] SVD skipped: only 1 numeric column (%s)',
+                        numeric_cols[0])
+
         # Get row count
         df_count = lf.select(pl.len()).collect(streaming=True)
         total_rows = df_count[0, 0] if df_count.height > 0 else 0
@@ -593,6 +637,7 @@ def _background_process(run_id, upload_dir):
             'row_count': total_rows, 'file_count': total_files, 'upload_dir': str(upload_dir),
             'csv_glob': str(upload_dir / '*.csv'),
             'sqlite_table': table_name,
+            'svd_components': svd_components,
         })
 
         # ── Phase 3: Data ready for filtering/clustering via MCP tools ──
@@ -730,7 +775,7 @@ def _run_pipeline_worker(run_id, analysis_fn, **ctx):
 # ── Shared clustering pipeline ─────────────────────────────────────────
 
 def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorithm,
-                              min_cluster_size, run_pca=True, head=None):
+                              min_cluster_size, run_umap=True, head=None):
     """Unified clustering pipeline: clustering → feature extraction → UMAP → ORM save.
 
     Args:
@@ -740,7 +785,7 @@ def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorith
         feature_columns: list of feature column names (None = auto-detect).
         algorithm: 'hdbscan' or 'kmeans'.
         min_cluster_size: int for HDBSCAN.
-        run_pca: bool, whether to compute and save UMAP-2D embeddings.
+        run_umap: bool, whether to compute and save UMAP-2D embeddings.
         head: optional int, downsample to at most this many rows (min 100).
     """
     import warnings
@@ -851,7 +896,7 @@ def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorith
             raise Exception(feat_result['error'])
 
         # ── Step 3: UMAP-2D embedding (sampled training for large data) ─────────
-        if run_pca:
+        if run_umap:
             try:
                 lf = entry['lazyframe']
                 # UMAP: sample max 10K for training, batch-transform rest in 1K batches
@@ -876,11 +921,11 @@ def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorith
                         run.progress_msg = f'UMAP 采样 {MAX_UMAP_TRAIN}/{n_total} 实体训练...'
                         run.save(update_fields=['progress_msg'])
                         df_sample = lf.sample(n=MAX_UMAP_TRAIN, seed=42).collect(streaming=True)
-                        df_pca = lf.collect(streaming=True)
+                        df_umap = lf.collect(streaming=True)
                     else:
-                        df_pca = lf.collect(streaming=True)
+                        df_umap = lf.collect(streaming=True)
 
-                    if len(df_pca) > 2:
+                    if len(df_umap) > 2:
                         if n_total > MAX_UMAP_TRAIN:
                             # Train UMAP on 10K sample, batch-transform full dataset
                             mat_sample = df_sample.select(num_cols).to_numpy()
@@ -892,23 +937,23 @@ def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorith
                             reducer.fit(mat_sample_scaled)
                             # Batch transform in 1K-row batches
                             coords_list = []
-                            for start in range(0, len(df_pca), BATCH_SIZE):
-                                end = min(start + BATCH_SIZE, len(df_pca))
+                            for start in range(0, len(df_umap), BATCH_SIZE):
+                                end = min(start + BATCH_SIZE, len(df_umap))
                                 mat_batch = scaler.transform(
-                                    df_pca[start:end].select(num_cols).to_numpy())
+                                    df_umap[start:end].select(num_cols).to_numpy())
                                 mat_batch = np.nan_to_num(mat_batch, nan=0.0)
                                 coords_list.append(reducer.transform(mat_batch))
                             coords = np.vstack(coords_list)
                         else:
                             mat = np.nan_to_num(StandardScaler().fit_transform(
-                                df_pca.select(num_cols).to_numpy()), nan=0.0)
+                                df_umap.select(num_cols).to_numpy()), nan=0.0)
                             reducer = umap.UMAP(n_components=2, random_state=42,
                                                 n_neighbors=15, min_dist=0.1,
                                                 metric='euclidean')
                             coords = reducer.fit_transform(mat)
 
-                        non_num = [c for c in df_pca.columns if c not in num_cols and not c.startswith('_')]
-                        ent_col = non_num[0] if non_num else df_pca.columns[0]
+                        non_num = [c for c in df_umap.columns if c not in num_cols and not c.startswith('_')]
+                        ent_col = non_num[0] if non_num else df_umap.columns[0]
 
                         cluster_entry = store.get_cluster_result(cluster_id)
                         labels_list = cluster_entry.get('labels', []) if cluster_entry else []
@@ -918,7 +963,7 @@ def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorith
 
                         profiles = []
                         cr_cache = {}
-                        for i, row in enumerate(df_pca.iter_rows(named=True)):
+                        for i, row in enumerate(df_umap.iter_rows(named=True)):
                             ev = str(row.get(ent_col, ''))
                             if not ev:
                                 continue
@@ -1019,7 +1064,7 @@ def manual_run_analysis(request):
                 run=run, store=store, entity_ds_id=ds_id,
                 feature_columns=feature_columns,
                 algorithm=algorithm, min_cluster_size=min_cluster_size,
-                run_pca=True, head=head,
+                run_umap=True, head=head,
             )
         except Exception:
             tb = traceback.format_exc()
@@ -1253,7 +1298,7 @@ def run_llm_analysis_view(request):
                         feature_columns=None,
                         algorithm='hdbscan',
                         min_cluster_size=5,
-                        run_pca=True,
+                        run_umap=True,
                         head=ctx.get('head'),
                     )
                 else:

templates/analysis/cluster_overview.html

@@ -28,7 +28,7 @@
 </div>
 
 <div class="card">
-    <h2>PCA-2D Embedding</h2>
+    <h2>UMAP-2D Embedding</h2>
     <div class="chart-wrapper">
         <div class="chart-container">
             <canvas id="scatterChart" class="scatter-canvas"></canvas>
@@ -189,7 +189,7 @@ function drawScatter(canvasId, data, xLabel, yLabel) {
     }
 }
 
-drawScatter('scatterChart', scatterData, 'PCA-1', 'PCA-2');
+drawScatter('scatterChart', scatterData, 'UMAP-1', 'UMAP-2');
 
 {% if geo_count > 0 %}
 const geoData = {{ geo_data_json|safe }};

templates/analysis/entity_profile.html

@@ -37,7 +37,7 @@
 
 {% if entity.embedding_x is not None and entity.embedding_y is not None %}
 <div class="card">
-    <h2>PCA Position</h2>
+    <h2>UMAP Position</h2>
     <p>X: {{ entity.embedding_x|floatformat:3 }}, Y: {{ entity.embedding_y|floatformat:3 }}</p>
     <canvas id="entityScatter" class="scatter-canvas" height="300"></canvas>
 </div>

templates/tianxuan/manual.html

@@ -194,7 +194,7 @@ function buildDatasetTable(filter) {
         return;
     }
 
-    let html = '<table class="ds-table"><thead><tr><th>ID</th><th>状态</th><th>行数</th><th>文件数</th><th></th></tr></thead><tbody>';
+    let html = '<table class="ds-table"><thead><tr><th>ID</th><th>状态</th><th>行数</th><th>文件数</th><th>SVD</th><th></th></tr></thead><tbody>';
     rows.forEach(ds => {
         const label = ds.status_label || '待分析';
         const badgeClass = (ds.run_status === 'completed') ? 'completed' :
@@ -203,11 +203,13 @@ function buildDatasetTable(filter) {
         const dispId = ds.display_id || ds.dataset_id;
         const rowCount = ds.row_count != null ? ds.row_count.toLocaleString() : '?';
         const fileCount = ds.file_count != null ? ds.file_count : '?';
+        const svdInfo = ds.svd_components ? `SVD: ${ds.svd_components}维` : '-';
         html += `<tr data-ds-id="${ds.dataset_id}" onclick="selectDataset('${ds.dataset_id}')">
             <td style="font-weight:600;">${dispId}</td>
             <td><span class="status-badge ${badgeClass}">${label}</span></td>
             <td>${rowCount}</td>
             <td>${fileCount}</td>
+            <td>${svdInfo}</td>
             <td><button class="btn" onclick="event.stopPropagation();selectDataset('${ds.dataset_id}')" style="font-size:0.75rem;padding:0.2rem 0.5rem;">选择</button></td>
         </tr>`;
     });

tests/test_e2e_batch.py

@@ -1,7 +1,7 @@
 """E2E batch upload test: upload 10 CSV files (2 batches of 5), cluster, verify."""
 import subprocess, time, os, sys, glob, json
 
-PROJ = r"C:\Users\25044\Desktop\Proj\天璇"
+PROJ = r"E:\hjq\天璇"
 PORT = 8765
 BASE = f"http://127.0.0.1:{PORT}"
 

tests/test_e2e_v1.1.4.py

@@ -1,229 +0,0 @@
-"""E2E test v1.1.4: upload 10 CSVs → cluster → verify globe.
-
-Full pipeline WITHOUT manual server start/stop.
-Port 8766 avoids conflicts with 8000.
-Uses stdlib urllib (no pip install needed).
-"""
-import subprocess
-import time
-import sys
-import os
-import glob
-import json
-import uuid
-import urllib.request
-import urllib.error
-
-PORT = 8766
-BASE = f"http://127.0.0.1:{PORT}"
-PROJ = r"C:\Users\25044\Desktop\Proj\天璇"
-
-
-def http_get(path, timeout=30):
-    """GET and return (status_code, body_bytes)."""
-    try:
-        r = urllib.request.urlopen(f"{BASE}{path}", timeout=timeout)
-        body = r.read()
-        code = r.status
-        r.close()
-        return code, body
-    except urllib.error.HTTPError as e:
-        return e.code, e.read()
-
-
-def http_post_json(path, data_dict, timeout=30):
-    """POST JSON and return (status_code, parsed_json)."""
-    body = json.dumps(data_dict).encode("utf-8")
-    req = urllib.request.Request(
-        f"{BASE}{path}", data=body, headers={"Content-Type": "application/json"}
-    )
-    try:
-        r = urllib.request.urlopen(req, timeout=timeout)
-        return r.status, json.loads(r.read())
-    except urllib.error.HTTPError as e:
-        return e.code, json.loads(e.read())
-
-
-def http_post_files(path, file_paths, timeout=120):
-    """POST multipart/form-data upload. Returns (status_code, parsed_json)."""
-    boundary = uuid.uuid4().hex
-    lines = []
-    for fp in file_paths:
-        fname = os.path.basename(fp)
-        with open(fp, "rb") as f:
-            filedata = f.read()
-        lines.append(f"--{boundary}".encode())
-        lines.append(
-            f'Content-Disposition: form-data; name="files"; filename="{fname}"'.encode()
-        )
-        lines.append(b"Content-Type: text/csv")
-        lines.append(b"")
-        lines.append(filedata)
-    lines.append(f"--{boundary}--".encode())
-    body = b"\r\n".join(lines)
-
-    req = urllib.request.Request(
-        f"{BASE}{path}",
-        data=body,
-        headers={"Content-Type": f"multipart/form-data; boundary={boundary}"},
-    )
-    try:
-        r = urllib.request.urlopen(req, timeout=timeout)
-        return r.status, json.loads(r.read())
-    except urllib.error.HTTPError as e:
-        return e.code, json.loads(e.read())
-
-
-# ── 1. Start server ──
-print("[1/6] Starting Django server...")
-import tempfile as _tmp
-_stderr_log = os.path.join(PROJ, "tests", "_server_stderr.log")
-_stderr_fh = open(_stderr_log, "w")
-server = subprocess.Popen(
-    [
-        os.path.join(PROJ, "runtime", "python", "python.exe"),
-        "manage.py",
-        "runserver",
-        f"127.0.0.1:{PORT}",
-        "--noreload",
-    ],
-    cwd=PROJ,
-    stdout=_stderr_fh,
-    stderr=subprocess.STDOUT,
-)
-time.sleep(8)
-
-# Verify server is up
-for attempt in range(10):
-    try:
-        urllib.request.urlopen(f"{BASE}/", timeout=3)
-        print("  Server is up.")
-        break
-    except Exception:
-        if attempt == 9:
-            print("  FAIL: Server did not start within 30 seconds")
-            server.terminate()
-            server.wait()
-            sys.exit(1)
-        time.sleep(2)
-
-# ── 2. Upload 10 test_*.csv files ──
-csv_files = sorted(glob.glob(os.path.join(PROJ, "data", "multi_upload", "test_*.csv")))
-print(f"[2/6] Found {len(csv_files)} CSV files. Uploading first 10...")
-batch = csv_files[:10]
-code, data = http_post_files("/upload/csv/", batch, timeout=120)
-
-if code != 200:
-    print(f"  FAIL: Upload returned HTTP {code}: {data}")
-    server.terminate()
-    server.wait()
-    sys.exit(1)
-
-run_id = data["run_id"]
-print(f"  Uploaded 10 files → run_id={run_id}, status={data['status']}")
-
-# ── 3. Poll until upload processing completes (status='ready') ──
-print("[3/6] Waiting for background processing...")
-s = {}
-for i in range(180):
-    code, body = http_get(f"/runs/{run_id}/status/", timeout=10)
-    s = json.loads(body) if code == 200 else {}
-    if s.get("status") in ("ready", "failed"):
-        break
-    if i % 10 == 0:
-        print(f"  Poll #{i}: {s.get('status')} ({s.get('progress_pct', 0)}%)")
-    time.sleep(2)
-print(f"  Status: {s.get('status')}, progress={s.get('progress_pct')}%")
-
-if s.get("status") == "failed":
-    print(f"  FAIL: {s.get('error_message', '')[:500]}")
-    server.terminate()
-    server.wait()
-    sys.exit(1)
-
-if s.get("status") != "ready":
-    print(f"  FAIL: Timed out waiting for 'ready', got '{s.get('status')}'")
-    server.terminate()
-    server.wait()
-    sys.exit(1)
-
-# ── 4. Run clustering ──
-print("[4/6] Starting clustering (HDBSCAN, min_cluster_size=5)...")
-code, resp = http_post_json(
-    "/analyze/run/",
-    {"run_id": run_id, "algorithm": "hdbscan", "min_cluster_size": 5, "cluster_mode": "raw"},
-    timeout=30,
-)
-print(f"  Response HTTP {code}: {resp.get('status')}")
-
-if code != 200:
-    print(f"  FAIL: Clustering start failed: {resp}")
-    server.terminate()
-    server.wait()
-    sys.exit(1)
-
-# ── 5. Poll until clustering completes ──
-print("[5/6] Waiting for clustering...")
-for i in range(120):
-    code, body = http_get(f"/runs/{run_id}/status/", timeout=10)
-    s = json.loads(body) if code == 200 else {}
-    if s.get("status") in ("completed", "failed"):
-        break
-    if i % 5 == 0:
-        print(
-            f"  Poll #{i}: {s.get('status')} ({s.get('progress_pct', 0)}%)"
-            f" - {s.get('progress_msg', '')}"
-        )
-    time.sleep(3)
-print(
-    f"  Final: status={s.get('status')}, entity_count={s.get('entity_count')},"
-    f" cluster_count={s.get('cluster_count')}"
-)
-
-# ── 6. Verify globe page ──
-print("[6/6] Verifying globe page...")
-code, body = http_get(f"/globe/?runs={run_id}", timeout=30)
-globe_text = body.decode("utf-8", errors="replace")
-has_arcs = "arcCount" in globe_text
-print(f"  Globe HTTP {code}, arcCount present: {has_arcs}")
-
-# ── Cleanup ──
-server.terminate()
-server.wait()
-_stderr_fh.close()
-
-# Print last 30 lines of server output on failure
-_need_server_log = False
-
-# ── Report ──
-errors = []
-
-if s.get("status") != "completed":
-    errors.append(f"Expected status='completed', got '{s.get('status')}'")
-    errors.append(f"error: {s.get('error_message', '')[:500]}")
-
-if s.get("cluster_count", 0) == 0:
-    errors.append("No clusters found (cluster_count=0)")
-
-if not has_arcs:
-    errors.append("Globe page missing 'arcCount' element")
-
-if errors:
-    print(f"\nFAIL ({len(errors)} errors):")
-    for e in errors:
-        print(f"  - {e}")
-    # Show server stderr tail
-    if os.path.exists(_stderr_log):
-        with open(_stderr_log, "r", encoding="utf-8", errors="replace") as f:
-            lines = f.readlines()
-            tail = lines[-30:] if len(lines) > 30 else lines
-            print(f"\n--- Server stderr (last {len(tail)} lines) ---")
-            for line in tail:
-                print(f"  {line.rstrip()}")
-    sys.exit(1)
-
-print("\n*** E2E v1.1.4 PASSED ***")
-print(
-    f"  run_id={run_id}, clusters={s.get('cluster_count')},"
-    f" entities={s.get('entity_count')}, globe_arcs={'yes' if has_arcs else 'no'}"
-)

tests/test_e2e_v1.1.6.py

@@ -1,237 +0,0 @@
-"""E2E test v1.1.6: generate 50 CSVs → upload 10 → cluster → verify globe.
-
-Self-contained: generates data, starts server, uploads, clusters, verifies globe, reports.
-Port 18766. Uses requests for HTTP (installed automatically if missing).
-"""
-import subprocess
-import time
-import sys
-import os
-import glob
-import json
-import urllib.request
-import urllib.error
-
-PORT = 18766
-BASE = f"http://127.0.0.1:{PORT}"
-PROJ = r"C:\Users\25044\Desktop\Proj\天璇"
-PYTHON = os.path.join(PROJ, "runtime", "python", "python.exe")
-
-# ── Ensure requests is installed ──
-try:
-    import requests
-except ImportError:
-    print("[0] Installing requests...")
-    subprocess.run([PYTHON, "-m", "pip", "install", "requests", "-q"], cwd=PROJ, check=True, timeout=120)
-    import requests
-
-
-def http_get(path, timeout=30):
-    """GET and return (status_code, parsed_json or body_bytes)."""
-    try:
-        r = requests.get(f"{BASE}{path}", timeout=timeout)
-        try:
-            return r.status_code, r.json()
-        except Exception:
-            return r.status_code, r.text
-    except Exception as e:
-        return 0, str(e)
-
-
-def http_post_json(path, data_dict, timeout=30):
-    """POST JSON and return (status_code, parsed_json)."""
-    try:
-        r = requests.post(f"{BASE}{path}", json=data_dict, timeout=timeout)
-        try:
-            return r.status_code, r.json()
-        except Exception:
-            return r.status_code, r.text
-    except Exception as e:
-        return 0, str(e)
-
-
-def http_post_files(path, file_paths, timeout=120):
-    """POST multipart/form-data upload. Returns (status_code, parsed_json)."""
-    files = [("files", (os.path.basename(fp), open(fp, "rb"), "text/csv")) for fp in file_paths]
-    try:
-        r = requests.post(f"{BASE}{path}", files=files, timeout=timeout)
-        return r.status_code, r.json()
-    except Exception as e:
-        return 0, str(e)
-
-
-def poll_status(run_id, target_status, max_wait_sec, description):
-    """Poll /runs/<run_id>/status/ until target_status or timeout. Returns status dict."""
-    s = {}
-    for i in range(max_wait_sec // 2):
-        code, body = http_get(f"/runs/{run_id}/status/", timeout=10)
-        s = body if isinstance(body, dict) else {}
-        if s.get("status") in (target_status, "failed"):
-            break
-        if i % 10 == 0:
-            print(f"  [{description}] Poll #{i}: {s.get('status')} ({s.get('progress_pct', 0)}%)")
-        time.sleep(2)
-    print(f"  [{description}] Final: status={s.get('status')}, progress={s.get('progress_pct', 0)}%")
-    return s
-
-
-# ═══════════════════════════════════════════════════════════════════════════
-# MAIN
-# ═══════════════════════════════════════════════════════════════════════════
-
-errors = []
-
-# ── 1. Generate 50 test CSV files ──
-print("[1/6] Generating 50 test CSV files (100 rows each)...")
-gen_result = subprocess.run(
-    [PYTHON, "scripts/gen_multi.py", "--files", "50", "--rows", "100"],
-    cwd=PROJ, capture_output=True, text=True, timeout=180,
-)
-if gen_result.returncode != 0:
-    print(f"  FAIL: gen_multi.py failed.\nSTDERR:\n{gen_result.stderr}")
-    sys.exit(1)
-print(gen_result.stdout.strip())
-
-csv_files = sorted(glob.glob(os.path.join(PROJ, "data", "multi_upload", "test_*.csv")))
-print(f"  Generated {len(csv_files)} CSV files.")
-
-# ── 2. Start Django server ──
-print(f"[2/6] Starting Django server on port {PORT}...")
-stderr_log = os.path.join(PROJ, "tests", "_e2e_server_v116.log")
-stderr_fh = open(stderr_log, "w")
-server = subprocess.Popen(
-    [PYTHON, "manage.py", "runserver", f"127.0.0.1:{PORT}", "--noreload"],
-    cwd=PROJ, stdout=stderr_fh, stderr=subprocess.STDOUT,
-)
-time.sleep(6)
-
-# Verify server is up
-server_up = False
-for attempt in range(15):
-    try:
-        urllib.request.urlopen(f"{BASE}/", timeout=3)
-        server_up = True
-        print("  Server is up.")
-        break
-    except Exception:
-        if attempt == 14:
-            errors.append("Server did not start within 45 seconds")
-        time.sleep(2)
-
-if not server_up:
-    server.terminate(); server.wait(); stderr_fh.close()
-    print("\nFAIL")
-    for e in errors: print(f"  - {e}")
-    sys.exit(1)
-
-# ── 3. Upload first 10 CSV files ──
-print("[3/6] Uploading first 10 CSV files...")
-batch = csv_files[:10]
-code, data = http_post_files("/upload/csv/", batch, timeout=180)
-
-if code != 200:
-    errors.append(f"Upload returned HTTP {code}: {str(data)[:300]}")
-    server.terminate(); server.wait(); stderr_fh.close()
-    print("\nFAIL")
-    for e in errors: print(f"  - {e}")
-    sys.exit(1)
-
-run_id = data["run_id"]
-print(f"  Uploaded {len(batch)} files → run_id={run_id}, status={data['status']}")
-
-# ── 4. Poll until background processing completes (status='ready') ──
-print("[4/6] Waiting for background upload processing (max 300s)...")
-s = poll_status(run_id, "ready", 300, "Upload processing")
-
-if s.get("status") == "failed":
-    errors.append(f"Upload processing failed: {s.get('error_message', '')[:500]}")
-
-if s.get("status") not in ("ready",):
-    errors.append(f"Expected status='ready', got '{s.get('status')}'")
-
-if errors:
-    server.terminate(); server.wait(); stderr_fh.close()
-    print("\nFAIL")
-    for e in errors: print(f"  - {e}")
-    sys.exit(1)
-
-# ── 5. Run clustering and poll ──
-print("[5/6] Starting clustering (HDBSCAN, min_cluster_size=5)...")
-code, resp = http_post_json(
-    "/analyze/run/",
-    {"run_id": run_id, "algorithm": "hdbscan", "min_cluster_size": 5, "cluster_mode": "raw"},
-    timeout=30,
-)
-print(f"  Response HTTP {code}: {resp if isinstance(resp, dict) else str(resp)[:200]}")
-
-if code != 200:
-    errors.append(f"Clustering start failed: {resp}")
-    server.terminate(); server.wait(); stderr_fh.close()
-    print("\nFAIL")
-    for e in errors: print(f"  - {e}")
-    sys.exit(1)
-
-s_cluster = poll_status(run_id, "completed", 600, "Clustering")
-
-if s_cluster.get("status") == "failed":
-    errors.append(f"Clustering failed: {s_cluster.get('error_message', '')[:500]}")
-
-if s_cluster.get("status") != "completed":
-    errors.append(f"Expected status='completed', got '{s_cluster.get('status')}'")
-
-# ── 6. Verify globe page ──
-print("[6/6] Verifying globe page...")
-code, globe_body = http_get(f"/globe/?runs={run_id}", timeout=30)
-globe_text = globe_body if isinstance(globe_body, str) else json.dumps(globe_body)
-has_arcs = "arcCount" in globe_text if isinstance(globe_text, str) else False
-print(f"  Globe HTTP {code}, arcCount present: {has_arcs}")
-
-# ── Cleanup ──
-server.terminate()
-server.wait()
-stderr_fh.close()
-
-# ── Collect metrics ──
-total_flows = s_cluster.get("total_flows", "?")
-entity_count = s_cluster.get("entity_count", 0)
-cluster_count = s_cluster.get("cluster_count", 0)
-status = s_cluster.get("status", "unknown")
-
-# ── Validate required metrics ──
-if not has_arcs:
-    errors.append("Globe page missing 'arcCount' element")
-if entity_count == 0:
-    errors.append("entity_count=0 (no entities detected)")
-if cluster_count == 0:
-    errors.append("cluster_count=0 (no clusters found)")
-
-# ── Show server log tail on failure ──
-if errors and os.path.exists(stderr_log):
-    with open(stderr_log, "r", encoding="utf-8", errors="replace") as f:
-        lines = f.readlines()
-        tail = lines[-40:] if len(lines) > 40 else lines
-        print(f"\n--- Server log tail ({len(tail)} lines) ---")
-        for line in tail:
-            print(f"  {line.rstrip()}")
-
-# ── Report ──
-print()
-print("=" * 60)
-print("  E2E v1.1.6 RESULTS")
-print("=" * 60)
-print(f"  run_id        : {run_id}")
-print(f"  status        : {status}")
-print(f"  total_flows   : {total_flows}")
-print(f"  entity_count  : {entity_count}")
-print(f"  cluster_count : {cluster_count}")
-print(f"  globe_arcs    : {'PASS' if has_arcs else 'FAIL'}")
-
-if errors:
-    print(f"\n  FAIL ({len(errors)} errors):")
-    for e in errors:
-        print(f"    - {e}")
-    sys.exit(1)
-else:
-    print(f"\n  *** E2E v1.1.6 PASSED ***")
-    print(f"  run_id={run_id}, clusters={cluster_count}, entities={entity_count}, globe_arcs=yes")
-    sys.exit(0)

tests/test_e2e_v1.1.6_full.py

@@ -1,70 +0,0 @@
-import subprocess, time, os, sys, glob, json
-
-PORT = 17766
-PROJ = r"C:\Users\25044\Desktop\Proj\天璇"
-os.chdir(PROJ)
-
-# 1. Start server
-print("Starting server...")
-server = subprocess.Popen(
-    [r"runtime\python\python.exe", "manage.py", "runserver", f"127.0.0.1:{PORT}", "--noreload"],
-    stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
-)
-time.sleep(6)
-
-import requests
-
-# 2. Upload 50 files
-print("Uploading 50 files...")
-csvs = sorted(glob.glob("data/multi_upload/[0-9]*.csv"))[:50]
-files = [("files", (os.path.basename(f), open(f, "rb"), "text/csv")) for f in csvs]
-r = requests.post(f"http://127.0.0.1:{PORT}/upload/csv/", files=files, timeout=300)
-data = r.json()
-run_id = data["run_id"]
-print(f"run_id={run_id}")
-
-# 3. Poll until ready
-for _ in range(180):
-    r = requests.get(f"http://127.0.0.1:{PORT}/runs/{run_id}/status/", timeout=10)
-    s = r.json()
-    st = s["status"]
-    if st in ("ready", "failed"):
-        break
-    time.sleep(2)
-print(f"Upload status={st}, flows={s.get('total_flows')}, error={s.get('error_message','')[:300]}")
-
-if st == "failed":
-    print(f"FAILED: {s.get('error_message','')}")
-    server.terminate(); sys.exit(1)
-
-# 4. Run clustering
-print("Running clustering...")
-r = requests.post(f"http://127.0.0.1:{PORT}/analyze/run/",
-    json={"run_id": run_id, "algorithm": "hdbscan", "min_cluster_size": 5, "cluster_mode": "raw"},
-    timeout=30)
-print(f"Cluster started: {r.json()}")
-
-# 5. Poll until completed
-for _ in range(120):
-    r = requests.get(f"http://127.0.0.1:{PORT}/runs/{run_id}/status/", timeout=10)
-    s = r.json()
-    st = s["status"]
-    if st in ("completed", "failed"):
-        break
-    time.sleep(5)
-print(f"Cluster status={st}, entity_count={s.get('entity_count')}, cluster_count={s.get('cluster_count')}")
-
-# 6. Verify globe
-r = requests.get(f"http://127.0.0.1:{PORT}/globe/?runs={run_id}", timeout=10)
-has_arcs = "arcCount" in r.text
-print(f"Globe arcCount present: {has_arcs}")
-
-# 7. Result
-server.terminate()
-server.wait()
-if st == "completed" and s.get("cluster_count", 0) > 0:
-    print(f"E2E PASSED: {s['entity_count']} entities, {s['cluster_count']} clusters")
-    sys.exit(0)
-else:
-    print(f"E2E FAILED")
-    sys.exit(1)

tianxuan/llm_orchestrator.py

@@ -19,7 +19,7 @@ TOOLS = [
         "description": "按实体列（源IP、SNI等）分组原始流。生成每个实体的特征（流数量、TLS版本、端口）。需要已概要分析的数据。",
         "parameters": {"type": "object", "properties": {"dataset_id": {"type": "string"}, "entity_column": {"type": "string"}, "entity_columns": {"type": "array", "items": {"type": "string"}}, "auto_detect": {"type": "boolean"}}, "required": ["dataset_id"]}}},
     {"type": "function", "function": {"name": "compute_scores",
-        "description": "使用PCA权重学习在实体数据上计算自适应代理/异常/威胁评分。在build_entity_profiles之后调用。添加proxy_score/risk_level/threat_score列。",
+        "description": "使用自适应权重学习在实体数据上计算自适应代理/异常/威胁评分。在build_entity_profiles之后调用。添加proxy_score/risk_level/threat_score列。",
         "parameters": {"type": "object", "properties": {"dataset_id": {"type": "string"}}, "required": ["dataset_id"]}}},
     {"type": "function", "function": {"name": "run_clustering",
         "description": "将实体数据聚类为行为组。需要实体数据集ID。探索性分析选择hdbscan（自动检测聚类数）；固定簇数选择kmeans。",
@@ -31,7 +31,7 @@ TOOLS = [
         "description": "运行孤立森林（Isolation Forest）检测异常实体。在compute_scores之后调用。返回异常数量和比率。",
         "parameters": {"type": "object", "properties": {"dataset_id": {"type": "string"}, "contamination": {"type": "number", "description": "Expected anomaly rate 0.01-0.1 (default 0.05)"}}, "required": ["dataset_id"]}}},
     {"type": "function", "function": {"name": "visualize_anomalies",
-        "description": "生成PCA-2D散点数据用于前端展示。按风险等级着色。在compute_scores或detect_anomalies之后调用。",
+        "description": "生成UMAP-2D散点数据用于前端展示。按风险等级着色。在compute_scores或detect_anomalies之后调用。",
         "parameters": {"type": "object", "properties": {"dataset_id": {"type": "string"}}, "required": ["dataset_id"]}}},
     # ── Diagnostic (call when pipeline fails or results look wrong) ──
     {"type": "function", "function": {"name": "validate_data",