chore: clean PCA references, integrate SVD pipeline, update AGENTS.md

2026-07-22 21:10:13 +08:00
parent 585dbb4ec4
commit 7b9084b33a
9 changed files with 78 additions and 30 deletions
@@ -62,7 +62,7 @@ class Command(BaseCommand):
                and schema[c].split('(')[0].strip() in numeric_types
            ][:10]

-            # ── Step 2: Clustering + Extraction + PCA (shared pipeline) ─────
+            # ── Step 2: Clustering + Extraction + UMAP (shared pipeline) ─────
            _self.stdout.write(f'[2/3] 聚类 ({algo})')
            _self.stdout.write(f'  → 聚类特征: {feature_cols}')
            run.total_flows = row_count
@@ -72,7 +72,7 @@ class Command(BaseCommand):
                run=run, store=store, entity_ds_id=ds_id,
                feature_columns=feature_cols,
                algorithm=algo, min_cluster_size=5,
-                run_pca=True,
+                run_umap=True,
            )
            if run.status == 'failed':
                _self.stderr.write(_self.style.ERROR(f'  ✗ 分析失败: {run.error_message}'))
@@ -94,8 +94,8 @@ class EntityProfile(models.Model):
                                 related_name='entities')
    feature_json = models.JSONField(default=dict, blank=True,
                                     help_text="Aggregated entity features as dict")
-    embedding_x = models.FloatField(null=True, blank=True, help_text="PCA-2D X coordinate")
-    embedding_y = models.FloatField(null=True, blank=True, help_text="PCA-2D Y coordinate")
+    embedding_x = models.FloatField(null=True, blank=True, help_text="UMAP-2D X coordinate")
+    embedding_y = models.FloatField(null=True, blank=True, help_text="UMAP-2D Y coordinate")

    class Meta:
        unique_together = ['run', 'entity_value']
@@ -252,6 +252,7 @@ class SessionStore:
                    'parent_id': entry.get('parent_id'),
                    'row_count': meta.get('row_count'),
                    'file_count': meta.get('file_count'),
+                    'svd_components': meta.get('svd_components', 0),
                    'columns': list(entry.get('schema', {}).keys()),
                    'column_count': len(entry.get('schema', {})),
                })
@@ -498,7 +498,7 @@ def get_tools_meta() -> list[Tool]:
        Tool(
            name="compute_scores",
            description=(
-                "在实体数据上计算自适应代理/异常/威胁评分。使用PCA从数据自动学习"
+                "在实体数据上计算自适应代理/异常/威胁评分。从数据自动学习"
                "特征权重，计算加权代理评分（proxy_score）、基于IQR的风险等级"
                "（normal/watch/suspicious/critical）和威胁评分（threat_score）。"
                "返回评分摘要与阈值。"
@@ -539,7 +539,7 @@ def get_tools_meta() -> list[Tool]:
        Tool(
            name="visualize_anomalies",
            description=(
-                "为高风险实体生成PCA-2D嵌入和聚类可视化数据。按风险等级和聚类着色。"
+                "为高风险实体生成UMAP-2D嵌入和聚类可视化数据。按风险等级和聚类着色。"
                "返回用于前端渲染的散点图JSON数据。"
            ),
            inputSchema={
@@ -608,7 +608,7 @@ def get_tools_meta() -> list[Tool]:
            name="diagnose_clustering",
            description=(
                "诊断工具：当聚类产生太少/太多聚类、全部为噪声或轮廓系数低时——诊断原因。"
-                "检查：特征方差、相关矩阵、PCA解释方差、数据稀疏性，并建议参数调整"
+                "检查：特征方差、相关矩阵、特征降维分析、数据稀疏性，并建议参数调整"
                "（min_cluster_size、算法选择、特征选择）。"
            ),
            inputSchema={
@@ -1598,7 +1598,7 @@ async def _handle_extract_features(
                                        metric='euclidean')
                    coords = reducer.fit_transform(data_scaled)
                non_numeric = [c for c in df.columns if c not in numeric_cols and not c.startswith('_')]
-                entity_col_pca = non_numeric[0] if non_numeric else df.columns[0]
+                entity_col_umap = non_numeric[0] if non_numeric else df.columns[0]
                labels_list = cluster_entry.get('labels', [])
                # Save: create EntityProfile records with coordinates
                from analysis.models import EntityProfile as EP
@@ -1616,7 +1616,7 @@ async def _handle_extract_features(
                    cr_cache = {}
                    for i, row in enumerate(df.iter_rows(named=True)):
                        if i >= len(coords): break
-                        ev = str(row.get(entity_col_pca, ''))
+                        ev = str(row.get(entity_col_umap, ''))
                        if not ev: continue
                        lbl = labels_list[i] if i < len(labels_list) else -1
                        cache_key = f'{run_id_for_ep}_{lbl}'
@@ -580,6 +580,50 @@ def _background_process(run_id, upload_dir):
                logger.info('[BACKGROUND] Restored numeric types for %d columns: %s',
                            len(type_map), list(type_map.keys())[:10])

+        # ── SVD dimensionality reduction ──
+        svd_components = 0
+        # Only operate on columns with known numeric types in the schema
+        numeric_dtypes = {'Int64', 'Float64', 'Int32', 'Float32',
+                          'Int8', 'Int16', 'UInt8', 'UInt16', 'UInt32', 'UInt64'}
+        numeric_cols = [c for c, dt in schema.items() if dt in numeric_dtypes]
+        n_numeric = len(numeric_cols)
+
+        if n_numeric >= 2:
+            n_components = min(50, n_numeric - 1)
+            try:
+                from sklearn.decomposition import TruncatedSVD  # fmt: skip
+                import numpy as np  # fmt: skip
+
+                # Collect numeric data for SVD (full materialise → transform → add back)
+                df_full = lf.select(numeric_cols).collect(streaming=True)
+                X = df_full.to_numpy()
+                X = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)
+
+                svd = TruncatedSVD(n_components=n_components, random_state=42)
+                X_svd = svd.fit_transform(X)
+                svd_components = n_components
+
+                # Build a new LazyFrame that includes _svd_* columns
+                # Strategy: collect full LazyFrame once, add SVD columns, re-wrap as lazy
+                df_all = lf.collect(streaming=True)
+                for i in range(n_components):
+                    col_name = f'_svd_{i}'
+                    schema[col_name] = 'Float64'
+                    df_all = df_all.with_columns(
+                        pl.Series(col_name, X_svd[:, i])
+                    )
+                lf = df_all.lazy()
+
+                logger.info('[BACKGROUND] SVD: %d components from %d numeric columns '
+                            '(explained variance ratio sum=%.4f)',
+                            svd_components, n_numeric,
+                            float(svd.explained_variance_ratio_.sum()))
+            except Exception as e:
+                logger.warning('[BACKGROUND] SVD failed (non-fatal): %s', e)
+        elif n_numeric == 1:
+            logger.info('[BACKGROUND] SVD skipped: only 1 numeric column (%s)',
+                        numeric_cols[0])
+
        # Get row count
        df_count = lf.select(pl.len()).collect(streaming=True)
        total_rows = df_count[0, 0] if df_count.height > 0 else 0
@@ -593,6 +637,7 @@ def _background_process(run_id, upload_dir):
            'row_count': total_rows, 'file_count': total_files, 'upload_dir': str(upload_dir),
            'csv_glob': str(upload_dir / '*.csv'),
            'sqlite_table': table_name,
+            'svd_components': svd_components,
        })

        # ── Phase 3: Data ready for filtering/clustering via MCP tools ──
@@ -730,7 +775,7 @@ def _run_pipeline_worker(run_id, analysis_fn, **ctx):
 # ── Shared clustering pipeline ─────────────────────────────────────────

 def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorithm,
-                              min_cluster_size, run_pca=True, head=None):
+                              min_cluster_size, run_umap=True, head=None):
    """Unified clustering pipeline: clustering → feature extraction → UMAP → ORM save.

    Args:
@@ -740,7 +785,7 @@ def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorith
        feature_columns: list of feature column names (None = auto-detect).
        algorithm: 'hdbscan' or 'kmeans'.
        min_cluster_size: int for HDBSCAN.
-        run_pca: bool, whether to compute and save UMAP-2D embeddings.
+        run_umap: bool, whether to compute and save UMAP-2D embeddings.
        head: optional int, downsample to at most this many rows (min 100).
    """
    import warnings
@@ -851,7 +896,7 @@ def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorith
            raise Exception(feat_result['error'])

        # ── Step 3: UMAP-2D embedding (sampled training for large data) ─────────
-        if run_pca:
+        if run_umap:
            try:
                lf = entry['lazyframe']
                # UMAP: sample max 10K for training, batch-transform rest in 1K batches
@@ -876,11 +921,11 @@ def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorith
                        run.progress_msg = f'UMAP 采样 {MAX_UMAP_TRAIN}/{n_total} 实体训练...'
                        run.save(update_fields=['progress_msg'])
                        df_sample = lf.sample(n=MAX_UMAP_TRAIN, seed=42).collect(streaming=True)
-                        df_pca = lf.collect(streaming=True)
+                        df_umap = lf.collect(streaming=True)
                    else:
-                        df_pca = lf.collect(streaming=True)
+                        df_umap = lf.collect(streaming=True)

-                    if len(df_pca) > 2:
+                    if len(df_umap) > 2:
                        if n_total > MAX_UMAP_TRAIN:
                            # Train UMAP on 10K sample, batch-transform full dataset
                            mat_sample = df_sample.select(num_cols).to_numpy()
@@ -892,23 +937,23 @@ def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorith
                            reducer.fit(mat_sample_scaled)
                            # Batch transform in 1K-row batches
                            coords_list = []
-                            for start in range(0, len(df_pca), BATCH_SIZE):
-                                end = min(start + BATCH_SIZE, len(df_pca))
+                            for start in range(0, len(df_umap), BATCH_SIZE):
+                                end = min(start + BATCH_SIZE, len(df_umap))
                                mat_batch = scaler.transform(
-                                    df_pca[start:end].select(num_cols).to_numpy())
+                                    df_umap[start:end].select(num_cols).to_numpy())
                                mat_batch = np.nan_to_num(mat_batch, nan=0.0)
                                coords_list.append(reducer.transform(mat_batch))
                            coords = np.vstack(coords_list)
                        else:
                            mat = np.nan_to_num(StandardScaler().fit_transform(
-                                df_pca.select(num_cols).to_numpy()), nan=0.0)
+                                df_umap.select(num_cols).to_numpy()), nan=0.0)
                            reducer = umap.UMAP(n_components=2, random_state=42,
                                                n_neighbors=15, min_dist=0.1,
                                                metric='euclidean')
                            coords = reducer.fit_transform(mat)

-                        non_num = [c for c in df_pca.columns if c not in num_cols and not c.startswith('_')]
-                        ent_col = non_num[0] if non_num else df_pca.columns[0]
+                        non_num = [c for c in df_umap.columns if c not in num_cols and not c.startswith('_')]
+                        ent_col = non_num[0] if non_num else df_umap.columns[0]

                        cluster_entry = store.get_cluster_result(cluster_id)
                        labels_list = cluster_entry.get('labels', []) if cluster_entry else []
@@ -918,7 +963,7 @@ def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorith

                        profiles = []
                        cr_cache = {}
-                        for i, row in enumerate(df_pca.iter_rows(named=True)):
+                        for i, row in enumerate(df_umap.iter_rows(named=True)):
                            ev = str(row.get(ent_col, ''))
                            if not ev:
                                continue
@@ -1019,7 +1064,7 @@ def manual_run_analysis(request):
                run=run, store=store, entity_ds_id=ds_id,
                feature_columns=feature_columns,
                algorithm=algorithm, min_cluster_size=min_cluster_size,
-                run_pca=True, head=head,
+                run_umap=True, head=head,
            )
        except Exception:
            tb = traceback.format_exc()
@@ -1253,7 +1298,7 @@ def run_llm_analysis_view(request):
                        feature_columns=None,
                        algorithm='hdbscan',
                        min_cluster_size=5,
-                        run_pca=True,
+                        run_umap=True,
                        head=ctx.get('head'),
                    )
                else:
@@ -28,7 +28,7 @@
 </div>

 <div class="card">
-    <h2>PCA-2D Embedding</h2>
+    <h2>UMAP-2D Embedding</h2>
    <div class="chart-wrapper">
        <div class="chart-container">
            <canvas id="scatterChart" class="scatter-canvas"></canvas>
@@ -189,7 +189,7 @@ function drawScatter(canvasId, data, xLabel, yLabel) {
    }
 }

-drawScatter('scatterChart', scatterData, 'PCA-1', 'PCA-2');
+drawScatter('scatterChart', scatterData, 'UMAP-1', 'UMAP-2');

 {% if geo_count > 0 %}
 const geoData = {{ geo_data_json|safe }};
@@ -37,7 +37,7 @@

 {% if entity.embedding_x is not None and entity.embedding_y is not None %}
 <div class="card">
-    <h2>PCA Position</h2>
+    <h2>UMAP Position</h2>
    <p>X: {{ entity.embedding_x|floatformat:3 }}, Y: {{ entity.embedding_y|floatformat:3 }}</p>
    <canvas id="entityScatter" class="scatter-canvas" height="300"></canvas>
 </div>
@@ -194,7 +194,7 @@ function buildDatasetTable(filter) {
        return;
    }

-    let html = '<table class="ds-table"><thead><tr><th>ID</th><th>状态</th><th>行数</th><th>文件数</th><th></th></tr></thead><tbody>';
+    let html = '<table class="ds-table"><thead><tr><th>ID</th><th>状态</th><th>行数</th><th>文件数</th><th>SVD</th><th></th></tr></thead><tbody>';
    rows.forEach(ds => {
        const label = ds.status_label || '待分析';
        const badgeClass = (ds.run_status === 'completed') ? 'completed' :
@@ -203,11 +203,13 @@ function buildDatasetTable(filter) {
        const dispId = ds.display_id || ds.dataset_id;
        const rowCount = ds.row_count != null ? ds.row_count.toLocaleString() : '?';
        const fileCount = ds.file_count != null ? ds.file_count : '?';
+        const svdInfo = ds.svd_components ? `SVD: ${ds.svd_components}维` : '-';
        html += `<tr data-ds-id="${ds.dataset_id}" onclick="selectDataset('${ds.dataset_id}')">
            <td style="font-weight:600;">${dispId}</td>
            <td><span class="status-badge ${badgeClass}">${label}</span></td>
            <td>${rowCount}</td>
            <td>${fileCount}</td>
+            <td>${svdInfo}</td>
            <td><button class="btn" onclick="event.stopPropagation();selectDataset('${ds.dataset_id}')" style="font-size:0.75rem;padding:0.2rem 0.5rem;">选择</button></td>
        </tr>`;
    });
@@ -19,7 +19,7 @@ TOOLS = [
        "description": "按实体列（源IP、SNI等）分组原始流。生成每个实体的特征（流数量、TLS版本、端口）。需要已概要分析的数据。",
        "parameters": {"type": "object", "properties": {"dataset_id": {"type": "string"}, "entity_column": {"type": "string"}, "entity_columns": {"type": "array", "items": {"type": "string"}}, "auto_detect": {"type": "boolean"}}, "required": ["dataset_id"]}}},
    {"type": "function", "function": {"name": "compute_scores",
-        "description": "使用PCA权重学习在实体数据上计算自适应代理/异常/威胁评分。在build_entity_profiles之后调用。添加proxy_score/risk_level/threat_score列。",
+        "description": "使用自适应权重学习在实体数据上计算自适应代理/异常/威胁评分。在build_entity_profiles之后调用。添加proxy_score/risk_level/threat_score列。",
        "parameters": {"type": "object", "properties": {"dataset_id": {"type": "string"}}, "required": ["dataset_id"]}}},
    {"type": "function", "function": {"name": "run_clustering",
        "description": "将实体数据聚类为行为组。需要实体数据集ID。探索性分析选择hdbscan（自动检测聚类数）；固定簇数选择kmeans。",
@@ -31,7 +31,7 @@ TOOLS = [
        "description": "运行孤立森林（Isolation Forest）检测异常实体。在compute_scores之后调用。返回异常数量和比率。",
        "parameters": {"type": "object", "properties": {"dataset_id": {"type": "string"}, "contamination": {"type": "number", "description": "Expected anomaly rate 0.01-0.1 (default 0.05)"}}, "required": ["dataset_id"]}}},
    {"type": "function", "function": {"name": "visualize_anomalies",
-        "description": "生成PCA-2D散点数据用于前端展示。按风险等级着色。在compute_scores或detect_anomalies之后调用。",
+        "description": "生成UMAP-2D散点数据用于前端展示。按风险等级着色。在compute_scores或detect_anomalies之后调用。",
        "parameters": {"type": "object", "properties": {"dataset_id": {"type": "string"}}, "required": ["dataset_id"]}}},
    # ── Diagnostic (call when pipeline fails or results look wrong) ──
    {"type": "function", "function": {"name": "validate_data",