refactor: Wave 3 - UMAP-2D replace PCA scatter, TruncatedSVD replace PCA preprocessing, update AGENTS.md

AGENTS.md

@@ -10,7 +10,7 @@
 | Python 版本 | 3.12 (embedded portable at `runtime/python/python.exe`) |
 | Polars 版本 | 1.42.1 (精确锁定) |
 | 数据框架 | Polars (LazyFrame + streaming) |
-| 机器学习 | scikit-learn (HDBSCAN, KMeans, PCA, StandardScaler) |
+| 机器学习 | scikit-learn (HDBSCAN, KMeans, UMAP, TruncatedSVD, StandardScaler) |
 | Web框架 | Django 4.2 + SQLite WAL |
 | 前端3D | Three.js (603KB, 离线, 地球贴图1.4MB) |
 | 前端图表 | 纯Canvas散点图(无Chart.js) |
@@ -449,3 +449,19 @@ Cleanup + minor fixes release.
 ### Pending
 - tests/ 需要更新 entity_detector 相关
 - 大文件上传存在内存泄漏需要进一步排查
+
+---
+
+## Operating Procedures
+
+1. **不要阻塞式启动后端** — Start Django backend in background (use `start /B` or threading), never block the terminal. Use `start /B runtime\python\python.exe manage.py runserver` to avoid hanging the shell.
+
+2. **全量测试** — After all changes, run full E2E test: generate 2000 CSVs, upload through frontend, verify complete pipeline. This validates the entire system end-to-end before considering work complete.
+
+3. **更新文档和Git** — After tests pass: update AGENTS.md, commit all changes, push, clean up temp files and running processes. Do not leave dirty state (orphan processes, temp files, uncommitted changes).
+
+4. **同步依赖** — Use `git pull` to sync latest code, then `runtime\python\python.exe -m pip install -r requirements.txt` to install any new packages. Do this before starting any new work session.
+
+5. **编码前缀** — Prefix ALL terminal commands with `chcp 65001` to prevent Opencode encoding errors. PowerShell/CMD encoding issues are the #1 source of inexplicable command failures.
+
+6. **Python UTF-8** — When running Python, always add `set PYTHONUTF8=1` before the command. This ensures consistent Unicode handling across all machines and prevents cross-machine encoding drift.

analysis/tool_registry.py

@@ -1283,12 +1283,12 @@ async def _handle_run_clustering(
                 data_scaled = data_scaled[:, keep]
                 feature_cols = [feature_cols[j] for j in keep]
 
-        # ── D14: PCA dimensionality reduction (if >50 features) ───────────
+        # ── D14: SVD dimensionality reduction (if >50 features) ───────────
         n_features = data_scaled.shape[1]
         if n_features > 50:
-            from sklearn.decomposition import PCA
+            from sklearn.decomposition import TruncatedSVD
-            pca = PCA(n_components=50, random_state=random_state)
+            svd = TruncatedSVD(n_components=50, random_state=random_state)
-            data_scaled = pca.fit_transform(data_scaled)
+            data_scaled = svd.fit_transform(data_scaled)
 
         # ── H17: HDBSCAN hyperparameter auto-tuning based on data size ────
         n = len(data_scaled)
@@ -1561,17 +1561,42 @@ async def _handle_extract_features(
             cluster_features.sort(key=lambda x: abs(x['distinguishing_score']), reverse=True)
             all_features.extend(cluster_features[:top_k])
 
-        # ── Compute PCA-2D embedding ──────────────────────────────────────
+        # ── Compute UMAP-2D embedding ──────────────────────────────────────
         if len(numeric_cols) >= 2 and len(df) > 2:
             try:
-                data_matrix = df.select(numeric_cols).to_numpy()
                 from sklearn.preprocessing import StandardScaler
-                from sklearn.decomposition import PCA
+                import umap
-                data_scaled = StandardScaler().fit_transform(data_matrix)
+                MAX_UMAP_TRAIN = 10000
-                # Handle NaN/Inf values (np is imported at module level)
+                BATCH_SIZE = 1000
-                data_scaled = np.nan_to_num(data_scaled, nan=0.0, posinf=0.0, neginf=0.0)
+                data_matrix = df.select(numeric_cols).to_numpy()
-                pca_model = PCA(n_components=2, random_state=42)
+                if len(df) > MAX_UMAP_TRAIN:
-                coords = pca_model.fit_transform(data_scaled)
+                    # Train UMAP on 10K sample, batch-transform full dataset
+                    idx_sample = np.random.RandomState(42).choice(
+                        len(df), size=MAX_UMAP_TRAIN, replace=False)
+                    mat_sample = data_matrix[idx_sample]
+                    scaler = StandardScaler().fit(mat_sample)
+                    mat_sample_scaled = np.nan_to_num(scaler.transform(mat_sample),
+                                                      nan=0.0, posinf=0.0, neginf=0.0)
+                    reducer = umap.UMAP(n_components=2, random_state=42,
+                                        n_neighbors=15, min_dist=0.1,
+                                        metric='euclidean')
+                    reducer.fit(mat_sample_scaled)
+                    # Batch transform in 1K-row batches
+                    coords_list = []
+                    for start in range(0, len(df), BATCH_SIZE):
+                        end = min(start + BATCH_SIZE, len(df))
+                        mat_batch = scaler.transform(data_matrix[start:end])
+                        mat_batch = np.nan_to_num(mat_batch, nan=0.0, posinf=0.0, neginf=0.0)
+                        coords_list.append(reducer.transform(mat_batch))
+                    coords = np.vstack(coords_list)
+                else:
+                    data_scaled = StandardScaler().fit_transform(data_matrix)
+                    # Handle NaN/Inf values (np is imported at module level)
+                    data_scaled = np.nan_to_num(data_scaled, nan=0.0, posinf=0.0, neginf=0.0)
+                    reducer = umap.UMAP(n_components=2, random_state=42,
+                                        n_neighbors=15, min_dist=0.1,
+                                        metric='euclidean')
+                    coords = reducer.fit_transform(data_scaled)
                 non_numeric = [c for c in df.columns if c not in numeric_cols and not c.startswith('_')]
                 entity_col_pca = non_numeric[0] if non_numeric else df.columns[0]
                 labels_list = cluster_entry.get('labels', [])
@@ -1606,7 +1631,7 @@ async def _handle_extract_features(
                     if profiles:
                         EP.objects.bulk_create(profiles, ignore_conflicts=True, batch_size=1000)
             except Exception:
-                import traceback, sys; print(f'[PCA_ERR] {traceback.format_exc()}', file=sys.stderr)
+                import traceback, sys; print(f'[UMAP_ERR] {traceback.format_exc()}', file=sys.stderr)
 
         # Persist to Django ORM
         db_saved = False
@@ -2029,12 +2054,12 @@ _ANOMALY_FEATURE_COLS = [
 
 
 def _add_adaptive_scores(lf: pl.LazyFrame, feature_names: list[str]) -> pl.LazyFrame:
-    """Add anomaly scores using data-driven weights (PCA + IQR + percentile).
+    """Add anomaly scores using data-driven weights (SVD + IQR + percentile).
 
     Steps:
     1. Find which anomaly feature columns exist in the aggregated data
-    2. Collect a sample (max 10K rows) for PCA weight learning
+    2. Collect a sample (max 10K rows) for SVD weight learning
-    3. Learn feature weights from PCA first component loadings
+    3. Learn feature weights from SVD first component loadings
     4. Compute weighted proxy_score across ALL rows (vectorized, no per-row ops)
     5. Add percentile rank calibration
     6. Infer risk thresholds from data distribution
@@ -2052,13 +2077,13 @@ def _add_adaptive_scores(lf: pl.LazyFrame, feature_names: list[str]) -> pl.LazyF
         mat = df_sample.to_numpy()
         mat = np.nan_to_num(mat, nan=0.0)
 
-        # Step 3: PCA weight learning
+        # Step 3: SVD weight learning
-        from sklearn.decomposition import PCA
+        from sklearn.decomposition import TruncatedSVD
         n_comp = min(1, min(mat.shape) - 1)
         if n_comp < 1 or mat.shape[1] < 2:
             return lf
-        pca = PCA(n_components=1, random_state=42).fit(mat)
+        svd = TruncatedSVD(n_components=1, random_state=42).fit(mat)
-        raw_weights = np.abs(pca.components_[0])
+        raw_weights = np.abs(svd.components_[0])
         # Normalise to sum = 1
         weights = dict(zip(avail, raw_weights / raw_weights.sum()))
 
@@ -2114,7 +2139,7 @@ def _add_adaptive_scores(lf: pl.LazyFrame, feature_names: list[str]) -> pl.LazyF
         import logging
         logger = logging.getLogger(__name__)
         logger.info(
-            '[SCORE] adaptive weights from PCA: %s | thresholds: p50=%.3f upper=%.3f extreme=%.3f | sample=%d/%d',
+            '[SCORE] adaptive weights from SVD: %s | thresholds: p50=%.3f upper=%.3f extreme=%.3f | sample=%d/%d',
             {c: f'{w:.3f}' for c, w in sorted(weights.items(), key=lambda x: -x[1])},
             float(q50), upper_fence, extreme_fence,
             min(n_rows, 10000), n_rows,
@@ -2315,11 +2340,11 @@ async def _handle_filter_and_cluster(
                 data_scaled = data_scaled[:, keep]
                 feature_cols = [feature_cols[j] for j in keep]
 
-        # PCA dimensionality reduction (if >50 features)
+        # SVD dimensionality reduction (if >50 features)
         n_features = data_scaled.shape[1]
         if n_features > 50:
-            from sklearn.decomposition import PCA
+            from sklearn.decomposition import TruncatedSVD
-            data_scaled = PCA(n_components=50, random_state=42).fit_transform(data_scaled)
+            data_scaled = TruncatedSVD(n_components=50, random_state=42).fit_transform(data_scaled)
 
         # Hyperparameter auto-tuning
         n = len(data_scaled)
@@ -2463,9 +2488,9 @@ async def _handle_detect_anomalies(dataset_id: str, contamination: float = 0.05)
 # ── 15. visualize_anomalies ───────────────────────────────────────────────
 
 async def _handle_visualize_anomalies(dataset_id: str) -> dict:
-    """Generate PCA-2D embedding for anomaly visualization."""
+    """Generate UMAP-2D embedding for anomaly visualization."""
     from sklearn.preprocessing import StandardScaler
-    from sklearn.decomposition import PCA
+    import umap
     store = SessionStore()
     entry = store.get_dataset(dataset_id)
     if entry is None:
@@ -2476,9 +2501,9 @@ async def _handle_visualize_anomalies(dataset_id: str) -> dict:
     numeric_types = {'Int8', 'Int16', 'Int32', 'Int64', 'UInt8', 'UInt16', 'UInt32', 'UInt64', 'Float32', 'Float64'}
     num_cols = [c for c in schema if schema[c].split('(')[0].strip() in numeric_types and not c.startswith('_')]
     if len(num_cols) < 2:
-        return {'error': 'Need at least 2 numeric columns for PCA', 'truncated': False}
+        return {'error': 'Need at least 2 numeric columns for UMAP', 'truncated': False}
 
-    # Sample for PCA visualisation
+    # Sample for UMAP visualisation (max 10K)
     try:
         n_total = lf.select(pl.len()).collect(streaming=True).item()
     except Exception:
@@ -2487,7 +2512,9 @@ async def _handle_visualize_anomalies(dataset_id: str) -> dict:
     df = sample_lf.select(num_cols).collect(streaming=True)
     mat = StandardScaler().fit_transform(df.to_numpy())
     mat = np.nan_to_num(mat, nan=0.0)
-    coords = PCA(n_components=2, random_state=42).fit_transform(mat)
+    reducer = umap.UMAP(n_components=2, random_state=42,
+                        n_neighbors=15, min_dist=0.1, metric='euclidean')
+    coords = reducer.fit_transform(mat)
 
     risk_col = next((c for c in schema if 'risk' in c.lower()), None)
     score_col = next((c for c in schema if 'proxy_score' in c.lower() or 'threat' in c.lower()), None)
@@ -2629,7 +2656,7 @@ async def _handle_find_outliers(dataset_id: str, columns: list[str] = None,
 
 async def _handle_diagnose_clustering(dataset_id: str, cluster_result_id: str) -> dict:
     """Diagnose clustering quality issues."""
-    from sklearn.decomposition import PCA
+    from sklearn.decomposition import TruncatedSVD
     store = SessionStore()
     entry = store.get_dataset(dataset_id)
     cluster_entry = store.get_cluster_result(cluster_result_id)
@@ -2661,16 +2688,17 @@ async def _handle_diagnose_clustering(dataset_id: str, cluster_result_id: str) -
     if len(feature_cols) < 3:
         diagnosis.append(f'Only {len(feature_cols)} features — add more columns for meaningful clusters')
 
-    # PCA variance check
+    # SVD variance check
     try:
         df_sample = lf.select(num_cols).head(5000).collect(streaming=True)
         mat = df_sample.to_numpy()
-        pca = PCA().fit(mat)
+        n_comp = min(mat.shape)
-        var_explained = pca.explained_variance_ratio_
+        svd = TruncatedSVD(n_components=n_comp, random_state=42).fit(mat)
+        var_explained = svd.explained_variance_ratio_
         if var_explained[0] > 0.95:
-            diagnosis.append(f'First PCA component explains {var_explained[0]:.0%} variance — data nearly 1D')
+            diagnosis.append(f'First SVD component explains {var_explained[0]:.0%} variance — data nearly 1D')
     except Exception:
-        pass  # PCA fit failed — likely constant or all-null columns
+        pass  # SVD fit failed — likely constant or all-null columns
 
     return {
         'diagnosis': diagnosis,

analysis/views.py

@@ -75,12 +75,12 @@ def _extract_lon(val):
 
 
 def cluster_overview(request, display_id):
-    """Overview of all clusters for a run, with PCA scatter data and geo scatter data."""
+    """Overview of all clusters for a run, with UMAP scatter data and geo scatter data."""
     run = get_object_or_404(AnalysisRun, display_id=display_id)
     clusters = run.clusters.exclude(cluster_label=-1).order_by('-size').prefetch_related('features')
     noise = run.clusters.filter(cluster_label=-1).first()
 
-    # Build PCA scatter data for Chart.js
+    # Build UMAP scatter data for Canvas
     entities = run.entities.exclude(embedding_x=None).exclude(embedding_y=None).only('embedding_x', 'embedding_y', 'cluster_label', 'entity_value')
     scatter_data = [
         {'x': e.embedding_x, 'y': e.embedding_y, 'label': e.cluster_label, 'entity': e.entity_value}
@@ -731,7 +731,7 @@ def _run_pipeline_worker(run_id, analysis_fn, **ctx):
 
 def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorithm,
                               min_cluster_size, run_pca=True, head=None):
-    """Unified clustering pipeline: clustering → feature extraction → PCA → ORM save.
+    """Unified clustering pipeline: clustering → feature extraction → UMAP → ORM save.
 
     Args:
         run: AnalysisRun ORM object.
@@ -740,7 +740,7 @@ def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorith
         feature_columns: list of feature column names (None = auto-detect).
         algorithm: 'hdbscan' or 'kmeans'.
         min_cluster_size: int for HDBSCAN.
-        run_pca: bool, whether to compute and save PCA-2D embeddings.
+        run_pca: bool, whether to compute and save UMAP-2D embeddings.
         head: optional int, downsample to at most this many rows (min 100).
     """
     import warnings
@@ -850,71 +850,96 @@ def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorith
         if 'error' in feat_result:
             raise Exception(feat_result['error'])
 
-        # ── Step 3: PCA-2D embedding (downsampled for large data) ─────────
+        # ── Step 3: UMAP-2D embedding (sampled training for large data) ─────────
         if run_pca:
             try:
                 lf = entry['lazyframe']
-                # Sample max 50K rows for PCA to avoid memory blowup on 10M entities
+                # UMAP: sample max 10K for training, batch-transform rest in 1K batches
-                MAX_PCA = 50000
+                MAX_UMAP_TRAIN = 10000
+                BATCH_SIZE = 1000
                 try:
                     n_total = lf.select(pl.len()).collect(streaming=True).item()
                 except Exception:
                     n_total = 0
-                if n_total > MAX_PCA:
-                    lf_pca = lf.sample(n=MAX_PCA, seed=42)
-                    run.progress_msg = f'PCA 采样 {MAX_PCA}/{n_total} 实体...'
-                    run.save(update_fields=['progress_msg'])
-                else:
-                    lf_pca = lf
-                df_pca = lf_pca.collect(streaming=True)
                 from sklearn.preprocessing import StandardScaler
-                from sklearn.decomposition import PCA
+                import umap
                 import numpy as np
 
-                num_cols = [c for c in df_pca.columns
+                live_schema = lf.collect_schema()
-                            if df_pca[c].dtype in (
+                num_cols = [name for name, dt in zip(live_schema.names(), live_schema.dtypes())
-                                pl.Int8, pl.Int16, pl.Int32, pl.Int64,
+                            if dt in (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
-                                pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
+                                      pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
-                                pl.Float32, pl.Float64)]
+                                      pl.Float32, pl.Float64) and not name.startswith('_')]
 
-                if len(num_cols) >= 2 and len(df_pca) > 2:
+                if len(num_cols) >= 2:
-                    mat = np.nan_to_num(StandardScaler().fit_transform(
+                    if n_total > MAX_UMAP_TRAIN:
-                        df_pca.select(num_cols).to_numpy()), nan=0.0)
+                        run.progress_msg = f'UMAP 采样 {MAX_UMAP_TRAIN}/{n_total} 实体训练...'
-                    coords = PCA(n_components=2, random_state=42).fit_transform(mat)
+                        run.save(update_fields=['progress_msg'])
+                        df_sample = lf.sample(n=MAX_UMAP_TRAIN, seed=42).collect(streaming=True)
+                        df_pca = lf.collect(streaming=True)
+                    else:
+                        df_pca = lf.collect(streaming=True)
 
-                    non_num = [c for c in df_pca.columns if c not in num_cols and not c.startswith('_')]
+                    if len(df_pca) > 2:
-                    ent_col = non_num[0] if non_num else df_pca.columns[0]
+                        if n_total > MAX_UMAP_TRAIN:
+                            # Train UMAP on 10K sample, batch-transform full dataset
+                            mat_sample = df_sample.select(num_cols).to_numpy()
+                            scaler = StandardScaler().fit(mat_sample)
+                            mat_sample_scaled = np.nan_to_num(scaler.transform(mat_sample), nan=0.0)
+                            reducer = umap.UMAP(n_components=2, random_state=42,
+                                                n_neighbors=15, min_dist=0.1,
+                                                metric='euclidean')
+                            reducer.fit(mat_sample_scaled)
+                            # Batch transform in 1K-row batches
+                            coords_list = []
+                            for start in range(0, len(df_pca), BATCH_SIZE):
+                                end = min(start + BATCH_SIZE, len(df_pca))
+                                mat_batch = scaler.transform(
+                                    df_pca[start:end].select(num_cols).to_numpy())
+                                mat_batch = np.nan_to_num(mat_batch, nan=0.0)
+                                coords_list.append(reducer.transform(mat_batch))
+                            coords = np.vstack(coords_list)
+                        else:
+                            mat = np.nan_to_num(StandardScaler().fit_transform(
+                                df_pca.select(num_cols).to_numpy()), nan=0.0)
+                            reducer = umap.UMAP(n_components=2, random_state=42,
+                                                n_neighbors=15, min_dist=0.1,
+                                                metric='euclidean')
+                            coords = reducer.fit_transform(mat)
 
-                    cluster_entry = store.get_cluster_result(cluster_id)
+                        non_num = [c for c in df_pca.columns if c not in num_cols and not c.startswith('_')]
-                    labels_list = cluster_entry.get('labels', []) if cluster_entry else []
+                        ent_col = non_num[0] if non_num else df_pca.columns[0]
 
-                    from analysis.models import EntityProfile as EP
+                        cluster_entry = store.get_cluster_result(cluster_id)
-                    from analysis.models import ClusterResult as CR
+                        labels_list = cluster_entry.get('labels', []) if cluster_entry else []
 
-                    profiles = []
+                        from analysis.models import EntityProfile as EP
-                    cr_cache = {}
+                        from analysis.models import ClusterResult as CR
-                    for i, row in enumerate(df_pca.iter_rows(named=True)):
-                        ev = str(row.get(ent_col, ''))
-                        if not ev:
-                            continue
-                        lbl = labels_list[i] if i < len(labels_list) else -1
-                        cache_key = f'{run.id}_{lbl}'
-                        if cache_key not in cr_cache:
-                            cr_cache[cache_key] = CR.objects.filter(run=run, cluster_label=lbl).first()
-                        profiles.append(EP(
-                            entity_value=ev, run=run, cluster_label=lbl,
-                            cluster=cr_cache[cache_key],
-                            embedding_x=float(coords[i, 0]) if i < len(coords) else None,
-                            embedding_y=float(coords[i, 1]) if i < len(coords) else None,
-                        ))
-                    if profiles:
-                        EP.objects.bulk_create(profiles, ignore_conflicts=True, batch_size=1000)
-                        run.entity_count = len(profiles)
-                        run.save(update_fields=['entity_count'])
-            except Exception as pca_err:
-                logger.warning(f'PCA-2D embedding skipped: {pca_err}')
 
-        # Fallback: if entity_count not set (PCA skipped), derive from cluster sizes
+                        profiles = []
+                        cr_cache = {}
+                        for i, row in enumerate(df_pca.iter_rows(named=True)):
+                            ev = str(row.get(ent_col, ''))
+                            if not ev:
+                                continue
+                            lbl = labels_list[i] if i < len(labels_list) else -1
+                            cache_key = f'{run.id}_{lbl}'
+                            if cache_key not in cr_cache:
+                                cr_cache[cache_key] = CR.objects.filter(run=run, cluster_label=lbl).first()
+                            profiles.append(EP(
+                                entity_value=ev, run=run, cluster_label=lbl,
+                                cluster=cr_cache[cache_key],
+                                embedding_x=float(coords[i, 0]) if i < len(coords) else None,
+                                embedding_y=float(coords[i, 1]) if i < len(coords) else None,
+                            ))
+                        if profiles:
+                            EP.objects.bulk_create(profiles, ignore_conflicts=True, batch_size=1000)
+                            run.entity_count = len(profiles)
+                            run.save(update_fields=['entity_count'])
+            except Exception as umap_err:
+                logger.warning(f'UMAP-2D embedding skipped: {umap_err}')
+
+        # Fallback: if entity_count not set (UMAP skipped), derive from cluster sizes
         if run.entity_count is None:
             from django.db.models import Sum
             total = run.clusters.aggregate(total=Sum('size'))['total'] or 0
@@ -988,7 +1013,7 @@ def manual_run_analysis(request):
 
             ds_id = upload_ds_id if store.get_dataset(upload_ds_id) else f'upload_{pk}'
 
-            # Use the unified clustering pipeline (clustering → extraction → PCA)
+            # Use the unified clustering pipeline (clustering → extraction → UMAP)
             from analysis.views import _run_clustering_pipeline
             _run_clustering_pipeline(
                 run=run, store=store, entity_ds_id=ds_id,