refactor: Wave 3 - UMAP-2D replace PCA scatter, TruncatedSVD replace PCA preprocessing, update AGENTS.md

AGENTS.md

@@ -10,7 +10,7 @@
 | Python 版本 | 3.12 (embedded portable at `runtime/python/python.exe`) |
 | Polars 版本 | 1.42.1 (精确锁定) |
 | 数据框架 | Polars (LazyFrame + streaming) |
-| 机器学习 | scikit-learn (HDBSCAN, KMeans, PCA, StandardScaler) |
+| 机器学习 | scikit-learn (HDBSCAN, KMeans, UMAP, TruncatedSVD, StandardScaler) |
 | Web框架 | Django 4.2 + SQLite WAL |
 | 前端3D | Three.js (603KB, 离线, 地球贴图1.4MB) |
 | 前端图表 | 纯Canvas散点图(无Chart.js) |
@@ -449,3 +449,19 @@ Cleanup + minor fixes release.
 ### Pending
 - tests/ 需要更新 entity_detector 相关
 - 大文件上传存在内存泄漏需要进一步排查
+
+---
+
+## Operating Procedures
+
+1. **不要阻塞式启动后端** — Start Django backend in background (use `start /B` or threading), never block the terminal. Use `start /B runtime\python\python.exe manage.py runserver` to avoid hanging the shell.
+
+2. **全量测试** — After all changes, run full E2E test: generate 2000 CSVs, upload through frontend, verify complete pipeline. This validates the entire system end-to-end before considering work complete.
+
+3. **更新文档和Git** — After tests pass: update AGENTS.md, commit all changes, push, clean up temp files and running processes. Do not leave dirty state (orphan processes, temp files, uncommitted changes).
+
+4. **同步依赖** — Use `git pull` to sync latest code, then `runtime\python\python.exe -m pip install -r requirements.txt` to install any new packages. Do this before starting any new work session.
+
+5. **编码前缀** — Prefix ALL terminal commands with `chcp 65001` to prevent Opencode encoding errors. PowerShell/CMD encoding issues are the #1 source of inexplicable command failures.
+
+6. **Python UTF-8** — When running Python, always add `set PYTHONUTF8=1` before the command. This ensures consistent Unicode handling across all machines and prevents cross-machine encoding drift.

analysis/session_store.py

@@ -155,7 +155,12 @@ class SessionStore:
                 continue
             try:
                 lf, schema_csv, row_count, file_count, memory_mb = load_csv_directory(csv_glob)
-                # Merge schema from CSV (may be more accurate than persisted schema)
+                # Preserve original metadata counts from disk — CSV re-scan may
+                # detect different file/row counts if files changed on disk.
+                if 'file_count' not in meta and file_count is not None:
+                    meta['file_count'] = file_count
+                if 'row_count' not in meta and row_count is not None:
+                    meta['row_count'] = row_count
                 self.store_dataset(ds_id, lf, schema=schema_csv, metadata=meta)
                 restored += 1
             except Exception:

analysis/tool_registry.py

@@ -1283,12 +1283,12 @@ async def _handle_run_clustering(
                 data_scaled = data_scaled[:, keep]
                 feature_cols = [feature_cols[j] for j in keep]
 
-        # ── D14: PCA dimensionality reduction (if >50 features) ───────────
+        # ── D14: SVD dimensionality reduction (if >50 features) ───────────
         n_features = data_scaled.shape[1]
         if n_features > 50:
-            from sklearn.decomposition import PCA
-            pca = PCA(n_components=50, random_state=random_state)
-            data_scaled = pca.fit_transform(data_scaled)
+            from sklearn.decomposition import TruncatedSVD
+            svd = TruncatedSVD(n_components=50, random_state=random_state)
+            data_scaled = svd.fit_transform(data_scaled)
 
         # ── H17: HDBSCAN hyperparameter auto-tuning based on data size ────
         n = len(data_scaled)
@@ -1561,17 +1561,42 @@ async def _handle_extract_features(
             cluster_features.sort(key=lambda x: abs(x['distinguishing_score']), reverse=True)
             all_features.extend(cluster_features[:top_k])
 
-        # ── Compute PCA-2D embedding ──────────────────────────────────────
+        # ── Compute UMAP-2D embedding ──────────────────────────────────────
         if len(numeric_cols) >= 2 and len(df) > 2:
             try:
-                data_matrix = df.select(numeric_cols).to_numpy()
                 from sklearn.preprocessing import StandardScaler
-                from sklearn.decomposition import PCA
-                data_scaled = StandardScaler().fit_transform(data_matrix)
-                # Handle NaN/Inf values (np is imported at module level)
-                data_scaled = np.nan_to_num(data_scaled, nan=0.0, posinf=0.0, neginf=0.0)
-                pca_model = PCA(n_components=2, random_state=42)
-                coords = pca_model.fit_transform(data_scaled)
+                import umap
+                MAX_UMAP_TRAIN = 10000
+                BATCH_SIZE = 1000
+                data_matrix = df.select(numeric_cols).to_numpy()
+                if len(df) > MAX_UMAP_TRAIN:
+                    # Train UMAP on 10K sample, batch-transform full dataset
+                    idx_sample = np.random.RandomState(42).choice(
+                        len(df), size=MAX_UMAP_TRAIN, replace=False)
+                    mat_sample = data_matrix[idx_sample]
+                    scaler = StandardScaler().fit(mat_sample)
+                    mat_sample_scaled = np.nan_to_num(scaler.transform(mat_sample),
+                                                      nan=0.0, posinf=0.0, neginf=0.0)
+                    reducer = umap.UMAP(n_components=2, random_state=42,
+                                        n_neighbors=15, min_dist=0.1,
+                                        metric='euclidean')
+                    reducer.fit(mat_sample_scaled)
+                    # Batch transform in 1K-row batches
+                    coords_list = []
+                    for start in range(0, len(df), BATCH_SIZE):
+                        end = min(start + BATCH_SIZE, len(df))
+                        mat_batch = scaler.transform(data_matrix[start:end])
+                        mat_batch = np.nan_to_num(mat_batch, nan=0.0, posinf=0.0, neginf=0.0)
+                        coords_list.append(reducer.transform(mat_batch))
+                    coords = np.vstack(coords_list)
+                else:
+                    data_scaled = StandardScaler().fit_transform(data_matrix)
+                    # Handle NaN/Inf values (np is imported at module level)
+                    data_scaled = np.nan_to_num(data_scaled, nan=0.0, posinf=0.0, neginf=0.0)
+                    reducer = umap.UMAP(n_components=2, random_state=42,
+                                        n_neighbors=15, min_dist=0.1,
+                                        metric='euclidean')
+                    coords = reducer.fit_transform(data_scaled)
                 non_numeric = [c for c in df.columns if c not in numeric_cols and not c.startswith('_')]
                 entity_col_pca = non_numeric[0] if non_numeric else df.columns[0]
                 labels_list = cluster_entry.get('labels', [])
@@ -1606,7 +1631,7 @@ async def _handle_extract_features(
                     if profiles:
                         EP.objects.bulk_create(profiles, ignore_conflicts=True, batch_size=1000)
             except Exception:
-                import traceback, sys; print(f'[PCA_ERR] {traceback.format_exc()}', file=sys.stderr)
+                import traceback, sys; print(f'[UMAP_ERR] {traceback.format_exc()}', file=sys.stderr)
 
         # Persist to Django ORM
         db_saved = False
@@ -2029,12 +2054,12 @@ _ANOMALY_FEATURE_COLS = [
 
 
 def _add_adaptive_scores(lf: pl.LazyFrame, feature_names: list[str]) -> pl.LazyFrame:
-    """Add anomaly scores using data-driven weights (PCA + IQR + percentile).
+    """Add anomaly scores using data-driven weights (SVD + IQR + percentile).
 
     Steps:
     1. Find which anomaly feature columns exist in the aggregated data
-    2. Collect a sample (max 10K rows) for PCA weight learning
-    3. Learn feature weights from PCA first component loadings
+    2. Collect a sample (max 10K rows) for SVD weight learning
+    3. Learn feature weights from SVD first component loadings
     4. Compute weighted proxy_score across ALL rows (vectorized, no per-row ops)
     5. Add percentile rank calibration
     6. Infer risk thresholds from data distribution
@@ -2052,13 +2077,13 @@ def _add_adaptive_scores(lf: pl.LazyFrame, feature_names: list[str]) -> pl.LazyF
         mat = df_sample.to_numpy()
         mat = np.nan_to_num(mat, nan=0.0)
 
-        # Step 3: PCA weight learning
-        from sklearn.decomposition import PCA
+        # Step 3: SVD weight learning
+        from sklearn.decomposition import TruncatedSVD
         n_comp = min(1, min(mat.shape) - 1)
         if n_comp < 1 or mat.shape[1] < 2:
             return lf
-        pca = PCA(n_components=1, random_state=42).fit(mat)
-        raw_weights = np.abs(pca.components_[0])
+        svd = TruncatedSVD(n_components=1, random_state=42).fit(mat)
+        raw_weights = np.abs(svd.components_[0])
         # Normalise to sum = 1
         weights = dict(zip(avail, raw_weights / raw_weights.sum()))
 
@@ -2114,7 +2139,7 @@ def _add_adaptive_scores(lf: pl.LazyFrame, feature_names: list[str]) -> pl.LazyF
         import logging
         logger = logging.getLogger(__name__)
         logger.info(
-            '[SCORE] adaptive weights from PCA: %s | thresholds: p50=%.3f upper=%.3f extreme=%.3f | sample=%d/%d',
+            '[SCORE] adaptive weights from SVD: %s | thresholds: p50=%.3f upper=%.3f extreme=%.3f | sample=%d/%d',
             {c: f'{w:.3f}' for c, w in sorted(weights.items(), key=lambda x: -x[1])},
             float(q50), upper_fence, extreme_fence,
             min(n_rows, 10000), n_rows,
@@ -2315,11 +2340,11 @@ async def _handle_filter_and_cluster(
                 data_scaled = data_scaled[:, keep]
                 feature_cols = [feature_cols[j] for j in keep]
 
-        # PCA dimensionality reduction (if >50 features)
+        # SVD dimensionality reduction (if >50 features)
         n_features = data_scaled.shape[1]
         if n_features > 50:
-            from sklearn.decomposition import PCA
-            data_scaled = PCA(n_components=50, random_state=42).fit_transform(data_scaled)
+            from sklearn.decomposition import TruncatedSVD
+            data_scaled = TruncatedSVD(n_components=50, random_state=42).fit_transform(data_scaled)
 
         # Hyperparameter auto-tuning
         n = len(data_scaled)
@@ -2463,9 +2488,9 @@ async def _handle_detect_anomalies(dataset_id: str, contamination: float = 0.05)
 # ── 15. visualize_anomalies ───────────────────────────────────────────────
 
 async def _handle_visualize_anomalies(dataset_id: str) -> dict:
-    """Generate PCA-2D embedding for anomaly visualization."""
+    """Generate UMAP-2D embedding for anomaly visualization."""
     from sklearn.preprocessing import StandardScaler
-    from sklearn.decomposition import PCA
+    import umap
     store = SessionStore()
     entry = store.get_dataset(dataset_id)
     if entry is None:
@@ -2476,9 +2501,9 @@ async def _handle_visualize_anomalies(dataset_id: str) -> dict:
     numeric_types = {'Int8', 'Int16', 'Int32', 'Int64', 'UInt8', 'UInt16', 'UInt32', 'UInt64', 'Float32', 'Float64'}
     num_cols = [c for c in schema if schema[c].split('(')[0].strip() in numeric_types and not c.startswith('_')]
     if len(num_cols) < 2:
-        return {'error': 'Need at least 2 numeric columns for PCA', 'truncated': False}
+        return {'error': 'Need at least 2 numeric columns for UMAP', 'truncated': False}
 
-    # Sample for PCA visualisation
+    # Sample for UMAP visualisation (max 10K)
     try:
         n_total = lf.select(pl.len()).collect(streaming=True).item()
     except Exception:
@@ -2487,7 +2512,9 @@ async def _handle_visualize_anomalies(dataset_id: str) -> dict:
     df = sample_lf.select(num_cols).collect(streaming=True)
     mat = StandardScaler().fit_transform(df.to_numpy())
     mat = np.nan_to_num(mat, nan=0.0)
-    coords = PCA(n_components=2, random_state=42).fit_transform(mat)
+    reducer = umap.UMAP(n_components=2, random_state=42,
+                        n_neighbors=15, min_dist=0.1, metric='euclidean')
+    coords = reducer.fit_transform(mat)
 
     risk_col = next((c for c in schema if 'risk' in c.lower()), None)
     score_col = next((c for c in schema if 'proxy_score' in c.lower() or 'threat' in c.lower()), None)
@@ -2629,7 +2656,7 @@ async def _handle_find_outliers(dataset_id: str, columns: list[str] = None,
 
 async def _handle_diagnose_clustering(dataset_id: str, cluster_result_id: str) -> dict:
     """Diagnose clustering quality issues."""
-    from sklearn.decomposition import PCA
+    from sklearn.decomposition import TruncatedSVD
     store = SessionStore()
     entry = store.get_dataset(dataset_id)
     cluster_entry = store.get_cluster_result(cluster_result_id)
@@ -2661,16 +2688,17 @@ async def _handle_diagnose_clustering(dataset_id: str, cluster_result_id: str) -
     if len(feature_cols) < 3:
         diagnosis.append(f'Only {len(feature_cols)} features — add more columns for meaningful clusters')
 
-    # PCA variance check
+    # SVD variance check
     try:
         df_sample = lf.select(num_cols).head(5000).collect(streaming=True)
         mat = df_sample.to_numpy()
-        pca = PCA().fit(mat)
-        var_explained = pca.explained_variance_ratio_
+        n_comp = min(mat.shape)
+        svd = TruncatedSVD(n_components=n_comp, random_state=42).fit(mat)
+        var_explained = svd.explained_variance_ratio_
         if var_explained[0] > 0.95:
-            diagnosis.append(f'First PCA component explains {var_explained[0]:.0%} variance — data nearly 1D')
+            diagnosis.append(f'First SVD component explains {var_explained[0]:.0%} variance — data nearly 1D')
     except Exception:
-        pass  # PCA fit failed — likely constant or all-null columns
+        pass  # SVD fit failed — likely constant or all-null columns
 
     return {
         'diagnosis': diagnosis,

analysis/views.py

@@ -75,12 +75,12 @@ def _extract_lon(val):
 
 
 def cluster_overview(request, display_id):
-    """Overview of all clusters for a run, with PCA scatter data and geo scatter data."""
+    """Overview of all clusters for a run, with UMAP scatter data and geo scatter data."""
     run = get_object_or_404(AnalysisRun, display_id=display_id)
     clusters = run.clusters.exclude(cluster_label=-1).order_by('-size').prefetch_related('features')
     noise = run.clusters.filter(cluster_label=-1).first()
 
-    # Build PCA scatter data for Chart.js
+    # Build UMAP scatter data for Canvas
     entities = run.entities.exclude(embedding_x=None).exclude(embedding_y=None).only('embedding_x', 'embedding_y', 'cluster_label', 'entity_value')
     scatter_data = [
         {'x': e.embedding_x, 'y': e.embedding_y, 'label': e.cluster_label, 'entity': e.entity_value}
@@ -731,7 +731,7 @@ def _run_pipeline_worker(run_id, analysis_fn, **ctx):
 
 def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorithm,
                               min_cluster_size, run_pca=True, head=None):
-    """Unified clustering pipeline: clustering → feature extraction → PCA → ORM save.
+    """Unified clustering pipeline: clustering → feature extraction → UMAP → ORM save.
 
     Args:
         run: AnalysisRun ORM object.
@@ -740,7 +740,7 @@ def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorith
         feature_columns: list of feature column names (None = auto-detect).
         algorithm: 'hdbscan' or 'kmeans'.
         min_cluster_size: int for HDBSCAN.
-        run_pca: bool, whether to compute and save PCA-2D embeddings.
+        run_pca: bool, whether to compute and save UMAP-2D embeddings.
         head: optional int, downsample to at most this many rows (min 100).
     """
     import warnings
@@ -850,71 +850,96 @@ def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorith
         if 'error' in feat_result:
             raise Exception(feat_result['error'])
 
-        # ── Step 3: PCA-2D embedding (downsampled for large data) ─────────
+        # ── Step 3: UMAP-2D embedding (sampled training for large data) ─────────
         if run_pca:
             try:
                 lf = entry['lazyframe']
-                # Sample max 50K rows for PCA to avoid memory blowup on 10M entities
-                MAX_PCA = 50000
+                # UMAP: sample max 10K for training, batch-transform rest in 1K batches
+                MAX_UMAP_TRAIN = 10000
+                BATCH_SIZE = 1000
                 try:
                     n_total = lf.select(pl.len()).collect(streaming=True).item()
                 except Exception:
                     n_total = 0
-                if n_total > MAX_PCA:
-                    lf_pca = lf.sample(n=MAX_PCA, seed=42)
-                    run.progress_msg = f'PCA 采样 {MAX_PCA}/{n_total} 实体...'
-                    run.save(update_fields=['progress_msg'])
-                else:
-                    lf_pca = lf
-                df_pca = lf_pca.collect(streaming=True)
                 from sklearn.preprocessing import StandardScaler
-                from sklearn.decomposition import PCA
+                import umap
                 import numpy as np
 
-                num_cols = [c for c in df_pca.columns
-                            if df_pca[c].dtype in (
-                                pl.Int8, pl.Int16, pl.Int32, pl.Int64,
-                                pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
-                                pl.Float32, pl.Float64)]
+                live_schema = lf.collect_schema()
+                num_cols = [name for name, dt in zip(live_schema.names(), live_schema.dtypes())
+                            if dt in (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
+                                      pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
+                                      pl.Float32, pl.Float64) and not name.startswith('_')]
 
-                if len(num_cols) >= 2 and len(df_pca) > 2:
-                    mat = np.nan_to_num(StandardScaler().fit_transform(
-                        df_pca.select(num_cols).to_numpy()), nan=0.0)
-                    coords = PCA(n_components=2, random_state=42).fit_transform(mat)
+                if len(num_cols) >= 2:
+                    if n_total > MAX_UMAP_TRAIN:
+                        run.progress_msg = f'UMAP 采样 {MAX_UMAP_TRAIN}/{n_total} 实体训练...'
+                        run.save(update_fields=['progress_msg'])
+                        df_sample = lf.sample(n=MAX_UMAP_TRAIN, seed=42).collect(streaming=True)
+                        df_pca = lf.collect(streaming=True)
+                    else:
+                        df_pca = lf.collect(streaming=True)
 
-                    non_num = [c for c in df_pca.columns if c not in num_cols and not c.startswith('_')]
-                    ent_col = non_num[0] if non_num else df_pca.columns[0]
+                    if len(df_pca) > 2:
+                        if n_total > MAX_UMAP_TRAIN:
+                            # Train UMAP on 10K sample, batch-transform full dataset
+                            mat_sample = df_sample.select(num_cols).to_numpy()
+                            scaler = StandardScaler().fit(mat_sample)
+                            mat_sample_scaled = np.nan_to_num(scaler.transform(mat_sample), nan=0.0)
+                            reducer = umap.UMAP(n_components=2, random_state=42,
+                                                n_neighbors=15, min_dist=0.1,
+                                                metric='euclidean')
+                            reducer.fit(mat_sample_scaled)
+                            # Batch transform in 1K-row batches
+                            coords_list = []
+                            for start in range(0, len(df_pca), BATCH_SIZE):
+                                end = min(start + BATCH_SIZE, len(df_pca))
+                                mat_batch = scaler.transform(
+                                    df_pca[start:end].select(num_cols).to_numpy())
+                                mat_batch = np.nan_to_num(mat_batch, nan=0.0)
+                                coords_list.append(reducer.transform(mat_batch))
+                            coords = np.vstack(coords_list)
+                        else:
+                            mat = np.nan_to_num(StandardScaler().fit_transform(
+                                df_pca.select(num_cols).to_numpy()), nan=0.0)
+                            reducer = umap.UMAP(n_components=2, random_state=42,
+                                                n_neighbors=15, min_dist=0.1,
+                                                metric='euclidean')
+                            coords = reducer.fit_transform(mat)
 
-                    cluster_entry = store.get_cluster_result(cluster_id)
-                    labels_list = cluster_entry.get('labels', []) if cluster_entry else []
+                        non_num = [c for c in df_pca.columns if c not in num_cols and not c.startswith('_')]
+                        ent_col = non_num[0] if non_num else df_pca.columns[0]
 
-                    from analysis.models import EntityProfile as EP
-                    from analysis.models import ClusterResult as CR
+                        cluster_entry = store.get_cluster_result(cluster_id)
+                        labels_list = cluster_entry.get('labels', []) if cluster_entry else []
 
-                    profiles = []
-                    cr_cache = {}
-                    for i, row in enumerate(df_pca.iter_rows(named=True)):
-                        ev = str(row.get(ent_col, ''))
-                        if not ev:
-                            continue
-                        lbl = labels_list[i] if i < len(labels_list) else -1
-                        cache_key = f'{run.id}_{lbl}'
-                        if cache_key not in cr_cache:
-                            cr_cache[cache_key] = CR.objects.filter(run=run, cluster_label=lbl).first()
-                        profiles.append(EP(
-                            entity_value=ev, run=run, cluster_label=lbl,
-                            cluster=cr_cache[cache_key],
-                            embedding_x=float(coords[i, 0]) if i < len(coords) else None,
-                            embedding_y=float(coords[i, 1]) if i < len(coords) else None,
-                        ))
-                    if profiles:
-                        EP.objects.bulk_create(profiles, ignore_conflicts=True, batch_size=1000)
-                        run.entity_count = len(profiles)
-                        run.save(update_fields=['entity_count'])
-            except Exception as pca_err:
-                logger.warning(f'PCA-2D embedding skipped: {pca_err}')
+                        from analysis.models import EntityProfile as EP
+                        from analysis.models import ClusterResult as CR
 
-        # Fallback: if entity_count not set (PCA skipped), derive from cluster sizes
+                        profiles = []
+                        cr_cache = {}
+                        for i, row in enumerate(df_pca.iter_rows(named=True)):
+                            ev = str(row.get(ent_col, ''))
+                            if not ev:
+                                continue
+                            lbl = labels_list[i] if i < len(labels_list) else -1
+                            cache_key = f'{run.id}_{lbl}'
+                            if cache_key not in cr_cache:
+                                cr_cache[cache_key] = CR.objects.filter(run=run, cluster_label=lbl).first()
+                            profiles.append(EP(
+                                entity_value=ev, run=run, cluster_label=lbl,
+                                cluster=cr_cache[cache_key],
+                                embedding_x=float(coords[i, 0]) if i < len(coords) else None,
+                                embedding_y=float(coords[i, 1]) if i < len(coords) else None,
+                            ))
+                        if profiles:
+                            EP.objects.bulk_create(profiles, ignore_conflicts=True, batch_size=1000)
+                            run.entity_count = len(profiles)
+                            run.save(update_fields=['entity_count'])
+            except Exception as umap_err:
+                logger.warning(f'UMAP-2D embedding skipped: {umap_err}')
+
+        # Fallback: if entity_count not set (UMAP skipped), derive from cluster sizes
         if run.entity_count is None:
             from django.db.models import Sum
             total = run.clusters.aggregate(total=Sum('size'))['total'] or 0
@@ -988,7 +1013,7 @@ def manual_run_analysis(request):
 
             ds_id = upload_ds_id if store.get_dataset(upload_ds_id) else f'upload_{pk}'
 
-            # Use the unified clustering pipeline (clustering → extraction → PCA)
+            # Use the unified clustering pipeline (clustering → extraction → UMAP)
             from analysis.views import _run_clustering_pipeline
             _run_clustering_pipeline(
                 run=run, store=store, entity_ds_id=ds_id,
@@ -1301,11 +1326,17 @@ def run_llm_analysis_view(request):
 
 def auto_page(request):
     """LLM-driven auto analysis page."""
-    runs = AnalysisRun.objects.filter(status='ready').order_by('-created_at')[:20]
     cfg = get_config()
+    selected_run = None
+    run_id = request.GET.get('run_id')
+    if run_id:
+        try:
+            selected_run = AnalysisRun.objects.get(display_id=int(run_id))
+        except (ValueError, AnalysisRun.DoesNotExist):
+            pass
     return render(request, 'tianxuan/auto.html', {
-        'runs': runs,
         'llm_configured': bool(cfg.llm.base_url and cfg.llm.api_key),
+        'selected_run': selected_run,
     })
 
 

templates/tianxuan/auto.html

@@ -116,40 +116,23 @@
     {% endif %}
 </div>
 
-<div class="card">
-    <h3>选择数据集</h3>
-    {% if runs %}
-    <table>
-        <thead><tr><th></th><th>ID</th><th>文件</th><th>状态</th></tr></thead>
-        <tbody>
-            {% for run in runs %}
-            <tr>
-                <td><input type="radio" name="auto_run_id" value="{{ run.display_id }}"></td>
-                <td>#{{ run.display_id }}</td>
-                <td style="max-width:200px;overflow:hidden;text-overflow:ellipsis;">{% if run.sqlite_table %}SQLite表 {{ run.sqlite_table }}{% else %}{{ run.csv_glob }}{% endif %}</td>
-                <td><span class="badge badge-success">{{ run.status }}</span></td>
-            </tr>
-            {% empty %}
-            <tr><td colspan="4" style="text-align:center;">暂无数据。先去 <a href="/upload/">上传 CSV</a>。</td></tr>
-            {% endfor %}
-        </tbody>
-    </table>
-    <button class="btn btn-primary" style="margin-top:1rem;" onclick="startAuto()" {% if not llm_configured %}disabled{% endif %}>开始自动分析</button>
-    {% endif %}
-    <div id="autoProgress" style="display:none;margin-top:1rem;">
-        <p>LLM 编排中... <span id="autoStatus"></span></p>
-        <div style="background:#eee;height:8px;border-radius:4px;overflow:hidden;">
-            <div id="autoBar" style="background:#4361ee;height:100%;width:0%;"></div>
-        </div>
-
-        <button id="cancelBtn" class="btn" style="background:#f72585;color:white;display:none;" onclick="cancelAnalysis()">取消分析</button>
-
-        <!-- Unified timeline: interleaved thinking + tool calls -->
-        <div id="timelinePanel" style="display:none;margin-top:1rem;">
-            <h4 style="margin-bottom:0.6rem;color:#333;">📋 分析步骤</h4>
-            <div id="timeline"></div>
-        </div>
+<div id="autoProgress" style="display:none;margin-top:1rem;">
+    <p>LLM 编排中... <span id="autoStatus"></span></p>
+    <div style="background:#eee;height:8px;border-radius:4px;overflow:hidden;">
+        <div id="autoBar" style="background:#4361ee;height:100%;width:0%;"></div>
     </div>
+
+    <button id="cancelBtn" class="btn" style="background:#f72585;color:white;display:none;" onclick="cancelAnalysis()">取消分析</button>
+
+    <!-- Unified timeline: interleaved thinking + tool calls -->
+    <div id="timelinePanel" style="display:none;margin-top:1rem;">
+        <h4 style="margin-bottom:0.6rem;color:#333;">📋 分析步骤</h4>
+        <div id="timeline"></div>
+    </div>
+</div>
+
+<div id="noRunMessage" style="background:#fff3cd;padding:1rem;border-radius:6px;margin:1rem 0;display:none;">
+    请从上传页面启动LLM分析
 </div>
 
 <script>
@@ -176,20 +159,8 @@ function formatTimestamp() {
 }
 
 function buildTimeline(llmThinking, toolCalls) {
-    // Parse llm_thinking into [{step, text}] entries
-    const thoughtEntries = [];
-    if (llmThinking) {
-        const parts = llmThinking.split(/\n(?=\[\d+\])/);
-        for (const part of parts) {
-            const m = part.match(/^\[(\d+)\]\s*(.*)/s);
-            if (m) {
-                thoughtEntries.push({ step: parseInt(m[1]), type: 'thought', text: m[2].trim() });
-            }
-        }
-    }
-
-    // tool_calls already have step, type, name, input, output
-    const toolEntries = (toolCalls || []).map(tc => ({
+    // Build tool entries from tool_calls_json (no separate thought entries)
+    const entries = (toolCalls || []).map(tc => ({
         step: tc.step,
         type: 'tool',
         name: tc.name,
@@ -197,9 +168,25 @@ function buildTimeline(llmThinking, toolCalls) {
         output: tc.output,
     }));
 
-    // Merge and sort by step
-    const merged = [...thoughtEntries, ...toolEntries].sort((a, b) => a.step - b.step);
-    return merged;
+    // Match llm_thinking text to tool entries by step number
+    if (llmThinking) {
+        const thinkingByStep = {};
+        const parts = llmThinking.split(/\n(?=\[\d+\])/);
+        for (const part of parts) {
+            const m = part.match(/^\[(\d+)\]\s*(.*)/s);
+            if (m) {
+                thinkingByStep[parseInt(m[1])] = m[2].trim();
+            }
+        }
+        for (const entry of entries) {
+            if (thinkingByStep[entry.step] !== undefined) {
+                entry.thinking = thinkingByStep[entry.step];
+            }
+        }
+    }
+
+    entries.sort((a, b) => a.step - b.step);
+    return entries;
 }
 
 function renderTimeline(entries) {
@@ -222,26 +209,26 @@ function renderTimeline(entries) {
             lastStep = entry.step;
         }
 
-        if (entry.type === 'thought') {
-            // LLM reasoning — collapsible
-            const card = document.createElement('div');
-            card.style.cssText = 'border:1px solid #d0d8f0;border-radius:6px;margin-bottom:0.3rem;overflow:hidden;background:#f8f9ff;';
-            card.innerHTML =
-                '<div class="tool-call-header" onclick="toggleStep(this)" style="background:#eef2ff;padding:0.4rem 0.8rem;">' +
-                    '<span><span style="background:#4361ee;color:#fff;border-radius:3px;padding:0 5px;font-size:0.7rem;font-weight:600;margin-right:6px;">💭</span>' +
-                    '<span style="font-weight:600;font-size:0.8rem;">LLM 推理</span></span>' +
-                    '<span class="arrow">▶</span>' +
-                '</div>' +
-                '<div class="tool-call-body" style="padding:0.6rem 0.8rem;font-size:0.78rem;line-height:1.6;color:#333;background:#fff;">' +
-                    escapeHtml(entry.text) +
-                '</div>';
-            container.appendChild(card);
-        } else if (entry.type === 'tool') {
-            // Tool call — collapsible with I/O
+        if (entry.type === 'tool') {
+            // Tool call — collapsible with I/O; thinking shown when output is null
             const card = document.createElement('div');
             card.style.cssText = 'border:1px solid #e0e0e0;border-radius:6px;margin-bottom:0.3rem;overflow:hidden;background:#fff;';
             const inputStr = prettyJson(entry.input);
+            const hasThinking = entry.thinking && (entry.output == null || entry.output === '' || entry.output === 'null' || entry.output === 'None');
             const outputStr = prettyJson(entry.output);
+            let bodyInner = '';
+            bodyInner +=
+                '<div style="font-weight:600;font-size:0.72rem;color:#555;margin-bottom:0.2rem;">输入 (INPUT):</div>' +
+                '<pre style="background:#1a1a2e;color:#e0e0e0;padding:0.5rem;border-radius:4px;font-size:0.7rem;max-height:200px;overflow:auto;margin:0 0 0.3rem 0;"><code>' + escapeHtml(inputStr) + '</code></pre>';
+            if (hasThinking) {
+                bodyInner +=
+                    '<div style="font-weight:600;font-size:0.72rem;color:#555;margin-bottom:0.2rem;">思考 (THINKING):</div>' +
+                    '<div style="background:#eef2ff;padding:0.6rem;border-radius:4px;font-size:0.78rem;line-height:1.6;color:#333;max-height:200px;overflow:auto;margin:0;white-space:pre-wrap;">' + escapeHtml(entry.thinking) + '</div>';
+            } else {
+                bodyInner +=
+                    '<div style="font-weight:600;font-size:0.72rem;color:#555;margin-bottom:0.2rem;">输出 (OUTPUT):</div>' +
+                    '<pre style="background:#1a1a2e;color:#e0e0e0;padding:0.5rem;border-radius:4px;font-size:0.7rem;max-height:200px;overflow:auto;margin:0;"><code>' + escapeHtml(outputStr) + '</code></pre>';
+            }
             card.innerHTML =
                 '<div class="tool-call-header" onclick="toggleStep(this)" style="padding:0.4rem 0.8rem;">' +
                     '<span><span style="background:#43aa8b;color:#fff;border-radius:3px;padding:0 5px;font-size:0.7rem;font-weight:600;margin-right:6px;">🔧</span>' +
@@ -249,10 +236,7 @@ function renderTimeline(entries) {
                     '<span class="arrow">▶</span>' +
                 '</div>' +
                 '<div class="tool-call-body" style="padding:0.5rem 0.8rem;background:#fafbfc;">' +
-                    '<div style="font-weight:600;font-size:0.72rem;color:#555;margin-bottom:0.2rem;">输入 (INPUT):</div>' +
-                    '<pre style="background:#1a1a2e;color:#e0e0e0;padding:0.5rem;border-radius:4px;font-size:0.7rem;max-height:200px;overflow:auto;margin:0 0 0.3rem 0;"><code>' + escapeHtml(inputStr) + '</code></pre>' +
-                    '<div style="font-weight:600;font-size:0.72rem;color:#555;margin-bottom:0.2rem;">输出 (OUTPUT):</div>' +
-                    '<pre style="background:#1a1a2e;color:#e0e0e0;padding:0.5rem;border-radius:4px;font-size:0.7rem;max-height:200px;overflow:auto;margin:0;"><code>' + escapeHtml(outputStr) + '</code></pre>' +
+                    bodyInner +
                 '</div>';
             container.appendChild(card);
         }
@@ -260,13 +244,18 @@ function renderTimeline(entries) {
 }
 
 async function startAuto() {
-    const selected = document.querySelector('input[name="auto_run_id"]:checked');
-    if (!selected) { showError('请先选择一个数据集'); return; }
+    const params = new URLSearchParams(window.location.search);
+    const runId = params.get('run_id');
+    if (!runId) {
+        document.getElementById('noRunMessage').style.display = 'block';
+        return;
+    }
 
     const timelineContainer = document.getElementById('timeline');
     const timelinePanel = document.getElementById('timelinePanel');
 
     // Reset UI
+    document.getElementById('noRunMessage').style.display = 'none';
     document.getElementById('autoProgress').style.display = 'block';
     document.getElementById('cancelBtn').style.display = 'inline-block';
     timelinePanel.style.display = 'none';
@@ -277,11 +266,10 @@ async function startAuto() {
         resp = await fetch('/analyze/llm/', {
             method: 'POST',
             headers: { 'Content-Type': 'application/json', 'X-CSRFToken': '{{ csrf_token }}' },
-            body: JSON.stringify({ run_id: parseInt(selected.value) }),
+            body: JSON.stringify({ run_id: parseInt(runId) }),
         });
     } catch (e) { showError('启动 LLM 分析失败: ' + e.message); return; }
     const data = await resp.json();
-    const runId = selected.value;
 
     let lastTimelineHash = '';
 
@@ -296,7 +284,7 @@ async function startAuto() {
 
             // Build and render unified timeline
             const entries = buildTimeline(s.llm_thinking, s.tool_calls);
-            const hash = JSON.stringify(entries.map(e => ({step:e.step,type:e.type,name:e.name,text:(e.text||'').slice(0,40)})));
+            const hash = JSON.stringify(entries.map(e => ({step:e.step,name:e.name,thinking:(e.thinking||'').slice(0,40)})));
             if (hash !== lastTimelineHash) {
                 renderTimeline(entries);
                 lastTimelineHash = hash;
@@ -324,6 +312,14 @@ function cancelAnalysis() {
     document.getElementById('cancelBtn').style.display = 'none';
     document.getElementById('timelinePanel').innerHTML += '\n--- 分析已取消 ---\n';
 }
+
+// Auto-start if run_id is provided via URL parameter
+(function() {
+    const params = new URLSearchParams(window.location.search);
+    if (params.get('run_id')) {
+        startAuto();
+    }
+})();
 </script>
 {% endblock %}
 <!--VERSION2-->

templates/tianxuan/manual.html

@@ -194,7 +194,7 @@ function buildDatasetTable(filter) {
         return;
     }
 
-    let html = '<table class="ds-table"><thead><tr><th>ID</th><th>状态</th><th>行数</th><th>文件</th><th></th></tr></thead><tbody>';
+    let html = '<table class="ds-table"><thead><tr><th>ID</th><th>状态</th><th>行数</th><th>文件数</th><th></th></tr></thead><tbody>';
     rows.forEach(ds => {
         const label = ds.status_label || '待分析';
         const badgeClass = (ds.run_status === 'completed') ? 'completed' :