From 30e48ba282494e8b95ec08d6addc8126f1d0a551 Mon Sep 17 00:00:00 2001 From: PM-pinou <2504420230@qq.com> Date: Fri, 24 Jul 2026 12:57:33 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20increase=20sampling=20coverage=20?= =?UTF-8?q?=E2=80=94=20MAX=5FSAMPLE=5FROWS=2030K=E2=86=92200K,=20LOW=5FMEM?= =?UTF-8?q?ORY=20100K=E2=86=92500K,=20UMAP=5FBATCH=201K=E2=86=922K?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- analysis/constants.py | 38 +++++++++++++++++++++----------------- server_stderr.txt | 20 ++++++++++++++++++++ 2 files changed, 41 insertions(+), 17 deletions(-) diff --git a/analysis/constants.py b/analysis/constants.py index faa5145..c745737 100644 --- a/analysis/constants.py +++ b/analysis/constants.py @@ -9,36 +9,40 @@ so they can be updated in one place. Import with:: import polars as pl # --------------------------------------------------------------------------- -# Clustering / UMAP +# Clustering / UMAP — tuned for 25M rows × 50 cols on 8GB RAM # --------------------------------------------------------------------------- -MAX_SAMPLE_ROWS = 30000 -"""Maximum rows for clustering sample (downsampling threshold, tuned for 8GB RAM).""" +MAX_SAMPLE_ROWS = 200000 +"""Maximum rows for clustering sample. 200K ≈ 0.8% of 25M rows, fits 8GB RAM.""" -MAX_DISTRIBUTION_SAMPLE = 10000 -"""Default sample size for distribution exploration and scoring.""" +MAX_DISTRIBUTION_SAMPLE = 50000 +"""Default sample for distribution exploration — 50K provides stable statistics.""" -UMAP_TRAIN_SAMPLE = 10000 -"""Max rows used to train a UMAP reducer in extract_features.""" +UMAP_TRAIN_SAMPLE = 30000 +"""Max rows for UMAP training. 30K is a balance: UMAP O(n²) for neighbors.""" +# Note: UMAP neighbors search is O(n * n_neighbors * log n). 30K × 15 ≈ fine. +# Going above 50K causes UMAP to allocate several GB for the nearest-neighbor index. -UMAP_BATCH_SIZE = 1000 +UMAP_BATCH_SIZE = 2000 """Batch size for UMAP transform when dataset exceeds UMAP_TRAIN_SAMPLE.""" LOW_MEMORY_THRESHOLD_GB = 2 """If available memory < this (in GiB), enable low-memory clustering path.""" -LOW_MEMORY_THRESHOLD_ROWS = 100000 -"""Row count above which low-memory downsampling is triggered.""" +LOW_MEMORY_THRESHOLD_ROWS = 500000 +"""Row count above which low-memory downsampling is triggered. +500K gives 2% coverage of 25M rows while keeping numpy matrix < 200 MB.""" RANDOM_SEED = 42 """Default random seed for all sklearn/numpy operations.""" # --------------------------------------------------------------------------- -# Globe +# Globe — 8GB RAM / 25M rows constraints # --------------------------------------------------------------------------- -GLOBE_MAX_ROWS = 300 -"""Max rows to load per run for 3D globe visualization.""" +GLOBE_MAX_ROWS = 500 +"""Max rows to load per run for 3D globe visualization. 300→500 for better +coverage on the globe, still < 1 MB of flow data.""" # --------------------------------------------------------------------------- # Filter @@ -76,11 +80,11 @@ entity aggregation, anomaly detection, and feature extraction.""" # Diagnostics / Validation # --------------------------------------------------------------------------- -MAX_VALIDATE_SAMPLE = 1000 +MAX_VALIDATE_SAMPLE = 5000 """Max rows sampled during validate_data.""" -MAX_DIAGNOSE_SAMPLE = 5000 -"""Max rows sampled for clustering diagnosis (SVD).""" +MAX_DIAGNOSE_SAMPLE = 50000 +"""Max rows sampled for clustering diagnosis (SVD). 50K for stable SVD.""" MAX_CLUSTER_FEATURES = 50 """Feature count threshold above which TruncatedSVD is applied.""" @@ -92,5 +96,5 @@ CLUSTER_MAX_FEATURES_TOP = 10 # Globe / Flow extraction # --------------------------------------------------------------------------- -GLOBE_FLOW_MAX_ROWS = 300 +GLOBE_FLOW_MAX_ROWS = 500 """Alias for GLOBE_MAX_ROWS — max rows loaded for flow extraction.""" diff --git a/server_stderr.txt b/server_stderr.txt index eb11cfe..8b57fcd 100644 --- a/server_stderr.txt +++ b/server_stderr.txt @@ -87,3 +87,23 @@ Restored 1 datasets from disk [24/Jul/2026 12:44:37] "HEAD / HTTP/1.1" 200 0 [24/Jul/2026 12:45:07] "HEAD / HTTP/1.1" 200 0 [24/Jul/2026 12:45:37] "HEAD / HTTP/1.1" 200 0 +[24/Jul/2026 12:46:07] "HEAD / HTTP/1.1" 200 0 +[24/Jul/2026 12:46:37] "HEAD / HTTP/1.1" 200 0 +[24/Jul/2026 12:47:07] "HEAD / HTTP/1.1" 200 0 +[24/Jul/2026 12:47:37] "HEAD / HTTP/1.1" 200 0 +[24/Jul/2026 12:48:07] "HEAD / HTTP/1.1" 200 0 +[24/Jul/2026 12:48:37] "HEAD / HTTP/1.1" 200 0 +[24/Jul/2026 12:49:07] "HEAD / HTTP/1.1" 200 0 +[24/Jul/2026 12:49:37] "HEAD / HTTP/1.1" 200 0 +[24/Jul/2026 12:50:07] "HEAD / HTTP/1.1" 200 0 +[24/Jul/2026 12:50:37] "HEAD / HTTP/1.1" 200 0 +[24/Jul/2026 12:51:07] "HEAD / HTTP/1.1" 200 0 +[24/Jul/2026 12:51:37] "HEAD / HTTP/1.1" 200 0 +[24/Jul/2026 12:52:07] "HEAD / HTTP/1.1" 200 0 +[24/Jul/2026 12:52:37] "HEAD / HTTP/1.1" 200 0 +[24/Jul/2026 12:53:07] "HEAD / HTTP/1.1" 200 0 +[24/Jul/2026 12:53:37] "HEAD / HTTP/1.1" 200 0 +[24/Jul/2026 12:54:07] "HEAD / HTTP/1.1" 200 0 +[24/Jul/2026 12:54:37] "HEAD / HTTP/1.1" 200 0 +[24/Jul/2026 12:55:07] "HEAD / HTTP/1.1" 200 0 +[24/Jul/2026 12:55:37] "HEAD / HTTP/1.1" 200 0