HJQ/tianxuan
2
1
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

perf: increase sampling coverage — MAX_SAMPLE_ROWS 30K→200K, LOW_MEMORY 100K→500K, UMAP_BATCH 1K→2K

Browse Source
This commit is contained in:
PM-pinou
2026-07-24 12:57:33 +08:00
parent df0f7d621f
commit 30e48ba282
2 changed files with 41 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
analysis/constants.py
+21 -17
View File
@@ -9,36 +9,40 @@ so they can be updated in one place. Import with::
import polars as pl import polars as pl
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Clustering / UMAP # Clustering / UMAP — tuned for 25M rows × 50 cols on 8GB RAM
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
MAX_SAMPLE_ROWS = 30000 MAX_SAMPLE_ROWS = 200000
"""Maximum rows for clustering sample (downsampling threshold, tuned for 8GB RAM).""" """Maximum rows for clustering sample. 200K ≈ 0.8% of 25M rows, fits 8GB RAM."""
MAX_DISTRIBUTION_SAMPLE = 10000 MAX_DISTRIBUTION_SAMPLE = 50000
"""Default sample size for distribution exploration and scoring.""" """Default sample for distribution exploration — 50K provides stable statistics."""
UMAP_TRAIN_SAMPLE = 10000 UMAP_TRAIN_SAMPLE = 30000
"""Max rows used to train a UMAP reducer in extract_features.""" """Max rows for UMAP training. 30K is a balance: UMAP O(n²) for neighbors."""
# Note: UMAP neighbors search is O(n * n_neighbors * log n). 30K × 15 ≈ fine.
# Going above 50K causes UMAP to allocate several GB for the nearest-neighbor index.
UMAP_BATCH_SIZE = 1000 UMAP_BATCH_SIZE = 2000
"""Batch size for UMAP transform when dataset exceeds UMAP_TRAIN_SAMPLE.""" """Batch size for UMAP transform when dataset exceeds UMAP_TRAIN_SAMPLE."""
LOW_MEMORY_THRESHOLD_GB = 2 LOW_MEMORY_THRESHOLD_GB = 2
"""If available memory < this (in GiB), enable low-memory clustering path.""" """If available memory < this (in GiB), enable low-memory clustering path."""
LOW_MEMORY_THRESHOLD_ROWS = 100000 LOW_MEMORY_THRESHOLD_ROWS = 500000
"""Row count above which low-memory downsampling is triggered.""" """Row count above which low-memory downsampling is triggered.
500K gives 2% coverage of 25M rows while keeping numpy matrix < 200 MB."""
RANDOM_SEED = 42 RANDOM_SEED = 42
"""Default random seed for all sklearn/numpy operations.""" """Default random seed for all sklearn/numpy operations."""
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Globe # Globe — 8GB RAM / 25M rows constraints
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
GLOBE_MAX_ROWS = 300 GLOBE_MAX_ROWS = 500
"""Max rows to load per run for 3D globe visualization.""" """Max rows to load per run for 3D globe visualization. 300→500 for better
coverage on the globe, still < 1 MB of flow data."""
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Filter # Filter
@@ -76,11 +80,11 @@ entity aggregation, anomaly detection, and feature extraction."""
# Diagnostics / Validation # Diagnostics / Validation
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
MAX_VALIDATE_SAMPLE = 1000 MAX_VALIDATE_SAMPLE = 5000
"""Max rows sampled during validate_data.""" """Max rows sampled during validate_data."""
MAX_DIAGNOSE_SAMPLE = 5000 MAX_DIAGNOSE_SAMPLE = 50000
"""Max rows sampled for clustering diagnosis (SVD).""" """Max rows sampled for clustering diagnosis (SVD). 50K for stable SVD."""
MAX_CLUSTER_FEATURES = 50 MAX_CLUSTER_FEATURES = 50
"""Feature count threshold above which TruncatedSVD is applied.""" """Feature count threshold above which TruncatedSVD is applied."""
@@ -92,5 +96,5 @@ CLUSTER_MAX_FEATURES_TOP = 10
# Globe / Flow extraction # Globe / Flow extraction
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
GLOBE_FLOW_MAX_ROWS = 300 GLOBE_FLOW_MAX_ROWS = 500
"""Alias for GLOBE_MAX_ROWS — max rows loaded for flow extraction.""" """Alias for GLOBE_MAX_ROWS — max rows loaded for flow extraction."""
server_stderr.txt
+20
View File
@@ -87,3 +87,23 @@ Restored 1 datasets from disk
[24/Jul/2026 12:44:37] "HEAD / HTTP/1.1" 200 0 [24/Jul/2026 12:44:37] "HEAD / HTTP/1.1" 200 0
[24/Jul/2026 12:45:07] "HEAD / HTTP/1.1" 200 0 [24/Jul/2026 12:45:07] "HEAD / HTTP/1.1" 200 0
[24/Jul/2026 12:45:37] "HEAD / HTTP/1.1" 200 0 [24/Jul/2026 12:45:37] "HEAD / HTTP/1.1" 200 0
[24/Jul/2026 12:46:07] "HEAD / HTTP/1.1" 200 0
[24/Jul/2026 12:46:37] "HEAD / HTTP/1.1" 200 0
[24/Jul/2026 12:47:07] "HEAD / HTTP/1.1" 200 0
[24/Jul/2026 12:47:37] "HEAD / HTTP/1.1" 200 0
[24/Jul/2026 12:48:07] "HEAD / HTTP/1.1" 200 0
[24/Jul/2026 12:48:37] "HEAD / HTTP/1.1" 200 0
[24/Jul/2026 12:49:07] "HEAD / HTTP/1.1" 200 0
[24/Jul/2026 12:49:37] "HEAD / HTTP/1.1" 200 0
[24/Jul/2026 12:50:07] "HEAD / HTTP/1.1" 200 0
[24/Jul/2026 12:50:37] "HEAD / HTTP/1.1" 200 0
[24/Jul/2026 12:51:07] "HEAD / HTTP/1.1" 200 0
[24/Jul/2026 12:51:37] "HEAD / HTTP/1.1" 200 0
[24/Jul/2026 12:52:07] "HEAD / HTTP/1.1" 200 0
[24/Jul/2026 12:52:37] "HEAD / HTTP/1.1" 200 0
[24/Jul/2026 12:53:07] "HEAD / HTTP/1.1" 200 0
[24/Jul/2026 12:53:37] "HEAD / HTTP/1.1" 200 0
[24/Jul/2026 12:54:07] "HEAD / HTTP/1.1" 200 0
[24/Jul/2026 12:54:37] "HEAD / HTTP/1.1" 200 0
[24/Jul/2026 12:55:07] "HEAD / HTTP/1.1" 200 0
[24/Jul/2026 12:55:37] "HEAD / HTTP/1.1" 200 0