PM-pinou
101 lines
3.8 KiB
Python
101 lines
3.8 KiB
Python
"""Centralized constants for the TianXuan analysis module.
|
||
|
||
All magic numbers from analysis/tools/ and analysis/views/ are defined here
|
||
so they can be updated in one place. Import with::
|
||
|
||
from analysis.constants import RANDOM_SEED, MAX_SAMPLE_ROWS, ...
|
||
"""
|
||
|
||
import polars as pl
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Clustering / UMAP — tuned for 25M rows × 50 cols on 8GB RAM
|
||
# ---------------------------------------------------------------------------
|
||
|
||
MAX_SAMPLE_ROWS = 200000
|
||
"""Maximum rows for clustering sample. 200K ≈ 0.8% of 25M rows, fits 8GB RAM."""
|
||
|
||
MAX_DISTRIBUTION_SAMPLE = 50000
|
||
"""Default sample for distribution exploration — 50K provides stable statistics."""
|
||
|
||
UMAP_TRAIN_SAMPLE = 30000
|
||
"""Max rows for UMAP training. 30K is a balance: UMAP O(n²) for neighbors."""
|
||
# Note: UMAP neighbors search is O(n * n_neighbors * log n). 30K × 15 ≈ fine.
|
||
# Going above 50K causes UMAP to allocate several GB for the nearest-neighbor index.
|
||
|
||
UMAP_BATCH_SIZE = 2000
|
||
"""Batch size for UMAP transform when dataset exceeds UMAP_TRAIN_SAMPLE."""
|
||
|
||
LOW_MEMORY_THRESHOLD_GB = 2
|
||
"""If available memory < this (in GiB), enable low-memory clustering path."""
|
||
|
||
LOW_MEMORY_THRESHOLD_ROWS = 500000
|
||
"""Row count above which low-memory downsampling is triggered.
|
||
500K gives 2% coverage of 25M rows while keeping numpy matrix < 200 MB."""
|
||
|
||
RANDOM_SEED = 42
|
||
"""Default random seed for all sklearn/numpy operations."""
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Globe — 8GB RAM / 25M rows constraints
|
||
# ---------------------------------------------------------------------------
|
||
|
||
GLOBE_MAX_ROWS = 500
|
||
"""Max rows to load per run for 3D globe visualization. 300→500 for better
|
||
coverage on the globe, still < 1 MB of flow data."""
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Filter
|
||
# ---------------------------------------------------------------------------
|
||
|
||
FILTER_MAX_ROWS = 5
|
||
"""Maximum visible filter rows in the manual analysis UI (deprecated / reserved)."""
|
||
|
||
MAX_COLUMNS_PROFILE = 200
|
||
"""Max columns to include in a dataset profile summary."""
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# File upload limits
|
||
# ---------------------------------------------------------------------------
|
||
|
||
FILE_UPLOAD_MAX_SIZE = 500 * 1024 * 1024 # 500 MB
|
||
"""Maximum size for a single uploaded file."""
|
||
|
||
TOTAL_UPLOAD_MAX_SIZE = 50 * 1024 * 1024 * 1024 # 50 GB
|
||
"""Maximum total upload size across all files."""
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Tool limits (numeric type tuples used across tools/)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
NUMERIC_DTYPES: tuple[type[pl.DataType], ...] = (
|
||
pl.Int8, pl.Int16, pl.Int32, pl.Int64,
|
||
pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
|
||
pl.Float32, pl.Float64,
|
||
)
|
||
"""Polars numeric data types — used for column selection in clustering,
|
||
entity aggregation, anomaly detection, and feature extraction."""
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Diagnostics / Validation
|
||
# ---------------------------------------------------------------------------
|
||
|
||
MAX_VALIDATE_SAMPLE = 5000
|
||
"""Max rows sampled during validate_data."""
|
||
|
||
MAX_DIAGNOSE_SAMPLE = 50000
|
||
"""Max rows sampled for clustering diagnosis (SVD). 50K for stable SVD."""
|
||
|
||
MAX_CLUSTER_FEATURES = 50
|
||
"""Feature count threshold above which TruncatedSVD is applied."""
|
||
|
||
CLUSTER_MAX_FEATURES_TOP = 10
|
||
"""Max feature columns selected by auto-detection in clustering tools."""
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Globe / Flow extraction
|
||
# ---------------------------------------------------------------------------
|
||
|
||
GLOBE_FLOW_MAX_ROWS = 500
|
||
"""Alias for GLOBE_MAX_ROWS — max rows loaded for flow extraction."""
|