PM-pinou
78 lines
2.5 KiB
Python
78 lines
2.5 KiB
Python
import os, sys, django, time
|
|
os.environ['DJANGO_SETTINGS_MODULE'] = 'tianxuan.settings'
|
|
os.environ['DJANGO_ALLOW_ASYNC_UNSAFE'] = 'true'
|
|
sys.path.insert(0, r'C:\Users\25044\Desktop\Proj\天璇')
|
|
django.setup()
|
|
|
|
import polars as pl, numpy as np
|
|
from analysis.data_loader import load_from_db
|
|
|
|
lf = load_from_db('_data_43')
|
|
if lf is None:
|
|
print('_data_43 not found')
|
|
sys.exit(1)
|
|
|
|
# Coerce key columns to numeric (simulating background process)
|
|
schema = lf.collect_schema()
|
|
names = schema.names()
|
|
print(f'Columns: {len(names)}')
|
|
|
|
# Pick numeric candidates
|
|
numeric_candidates = []
|
|
utf8_cols = []
|
|
for n, d in zip(names, list(schema.dtypes())):
|
|
if d == pl.Utf8 and not n.startswith('_'):
|
|
utf8_cols.append(n)
|
|
|
|
# Sample to check castability
|
|
sample = lf.select(utf8_cols).head(500).collect(streaming=True)
|
|
type_map = {}
|
|
for col in utf8_cols:
|
|
try:
|
|
casted = sample[col].cast(pl.Float64, strict=False)
|
|
nn = casted.is_not_null().sum()
|
|
if nn > len(casted) * 0.5:
|
|
type_map[col] = pl.Float64
|
|
except:
|
|
pass
|
|
|
|
print(f'Coercible columns: {len(type_map)}: {list(type_map.keys())[:8]}')
|
|
|
|
# Apply casts
|
|
casts = [pl.col(c).cast(pl.Float64, strict=False) for c in type_map]
|
|
lf = lf.with_columns(casts)
|
|
|
|
# Find numeric columns
|
|
numeric_types = (pl.Int8, pl.Int16, pl.Int32, pl.Int64, pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64, pl.Float32, pl.Float64)
|
|
schema2 = lf.collect_schema()
|
|
feature_cols = [schema2.names()[i] for i, dt in enumerate(list(schema2.dtypes()))
|
|
if dt in numeric_types and not schema2.names()[i].startswith('_')][:10]
|
|
print(f'Feature cols ({len(feature_cols)}): {feature_cols}')
|
|
|
|
# Collect and run
|
|
t0 = time.time()
|
|
df = lf.select(feature_cols).collect(streaming=True)
|
|
data = df.to_numpy().astype(np.float64)
|
|
print(f'Data shape: {data.shape}, nans: {np.isnan(data).sum()}')
|
|
|
|
col_mean = np.nanmean(data, axis=0)
|
|
col_mean = np.nan_to_num(col_mean, nan=0.0)
|
|
data = np.where(np.isnan(data), col_mean, data)
|
|
|
|
if len(data) > 50000:
|
|
rng = np.random.default_rng(42)
|
|
data = data[rng.choice(len(data), 50000, replace=False)]
|
|
print(f'After sample: {data.shape}')
|
|
|
|
from sklearn.preprocessing import StandardScaler
|
|
print('Scaling...')
|
|
data_scaled = StandardScaler().fit_transform(data)
|
|
print(f'Scaled shape: {data_scaled.shape}')
|
|
|
|
from sklearn.cluster import HDBSCAN
|
|
print('HDBSCAN...')
|
|
model = HDBSCAN(min_cluster_size=5, min_samples=5)
|
|
labels = model.fit_predict(data_scaled)
|
|
nc = len(set(labels)) - (1 if -1 in labels else 0)
|
|
print(f'DONE in {time.time()-t0:.1f}s: {nc} clusters, {(labels==-1).sum()} noise')
|