import os, sys, django, time
os.environ['DJANGO_SETTINGS_MODULE'] = 'tianxuan.settings'
os.environ['DJANGO_ALLOW_ASYNC_UNSAFE'] = 'true'
sys.path.insert(0, r'C:\Users\25044\Desktop\Proj\天璇')
django.setup()

import polars as pl, numpy as np
from analysis.data_loader import load_from_db

lf = load_from_db('_data_43')
if lf is None:
    print('_data_43 not found')
    sys.exit(1)

# Coerce key columns to numeric (simulating background process)
schema = lf.collect_schema()
names = schema.names()
print(f'Columns: {len(names)}')

# Pick numeric candidates
numeric_candidates = []
utf8_cols = []
for n, d in zip(names, list(schema.dtypes())):
    if d == pl.Utf8 and not n.startswith('_'):
        utf8_cols.append(n)

# Sample to check castability
sample = lf.select(utf8_cols).head(500).collect(streaming=True)
type_map = {}
for col in utf8_cols:
    try:
        casted = sample[col].cast(pl.Float64, strict=False)
        nn = casted.is_not_null().sum()
        if nn > len(casted) * 0.5:
            type_map[col] = pl.Float64
    except:
        pass

print(f'Coercible columns: {len(type_map)}: {list(type_map.keys())[:8]}')

# Apply casts
casts = [pl.col(c).cast(pl.Float64, strict=False) for c in type_map]
lf = lf.with_columns(casts)

# Find numeric columns
numeric_types = (pl.Int8, pl.Int16, pl.Int32, pl.Int64, pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64, pl.Float32, pl.Float64)
schema2 = lf.collect_schema()
feature_cols = [schema2.names()[i] for i, dt in enumerate(list(schema2.dtypes())) 
                if dt in numeric_types and not schema2.names()[i].startswith('_')][:10]
print(f'Feature cols ({len(feature_cols)}): {feature_cols}')

# Collect and run
t0 = time.time()
df = lf.select(feature_cols).collect(streaming=True)
data = df.to_numpy().astype(np.float64)
print(f'Data shape: {data.shape}, nans: {np.isnan(data).sum()}')

col_mean = np.nanmean(data, axis=0)
col_mean = np.nan_to_num(col_mean, nan=0.0)
data = np.where(np.isnan(data), col_mean, data)

if len(data) > 50000:
    rng = np.random.default_rng(42)
    data = data[rng.choice(len(data), 50000, replace=False)]
print(f'After sample: {data.shape}')

from sklearn.preprocessing import StandardScaler
print('Scaling...')
data_scaled = StandardScaler().fit_transform(data)
print(f'Scaled shape: {data_scaled.shape}')

from sklearn.cluster import HDBSCAN
print('HDBSCAN...')
model = HDBSCAN(min_cluster_size=5, min_samples=5)
labels = model.fit_predict(data_scaled)
nc = len(set(labels)) - (1 if -1 in labels else 0)
print(f'DONE in {time.time()-t0:.1f}s: {nc} clusters, {(labels==-1).sum()} noise')