PM-pinou
52 lines
2.4 KiB
Python
52 lines
2.4 KiB
Python
"""Test PCA embedding in _handle_extract_features"""
|
|
import sys, os
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
os.environ['DJANGO_SETTINGS_MODULE'] = 'tianxuan.settings'
|
|
import django; django.setup()
|
|
import asyncio
|
|
from analysis.session_store import SessionStore
|
|
from analysis.data_loader import load_csv_directory
|
|
from analysis.entity_detector import detect_entity_column
|
|
from analysis.entity_aggregator import aggregate_by_entity
|
|
from analysis.tool_registry import _handle_run_clustering, _handle_extract_features
|
|
|
|
store = SessionStore(); store.drop_all()
|
|
csv = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'data', 'complex_test.csv')
|
|
lf, schema, rc, fc, mem = load_csv_directory(csv)
|
|
store.store_dataset('ds', lf, schema=schema, metadata={'row_count': rc, 'csv_glob': csv})
|
|
entity_col = detect_entity_column('ds')['recommended']
|
|
print(f'Entity col: {entity_col}')
|
|
agg_lf, features = aggregate_by_entity(lf, entity_col, schema)
|
|
df_agg = agg_lf.collect()
|
|
store.store_dataset('entity_data', agg_lf, schema=dict(zip(features, ['']*len(features))),
|
|
metadata={'row_count': len(df_agg), 'csv_glob': csv})
|
|
print(f'Aggregated: {len(df_agg)} rows, {len(features)} features')
|
|
|
|
# Create a dummy AnalysisRun so PCA embedding can save EntityProfiles
|
|
from analysis.models import AnalysisRun
|
|
run_for_pca, _ = AnalysisRun.objects.get_or_create(csv_glob=csv, defaults={'status': 'running'})
|
|
run_for_pca.status = 'running'
|
|
run_for_pca.save()
|
|
print(f'AnalysisRun: #{run_for_pca.id}')
|
|
|
|
clust = asyncio.run(_handle_run_clustering(dataset_id='entity_data',
|
|
cluster_columns=[c for c in features if not c.startswith('_')][:8],
|
|
algorithm='hdbscan', params={}, random_state=42))
|
|
cid = clust.get('cluster_result_id', '')
|
|
print(f'Cluster result: {cid}')
|
|
|
|
feat = asyncio.run(_handle_extract_features(dataset_id='entity_data', cluster_result_id=cid,
|
|
top_k=10, method='zscore', save_to_db=True))
|
|
print(f'Result keys: {list(feat.keys())}')
|
|
if 'error' in feat:
|
|
print(f'ERROR: {feat["error"]}')
|
|
print(f'n_features={feat.get("n_features")} db_saved={feat.get("db_saved")}')
|
|
|
|
from analysis.models import EntityProfile
|
|
cnt = EntityProfile.objects.exclude(embedding_x=None).count()
|
|
total = EntityProfile.objects.count()
|
|
print(f'EntityProfile with PCA coords: {cnt}/{total}')
|
|
if cnt > 0:
|
|
ep = EntityProfile.objects.exclude(embedding_x=None).first()
|
|
print(f'Sample: {ep.entity_value} ({ep.embedding_x}, {ep.embedding_y}) cluster={ep.cluster_label}')
|