PM-pinou
56 lines
2.2 KiB
Python
56 lines
2.2 KiB
Python
"""Debug database persistence"""
|
|
import sys, os
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
os.environ['DJANGO_SETTINGS_MODULE'] = 'tianxuan.settings'
|
|
|
|
import django
|
|
django.setup()
|
|
|
|
from analysis.session_store import SessionStore
|
|
from analysis.data_loader import load_csv_directory
|
|
from analysis.entity_aggregator import aggregate_by_entity
|
|
from analysis.entity_detector import detect_entity_column
|
|
import asyncio
|
|
from analysis.tool_registry import _handle_run_clustering, _handle_extract_features, _save_features_to_db
|
|
|
|
store = SessionStore()
|
|
store.drop_all()
|
|
csv_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'data', 'test_flows.csv')
|
|
|
|
lf, schema, rc, fc, mem = load_csv_directory(csv_path)
|
|
store.store_dataset('ds', lf, schema=schema, metadata={'row_count': rc, 'csv_glob': csv_path})
|
|
|
|
entity_col = detect_entity_column('ds')['recommended']
|
|
agg_lf, features = aggregate_by_entity(lf, entity_col, schema)
|
|
df_agg = agg_lf.collect(streaming=True)
|
|
store.store_dataset('entity', agg_lf, schema=dict(zip(features, ['']*len(features))),
|
|
metadata={'row_count': len(df_agg), 'csv_glob': csv_path})
|
|
|
|
clust_result = asyncio.run(_handle_run_clustering(dataset_id='entity',
|
|
cluster_columns=[c for c in features if not c.startswith('_')][:8],
|
|
algorithm='hdbscan', params={}, random_state=42))
|
|
cid = clust_result['cluster_result_id']
|
|
|
|
feat_result = asyncio.run(_handle_extract_features(
|
|
dataset_id='entity', cluster_result_id=cid,
|
|
top_k=10, method='zscore', save_to_db=True
|
|
))
|
|
|
|
feats = feat_result.get('features', [])
|
|
print(f'n_features={feat_result.get("n_features")}')
|
|
labels = set(f.get('cluster_label') for f in feats)
|
|
print(f'unique_labels={labels}')
|
|
if feats:
|
|
print(f'sample feature: {feats[0]}')
|
|
print(f'sample keys: {list(feats[0].keys())}')
|
|
|
|
print(f'db_saved={feat_result.get("db_saved")}')
|
|
|
|
from analysis.models import ClusterResult, ClusterFeature, EntityProfile
|
|
print(f'ClusterResult: {ClusterResult.objects.count()}')
|
|
for cr in ClusterResult.objects.all():
|
|
print(f' label={cr.cluster_label} size={cr.size} feats={cr.features.count()}')
|
|
|
|
print(f'ClusterFeature: {ClusterFeature.objects.count()}')
|
|
print(f'EntityProfile: {EntityProfile.objects.count()}')
|