PM-pinou
69 lines
2.5 KiB
Python
69 lines
2.5 KiB
Python
"""
|
|
Debug: test pipeline data persistence
|
|
"""
|
|
import sys, os
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
os.environ['DJANGO_SETTINGS_MODULE'] = 'tianxuan.settings'
|
|
|
|
import django
|
|
django.setup()
|
|
|
|
from analysis.session_store import SessionStore
|
|
from analysis.data_loader import load_csv_directory
|
|
from analysis.entity_aggregator import aggregate_by_entity
|
|
from analysis.entity_detector import detect_entity_column
|
|
import asyncio
|
|
from analysis.tool_registry import _handle_run_clustering, _handle_extract_features
|
|
|
|
store = SessionStore()
|
|
store.drop_all()
|
|
|
|
csv_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'data', 'test_flows.csv')
|
|
|
|
# Step 1: Load
|
|
lf, schema, rc, fc, mem = load_csv_directory(csv_path)
|
|
store.store_dataset('ds', lf, schema=schema, metadata={'row_count': rc, 'csv_glob': csv_path})
|
|
print(f'1. Loaded {rc} rows')
|
|
|
|
# Step 2: Detect entity
|
|
result = detect_entity_column('ds')
|
|
entity_col = result['recommended']
|
|
print(f'2. Entity col: {entity_col}')
|
|
|
|
# Step 3: Aggregate
|
|
agg_lf, features = aggregate_by_entity(lf, entity_col, schema)
|
|
df_agg = agg_lf.collect(streaming=True)
|
|
store.store_dataset('entity', agg_lf, schema=dict(zip(features, ['']*len(features))), metadata={'row_count': len(df_agg)})
|
|
print(f'3. Aggregated {len(df_agg)} entities')
|
|
|
|
# Step 4: Cluster
|
|
feature_cols = [c for c in features if not c.startswith('_')][:8]
|
|
clust_result = asyncio.run(_handle_run_clustering(
|
|
dataset_id='entity', cluster_columns=feature_cols,
|
|
algorithm='hdbscan', params={}, random_state=42
|
|
))
|
|
print(f'4. Clustering: n_clusters={clust_result.get("n_clusters")}')
|
|
if 'error' in clust_result:
|
|
print(f' ERROR: {clust_result["error"]}')
|
|
sys.exit(1)
|
|
|
|
cid = clust_result.get('cluster_result_id', '')
|
|
print(f' cluster_result_id={cid}')
|
|
entry = store.get_cluster_result(cid)
|
|
print(f' store entry exists={entry is not None}')
|
|
if entry:
|
|
print(f' n_clusters in store={entry.get("n_clusters")}')
|
|
|
|
# Step 5: Extract features + save to DB
|
|
feat_result = asyncio.run(_handle_extract_features(
|
|
dataset_id='entity', cluster_result_id=cid,
|
|
top_k=10, method='zscore', save_to_db=True
|
|
))
|
|
print(f'5. Features: n={feat_result.get("n_features")} db_saved={feat_result.get("db_saved")}')
|
|
if 'error' in feat_result:
|
|
print(f' ERROR: {feat_result["error"]}')
|
|
|
|
# Step 6: Check DB
|
|
from analysis.models import AnalysisRun, ClusterResult, ClusterFeature, EntityProfile
|
|
print(f'6. DB: ClusterResult={ClusterResult.objects.count()} ClusterFeature={ClusterFeature.objects.count()} EntityProfile={EntityProfile.objects.count()}')
|