PM-pinou
129 lines
6.0 KiB
Python
129 lines
6.0 KiB
Python
"""天璇完整流程端到端测试"""
|
|
import sys, os, importlib
|
|
# 项目根目录 = 脚本目录的父目录
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
os.environ['DJANGO_SETTINGS_MODULE'] = 'tianxuan.settings'
|
|
import django; django.setup()
|
|
|
|
OK = '[OK]'
|
|
FAIL = '[!!]'
|
|
|
|
results = []
|
|
|
|
def check(name, ok, detail=""):
|
|
icon = OK if ok else FAIL
|
|
results.append((name, ok))
|
|
print(f' {icon} {name} {detail}')
|
|
|
|
print('=' * 60)
|
|
print('天璇 完整流程端到端测试')
|
|
print('=' * 60)
|
|
|
|
# ── 1. 生成复杂测试数据 ──────────────────────────────────
|
|
print('\n[1/7] 生成测试数据')
|
|
import importlib.util
|
|
spec = importlib.util.spec_from_file_location('gen_complex_test',
|
|
os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'scripts', 'gen_complex_test.py'))
|
|
mod = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(mod)
|
|
df = mod.gen_flows(n_rows=2000)
|
|
csv_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'data', 'e2e_full.csv')
|
|
df.write_csv(csv_path, null_value='')
|
|
check('gen_complex_test', os.path.exists(csv_path), f'rows={len(df)} cols={len(df.columns)}')
|
|
|
|
# ── 2. 加载 + 清洗 ────────────────────────────────────────
|
|
print('\n[2/7] 加载 CSV + 自动清洗')
|
|
from analysis.data_loader import load_csv_directory
|
|
from analysis.session_store import SessionStore
|
|
|
|
store = SessionStore()
|
|
store.drop_all()
|
|
lf, schema, rc, fc, mem = load_csv_directory(csv_path)
|
|
store.store_dataset('ds', lf, schema=schema, metadata={'row_count': rc, 'csv_glob': csv_path})
|
|
|
|
# 检查旧经纬度列是否被丢弃
|
|
dropped_lat = not any('latitude' in c.lower() or 'longitude' in c.lower() for c in schema)
|
|
check('丢弃旧经纬度列', dropped_lat, f'columns={len(schema)}')
|
|
|
|
# 检查 HEX 列是否被预处理
|
|
schema_names = list(schema.keys())
|
|
check('schema 有效', len(schema_names) > 10, f'{len(schema_names)} columns')
|
|
|
|
# ── 3. 数据校验 ────────────────────────────────────────────
|
|
print('\n[3/7] 数据校验')
|
|
from analysis.data_validator import validate
|
|
v_result = validate('ds')
|
|
n_risks = len(v_result.get('risks', []))
|
|
check('数据校验完成', v_result.get('valid') is not None, f'valid={v_result["valid"]} risks={n_risks}')
|
|
|
|
# ── 4. 实体检测 ────────────────────────────────────────────
|
|
print('\n[4/7] 实体检测')
|
|
from analysis.entity_detector import detect_entity_column
|
|
detect_result = detect_entity_column('ds')
|
|
entity_col = detect_result.get('recommended', '')
|
|
n_candidates = len(detect_result.get('candidates', []))
|
|
check('实体检测', bool(entity_col), f'entity={entity_col} candidates={n_candidates}')
|
|
|
|
# ── 5. 类型分类 + 聚合 ─────────────────────────────────────
|
|
print('\n[5/7] 类型感知聚合')
|
|
from analysis.type_classifier import classify_schema, DataType
|
|
from analysis.entity_aggregator import aggregate_by_entity
|
|
|
|
# 类型分类
|
|
type_map = classify_schema(lf.head(500))
|
|
types_found = set(t.name for t in type_map.values())
|
|
check('类型分类', len(type_map) > 0, f'types={types_found}')
|
|
|
|
# 聚合
|
|
agg_lf, features = aggregate_by_entity(lf, entity_col, schema)
|
|
df_agg = agg_lf.collect()
|
|
entity_count = len(df_agg)
|
|
store.store_dataset('entity', agg_lf, schema=dict(zip(features, ['']*len(features))),
|
|
metadata={'row_count': entity_count, 'entity_column': entity_col})
|
|
check('实体聚合', entity_count > 0, f'{entity_count} entities, {len(features)} features')
|
|
|
|
# 检查 GeoIP 和类型特征
|
|
check('有特征列', len(features) > 5, f'features={features[:8]}')
|
|
|
|
# ── 6. 聚类 + 特征提取 ────────────────────────────────────
|
|
print('\n[6/7] 聚类 + 特征提取')
|
|
import asyncio
|
|
from analysis.tool_registry import _handle_run_clustering, _handle_extract_features
|
|
|
|
feature_cols = [c for c in features if not c.startswith('_')][:10]
|
|
clust_result = asyncio.run(_handle_run_clustering(
|
|
dataset_id='entity', cluster_columns=feature_cols,
|
|
algorithm='hdbscan', params={}, random_state=42
|
|
))
|
|
n_clusters = clust_result.get('n_clusters', 0)
|
|
check('HDBSCAN 聚类', n_clusters >= 0, f'n_clusters={n_clusters}')
|
|
if 'error' not in clust_result:
|
|
quality = clust_result.get('quality_metrics', {})
|
|
check('聚类质量', quality.get('silhouette_score') is not None, f'silhouette={quality.get("silhouette_score")}')
|
|
|
|
cid = clust_result.get('cluster_result_id', '')
|
|
if cid and 'error' not in clust_result:
|
|
feat_result = asyncio.run(_handle_extract_features(
|
|
dataset_id='entity', cluster_result_id=cid,
|
|
top_k=10, method='zscore', save_to_db=True
|
|
))
|
|
n_feats = feat_result.get('n_features', 0)
|
|
check('特征提取 + DB持久化', feat_result.get('db_saved'), f'{n_feats} features')
|
|
|
|
# ── 7. 验证 ORM 数据 ──────────────────────────────────────
|
|
print('\n[7/7] ORM 数据验证')
|
|
from analysis.models import ClusterResult, ClusterFeature, EntityProfile
|
|
n_clusters_db = ClusterResult.objects.count()
|
|
n_features_db = ClusterFeature.objects.count()
|
|
n_entities_db = EntityProfile.objects.count()
|
|
check('ClusterResult 表', n_clusters_db > 0, f'{n_clusters_db} records')
|
|
check('ClusterFeature 表', n_features_db > 0, f'{n_features_db} records')
|
|
check('EntityProfile 表', n_entities_db >= 0, f'{n_entities_db} records')
|
|
|
|
# ── 结论 ──────────────────────────────────────────────────
|
|
print('\n' + '=' * 60)
|
|
passed = sum(1 for _, ok in results if ok)
|
|
total = len(results)
|
|
print(f'\n结果: {passed}/{total} 通过')
|
|
print('=' * 60)
|