PM-pinou
66f68a2062
Move _run_clustering_pipeline (~295 lines) from analysis/views/clustering.py into analysis/services/clustering.py as three clean functions: - run_clustering_pipeline: main pipeline orchestration - compute_umap_embedding: UMAP 3D→2D fallback computation - save_entity_profiles: persist cluster labels + UMAP coords to ORM Views/clustering.py now delegates via thin wrapper. Callers (auto.py, manual.py, run_pipeline.py) import directly from services.clustering.
192 lines
7.7 KiB
Python
192 lines
7.7 KiB
Python
"""Manual analysis page: workflow builder and run execution."""
|
|
import json
|
|
import threading
|
|
import traceback
|
|
import logging
|
|
|
|
from django.shortcuts import render, get_object_or_404
|
|
from django.http import JsonResponse
|
|
from django.views.decorators.csrf import csrf_exempt
|
|
|
|
from analysis.models import AnalysisRun
|
|
from .pipeline import _run_pipeline_worker
|
|
from analysis.services.clustering import run_clustering_pipeline
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def manual_page(request):
|
|
"""Workflow builder: select dataset, add MCP tool steps, execute."""
|
|
from analysis.session_store import SessionStore
|
|
from analysis.tool_registry import get_tools_meta
|
|
import json as _json
|
|
|
|
# Get all tools metadata
|
|
tools = get_tools_meta()
|
|
core_names = {'profile_data', 'build_entity_profiles', 'compute_scores',
|
|
'run_clustering', 'extract_features', 'detect_anomalies',
|
|
'visualize_anomalies'}
|
|
diag_names = {'validate_data', 'explore_distributions', 'find_outliers',
|
|
'diagnose_clustering', 'compare_datasets', 'export_debug_sample',
|
|
'repair_schema'}
|
|
analysis_names = {'analyze_patterns', 'analyze_temporal', 'analyze_tls_health',
|
|
'analyze_geo_distribution', 'analyze_entity_detail'}
|
|
core_tools, diag_tools, analysis_tools, other_tools = [], [], [], []
|
|
for t in tools:
|
|
if t.name in core_names: core_tools.append(t)
|
|
elif t.name in diag_names: diag_tools.append(t)
|
|
elif t.name in analysis_names: analysis_tools.append(t)
|
|
else: other_tools.append(t)
|
|
|
|
meta_list = [{'name': t.name, 'description': t.description, 'inputSchema': t.inputSchema} for t in tools]
|
|
|
|
# Get datasets from SessionStore
|
|
store = SessionStore()
|
|
session_datasets = store.list_datasets() if hasattr(store, 'list_datasets') else []
|
|
session_ds_map = {ds['dataset_id']: ds for ds in session_datasets}
|
|
|
|
# Build dataset entries from ALL AnalysisRun records (upload/manual/auto, any status)
|
|
from analysis.models import AnalysisRun
|
|
all_runs = AnalysisRun.objects.all().order_by('-created_at')[:100]
|
|
datasets_with_status = []
|
|
|
|
for run in all_runs:
|
|
# Determine how the dataset was keyed in SessionStore
|
|
if run.run_type == 'upload':
|
|
ds_id = f'upload_{run.display_id}'
|
|
else:
|
|
ds_id = f'run_{run.display_id}'
|
|
|
|
session_entry = session_ds_map.get(ds_id)
|
|
|
|
if session_entry:
|
|
# Full data available in SessionStore — merge with run status
|
|
datasets_with_status.append({
|
|
**session_entry,
|
|
'dataset_id': ds_id,
|
|
'run_status': run.status,
|
|
'total_flows': run.total_flows,
|
|
'display_id': run.display_id,
|
|
'needs_reload': False,
|
|
})
|
|
elif run.csv_glob or run.sqlite_table:
|
|
# Data not in SessionStore but recoverable from disk/DB
|
|
datasets_with_status.append({
|
|
'dataset_id': ds_id,
|
|
'run_status': run.status,
|
|
'display_id': run.display_id,
|
|
'total_flows': run.total_flows,
|
|
'row_count': run.total_flows,
|
|
'file_count': None,
|
|
'column_count': 0,
|
|
'columns': [],
|
|
'svd_components': 0,
|
|
'needs_reload': True,
|
|
'csv_glob': run.csv_glob,
|
|
'sqlite_table': run.sqlite_table or '',
|
|
})
|
|
|
|
# Map statuses to user-facing labels for the filter tabs
|
|
# ready → 待分析, completed → 已分析, failed → 错误
|
|
# Everything else (pending/loading/profiling/aggregating/clustering/extracting) → analyzing → 分析中
|
|
for ds_info in datasets_with_status:
|
|
s = ds_info['run_status']
|
|
if s in ('completed',):
|
|
ds_info['status_label'] = '已分析'
|
|
ds_info['badge_class'] = 'completed'
|
|
elif s in ('failed',):
|
|
ds_info['status_label'] = '错误'
|
|
ds_info['badge_class'] = 'failed'
|
|
elif s in ('pending', 'loading', 'profiling', 'aggregating', 'clustering', 'extracting'):
|
|
ds_info['status_label'] = '分析中'
|
|
ds_info['badge_class'] = 'analyzing'
|
|
else:
|
|
ds_info['status_label'] = '待分析'
|
|
ds_info['badge_class'] = 'ready'
|
|
|
|
return render(request, 'tianxuan/manual.html', {
|
|
'core_tools': core_tools,
|
|
'diag_tools': diag_tools,
|
|
'analysis_tools': analysis_tools,
|
|
'other_tools': other_tools,
|
|
'tools_meta': _json.dumps(meta_list, ensure_ascii=False),
|
|
'datasets': datasets_with_status,
|
|
'datasets_with_status_json': _json.dumps(datasets_with_status, ensure_ascii=False),
|
|
})
|
|
|
|
|
|
@csrf_exempt
|
|
def manual_run_analysis(request):
|
|
"""AJAX POST: run clustering pipeline on the upload dataset."""
|
|
if request.method != 'POST':
|
|
return JsonResponse({'error': 'POST required'}, status=405)
|
|
import json as _json
|
|
body = _json.loads(request.body)
|
|
display_id = body.get('run_id')
|
|
run = get_object_or_404(AnalysisRun, display_id=display_id)
|
|
pk = run.id
|
|
run.run_type = 'manual'
|
|
run.save(update_fields=['run_type'])
|
|
feature_columns = body.get('feature_columns')
|
|
algorithm = body.get('algorithm', 'agglomerative')
|
|
min_cluster_size = int(body.get('min_cluster_size', 5))
|
|
head = body.get('head')
|
|
cluster_mode = body.get('cluster_mode', 'raw')
|
|
|
|
def _analysis_fn(run, ctx):
|
|
from analysis.session_store import SessionStore
|
|
|
|
feature_columns = ctx['feature_columns']
|
|
algorithm = ctx['algorithm']
|
|
min_cluster_size = ctx['min_cluster_size']
|
|
head = ctx.get('head')
|
|
pk = ctx['pk']
|
|
cluster_mode = ctx.get('cluster_mode', 'raw')
|
|
|
|
store = SessionStore()
|
|
try:
|
|
# Use display_id for dataset key (consistent with _background_process)
|
|
display_id = ctx.get('display_id', pk)
|
|
upload_ds_id = f'upload_{display_id}'
|
|
entry = store.get_dataset(upload_ds_id)
|
|
if entry is None:
|
|
# Fallback: try PK-based key (old pipeline)
|
|
entry = store.get_dataset(f'upload_{pk}')
|
|
if entry is None:
|
|
# Fallback: try entity dataset (from older pipeline)
|
|
entry = store.get_dataset(f'entity_{pk}')
|
|
if entry is None:
|
|
run.error_message = '请先上传并等待预处理完成'
|
|
run.status = 'failed'
|
|
run.save(update_fields=['status', 'error_message'])
|
|
return
|
|
|
|
ds_id = upload_ds_id if store.get_dataset(upload_ds_id) else f'upload_{pk}'
|
|
|
|
# Use the unified clustering pipeline (clustering → extraction → UMAP)
|
|
run_clustering_pipeline(
|
|
run=run, store=store, entity_ds_id=ds_id,
|
|
feature_columns=feature_columns,
|
|
algorithm=algorithm, min_cluster_size=min_cluster_size,
|
|
run_umap=True, head=head,
|
|
)
|
|
except Exception:
|
|
tb = traceback.format_exc()
|
|
logger.error(tb)
|
|
run.status = 'failed'
|
|
run.error_message = tb
|
|
run.progress_msg = f'失败: {tb}'
|
|
run.run_log += f'\n[ERROR] {tb}'
|
|
run.save(update_fields=['status', 'error_message', 'progress_msg', 'run_log'])
|
|
|
|
t = threading.Thread(
|
|
target=_run_pipeline_worker,
|
|
args=(pk, _analysis_fn),
|
|
kwargs=dict(feature_columns=feature_columns, algorithm=algorithm,
|
|
min_cluster_size=min_cluster_size, head=head, pk=pk,
|
|
display_id=run.display_id, cluster_mode=cluster_mode),
|
|
daemon=True,
|
|
)
|
|
t.start()
|
|
return JsonResponse({'status': 'started', 'redirect': f'/runs/{run.display_id}/'})
|