"""Manual analysis page: workflow builder and run execution.""" import json import threading import traceback import logging from django.shortcuts import render, get_object_or_404 from django.http import JsonResponse from django.views.decorators.csrf import csrf_exempt from analysis.models import AnalysisRun from .pipeline import _run_pipeline_worker from analysis.services.clustering import run_clustering_pipeline logger = logging.getLogger(__name__) def manual_page(request): """Workflow builder: select dataset, add MCP tool steps, execute.""" from analysis.session_store import SessionStore from analysis.tool_registry import get_tools_meta import json as _json # Get all tools metadata tools = get_tools_meta() core_names = {'profile_data', 'build_entity_profiles', 'compute_scores', 'run_clustering', 'extract_features', 'detect_anomalies', 'visualize_anomalies'} diag_names = {'validate_data', 'explore_distributions', 'find_outliers', 'diagnose_clustering', 'compare_datasets', 'export_debug_sample', 'repair_schema'} analysis_names = {'analyze_patterns', 'analyze_temporal', 'analyze_tls_health', 'analyze_geo_distribution', 'analyze_entity_detail'} core_tools, diag_tools, analysis_tools, other_tools = [], [], [], [] for t in tools: if t.name in core_names: core_tools.append(t) elif t.name in diag_names: diag_tools.append(t) elif t.name in analysis_names: analysis_tools.append(t) else: other_tools.append(t) meta_list = [{'name': t.name, 'description': t.description, 'inputSchema': t.inputSchema} for t in tools] # Get datasets from SessionStore store = SessionStore() session_datasets = store.list_datasets() if hasattr(store, 'list_datasets') else [] session_ds_map = {ds['dataset_id']: ds for ds in session_datasets} # Build dataset entries from ALL AnalysisRun records (upload/manual/auto, any status) from analysis.models import AnalysisRun all_runs = AnalysisRun.objects.all().order_by('-created_at')[:100] datasets_with_status = [] for run in all_runs: # Determine how the dataset was keyed in SessionStore if run.run_type == 'upload': ds_id = f'upload_{run.display_id}' else: ds_id = f'run_{run.display_id}' session_entry = session_ds_map.get(ds_id) if session_entry: # Full data available in SessionStore — merge with run status datasets_with_status.append({ **session_entry, 'dataset_id': ds_id, 'run_status': run.status, 'total_flows': run.total_flows, 'display_id': run.display_id, 'needs_reload': False, }) elif run.csv_glob or run.sqlite_table: # Data not in SessionStore but recoverable from disk/DB datasets_with_status.append({ 'dataset_id': ds_id, 'run_status': run.status, 'display_id': run.display_id, 'total_flows': run.total_flows, 'row_count': run.total_flows, 'file_count': None, 'column_count': 0, 'columns': [], 'svd_components': 0, 'needs_reload': True, 'csv_glob': run.csv_glob, 'sqlite_table': run.sqlite_table or '', }) # Map statuses to user-facing labels for the filter tabs # ready → 待分析, completed → 已分析, failed → 错误 # Everything else (pending/loading/profiling/aggregating/clustering/extracting) → analyzing → 分析中 for ds_info in datasets_with_status: s = ds_info['run_status'] if s in ('completed',): ds_info['status_label'] = '已分析' ds_info['badge_class'] = 'completed' elif s in ('failed',): ds_info['status_label'] = '错误' ds_info['badge_class'] = 'failed' elif s in ('pending', 'loading', 'profiling', 'aggregating', 'clustering', 'extracting'): ds_info['status_label'] = '分析中' ds_info['badge_class'] = 'analyzing' else: ds_info['status_label'] = '待分析' ds_info['badge_class'] = 'ready' return render(request, 'tianxuan/manual.html', { 'core_tools': core_tools, 'diag_tools': diag_tools, 'analysis_tools': analysis_tools, 'other_tools': other_tools, 'tools_meta': _json.dumps(meta_list, ensure_ascii=False), 'datasets': datasets_with_status, 'datasets_with_status_json': _json.dumps(datasets_with_status, ensure_ascii=False), }) @csrf_exempt def manual_run_analysis(request): """AJAX POST: run clustering pipeline on the upload dataset.""" if request.method != 'POST': return JsonResponse({'error': 'POST required'}, status=405) import json as _json body = _json.loads(request.body) display_id = body.get('run_id') run = get_object_or_404(AnalysisRun, display_id=display_id) pk = run.id run.run_type = 'manual' run.save(update_fields=['run_type']) feature_columns = body.get('feature_columns') algorithm = body.get('algorithm', 'agglomerative') min_cluster_size = int(body.get('min_cluster_size', 5)) head = body.get('head') cluster_mode = body.get('cluster_mode', 'raw') def _analysis_fn(run, ctx): from analysis.session_store import SessionStore feature_columns = ctx['feature_columns'] algorithm = ctx['algorithm'] min_cluster_size = ctx['min_cluster_size'] head = ctx.get('head') pk = ctx['pk'] cluster_mode = ctx.get('cluster_mode', 'raw') store = SessionStore() try: # Use display_id for dataset key (consistent with _background_process) display_id = ctx.get('display_id', pk) upload_ds_id = f'upload_{display_id}' entry = store.get_dataset(upload_ds_id) if entry is None: # Fallback: try PK-based key (old pipeline) entry = store.get_dataset(f'upload_{pk}') if entry is None: # Fallback: try entity dataset (from older pipeline) entry = store.get_dataset(f'entity_{pk}') if entry is None: run.error_message = '请先上传并等待预处理完成' run.status = 'failed' run.save(update_fields=['status', 'error_message']) return ds_id = upload_ds_id if store.get_dataset(upload_ds_id) else f'upload_{pk}' # Use the unified clustering pipeline (clustering → extraction → UMAP) run_clustering_pipeline( run=run, store=store, entity_ds_id=ds_id, feature_columns=feature_columns, algorithm=algorithm, min_cluster_size=min_cluster_size, run_umap=True, head=head, ) except Exception: tb = traceback.format_exc() logger.error(tb) run.status = 'failed' run.error_message = tb run.progress_msg = f'失败: {tb}' run.run_log += f'\n[ERROR] {tb}' run.save(update_fields=['status', 'error_message', 'progress_msg', 'run_log']) t = threading.Thread( target=_run_pipeline_worker, args=(pk, _analysis_fn), kwargs=dict(feature_columns=feature_columns, algorithm=algorithm, min_cluster_size=min_cluster_size, head=head, pk=pk, display_id=run.display_id, cluster_mode=cluster_mode), daemon=True, ) t.start() return JsonResponse({'status': 'started', 'redirect': f'/runs/{run.display_id}/'})