"""LLM auto analysis views.""" import json import asyncio import threading import traceback import logging from django.shortcuts import render, get_object_or_404 from django.http import JsonResponse from django.views.decorators.csrf import csrf_exempt from config import get_config from analysis.models import AnalysisRun from .pipeline import _run_pipeline_worker from analysis.services.clustering import run_clustering_pipeline from .helpers import _PLANS_DIR, _add_to_auto_index logger = logging.getLogger(__name__) def auto_page(request): """LLM-driven auto analysis page.""" from analysis.session_store import SessionStore import json as _json cfg = get_config() # Get datasets from SessionStore + ALL runs from DB store = SessionStore() session_datasets = store.list_datasets() if hasattr(store, 'list_datasets') else [] session_ds_map = {ds['dataset_id']: ds for ds in session_datasets} # Build dataset entries from ALL AnalysisRun records all_runs = AnalysisRun.objects.all().order_by('-created_at')[:100] datasets_with_status = [] for run in all_runs: if run.run_type == 'upload': ds_id = f'upload_{run.display_id}' else: ds_id = f'run_{run.display_id}' session_entry = session_ds_map.get(ds_id) if session_entry: datasets_with_status.append({ **session_entry, 'dataset_id': ds_id, 'run_status': run.status, 'total_flows': run.total_flows, 'display_id': run.display_id, 'needs_reload': False, 'csv_glob': run.csv_glob or '', 'sqlite_table': run.sqlite_table or '', }) elif run.csv_glob or run.sqlite_table: datasets_with_status.append({ 'dataset_id': ds_id, 'run_status': run.status, 'display_id': run.display_id, 'total_flows': run.total_flows, 'row_count': run.total_flows, 'file_count': None, 'columns': [], 'column_count': 0, 'svd_components': 0, 'needs_reload': True, 'csv_glob': run.csv_glob or '', 'sqlite_table': run.sqlite_table or '', }) for ds_info in datasets_with_status: s = ds_info['run_status'] if s in ('completed',): ds_info['status_label'] = '已分析' ds_info['badge_class'] = 'completed' elif s in ('failed',): ds_info['status_label'] = '错误' ds_info['badge_class'] = 'failed' elif s in ('pending', 'loading', 'profiling', 'aggregating', 'clustering', 'extracting'): ds_info['status_label'] = '分析中' ds_info['badge_class'] = 'analyzing' else: ds_info['status_label'] = '待分析' ds_info['badge_class'] = 'ready' return render(request, 'tianxuan/auto.html', { 'llm_configured': bool(cfg.llm.base_url and cfg.llm.api_key), 'datasets_with_status_json': _json.dumps(datasets_with_status, ensure_ascii=False), }) @csrf_exempt def run_llm_analysis_view(request): """AJAX POST: run LLM-driven analysis pipeline.""" if request.method != 'POST': return JsonResponse({'error': 'POST required'}, status=405) body = json.loads(request.body) display_id = body.get('run_id') run = get_object_or_404(AnalysisRun, display_id=display_id) pk = run.id run.run_type = 'auto' run.save(update_fields=['run_type']) head = body.get('head') from analysis.session_store import SessionStore store = SessionStore() # Determine dataset key format (consistent with manual_page) upload_ds_id = f'upload_{display_id}' run_ds_id = f'run_{display_id}' entry = store.get_dataset(upload_ds_id) if entry is None: entry = store.get_dataset(run_ds_id) if entry is None: # Try primary key fallback upload_pk_id = f'upload_{pk}' entity_pk_id = f'entity_{pk}' entry = store.get_dataset(upload_pk_id) if entry is None: entry = store.get_dataset(entity_pk_id) if entry is None: # Auto-reload from sqlite_table or csv_glob from analysis.data_loader import load_csv_directory, load_from_db lf = None schema = {} if run.sqlite_table: try: lf = load_from_db(run.sqlite_table) except Exception: logger.error("unknown failed: {}".format(traceback.format_exc())) pass if lf is None and run.csv_glob: try: lf, schema, _, _, _ = load_csv_directory(run.csv_glob) except Exception: logger.error("unknown failed: {}".format(traceback.format_exc())) pass if lf is None: return JsonResponse({'error': '数据集不存在，请先上传并等待预处理完成'}) entry_key = upload_ds_id if run.run_type == 'upload' else run_ds_id store.store_dataset(entry_key, lf, schema=schema, metadata={ 'row_count': run.total_flows, 'csv_glob': run.csv_glob or '', 'sqlite_table': run.sqlite_table or '', }) # Re-fetch entry entry = store.get_dataset(entry_key) # Determine the active dataset_id for candidate in [upload_ds_id, run_ds_id, f'upload_{pk}', f'entity_{pk}']: if store.get_dataset(candidate): dataset_id = candidate break else: return JsonResponse({'error': '数据集不存在，请先上传并等待预处理完成'}) # ── Optional: apply filter before LLM pipeline ── filters = body.get('filters', []) if filters and len(filters) > 0: logic = body.get('logic', 'and') try: from analysis.tool_registry import _handle_filter_data filter_result = asyncio.run(_handle_filter_data( dataset_id=dataset_id, filters=filters, logic=logic, )) if 'error' not in filter_result: filtered_id = filter_result.get('filtered_dataset_id', '') if filtered_id and store.get_dataset(filtered_id): dataset_id = filtered_id except Exception: logger.warning(f'[LLM auto] Filter failed (non-fatal): {traceback.format_exc()}') def _analysis_fn(run, ctx): from tianxuan.llm_orchestrator import run_llm_pipeline, LLMConfig dataset_id = ctx['dataset_id'] pk = ctx['pk'] def progress_callback(step, tool_name, result=None, meta=None): nonlocal run try: run = AnalysisRun.objects.get(id=pk) pct = min(10 + step * 6, 95) run.progress_pct = pct run.progress_msg = f'步骤 {step}: {tool_name}' save_fields = ['progress_pct', 'progress_msg'] if tool_name == 'thinking': # Accumulate LLM thinking text (appended to llm_thinking) if result: run.llm_thinking = (run.llm_thinking or '') + f'\n[{step}] {result}' save_fields = ['progress_pct', 'progress_msg', 'llm_thinking'] elif tool_name == 'complete': pass # No extra tracking needed else: # Single tool call — always has result if isinstance(result, dict): summary = json.dumps(result, default=str, ensure_ascii=False)[:500] else: summary = str(result)[:500] run.run_log += f'[{step}] {tool_name}: {summary}...\n' # One entry per tool call if meta and isinstance(meta, dict) and 'args' in meta: tool_calls = list(run.tool_calls_json or []) tool_calls.append({ 'step': step, 'name': tool_name, 'input': meta['args'], 'output': result, }) run.tool_calls_json = tool_calls save_fields = ['progress_pct', 'progress_msg', 'run_log', 'tool_calls_json'] run.save(update_fields=save_fields) except Exception as e: logger.warning('progress_callback error: %s', e) try: from config import get_config cfg = get_config() llm_cfg = LLMConfig(cfg.llm.base_url, cfg.llm.api_key, cfg.llm.model) run.run_log = '启动 LLM 分析...\n' run.status = 'loading' run.progress_pct = 5 run.progress_msg = 'LLM 初始化中...' run.save(update_fields=['status', 'progress_pct', 'progress_msg', 'run_log']) result = run_llm_pipeline(dataset_id, config=llm_cfg, callback=progress_callback) if 'error' in result: run.status = 'failed' run.progress_msg = f'失败: {result["error"]}' run.run_log += f'\n[ERROR] {result["error"]}' run.save(update_fields=['status', 'progress_pct', 'progress_msg', 'run_log']) else: run.progress_pct = 55 run.progress_msg = f'LLM分析完成: {result.get("result", "")[:200]}' run.run_log += f'\n[完成] {result.get("result", "")}\n[INFO] 开始自动聚类流程...' run.save(update_fields=['progress_pct', 'progress_msg', 'run_log']) # ── Auto-run clustering pipeline after LLM analysis ── entity_ds_id = None tool_calls = run.tool_calls_json or [] for tc in tool_calls: tool_name = tc.get('name', '') tool_input = tc.get('input', {}) if tool_name in ('build_entity_profiles', 'run_clustering'): entity_ds_id = tool_input.get('dataset_id', '') if entity_ds_id and store.get_dataset(entity_ds_id): break entity_ds_id = None # Fallback: try common dataset ID patterns if not entity_ds_id: candidates = [ ctx.get('dataset_id', ''), f'entity_{pk}', f'entity_{run.display_id}', f'upload_{run.display_id}', ] for cid in candidates: if cid and store.get_dataset(cid): entity_ds_id = cid break if entity_ds_id: run_clustering_pipeline( run=run, store=store, entity_ds_id=entity_ds_id, feature_columns=None, algorithm='agglomerative', min_cluster_size=5, run_umap=True, head=ctx.get('head'), ) else: logger.warning( f'[LLM auto] No entity dataset found for clustering. ' f'Tool calls: {[tc.get("name") for tc in tool_calls]}. ' f'Marking run as completed without clustering.' ) run.status = 'completed' run.progress_pct = 100 run.progress_msg = 'LLM分析完成（无实体数据，跳过聚类）' run.save(update_fields=['status', 'progress_pct', 'progress_msg']) # ── Auto-save workflow as loadable plan ── try: tool_calls_data = run.tool_calls_json or [] steps = [] for tc in tool_calls_data: tc_name = tc.get('name', '') if tc_name == 'thinking': continue steps.append({ 'tool_name': tc_name, 'params': tc.get('input', {}), }) if steps: plan_data = { 'name': f'_auto_{run.display_id}', 'steps': steps, 'auto': True, 'display_id': run.display_id, 'created_at': str(run.created_at) if run.created_at else '', } plan_path = _PLANS_DIR / f'_auto_{run.display_id}.json' plan_path.write_text( json.dumps(plan_data, ensure_ascii=False, indent=2), encoding='utf-8', ) _add_to_auto_index(f'_auto_{run.display_id}.json') logger.info( '[LLM auto] Saved auto plan %s (%d steps)', plan_path.name, len(steps), ) except Exception as save_err: logger.warning('[LLM auto] Failed to save auto plan: %s', save_err) except Exception: tb = traceback.format_exc() logger.error(tb) try: run = AnalysisRun.objects.get(id=pk) run.status = 'failed' run.progress_msg = f'失败: {tb}' run.run_log += f'\n[ERROR] {tb}' run.error_message = tb run.save(update_fields=['status', 'progress_msg', 'run_log', 'error_message']) except Exception as e: logger.warning('save failed after pipeline error: %s', e) t = threading.Thread( target=_run_pipeline_worker, args=(pk, _analysis_fn), kwargs=dict(dataset_id=dataset_id, head=head, pk=pk), daemon=True, ) t.start() return JsonResponse({'status': 'started', 'run_id': run.display_id})