PM-pinou
349 lines
14 KiB
Python
349 lines
14 KiB
Python
"""LLM auto analysis views."""
|
|
import json
|
|
import asyncio
|
|
import threading
|
|
import traceback
|
|
import logging
|
|
|
|
from django.shortcuts import render, get_object_or_404
|
|
from django.http import JsonResponse
|
|
from django.views.decorators.csrf import csrf_exempt
|
|
|
|
from config import get_config
|
|
from analysis.models import AnalysisRun
|
|
from .pipeline import _run_pipeline_worker
|
|
from analysis.services.clustering import run_clustering_pipeline
|
|
from .helpers import _PLANS_DIR, _add_to_auto_index
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def auto_page(request):
|
|
"""LLM-driven auto analysis page."""
|
|
from analysis.session_store import SessionStore
|
|
import json as _json
|
|
|
|
cfg = get_config()
|
|
|
|
# Get datasets from SessionStore + ALL runs from DB
|
|
store = SessionStore()
|
|
session_datasets = store.list_datasets() if hasattr(store, 'list_datasets') else []
|
|
session_ds_map = {ds['dataset_id']: ds for ds in session_datasets}
|
|
|
|
# Build dataset entries from ALL AnalysisRun records
|
|
all_runs = AnalysisRun.objects.all().order_by('-created_at')[:100]
|
|
datasets_with_status = []
|
|
|
|
for run in all_runs:
|
|
if run.run_type == 'upload':
|
|
ds_id = f'upload_{run.display_id}'
|
|
else:
|
|
ds_id = f'run_{run.display_id}'
|
|
|
|
session_entry = session_ds_map.get(ds_id)
|
|
|
|
if session_entry:
|
|
datasets_with_status.append({
|
|
**session_entry,
|
|
'dataset_id': ds_id,
|
|
'run_status': run.status,
|
|
'total_flows': run.total_flows,
|
|
'display_id': run.display_id,
|
|
'needs_reload': False,
|
|
'csv_glob': run.csv_glob or '',
|
|
'sqlite_table': run.sqlite_table or '',
|
|
})
|
|
elif run.csv_glob or run.sqlite_table:
|
|
datasets_with_status.append({
|
|
'dataset_id': ds_id,
|
|
'run_status': run.status,
|
|
'display_id': run.display_id,
|
|
'total_flows': run.total_flows,
|
|
'row_count': run.total_flows,
|
|
'file_count': None,
|
|
'columns': [],
|
|
'column_count': 0,
|
|
'svd_components': 0,
|
|
'needs_reload': True,
|
|
'csv_glob': run.csv_glob or '',
|
|
'sqlite_table': run.sqlite_table or '',
|
|
})
|
|
|
|
for ds_info in datasets_with_status:
|
|
s = ds_info['run_status']
|
|
if s in ('completed',):
|
|
ds_info['status_label'] = '已分析'
|
|
ds_info['badge_class'] = 'completed'
|
|
elif s in ('failed',):
|
|
ds_info['status_label'] = '错误'
|
|
ds_info['badge_class'] = 'failed'
|
|
elif s in ('pending', 'loading', 'profiling', 'aggregating', 'clustering', 'extracting'):
|
|
ds_info['status_label'] = '分析中'
|
|
ds_info['badge_class'] = 'analyzing'
|
|
else:
|
|
ds_info['status_label'] = '待分析'
|
|
ds_info['badge_class'] = 'ready'
|
|
|
|
return render(request, 'tianxuan/auto.html', {
|
|
'llm_configured': bool(cfg.llm.base_url and cfg.llm.api_key),
|
|
'datasets_with_status_json': _json.dumps(datasets_with_status, ensure_ascii=False),
|
|
})
|
|
|
|
|
|
@csrf_exempt
|
|
def run_llm_analysis_view(request):
|
|
"""AJAX POST: run LLM-driven analysis pipeline."""
|
|
if request.method != 'POST':
|
|
return JsonResponse({'error': 'POST required'}, status=405)
|
|
body = json.loads(request.body)
|
|
display_id = body.get('run_id')
|
|
run = get_object_or_404(AnalysisRun, display_id=display_id)
|
|
pk = run.id
|
|
run.run_type = 'auto'
|
|
run.save(update_fields=['run_type'])
|
|
head = body.get('head')
|
|
|
|
from analysis.session_store import SessionStore
|
|
store = SessionStore()
|
|
|
|
# Determine dataset key format (consistent with manual_page)
|
|
upload_ds_id = f'upload_{display_id}'
|
|
run_ds_id = f'run_{display_id}'
|
|
entry = store.get_dataset(upload_ds_id)
|
|
if entry is None:
|
|
entry = store.get_dataset(run_ds_id)
|
|
|
|
if entry is None:
|
|
# Try primary key fallback
|
|
upload_pk_id = f'upload_{pk}'
|
|
entity_pk_id = f'entity_{pk}'
|
|
entry = store.get_dataset(upload_pk_id)
|
|
if entry is None:
|
|
entry = store.get_dataset(entity_pk_id)
|
|
|
|
if entry is None:
|
|
# Auto-reload from sqlite_table or csv_glob
|
|
from analysis.data_loader import load_csv_directory, load_from_db
|
|
lf = None
|
|
schema = {}
|
|
if run.sqlite_table:
|
|
try:
|
|
lf = load_from_db(run.sqlite_table)
|
|
except Exception:
|
|
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
|
pass
|
|
if lf is None and run.csv_glob:
|
|
try:
|
|
lf, schema, _, _, _ = load_csv_directory(run.csv_glob)
|
|
except Exception:
|
|
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
|
pass
|
|
if lf is None:
|
|
return JsonResponse({'error': '数据集不存在，请先上传并等待预处理完成'})
|
|
entry_key = upload_ds_id if run.run_type == 'upload' else run_ds_id
|
|
store.store_dataset(entry_key, lf, schema=schema, metadata={
|
|
'row_count': run.total_flows,
|
|
'csv_glob': run.csv_glob or '',
|
|
'sqlite_table': run.sqlite_table or '',
|
|
})
|
|
# Re-fetch entry
|
|
entry = store.get_dataset(entry_key)
|
|
|
|
# Determine the active dataset_id
|
|
for candidate in [upload_ds_id, run_ds_id, f'upload_{pk}', f'entity_{pk}']:
|
|
if store.get_dataset(candidate):
|
|
dataset_id = candidate
|
|
break
|
|
else:
|
|
return JsonResponse({'error': '数据集不存在，请先上传并等待预处理完成'})
|
|
|
|
# ── Optional: apply filter before LLM pipeline ──
|
|
filters = body.get('filters', [])
|
|
if filters and len(filters) > 0:
|
|
logic = body.get('logic', 'and')
|
|
try:
|
|
from analysis.tool_registry import _handle_filter_data
|
|
filter_result = asyncio.run(_handle_filter_data(
|
|
dataset_id=dataset_id,
|
|
filters=filters,
|
|
logic=logic,
|
|
))
|
|
if 'error' not in filter_result:
|
|
filtered_id = filter_result.get('filtered_dataset_id', '')
|
|
if filtered_id and store.get_dataset(filtered_id):
|
|
dataset_id = filtered_id
|
|
except Exception:
|
|
logger.warning(f'[LLM auto] Filter failed (non-fatal): {traceback.format_exc()}')
|
|
|
|
def _analysis_fn(run, ctx):
|
|
from tianxuan.llm_orchestrator import run_llm_pipeline, LLMConfig
|
|
dataset_id = ctx['dataset_id']
|
|
pk = ctx['pk']
|
|
|
|
def progress_callback(step, tool_name, result=None, meta=None):
|
|
nonlocal run
|
|
try:
|
|
run = AnalysisRun.objects.get(id=pk)
|
|
pct = min(10 + step * 6, 95)
|
|
run.progress_pct = pct
|
|
run.progress_msg = f'步骤 {step}: {tool_name}'
|
|
|
|
save_fields = ['progress_pct', 'progress_msg']
|
|
if tool_name == 'thinking':
|
|
# Accumulate LLM thinking text (appended to llm_thinking)
|
|
thought = (result or '').strip()
|
|
if not thought:
|
|
thought = f'步骤 {step} 分析中...'
|
|
run.llm_thinking = (run.llm_thinking or '') + f'\n[{step}] {thought}'
|
|
save_fields = ['progress_pct', 'progress_msg', 'llm_thinking']
|
|
elif tool_name == 'complete':
|
|
pass # No extra tracking needed
|
|
else:
|
|
# Single tool call — always has result
|
|
if isinstance(result, dict):
|
|
summary = json.dumps(result, default=str, ensure_ascii=False)[:500]
|
|
else:
|
|
summary = str(result)[:500]
|
|
run.run_log += f'[{step}] {tool_name}: {summary}...\n'
|
|
|
|
# One entry per tool call
|
|
if meta and isinstance(meta, dict) and 'args' in meta:
|
|
tool_calls = list(run.tool_calls_json or [])
|
|
tool_calls.append({
|
|
'step': step,
|
|
'name': tool_name,
|
|
'input': meta['args'],
|
|
'output': result,
|
|
})
|
|
run.tool_calls_json = tool_calls
|
|
save_fields = ['progress_pct', 'progress_msg', 'run_log', 'tool_calls_json']
|
|
|
|
run.save(update_fields=save_fields)
|
|
except Exception as e:
|
|
logger.warning('progress_callback error: %s', e)
|
|
|
|
try:
|
|
from config import get_config
|
|
cfg = get_config()
|
|
llm_cfg = LLMConfig(cfg.llm.base_url, cfg.llm.api_key, cfg.llm.model)
|
|
run.run_log = '启动 LLM 分析...\n'
|
|
run.status = 'loading'
|
|
run.progress_pct = 5
|
|
run.progress_msg = 'LLM 初始化中...'
|
|
run.save(update_fields=['status', 'progress_pct', 'progress_msg', 'run_log'])
|
|
|
|
result = run_llm_pipeline(dataset_id, config=llm_cfg, callback=progress_callback)
|
|
|
|
if 'error' in result:
|
|
run.status = 'failed'
|
|
run.progress_msg = f'失败: {result["error"]}'
|
|
run.run_log += f'\n[ERROR] {result["error"]}'
|
|
run.save(update_fields=['status', 'progress_pct', 'progress_msg', 'run_log'])
|
|
else:
|
|
run.progress_pct = 55
|
|
run.progress_msg = f'LLM分析完成: {result.get("result", "")[:200]}'
|
|
run.run_log += f'\n[完成] {result.get("result", "")}\n[INFO] 开始自动聚类流程...'
|
|
run.save(update_fields=['progress_pct', 'progress_msg', 'run_log'])
|
|
|
|
# ── Auto-run clustering pipeline after LLM analysis ──
|
|
entity_ds_id = None
|
|
tool_calls = run.tool_calls_json or []
|
|
for tc in tool_calls:
|
|
tool_name = tc.get('name', '')
|
|
tool_input = tc.get('input', {})
|
|
if tool_name in ('build_entity_profiles', 'run_clustering'):
|
|
entity_ds_id = tool_input.get('dataset_id', '')
|
|
if entity_ds_id and store.get_dataset(entity_ds_id):
|
|
break
|
|
entity_ds_id = None
|
|
|
|
# Fallback: try common dataset ID patterns
|
|
if not entity_ds_id:
|
|
candidates = [
|
|
ctx.get('dataset_id', ''),
|
|
f'entity_{pk}',
|
|
f'entity_{run.display_id}',
|
|
f'upload_{run.display_id}',
|
|
]
|
|
for cid in candidates:
|
|
if cid and store.get_dataset(cid):
|
|
entity_ds_id = cid
|
|
break
|
|
|
|
if entity_ds_id:
|
|
run_clustering_pipeline(
|
|
run=run,
|
|
store=store,
|
|
ds_id=entity_ds_id,
|
|
feature_columns=None,
|
|
algorithm='agglomerative',
|
|
min_cluster_size=5,
|
|
run_umap=True,
|
|
head=ctx.get('head'),
|
|
)
|
|
else:
|
|
logger.warning(
|
|
f'[LLM auto] No entity dataset found for clustering. '
|
|
f'Tool calls: {[tc.get("name") for tc in tool_calls]}. '
|
|
f'Marking run as completed without clustering.'
|
|
)
|
|
run.status = 'completed'
|
|
run.progress_pct = 100
|
|
run.progress_msg = 'LLM分析完成（无实体数据，跳过聚类）'
|
|
run.save(update_fields=['status', 'progress_pct', 'progress_msg'])
|
|
|
|
# ── Auto-save workflow as loadable plan ──
|
|
try:
|
|
tool_calls_data = run.tool_calls_json or []
|
|
steps = []
|
|
for tc in tool_calls_data:
|
|
tc_name = tc.get('name', '')
|
|
if tc_name == 'thinking':
|
|
continue
|
|
steps.append({
|
|
'tool_name': tc_name,
|
|
'params': tc.get('input', {}),
|
|
})
|
|
if steps:
|
|
plan_data = {
|
|
'name': f'_auto_{run.display_id}',
|
|
'steps': steps,
|
|
'auto': True,
|
|
'display_id': run.display_id,
|
|
'created_at': str(run.created_at) if run.created_at else '',
|
|
}
|
|
plan_path = _PLANS_DIR / f'_auto_{run.display_id}.json'
|
|
plan_path.write_text(
|
|
json.dumps(plan_data, ensure_ascii=False, indent=2),
|
|
encoding='utf-8',
|
|
)
|
|
_add_to_auto_index(f'_auto_{run.display_id}.json')
|
|
logger.info(
|
|
'[LLM auto] Saved auto plan %s (%d steps)',
|
|
plan_path.name, len(steps),
|
|
)
|
|
except Exception as save_err:
|
|
logger.warning('[LLM auto] Failed to save auto plan: %s', save_err)
|
|
|
|
except Exception:
|
|
tb = traceback.format_exc()
|
|
logger.error(tb)
|
|
try:
|
|
run = AnalysisRun.objects.get(id=pk)
|
|
run.status = 'failed'
|
|
run.progress_msg = f'失败: {tb}'
|
|
run.run_log += f'\n[ERROR] {tb}'
|
|
run.error_message = tb
|
|
run.save(update_fields=['status', 'progress_msg', 'run_log', 'error_message'])
|
|
except Exception as e:
|
|
logger.warning('save failed after pipeline error: %s', e)
|
|
|
|
t = threading.Thread(
|
|
target=_run_pipeline_worker,
|
|
args=(pk, _analysis_fn),
|
|
kwargs=dict(dataset_id=dataset_id, head=head, pk=pk),
|
|
daemon=True,
|
|
)
|
|
t.start()
|
|
return JsonResponse({'status': 'started', 'run_id': run.display_id})
|