PM-pinou
9314ac6942
- New timeline UI: interleaved LLM reasoning + tool call display - Fix: run_log truncation for large tool outputs - Fix: lat/lon + values reduced to ~1% (was 99%) - Fix: @csrf_exempt on upload/delete endpoints - Fix: DATA_UPLOAD_MAX_NUMBER_FILES for 1998-file upload - Fix: dtype alignment for multi-batch CSV processing - New: tests/test_e2e_full.py with real DeepSeek LLM - New: 1998-file test data generator
1463 lines
59 KiB
Python
1463 lines
59 KiB
Python
import json
|
|
import time
|
|
import urllib.request
|
|
import urllib.error
|
|
import threading
|
|
import os
|
|
from pathlib import Path
|
|
|
|
from django.shortcuts import render, get_object_or_404, redirect
|
|
from django.http import HttpResponse, JsonResponse
|
|
from django.views.decorators.csrf import csrf_exempt
|
|
|
|
from config import get_config, save_config, Config
|
|
from .models import AnalysisRun, ClusterResult, EntityProfile
|
|
|
|
|
|
def dashboard(request):
|
|
"""Home page: recent runs overview."""
|
|
runs = AnalysisRun.objects.all()[:20]
|
|
return render(request, 'analysis/dashboard.html', {'runs': runs})
|
|
|
|
|
|
def run_list(request):
|
|
"""List all analysis runs with pagination and optional ?type= filter."""
|
|
run_type_filter = request.GET.get('type')
|
|
page = int(request.GET.get('page', 1))
|
|
per_page = 20
|
|
|
|
qs = AnalysisRun.objects.all()
|
|
if run_type_filter in dict(AnalysisRun.RUN_TYPE_CHOICES):
|
|
qs = qs.filter(run_type=run_type_filter)
|
|
|
|
total = qs.count()
|
|
runs = qs[(page - 1) * per_page: page * per_page]
|
|
return render(request, 'analysis/run_list.html', {
|
|
'runs': runs,
|
|
'page': page,
|
|
'total_pages': (total + per_page - 1) // per_page if total else 1,
|
|
'current_type': run_type_filter or '',
|
|
})
|
|
|
|
|
|
def run_detail(request, display_id):
|
|
"""Details for a single analysis run."""
|
|
run = get_object_or_404(AnalysisRun, display_id=display_id)
|
|
clusters = run.clusters.all().order_by('-size')
|
|
return render(request, 'analysis/run_detail.html', {
|
|
'run': run,
|
|
'clusters': clusters,
|
|
})
|
|
|
|
|
|
def _extract_lat(val):
|
|
"""Try to extract a latitude from *val* (number or str)."""
|
|
if val is None:
|
|
return None
|
|
try:
|
|
v = float(val)
|
|
return None if v < -90 or v > 90 else v
|
|
except (ValueError, TypeError):
|
|
return None
|
|
|
|
|
|
def _extract_lon(val):
|
|
"""Try to extract a longitude from *val* (number or str)."""
|
|
if val is None:
|
|
return None
|
|
try:
|
|
v = float(val)
|
|
return None if v < -180 or v > 180 else v
|
|
except (ValueError, TypeError):
|
|
return None
|
|
|
|
|
|
def cluster_overview(request, display_id):
|
|
"""Overview of all clusters for a run, with PCA scatter data and geo scatter data."""
|
|
run = get_object_or_404(AnalysisRun, display_id=display_id)
|
|
clusters = run.clusters.exclude(cluster_label=-1).order_by('-size').prefetch_related('features')
|
|
noise = run.clusters.filter(cluster_label=-1).first()
|
|
|
|
# Build PCA scatter data for Chart.js
|
|
entities = run.entities.exclude(embedding_x=None).exclude(embedding_y=None).only('embedding_x', 'embedding_y', 'cluster_label', 'entity_value')
|
|
scatter_data = [
|
|
{'x': e.embedding_x, 'y': e.embedding_y, 'label': e.cluster_label, 'entity': e.entity_value}
|
|
for e in entities
|
|
]
|
|
|
|
# Build geo scatter data from feature_json (lat/lon)
|
|
geo_data = []
|
|
geo_skipped = 0
|
|
for e in run.entities.all().only('feature_json', 'cluster_label', 'entity_value'):
|
|
fj = e.feature_json
|
|
if not fj:
|
|
continue
|
|
lat = _extract_lat(fj.get('avg_latitude'))
|
|
lon = _extract_lon(fj.get('avg_longitude'))
|
|
if lat is not None and lon is not None:
|
|
geo_data.append({
|
|
'x': lon,
|
|
'y': lat,
|
|
'label': e.cluster_label if e.cluster_label is not None else -1,
|
|
'entity': e.entity_value,
|
|
})
|
|
else:
|
|
geo_skipped += 1
|
|
|
|
# Cluster sizes for bar chart
|
|
cluster_sizes = [{'label': c.cluster_label, 'size': c.size, 'silhouette': c.silhouette_score}
|
|
for c in clusters]
|
|
|
|
# Top features per cluster from ClusterFeature
|
|
top_features = {}
|
|
for c in clusters:
|
|
features_qs = c.features.all().order_by('-distinguishing_score')[:10]
|
|
top_features[str(c.cluster_label)] = [
|
|
{'feature_name': f.feature_name, 'score': f.distinguishing_score, 'mean': f.mean}
|
|
for f in features_qs
|
|
]
|
|
|
|
return render(request, 'analysis/cluster_overview.html', {
|
|
'run': run,
|
|
'clusters': clusters,
|
|
'noise': noise,
|
|
'scatter_data_json': json.dumps(scatter_data),
|
|
'geo_data_json': json.dumps(geo_data),
|
|
'geo_skipped': geo_skipped,
|
|
'geo_count': len(geo_data),
|
|
'cluster_sizes_json': json.dumps(cluster_sizes),
|
|
'top_features_json': json.dumps(top_features),
|
|
})
|
|
|
|
|
|
def cluster_detail(request, display_id, cluster_label):
|
|
"""Detailed view of a single cluster with entity list."""
|
|
run = get_object_or_404(AnalysisRun, display_id=display_id)
|
|
cluster = get_object_or_404(ClusterResult, run=run, cluster_label=cluster_label)
|
|
features = cluster.features.all().order_by('-distinguishing_score')[:50]
|
|
entities = cluster.entities.all()[:50]
|
|
|
|
return render(request, 'analysis/cluster_detail.html', {
|
|
'run': run,
|
|
'cluster': cluster,
|
|
'features': features,
|
|
'entities': entities,
|
|
})
|
|
|
|
|
|
def entity_profile(request, entity_id):
|
|
"""Detailed profile for a single entity."""
|
|
entity = get_object_or_404(EntityProfile, id=entity_id)
|
|
run = entity.run
|
|
|
|
# Compute deviation from cluster mean
|
|
deviations = {}
|
|
if entity.feature_json and entity.cluster:
|
|
cluster_features = {
|
|
f.feature_name: {'mean': f.mean, 'std': f.std}
|
|
for f in entity.cluster.features.all()
|
|
}
|
|
for col, val in entity.feature_json.items():
|
|
if col in cluster_features and cluster_features[col].get('std') and cluster_features[col]['std'] > 1e-10:
|
|
deviations[col] = {
|
|
'value': val,
|
|
'cluster_mean': cluster_features[col]['mean'],
|
|
'zscore': (val - cluster_features[col]['mean']) / cluster_features[col]['std'],
|
|
}
|
|
|
|
return render(request, 'analysis/entity_profile.html', {
|
|
'entity': entity,
|
|
'run': run,
|
|
'deviations': deviations,
|
|
})
|
|
|
|
|
|
def start_analysis(request):
|
|
"""Placeholder: API endpoint to trigger analysis via MCP."""
|
|
return JsonResponse({'status': 'not_implemented', 'message': 'Use MCP tools to trigger analysis'})
|
|
|
|
|
|
def config_view(request):
|
|
"""Config page: GET renders current config, POST saves updates."""
|
|
cfg = get_config()
|
|
|
|
if request.method == 'POST':
|
|
# Build a new Config from POST data (fall back to current values)
|
|
server = Config.Server(
|
|
host=request.POST.get('server_host', cfg.server.host),
|
|
port=int(request.POST.get('server_port', cfg.server.port)),
|
|
debug=request.POST.get('server_debug') == 'true',
|
|
)
|
|
data = Config.Data(
|
|
schema_strict=request.POST.get('data_schema_strict') == 'true',
|
|
recursive=request.POST.get('data_recursive') == 'true',
|
|
)
|
|
clustering = Config.Clustering(
|
|
algorithm=request.POST.get('clustering_algorithm', cfg.clustering.algorithm),
|
|
min_cluster_size=int(request.POST.get('clustering_min_cluster_size', cfg.clustering.min_cluster_size)),
|
|
random_state=cfg.clustering.random_state,
|
|
)
|
|
llm = Config.LLM(
|
|
enabled=request.POST.get('llm_enabled') == 'true',
|
|
base_url=request.POST.get('llm_base_url', cfg.llm.base_url),
|
|
api_key=request.POST.get('llm_api_key', cfg.llm.api_key),
|
|
model=request.POST.get('llm_model', cfg.llm.model),
|
|
)
|
|
cfg = Config(server=server, data=data, clustering=clustering, llm=llm)
|
|
save_config(cfg)
|
|
# Re-render with saved flag
|
|
return render(request, 'tianxuan/config.html', {'config': cfg, 'saved': True})
|
|
|
|
return render(request, 'tianxuan/config.html', {'config': cfg})
|
|
|
|
|
|
@csrf_exempt
|
|
def llm_test(request):
|
|
"""LLM connectivity test: POST JSON {base_url, api_key, model} → chat/completions."""
|
|
if request.method != 'POST':
|
|
return JsonResponse({'success': False, 'message': '仅支持 POST 请求', 'latency_ms': 0}, status=405)
|
|
|
|
try:
|
|
body = json.loads(request.body)
|
|
except json.JSONDecodeError:
|
|
return JsonResponse({'success': False, 'message': '无效的 JSON 请求体', 'latency_ms': 0})
|
|
|
|
base_url = (body.get('base_url') or '').rstrip('/')
|
|
api_key = body.get('api_key') or ''
|
|
model = body.get('model') or ''
|
|
|
|
if not base_url:
|
|
return JsonResponse({'success': False, 'message': 'Base URL 不能为空', 'latency_ms': 0})
|
|
if not api_key:
|
|
return JsonResponse({'success': False, 'message': 'API Key 不能为空', 'latency_ms': 0})
|
|
if not model:
|
|
return JsonResponse({'success': False, 'message': 'Model 不能为空', 'latency_ms': 0})
|
|
|
|
url = f'{base_url}/chat/completions'
|
|
payload = json.dumps({
|
|
'model': model,
|
|
'messages': [{'role': 'user', 'content': 'respond with ok'}],
|
|
'max_tokens': 5,
|
|
}).encode('utf-8')
|
|
|
|
req = urllib.request.Request(
|
|
url,
|
|
data=payload,
|
|
headers={
|
|
'Content-Type': 'application/json',
|
|
'Authorization': f'Bearer {api_key}',
|
|
},
|
|
method='POST',
|
|
)
|
|
|
|
start = time.monotonic()
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=5) as resp:
|
|
elapsed = int((time.monotonic() - start) * 1000)
|
|
data = json.loads(resp.read().decode('utf-8'))
|
|
if 'choices' in data and len(data['choices']) > 0:
|
|
logger.info(f'[LLM_TEST] base_url={base_url} model={model} success=True latency={elapsed}ms')
|
|
return JsonResponse({
|
|
'success': True,
|
|
'message': f'连接成功，模型 {model} 返回正常',
|
|
'latency_ms': elapsed,
|
|
})
|
|
else:
|
|
logger.info(f'[LLM_TEST] base_url={base_url} model={model} success=True latency={elapsed}ms')
|
|
return JsonResponse({
|
|
'success': True,
|
|
'message': f'连接成功，但响应中无 choices（原始响应已记录）',
|
|
'latency_ms': elapsed,
|
|
})
|
|
except urllib.error.HTTPError as e:
|
|
elapsed = int((time.monotonic() - start) * 1000)
|
|
try:
|
|
detail = json.loads(e.read().decode('utf-8', errors='replace'))
|
|
msg = detail.get('error', {}).get('message', str(e))
|
|
except Exception:
|
|
msg = str(e)
|
|
logger.info(f'[LLM_TEST] base_url={base_url} model={model} success=False latency={elapsed}ms')
|
|
return JsonResponse({'success': False, 'message': f'HTTP {e.code}: {msg}', 'latency_ms': elapsed})
|
|
except urllib.error.URLError as e:
|
|
elapsed = int((time.monotonic() - start) * 1000)
|
|
reason = str(e.reason) if e.reason else '连接失败'
|
|
logger.info(f'[LLM_TEST] base_url={base_url} model={model} success=False latency={elapsed}ms')
|
|
return JsonResponse({'success': False, 'message': f'连接失败: {reason}', 'latency_ms': elapsed})
|
|
except TimeoutError:
|
|
elapsed = int((time.monotonic() - start) * 1000)
|
|
logger.info(f'[LLM_TEST] base_url={base_url} model={model} success=False latency={elapsed}ms')
|
|
return JsonResponse({'success': False, 'message': '请求超时（5秒）', 'latency_ms': elapsed})
|
|
except Exception as e:
|
|
elapsed = int((time.monotonic() - start) * 1000)
|
|
logger.info(f'[LLM_TEST] base_url={base_url} model={model} success=False latency={elapsed}ms')
|
|
return JsonResponse({'success': False, 'message': f'未知错误: {str(e)}', 'latency_ms': elapsed})
|
|
|
|
|
|
# ── Upload page ─────────────────────────────────────────────────────────
|
|
|
|
def upload_page(request):
|
|
"""CSV upload page."""
|
|
runs = AnalysisRun.objects.all().order_by('-created_at')[:20]
|
|
return render(request, 'tianxuan/upload.html', {'runs': runs})
|
|
|
|
|
|
@csrf_exempt
|
|
def upload_csv(request):
|
|
"""Handle CSV upload AJAX POST."""
|
|
if request.method != 'POST':
|
|
return JsonResponse({'error': 'POST required'}, status=405)
|
|
files = request.FILES.getlist('files')
|
|
if not files:
|
|
return JsonResponse({'error': 'No files uploaded'})
|
|
for f in files:
|
|
if f.size > 500 * 1024 * 1024:
|
|
return JsonResponse({'warning': f'文件 {f.name} 超过 500MB，建议分割后上传'})
|
|
total_size = sum(f.size for f in files)
|
|
if total_size > 50 * 1024 * 1024 * 1024:
|
|
return JsonResponse({'error': '总大小超过 50GB 限制'})
|
|
from django.utils import timezone
|
|
ts = timezone.now().strftime('%Y%m%d_%H%M%S')
|
|
from analysis.appdata import ensure_appdata_dir
|
|
base = ensure_appdata_dir() / 'data' / 'uploads'
|
|
upload_dir = base / ts
|
|
upload_dir.mkdir(parents=True, exist_ok=True)
|
|
saved = []
|
|
for f in files:
|
|
path = upload_dir / f.name
|
|
with open(path, 'wb+') as dst:
|
|
for chunk in f.chunks():
|
|
dst.write(chunk)
|
|
saved.append(str(path))
|
|
# Auto-process in background
|
|
run = AnalysisRun.objects.create(
|
|
csv_glob=str(upload_dir / '*.csv'),
|
|
status='loading',
|
|
run_type='upload',
|
|
)
|
|
t = threading.Thread(target=_background_process, args=(run.id, upload_dir), daemon=True)
|
|
t.start()
|
|
return JsonResponse({'run_id': run.display_id, 'files': saved, 'status': 'loading'})
|
|
|
|
|
|
@csrf_exempt
|
|
def delete_upload(request, display_id):
|
|
"""Delete uploaded files, SQLite data table, and AnalysisRun.
|
|
|
|
Handles all run statuses (loading, profiling, clustering, completed, failed).
|
|
Gracefully tolerates null csv_glob and missing/errored SQLite tables.
|
|
"""
|
|
run = get_object_or_404(AnalysisRun, display_id=display_id)
|
|
import logging
|
|
import shutil
|
|
|
|
_logger = logging.getLogger(__name__)
|
|
|
|
# 1. Drop the SQLite data table if one exists
|
|
if run.sqlite_table:
|
|
try:
|
|
from analysis.data_loader import drop_sqlite_table # fmt: skip
|
|
drop_sqlite_table(run.sqlite_table)
|
|
except Exception as exc:
|
|
_logger.warning('[DELETE] failed to drop SQLite table %s: %s', run.sqlite_table, exc)
|
|
|
|
# 2. Remove the upload directory if csv_glob is set
|
|
if run.csv_glob:
|
|
p = Path(run.csv_glob).parent
|
|
if p.exists():
|
|
shutil.rmtree(p, ignore_errors=True)
|
|
|
|
# 3. Delete the ORM record
|
|
run.delete()
|
|
return JsonResponse({'deleted': True})
|
|
|
|
|
|
import traceback, logging
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def _background_process(run_id, upload_dir):
|
|
"""Background: stream-load CSVs → save to SQLite → ready for analysis."""
|
|
import django
|
|
os.environ['DJANGO_SETTINGS_MODULE'] = 'tianxuan.settings'
|
|
django.setup()
|
|
from analysis.models import AnalysisRun
|
|
from analysis.session_store import SessionStore
|
|
|
|
run = AnalysisRun.objects.get(id=run_id)
|
|
store = SessionStore()
|
|
try:
|
|
# Check upload directory still exists (may have been temp-cleaned)
|
|
if not upload_dir.is_dir():
|
|
msg = '上传数据目录已不存在，请重新上传'
|
|
logger.error('[BACKGROUND] %s (run_id=%s, upload_dir=%s)', msg, run_id, upload_dir)
|
|
run.status = 'failed'
|
|
run.error_message = msg
|
|
run.save(update_fields=['status', 'error_message'])
|
|
return
|
|
|
|
import glob
|
|
import polars as pl
|
|
csv_paths = sorted(glob.glob(str(upload_dir / '*.csv')))
|
|
if not csv_paths:
|
|
msg = '没有找到 CSV 文件'
|
|
logger.error('[BACKGROUND] %s (run_id=%s)', msg, run_id)
|
|
run.status = 'failed'
|
|
run.error_message = msg
|
|
run.save(update_fields=['status', 'error_message'])
|
|
return
|
|
|
|
# ── Detect lat/lon columns for schema override ──
|
|
import csv as _csv
|
|
_LAT_LON_KW = ('latd', 'lond', 'latitude', 'longitude')
|
|
lat_lon_overrides = {}
|
|
try:
|
|
with open(csv_paths[0], 'r', encoding='utf-8') as _f:
|
|
_reader = _csv.reader(_f)
|
|
_headers = next(_reader, [])
|
|
for _col in _headers:
|
|
_norm = _col.lower().replace('-', '_').replace('.', '_').replace(' ', '_')
|
|
if any(kw in _norm for kw in _LAT_LON_KW):
|
|
lat_lon_overrides[_col] = pl.Utf8
|
|
logger.info('[BACKGROUND] Forcing lat/lon column %r to Utf8 for safe parsing', _col)
|
|
except Exception:
|
|
logger.warning('[BACKGROUND] Failed to detect lat/lon columns', exc_info=True)
|
|
|
|
total_files = len(csv_paths)
|
|
# Adaptive batch size: aim for ~20 batches total, cap at 500 files per batch
|
|
batch_size = max(1, min(500, total_files // 20 or 1))
|
|
total_batches = (total_files + batch_size - 1) // batch_size
|
|
|
|
run.status = 'loading'; run.progress_pct = 10
|
|
run.progress_msg = f'正在流式加载 {total_files} 个文件（共 {total_batches} 批）...'
|
|
run.save(update_fields=['status', 'progress_pct', 'progress_msg'])
|
|
|
|
from analysis.data_loader import save_to_db, drop_sqlite_table, load_from_db
|
|
|
|
table_name = f'_data_{run_id}'
|
|
|
|
# ── Phase 1: Scan first batch ONLY for schema consistency ──
|
|
# Use large infer_schema_length to catch mixed-type columns early
|
|
first_batch = csv_paths[:batch_size]
|
|
first_lf = pl.scan_csv(first_batch, infer_schema_length=100000,
|
|
schema_overrides=lat_lon_overrides)
|
|
first_schema = dict(first_lf.collect_schema())
|
|
schema = {name: str(dtype) for name, dtype in first_schema.items()}
|
|
reference_columns = list(schema.keys()) # canonical column order & set
|
|
|
|
# ── Phase 2: Batched collection → SQLite write ──
|
|
table_created = False
|
|
completed_batches = 0
|
|
total_rows = 0
|
|
for i in range(0, total_files, batch_size):
|
|
batch = csv_paths[i:i+batch_size]
|
|
batch_lf = pl.scan_csv(batch, infer_schema_length=100000,
|
|
schema_overrides=lat_lon_overrides)
|
|
df_batch = batch_lf.collect(streaming=True)
|
|
|
|
# Schema alignment: ensure columns match reference order
|
|
for col in reference_columns:
|
|
if col not in df_batch.columns:
|
|
df_batch = df_batch.with_columns(pl.lit(None).alias(col))
|
|
extra = [c for c in df_batch.columns if c not in reference_columns]
|
|
if extra:
|
|
df_batch = df_batch.drop(extra)
|
|
df_batch = df_batch.select(reference_columns)
|
|
|
|
# Dtype alignment: cast each column to the reference dtype
|
|
# to prevent mixed-type columns when loading from SQLite later
|
|
casts = {}
|
|
for col in reference_columns:
|
|
ref_dtype_str = schema.get(col)
|
|
if ref_dtype_str and col in df_batch.columns:
|
|
# Parse the polars dtype string back to a pl.DataType
|
|
_base = ref_dtype_str.split('(')[0].strip()
|
|
_ref_dtype = {'Float64': pl.Float64, 'Float32': pl.Float32,
|
|
'Int64': pl.Int64, 'Int32': pl.Int32,
|
|
'Utf8': pl.Utf8, 'String': pl.Utf8,
|
|
'Boolean': pl.Boolean, 'Bool': pl.Boolean}.get(_base)
|
|
if _ref_dtype and df_batch[col].dtype != _ref_dtype:
|
|
try:
|
|
casts[col] = pl.col(col).cast(_ref_dtype)
|
|
except Exception:
|
|
# If cast fails (e.g. "A" → Int64), force to Utf8
|
|
casts[col] = pl.col(col).cast(pl.Utf8)
|
|
if casts:
|
|
df_batch = df_batch.with_columns(list(casts.values()))
|
|
|
|
mode = 'replace' if not table_created else 'append'
|
|
save_to_db(run.id, df_batch.lazy(), mode=mode)
|
|
table_created = True
|
|
|
|
# Delete temp files after successful SQLite write
|
|
for f in batch:
|
|
try:
|
|
os.unlink(f)
|
|
except OSError:
|
|
pass
|
|
|
|
completed_batches += 1
|
|
run.progress_pct = int((completed_batches / total_batches) * 80 + 10)
|
|
run.progress_msg = f'正在加载数据... ({completed_batches}/{total_batches} 批)'
|
|
run.save(update_fields=['progress_pct', 'progress_msg'])
|
|
|
|
# ── Phase 3: Reload from SQLite with correct types ──
|
|
lf = load_from_db(table_name, schema_overrides=schema)
|
|
if lf is None:
|
|
raise RuntimeError(f'SQLite 表 {table_name} 加载失败')
|
|
|
|
# Apply _coerce_to_float on lat/lon columns (restore Float64 from Utf8 storage)
|
|
from analysis.data_loader import _coerce_to_float as _coerce_to_float_fn
|
|
for lat_lon_col in lat_lon_overrides:
|
|
if lat_lon_col in lf.columns:
|
|
lf = lf.with_columns(
|
|
pl.col(lat_lon_col).map_batches(_coerce_to_float_fn, return_dtype=pl.Float64).alias(lat_lon_col)
|
|
)
|
|
|
|
# Get row count
|
|
df_count = lf.select(pl.len()).collect(streaming=True)
|
|
total_rows = df_count[0, 0] if df_count.height > 0 else 0
|
|
|
|
run.sqlite_table = table_name
|
|
run.total_flows = total_rows
|
|
run.save(update_fields=['sqlite_table', 'total_flows'])
|
|
|
|
ds_id = f'upload_{run.display_id}'
|
|
store.store_dataset(ds_id, lf, schema=schema, metadata={
|
|
'row_count': total_rows, 'file_count': total_files, 'upload_dir': str(upload_dir),
|
|
'csv_glob': str(upload_dir / '*.csv'),
|
|
'sqlite_table': table_name,
|
|
})
|
|
|
|
# ── Phase 3: Data ready for filtering/clustering via MCP tools ──
|
|
run.status = 'ready'; run.progress_pct = 60; run.progress_msg = '数据准备就绪'
|
|
run.save(update_fields=['status', 'progress_pct', 'progress_msg'])
|
|
except Exception:
|
|
tb = traceback.format_exc()
|
|
logger.error(tb)
|
|
# Drop partial SQLite table on failure (idempotent)
|
|
try:
|
|
from analysis.data_loader import drop_sqlite_table # fmt: skip
|
|
drop_sqlite_table(f'_data_{run_id}')
|
|
except Exception:
|
|
pass
|
|
run.status = 'failed'
|
|
run.error_message = tb
|
|
run.run_log += f'\n[ERROR] {tb}'
|
|
run.save(update_fields=['status', 'error_message', 'run_log'])
|
|
|
|
|
|
# ── Manual analysis page ───────────────────────────────────────────────
|
|
|
|
def manual_page(request):
|
|
"""Workflow builder: select dataset, add MCP tool steps, execute."""
|
|
from analysis.session_store import SessionStore
|
|
from analysis.tool_registry import get_tools_meta
|
|
import json as _json
|
|
|
|
# Get all tools metadata
|
|
tools = get_tools_meta()
|
|
core_names = {'profile_data', 'build_entity_profiles', 'compute_scores',
|
|
'run_clustering', 'extract_features', 'detect_anomalies',
|
|
'visualize_anomalies'}
|
|
diag_names = {'validate_data', 'explore_distributions', 'find_outliers',
|
|
'diagnose_clustering', 'compare_datasets', 'export_debug_sample',
|
|
'repair_schema'}
|
|
analysis_names = {'analyze_patterns', 'analyze_temporal', 'analyze_tls_health',
|
|
'analyze_geo_distribution', 'analyze_entity_detail'}
|
|
core_tools, diag_tools, analysis_tools, other_tools = [], [], [], []
|
|
for t in tools:
|
|
if t.name in core_names: core_tools.append(t)
|
|
elif t.name in diag_names: diag_tools.append(t)
|
|
elif t.name in analysis_names: analysis_tools.append(t)
|
|
else: other_tools.append(t)
|
|
|
|
meta_list = [{'name': t.name, 'description': t.description, 'inputSchema': t.inputSchema} for t in tools]
|
|
|
|
# Get datasets from SessionStore
|
|
store = SessionStore()
|
|
datasets = store.list_datasets() if hasattr(store, 'list_datasets') else []
|
|
|
|
return render(request, 'tianxuan/manual.html', {
|
|
'core_tools': core_tools,
|
|
'diag_tools': diag_tools,
|
|
'analysis_tools': analysis_tools,
|
|
'other_tools': other_tools,
|
|
'tools_meta': _json.dumps(meta_list, ensure_ascii=False),
|
|
'datasets': datasets,
|
|
})
|
|
|
|
|
|
# ── Pipeline worker ─────────────────────────────────────────────────────────
|
|
|
|
|
|
def _run_pipeline_worker(run_id, analysis_fn, **ctx):
|
|
"""Unified background worker: setup / ORM / progress / error handling."""
|
|
import django
|
|
os.environ['DJANGO_SETTINGS_MODULE'] = 'tianxuan.settings'
|
|
os.environ['DJANGO_ALLOW_ASYNC_UNSAFE'] = 'true'
|
|
django.setup()
|
|
|
|
from analysis.db_utils import retry_on_lock
|
|
from django.db import close_old_connections
|
|
|
|
close_old_connections()
|
|
|
|
@retry_on_lock()
|
|
def _get_run():
|
|
return AnalysisRun.objects.get(id=run_id)
|
|
|
|
run = _get_run()
|
|
try:
|
|
analysis_fn(run, ctx)
|
|
except Exception:
|
|
import traceback
|
|
tb = traceback.format_exc()
|
|
logger.error(tb)
|
|
run.status = 'failed'
|
|
run.error_message = tb
|
|
run.run_log += f'\n[FATAL] {tb}'
|
|
retry_on_lock()(lambda: run.save(update_fields=['status', 'error_message', 'run_log']))()
|
|
|
|
|
|
# ── Shared clustering pipeline ─────────────────────────────────────────
|
|
|
|
def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorithm,
|
|
min_cluster_size, run_pca=True, head=None):
|
|
"""Unified clustering pipeline: clustering → feature extraction → PCA → ORM save.
|
|
|
|
Args:
|
|
run: AnalysisRun ORM object.
|
|
store: SessionStore instance.
|
|
entity_ds_id: dataset ID in SessionStore pointing to entity-aggregated data.
|
|
feature_columns: list of feature column names (None = auto-detect).
|
|
algorithm: 'hdbscan' or 'kmeans'.
|
|
min_cluster_size: int for HDBSCAN.
|
|
run_pca: bool, whether to compute and save PCA-2D embeddings.
|
|
head: optional int, downsample to at most this many rows (min 100).
|
|
"""
|
|
import warnings
|
|
warnings.filterwarnings('ignore', category=RuntimeWarning, module='sklearn')
|
|
import asyncio, traceback, logging
|
|
logger = logging.getLogger(__name__)
|
|
|
|
try:
|
|
from analysis.tool_registry import _handle_run_clustering, _handle_extract_features
|
|
|
|
entry = store.get_dataset(entity_ds_id)
|
|
if entry is None:
|
|
run.status = 'failed'
|
|
run.error_message = 'Entity dataset not found in session store'
|
|
run.save(update_fields=['status', 'error_message'])
|
|
return
|
|
|
|
schema = entry.get('schema', {})
|
|
|
|
# ── Downsampling ──
|
|
if head is not None and head > 0:
|
|
lf = entry['lazyframe']
|
|
import polars as pl
|
|
try:
|
|
row_count = lf.select(pl.len()).collect(streaming=True).item()
|
|
except Exception:
|
|
row_count = 0
|
|
|
|
if row_count > head:
|
|
head = max(head, 100) # ensure minimum rows for clustering
|
|
run.progress_msg = f'正在采样 {head}/{row_count} 行...'
|
|
run.save(update_fields=['progress_msg'])
|
|
lf = lf.head(head)
|
|
store.store_dataset(
|
|
entity_ds_id, lf,
|
|
schema=schema,
|
|
metadata=entry.get('metadata', {}),
|
|
)
|
|
entry = store.get_dataset(entity_ds_id) # refresh
|
|
row_count = head
|
|
|
|
# Resolve feature columns
|
|
numeric_types = {'Int8', 'Int16', 'Int32', 'Int64',
|
|
'UInt8', 'UInt16', 'UInt32', 'UInt64',
|
|
'Float32', 'Float64'}
|
|
|
|
if feature_columns:
|
|
feature_cols = [c for c in feature_columns if c in schema]
|
|
else:
|
|
feature_cols = [
|
|
c for c in schema.keys()
|
|
if not c.startswith('_')
|
|
and schema[c].split('(')[0].strip() in numeric_types
|
|
][:10]
|
|
|
|
if not feature_cols:
|
|
raise Exception('No valid feature columns available for clustering')
|
|
|
|
# ── Step 1: Clustering ──────────────────────────────────────────────
|
|
run.status = 'clustering'
|
|
run.progress_pct = 70
|
|
run.progress_msg = '正在进行聚类分析...'
|
|
run.save(update_fields=['status', 'progress_pct', 'progress_msg'])
|
|
|
|
result = asyncio.run(_handle_run_clustering(
|
|
dataset_id=entity_ds_id,
|
|
cluster_columns=feature_cols,
|
|
algorithm=algorithm,
|
|
params={'min_cluster_size': min_cluster_size},
|
|
random_state=42,
|
|
))
|
|
|
|
if 'error' in result:
|
|
err = result['error']
|
|
if 'numeric' in err.lower():
|
|
available = [
|
|
f"{k}({v})" for k, v in schema.items()
|
|
if v.split('(')[0].strip() in numeric_types and not k.startswith('_')
|
|
]
|
|
err += f"。可用数值列: {available}"
|
|
raise Exception(err)
|
|
|
|
cluster_id = result.get('cluster_result_id', '')
|
|
run.cluster_count = result.get('n_clusters', 0)
|
|
run.save(update_fields=['cluster_count'])
|
|
|
|
# ── Step 2: Feature extraction ──────────────────────────────────────
|
|
run.status = 'extracting'
|
|
run.progress_pct = 90
|
|
run.progress_msg = '正在提取聚类特征...'
|
|
run.save(update_fields=['status', 'progress_pct', 'progress_msg'])
|
|
|
|
feat_result = asyncio.run(_handle_extract_features(
|
|
dataset_id=entity_ds_id,
|
|
cluster_result_id=cluster_id,
|
|
top_k=10, method='zscore', save_to_db=True,
|
|
run_id=run.id,
|
|
))
|
|
|
|
if 'error' in feat_result:
|
|
raise Exception(feat_result['error'])
|
|
|
|
# ── Step 3: PCA-2D embedding (downsampled for large data) ─────────
|
|
if run_pca:
|
|
try:
|
|
import polars as pl
|
|
lf = entry['lazyframe']
|
|
# Sample max 50K rows for PCA to avoid memory blowup on 10M entities
|
|
MAX_PCA = 50000
|
|
try:
|
|
n_total = lf.select(pl.len()).collect(streaming=True).item()
|
|
except Exception:
|
|
n_total = 0
|
|
if n_total > MAX_PCA:
|
|
lf_pca = lf.sample(n=MAX_PCA, seed=42)
|
|
run.progress_msg = f'PCA 采样 {MAX_PCA}/{n_total} 实体...'
|
|
run.save(update_fields=['progress_msg'])
|
|
else:
|
|
lf_pca = lf
|
|
df_pca = lf_pca.collect(streaming=True)
|
|
from sklearn.preprocessing import StandardScaler
|
|
from sklearn.decomposition import PCA
|
|
import numpy as np
|
|
|
|
num_cols = [c for c in df_pca.columns
|
|
if df_pca[c].dtype in (
|
|
pl.Int8, pl.Int16, pl.Int32, pl.Int64,
|
|
pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
|
|
pl.Float32, pl.Float64)]
|
|
|
|
if len(num_cols) >= 2 and len(df_pca) > 2:
|
|
mat = np.nan_to_num(StandardScaler().fit_transform(
|
|
df_pca.select(num_cols).to_numpy()), nan=0.0)
|
|
coords = PCA(n_components=2, random_state=42).fit_transform(mat)
|
|
|
|
non_num = [c for c in df_pca.columns if c not in num_cols and not c.startswith('_')]
|
|
ent_col = non_num[0] if non_num else df_pca.columns[0]
|
|
|
|
cluster_entry = store.get_cluster_result(cluster_id)
|
|
labels_list = cluster_entry.get('labels', []) if cluster_entry else []
|
|
|
|
from analysis.models import EntityProfile as EP
|
|
from analysis.models import ClusterResult as CR
|
|
|
|
profiles = []
|
|
cr_cache = {}
|
|
for i, row in enumerate(df_pca.iter_rows(named=True)):
|
|
ev = str(row.get(ent_col, ''))
|
|
if not ev:
|
|
continue
|
|
lbl = labels_list[i] if i < len(labels_list) else -1
|
|
cache_key = f'{run.id}_{lbl}'
|
|
if cache_key not in cr_cache:
|
|
cr_cache[cache_key] = CR.objects.filter(run=run, cluster_label=lbl).first()
|
|
profiles.append(EP(
|
|
entity_value=ev, run=run, cluster_label=lbl,
|
|
cluster=cr_cache[cache_key],
|
|
embedding_x=float(coords[i, 0]) if i < len(coords) else None,
|
|
embedding_y=float(coords[i, 1]) if i < len(coords) else None,
|
|
))
|
|
if profiles:
|
|
EP.objects.bulk_create(profiles, ignore_conflicts=True, batch_size=1000)
|
|
except Exception as pca_err:
|
|
logger.warning(f'PCA-2D embedding skipped: {pca_err}')
|
|
|
|
# ── Done ──
|
|
run.status = 'completed'
|
|
run.progress_pct = 100
|
|
run.progress_msg = '分析完成'
|
|
run.save(update_fields=['status', 'progress_pct', 'progress_msg'])
|
|
|
|
except Exception:
|
|
tb = traceback.format_exc()
|
|
logger.error(tb)
|
|
try:
|
|
run.status = 'failed'
|
|
run.error_message = tb
|
|
run.progress_msg = f'失败: {tb}'
|
|
run.run_log += f'\n[ERROR] {tb}'
|
|
run.save(update_fields=['status', 'error_message', 'progress_msg', 'run_log'])
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
# ── Manual analysis page ───────────────────────────────────────────────
|
|
|
|
@csrf_exempt
|
|
def run_analysis(request):
|
|
"""AJAX POST: run clustering pipeline on the upload dataset."""
|
|
if request.method != 'POST':
|
|
return JsonResponse({'error': 'POST required'}, status=405)
|
|
import json
|
|
body = json.loads(request.body)
|
|
display_id = body.get('run_id')
|
|
run = get_object_or_404(AnalysisRun, display_id=display_id)
|
|
pk = run.id
|
|
run.run_type = 'manual'
|
|
run.save(update_fields=['run_type'])
|
|
feature_columns = body.get('feature_columns')
|
|
algorithm = body.get('algorithm', 'hdbscan')
|
|
min_cluster_size = int(body.get('min_cluster_size', 5))
|
|
head = body.get('head')
|
|
|
|
def _analysis_fn(run, ctx):
|
|
from analysis.session_store import SessionStore
|
|
|
|
feature_columns = ctx['feature_columns']
|
|
algorithm = ctx['algorithm']
|
|
min_cluster_size = ctx['min_cluster_size']
|
|
head = ctx.get('head')
|
|
pk = ctx['pk']
|
|
|
|
store = SessionStore()
|
|
try:
|
|
# Use display_id for dataset key (consistent with _background_process)
|
|
display_id = ctx.get('display_id', pk)
|
|
upload_ds_id = f'upload_{display_id}'
|
|
entry = store.get_dataset(upload_ds_id)
|
|
if entry is None:
|
|
# Fallback: try PK-based key (old pipeline)
|
|
entry = store.get_dataset(f'upload_{pk}')
|
|
if entry is None:
|
|
# Fallback: try entity dataset (from older pipeline)
|
|
entry = store.get_dataset(f'entity_{pk}')
|
|
if entry is None:
|
|
run.error_message = '请先上传并等待预处理完成'
|
|
run.status = 'failed'
|
|
run.save(update_fields=['status', 'error_message'])
|
|
return
|
|
|
|
ds_id = upload_ds_id if store.get_dataset(upload_ds_id) else f'upload_{pk}'
|
|
|
|
# Use the unified clustering pipeline (clustering → extraction → PCA)
|
|
from analysis.views import _run_clustering_pipeline
|
|
_run_clustering_pipeline(
|
|
run=run, store=store, entity_ds_id=ds_id,
|
|
feature_columns=feature_columns,
|
|
algorithm=algorithm, min_cluster_size=min_cluster_size,
|
|
run_pca=True, head=head,
|
|
)
|
|
except Exception:
|
|
tb = traceback.format_exc()
|
|
logger.error(tb)
|
|
run.status = 'failed'
|
|
run.error_message = tb
|
|
run.progress_msg = f'失败: {tb}'
|
|
run.run_log += f'\n[ERROR] {tb}'
|
|
run.save(update_fields=['status', 'error_message', 'progress_msg', 'run_log'])
|
|
|
|
t = threading.Thread(
|
|
target=_run_pipeline_worker,
|
|
args=(run_id, _analysis_fn),
|
|
kwargs=dict(feature_columns=feature_columns, algorithm=algorithm,
|
|
min_cluster_size=min_cluster_size, head=head, pk=pk,
|
|
display_id=run.display_id),
|
|
daemon=True,
|
|
)
|
|
t.start()
|
|
return JsonResponse({'status': 'started', 'redirect': f'/runs/{run.display_id}/'})
|
|
|
|
|
|
|
|
|
|
# ── LLM auto analysis page ─────────────────────────────────────────────
|
|
|
|
|
|
@csrf_exempt
|
|
def run_llm_analysis_view(request):
|
|
"""AJAX POST: run LLM-driven analysis pipeline."""
|
|
if request.method != 'POST':
|
|
return JsonResponse({'error': 'POST required'}, status=405)
|
|
import json
|
|
body = json.loads(request.body)
|
|
display_id = body.get('run_id')
|
|
run = get_object_or_404(AnalysisRun, display_id=display_id)
|
|
pk = run.id
|
|
run.run_type = 'auto'
|
|
run.save(update_fields=['run_type'])
|
|
head = body.get('head')
|
|
|
|
from analysis.session_store import SessionStore
|
|
store = SessionStore()
|
|
# Use display_id for dataset key (consistent with _background_process)
|
|
upload_ds_id = f'upload_{display_id}'
|
|
entry = store.get_dataset(upload_ds_id)
|
|
if entry is None:
|
|
# Fallback: try PK-based key (old pipeline)
|
|
upload_ds_id = f'upload_{pk}'
|
|
entry = store.get_dataset(upload_ds_id)
|
|
|
|
if entry is None:
|
|
# Try entity dataset as fallback
|
|
entity_ds_id = f'entity_{pk}'
|
|
entry = store.get_dataset(entity_ds_id)
|
|
if entry is None:
|
|
return JsonResponse({'error': '数据集不存在，请先上传并等待预处理完成'})
|
|
|
|
dataset_id = upload_ds_id if store.get_dataset(upload_ds_id) else f'entity_{pk}'
|
|
|
|
def _analysis_fn(run, ctx):
|
|
from tianxuan.llm_orchestrator import run_llm_pipeline, LLMConfig
|
|
dataset_id = ctx['dataset_id']
|
|
pk = ctx['pk']
|
|
|
|
def progress_callback(step, tool_name, result=None, meta=None):
|
|
nonlocal run
|
|
try:
|
|
run = AnalysisRun.objects.get(id=pk)
|
|
pct = min(10 + step * 6, 95)
|
|
run.progress_pct = pct
|
|
run.progress_msg = f'步骤 {step}: {tool_name}'
|
|
|
|
if tool_name == 'thinking':
|
|
# Accumulate LLM thinking text
|
|
if result:
|
|
run.llm_thinking = (run.llm_thinking or '') + f'\n[{step}] {result}'
|
|
save_fields = ['progress_pct', 'progress_msg', 'llm_thinking']
|
|
elif tool_name == 'complete':
|
|
pass # No extra tracking needed
|
|
else:
|
|
# Real tool call — log it
|
|
if result:
|
|
if isinstance(result, dict):
|
|
# Format dict as JSON, truncate large values
|
|
import json as _json
|
|
summary = _json.dumps(result, default=str, ensure_ascii=False)[:500]
|
|
else:
|
|
summary = str(result)[:500]
|
|
run.run_log += f'[{step}] {tool_name}: {summary}...\n'
|
|
else:
|
|
run.run_log += f'[{step}] {tool_name}\n'
|
|
|
|
# Capture structured tool call data
|
|
if meta and isinstance(meta, dict) and 'args' in meta:
|
|
tool_calls = list(run.tool_calls_json or [])
|
|
tool_calls.append({
|
|
'step': step,
|
|
'name': tool_name,
|
|
'input': meta['args'],
|
|
'output': result,
|
|
})
|
|
run.tool_calls_json = tool_calls
|
|
save_fields = ['progress_pct', 'progress_msg', 'run_log', 'tool_calls_json']
|
|
|
|
run.save(update_fields=save_fields)
|
|
except Exception:
|
|
pass
|
|
|
|
try:
|
|
from config import get_config
|
|
cfg = get_config()
|
|
llm_cfg = LLMConfig(cfg.llm.base_url, cfg.llm.api_key, cfg.llm.model)
|
|
run.run_log = '启动 LLM 分析...\n'
|
|
run.status = 'loading'
|
|
run.progress_pct = 5
|
|
run.progress_msg = 'LLM 初始化中...'
|
|
run.save(update_fields=['status', 'progress_pct', 'progress_msg', 'run_log'])
|
|
|
|
result = run_llm_pipeline(dataset_id, config=llm_cfg, callback=progress_callback)
|
|
|
|
if 'error' in result:
|
|
run.status = 'failed'
|
|
run.progress_msg = f'失败: {result["error"]}'
|
|
run.run_log += f'\n[ERROR] {result["error"]}'
|
|
else:
|
|
run.status = 'completed'
|
|
run.progress_pct = 100
|
|
run.progress_msg = 'LLM 分析完成'
|
|
run.run_log += f'\n[完成] {result.get("result", "")}'
|
|
run.save(update_fields=['status', 'progress_pct', 'progress_msg', 'run_log'])
|
|
except Exception:
|
|
tb = traceback.format_exc()
|
|
logger.error(tb)
|
|
try:
|
|
run = AnalysisRun.objects.get(id=pk)
|
|
run.status = 'failed'
|
|
run.progress_msg = f'失败: {tb}'
|
|
run.run_log += f'\n[ERROR] {tb}'
|
|
run.error_message = tb
|
|
run.save(update_fields=['status', 'progress_msg', 'run_log', 'error_message'])
|
|
except Exception:
|
|
pass
|
|
|
|
t = threading.Thread(
|
|
target=_run_pipeline_worker,
|
|
args=(pk, _analysis_fn),
|
|
kwargs=dict(dataset_id=dataset_id, head=head, pk=pk),
|
|
daemon=True,
|
|
)
|
|
t.start()
|
|
return JsonResponse({'status': 'started', 'run_id': run.display_id})
|
|
|
|
|
|
def auto_page(request):
|
|
"""LLM-driven auto analysis page."""
|
|
runs = AnalysisRun.objects.filter(status='ready').order_by('-created_at')[:20]
|
|
cfg = get_config()
|
|
return render(request, 'tianxuan/auto.html', {
|
|
'runs': runs,
|
|
'llm_configured': bool(cfg.llm.base_url and cfg.llm.api_key),
|
|
})
|
|
|
|
|
|
# ── Progress API ───────────────────────────────────────────────────────
|
|
|
|
def run_status_api(request, display_id):
|
|
"""Return current run status as JSON."""
|
|
run = get_object_or_404(AnalysisRun, display_id=display_id)
|
|
return JsonResponse({
|
|
'id': run.display_id,
|
|
'status': run.status,
|
|
'entity_count': run.entity_count,
|
|
'cluster_count': run.cluster_count,
|
|
'error_message': run.error_message,
|
|
'progress_pct': run.progress_pct,
|
|
'progress_msg': run.progress_msg,
|
|
'run_log': run.run_log[-2000:], # last 2000 chars
|
|
'llm_thinking': run.llm_thinking[-5000:], # last 5K chars
|
|
'tool_calls': run.tool_calls_json,
|
|
})
|
|
|
|
|
|
def log_viewer(request):
|
|
"""Display last 200 lines of the app log (DEBUG only).
|
|
With ?pos=N, returns text: line_count\\nnew_content for AJAX polling.
|
|
"""
|
|
from django.conf import settings
|
|
if not settings.DEBUG:
|
|
return HttpResponse('Log viewer only available in DEBUG mode', status=403)
|
|
log_path = settings.BASE_DIR / 'logs' / 'tianxuan.log'
|
|
pos = request.GET.get('pos')
|
|
if pos is not None:
|
|
# AJAX polling mode: return new content since byte position
|
|
try:
|
|
pos = int(pos)
|
|
except ValueError:
|
|
pos = 0
|
|
if log_path.exists():
|
|
with open(log_path, 'r', encoding='utf-8') as f:
|
|
total = f.seek(0, 2) # seek to end
|
|
if pos >= total:
|
|
return HttpResponse(f'{total}\n', content_type='text/plain')
|
|
f.seek(pos)
|
|
new_content = f.read()
|
|
return HttpResponse(f'{total}\n{new_content}', content_type='text/plain')
|
|
return HttpResponse('0\n', content_type='text/plain')
|
|
|
|
# HTML page mode
|
|
lines = []
|
|
if log_path.exists():
|
|
with open(log_path, 'r', encoding='utf-8') as f:
|
|
lines = f.readlines()[-200:]
|
|
return render(request, 'tianxuan/log_viewer.html', {'lines': lines})
|
|
|
|
# ── 3D Globe view ─────────────────────────────────────────────────────────
|
|
|
|
def _extract_flows_from_df(df, MAX_ROWS):
|
|
"""Extract flow data (lat/lon/TLS/bytes) from a Polars DataFrame."""
|
|
from analysis.geoip import lookup as geo_lookup
|
|
TLS_HEX_MAP = {'0303': 'TLSv1.2', '0304': 'TLSv1.3', '0302': 'TLSv1.1', '0301': 'TLSv1.0'}
|
|
# NOTE: Duplicated in type_classifier.py — keep both in sync.
|
|
|
|
src_lat_col = next((c for c in df.columns if c.lower() in (
|
|
':ips.latd', ':ips_latd', ':ips_lat', 'src_latitude', 'src_lat', 'source_latitude', 'source_lat')), None)
|
|
src_lon_col = next((c for c in df.columns if c.lower() in (
|
|
':ips.lond', ':ips_lond', ':ips_lon', 'src_longitude', 'src_lon', 'source_longitude', 'source_lon')), None)
|
|
dst_lat_col = next((c for c in df.columns if c.lower() in (
|
|
':ipd.latd', ':ipd_latd', ':ipd_lat', 'dst_latitude', 'dst_lat', 'dest_latitude', 'dest_lat')), None)
|
|
dst_lon_col = next((c for c in df.columns if c.lower() in (
|
|
':ipd.lond', ':ipd_lond', ':ipd_lon', 'dst_longitude', 'dst_lon', 'dest_longitude', 'dest_lon')), None)
|
|
src_ip_col = next((c for c in df.columns if 'src_ip' in c.lower() or ':ips' in c.lower()), None)
|
|
dst_ip_col = next((c for c in df.columns if 'dst_ip' in c.lower() or ':ipd' in c.lower()), None)
|
|
tls_col = next((c for c in df.columns if c.lower() in ('0ver', 'tls_version', '_tlsver', 'version')), None)
|
|
bytes_col = next((c for c in df.columns if c.lower() in ('8ack', '8pak', '8byt', 'bytes_sent', 'bytes')), None)
|
|
time_col = next((c for c in df.columns if c.lower() in (
|
|
'timestamp', 'time', '_time', 'ts', 'datetime', 'epoch',
|
|
'created_at', 'created', 'event_time')), None)
|
|
|
|
flows = []
|
|
for row in df.iter_rows(named=True):
|
|
slat = slon = dlat = dlon = None
|
|
|
|
if src_lat_col and src_lon_col:
|
|
slat = _extract_lat(row.get(src_lat_col))
|
|
slon = _extract_lon(row.get(src_lon_col))
|
|
if dst_lat_col and dst_lon_col:
|
|
dlat = _extract_lat(row.get(dst_lat_col))
|
|
dlon = _extract_lon(row.get(dst_lon_col))
|
|
|
|
if (slat is None or slon is None) and src_ip_col:
|
|
sg = geo_lookup(str(row.get(src_ip_col, '')))
|
|
if sg:
|
|
slat, slon = sg['lat'], sg['lon']
|
|
if (dlat is None or dlon is None) and dst_ip_col:
|
|
dg = geo_lookup(str(row.get(dst_ip_col, '')))
|
|
if dg:
|
|
dlat, dlon = dg['lat'], dg['lon']
|
|
|
|
if slat is None or slon is None or dlat is None or dlon is None:
|
|
continue
|
|
|
|
tls_val = str(row.get(tls_col or '', ''))
|
|
# Check hex format first (e.g. "03 03" or "0303")
|
|
hex_key = tls_val.replace(' ', '').strip()
|
|
if hex_key in TLS_HEX_MAP:
|
|
tls_label = TLS_HEX_MAP[hex_key]
|
|
elif '1.3' in tls_val:
|
|
tls_label = 'TLSv1.3'
|
|
elif '1.2' in tls_val:
|
|
tls_label = 'TLSv1.2'
|
|
else:
|
|
tls_label = 'other'
|
|
|
|
bs = float(row.get(bytes_col or '', 0) or 0)
|
|
|
|
# Extract timestamp for animation ordering
|
|
t_raw = row.get(time_col) if time_col else None
|
|
t_val = None
|
|
if t_raw is not None:
|
|
if isinstance(t_raw, (int, float)):
|
|
t_val = float(t_raw)
|
|
elif hasattr(t_raw, 'timestamp'):
|
|
t_val = t_raw.timestamp()
|
|
else:
|
|
try:
|
|
t_val = float(t_raw)
|
|
except (ValueError, TypeError):
|
|
pass
|
|
|
|
flows.append({
|
|
'slat': slat, 'slon': slon,
|
|
'dlat': dlat, 'dlon': dlon,
|
|
'tls': tls_label,
|
|
'bytes': bs,
|
|
'time': t_val,
|
|
})
|
|
|
|
# Normalize timestamps to 0-1 and sort flows by time
|
|
timed = [f for f in flows if f['time'] is not None]
|
|
if len(timed) > 1:
|
|
timed.sort(key=lambda f: f['time'])
|
|
t_min, t_max = timed[0]['time'], timed[-1]['time']
|
|
t_range = max(t_max - t_min, 1)
|
|
for f in timed:
|
|
f['time'] = (f['time'] - t_min) / t_range
|
|
# Untimed flows get evenly spaced after timed ones
|
|
untimed = [f for f in flows if f['time'] is None]
|
|
for i, f in enumerate(untimed):
|
|
f['time'] = min(1.0, (len(timed) + i) / max(len(flows) - 1, 1))
|
|
flows = timed + untimed
|
|
else:
|
|
for i, f in enumerate(flows):
|
|
f['time'] = i / max(len(flows) - 1, 1)
|
|
|
|
return flows
|
|
|
|
|
|
def globe_view(request):
|
|
import json
|
|
from analysis.geoip import lookup as geo_lookup
|
|
from analysis.data_loader import load_csv_directory
|
|
from analysis.models import AnalysisRun
|
|
from analysis.session_store import SessionStore
|
|
|
|
MAX_ROWS_PER_RUN = 300 # Limit per run to prevent browser overload
|
|
|
|
# Check for ?data=dataset_id parameter (direct data loading from session store)
|
|
data_param = request.GET.get('data')
|
|
if data_param:
|
|
store = SessionStore()
|
|
flows = []
|
|
all_runs = list(AnalysisRun.objects.filter(status='completed').order_by('-id').values('display_id', 'csv_glob'))
|
|
try:
|
|
entry = store.get_dataset(data_param)
|
|
if entry is not None:
|
|
lf = entry['lazyframe']
|
|
df = lf.head(MAX_ROWS_PER_RUN).collect()
|
|
flows = _extract_flows_from_df(df, MAX_ROWS_PER_RUN)
|
|
except Exception as exc:
|
|
import logging
|
|
logging.getLogger(__name__).warning(f'globe ?data= param failed: {exc}')
|
|
return render(request, 'tianxuan/globe.html', {
|
|
'geo_flows': json.dumps(flows),
|
|
'all_runs': all_runs,
|
|
'selected_ids': [],
|
|
})
|
|
|
|
# Get all completed runs for the selector
|
|
all_runs = list(AnalysisRun.objects.filter(status='completed').order_by('-id').values('display_id', 'csv_glob'))
|
|
|
|
# Get selected run IDs from query params (multiple checkboxes)
|
|
selected_ids_list = request.GET.getlist('runs')
|
|
selected_runs = []
|
|
if selected_ids_list:
|
|
for sid in selected_ids_list:
|
|
sid = sid.strip()
|
|
if sid.isdigit():
|
|
try:
|
|
r = AnalysisRun.objects.get(display_id=int(sid), status='completed')
|
|
selected_runs.append(r)
|
|
except AnalysisRun.DoesNotExist:
|
|
pass
|
|
|
|
# Check if user-selected runs have valid files; if not, fall through
|
|
all_selected_failed = bool(selected_ids_list)
|
|
fallback_run = None
|
|
|
|
if not selected_runs:
|
|
all_selected_failed = False
|
|
|
|
flows = []
|
|
|
|
for run in selected_runs:
|
|
try:
|
|
lf, schema, rc, fc, mem = load_csv_directory(run.csv_glob)
|
|
df = lf.head(MAX_ROWS_PER_RUN).collect()
|
|
flows.extend(_extract_flows_from_df(df, MAX_ROWS_PER_RUN))
|
|
all_selected_failed = False
|
|
break # stop after first successful load
|
|
except Exception as exc:
|
|
logger.warning('[GLOBE] Failed to load run %s (%s): %s', run.display_id, run.csv_glob, exc)
|
|
continue
|
|
|
|
if all_selected_failed or not flows:
|
|
# Fallback: latest completed run with valid files
|
|
for latest in AnalysisRun.objects.filter(status='completed').order_by('-id'):
|
|
if not latest.csv_glob:
|
|
continue
|
|
glob_path = Path(latest.csv_glob)
|
|
try:
|
|
if '/*' in latest.csv_glob:
|
|
if glob_path.parent.is_dir():
|
|
lf, schema, rc, fc, mem = load_csv_directory(latest.csv_glob)
|
|
df = lf.head(MAX_ROWS_PER_RUN).collect()
|
|
flows.extend(_extract_flows_from_df(df, MAX_ROWS_PER_RUN))
|
|
selected_runs = [latest]
|
|
break
|
|
elif glob_path.exists():
|
|
lf, schema, rc, fc, mem = load_csv_directory(latest.csv_glob)
|
|
df = lf.head(MAX_ROWS_PER_RUN).collect()
|
|
flows.extend(_extract_flows_from_df(df, MAX_ROWS_PER_RUN))
|
|
selected_runs = [latest]
|
|
break
|
|
except Exception:
|
|
continue
|
|
|
|
resp = render(request, 'tianxuan/globe.html', {
|
|
'geo_flows': json.dumps(flows),
|
|
'all_runs': all_runs,
|
|
'selected_ids': [r.display_id for r in selected_runs],
|
|
})
|
|
resp['Cache-Control'] = 'no-cache, no-store, must-revalidate'
|
|
resp['Pragma'] = 'no-cache'
|
|
resp['Expires'] = '0'
|
|
return resp
|
|
|
|
|
|
# ── MCP Tool Lab — manual tool execution ─────────────────────────────
|
|
|
|
|
|
def tool_lab(request):
|
|
"""List all available MCP tools with descriptions and input schemas."""
|
|
from analysis.tool_registry import get_tools_meta
|
|
tools = get_tools_meta()
|
|
# Group tools by category
|
|
core_names = {'profile_data', 'build_entity_profiles', 'compute_scores',
|
|
'run_clustering', 'extract_features', 'detect_anomalies',
|
|
'visualize_anomalies'}
|
|
diag_names = {'validate_data', 'explore_distributions', 'find_outliers',
|
|
'diagnose_clustering', 'compare_datasets', 'export_debug_sample',
|
|
'repair_schema'}
|
|
analysis_names = {'analyze_patterns', 'analyze_temporal', 'analyze_tls_health',
|
|
'analyze_geo_distribution', 'analyze_entity_detail'}
|
|
|
|
core_tools, diag_tools, analysis_tools, other_tools = [], [], [], []
|
|
for t in tools:
|
|
if t.name in core_names:
|
|
core_tools.append(t)
|
|
elif t.name in diag_names:
|
|
diag_tools.append(t)
|
|
elif t.name in analysis_names:
|
|
analysis_tools.append(t)
|
|
else:
|
|
other_tools.append(t)
|
|
|
|
# Get list of datasets for parameter autofill
|
|
from analysis.session_store import SessionStore
|
|
store = SessionStore()
|
|
ds_list = store.list_datasets() if hasattr(store, 'list_datasets') else []
|
|
|
|
import json as _json
|
|
# Serialize tools metadata for JS
|
|
meta_list = []
|
|
for t in tools:
|
|
meta_list.append({
|
|
'name': t.name,
|
|
'description': t.description,
|
|
'inputSchema': t.inputSchema,
|
|
})
|
|
return render(request, 'analysis/tool_lab.html', {
|
|
'core_tools': core_tools,
|
|
'diag_tools': diag_tools,
|
|
'analysis_tools': analysis_tools,
|
|
'other_tools': other_tools,
|
|
'datasets': ds_list,
|
|
'tools_meta': _json.dumps(meta_list, ensure_ascii=False),
|
|
})
|
|
|
|
|
|
@csrf_exempt
|
|
def tool_lab_run(request):
|
|
"""Execute a single MCP tool via AJAX POST."""
|
|
if request.method != 'POST':
|
|
return JsonResponse({'error': 'POST required'}, status=405)
|
|
import json as _json
|
|
try:
|
|
body = _json.loads(request.body)
|
|
except Exception:
|
|
return JsonResponse({'error': 'Invalid JSON body'}, status=400)
|
|
|
|
tool_name = body.get('tool')
|
|
params = body.get('params', {})
|
|
if not tool_name:
|
|
return JsonResponse({'error': 'Missing tool name'}, status=400)
|
|
|
|
from analysis.tool_registry import handle_call
|
|
import asyncio
|
|
try:
|
|
result = asyncio.run(handle_call(tool_name, params))
|
|
return JsonResponse({'status': 'ok', 'result': result})
|
|
except ValueError as e:
|
|
return JsonResponse({'error': str(e)}, status=400)
|
|
except Exception as e:
|
|
import traceback
|
|
return JsonResponse({'error': str(e), 'traceback': traceback.format_exc()}, status=500)
|
|
|
|
|
|
# ── Save / Load / Delete analysis plans ─────────────────────────────
|
|
|
|
from pathlib import Path
|
|
_PLANS_DIR = Path(__file__).resolve().parent.parent / '.omo' / 'plans'
|
|
_PLANS_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
|
@csrf_exempt
|
|
def tool_plan(request):
|
|
"""Save, list, load, or delete an analysis plan."""
|
|
import json as _j
|
|
if request.method == 'GET':
|
|
name = request.GET.get('name')
|
|
if name:
|
|
path = _PLANS_DIR / f'{name}.json'
|
|
if not path.exists():
|
|
return JsonResponse({'error': f'Plan "{name}" not found'}, status=404)
|
|
try:
|
|
data = _j.loads(path.read_text(encoding='utf-8'))
|
|
return JsonResponse(data)
|
|
except Exception as e:
|
|
return JsonResponse({'error': str(e)}, status=500)
|
|
# List all plans
|
|
plans = []
|
|
for p in sorted(_PLANS_DIR.glob('*.json')):
|
|
try:
|
|
content = _j.loads(p.read_text(encoding='utf-8'))
|
|
steps = len(content.get('steps', []))
|
|
plans.append({'name': p.stem, 'steps': steps})
|
|
except Exception:
|
|
plans.append({'name': p.stem, 'steps': 0})
|
|
return JsonResponse({'plans': plans})
|
|
|
|
elif request.method == 'POST':
|
|
try:
|
|
body = _j.loads(request.body)
|
|
name = body.get('name', '').strip()
|
|
steps = body.get('steps', [])
|
|
if not name:
|
|
return JsonResponse({'error': 'Missing plan name'}, status=400)
|
|
path = _PLANS_DIR / f'{name}.json'
|
|
path.write_text(_j.dumps({'name': name, 'steps': steps}, ensure_ascii=False, indent=2), encoding='utf-8')
|
|
return JsonResponse({'status': 'saved', 'name': name, 'steps': len(steps)})
|
|
except Exception as e:
|
|
return JsonResponse({'error': str(e)}, status=500)
|
|
|
|
elif request.method == 'DELETE':
|
|
name = request.GET.get('name', '').strip()
|
|
if not name:
|
|
return JsonResponse({'error': 'Missing plan name'}, status=400)
|
|
path = _PLANS_DIR / f'{name}.json'
|
|
if path.exists():
|
|
path.unlink()
|
|
return JsonResponse({'status': 'deleted', 'name': name})
|
|
|
|
return JsonResponse({'error': 'Method not allowed'}, status=405)
|
|
|
|
|
|
@csrf_exempt
|
|
def retry_run(request, display_id):
|
|
"""Reset a failed run and re-trigger based on run_type."""
|
|
if request.method != 'POST':
|
|
return JsonResponse({'error': 'POST required'}, status=405)
|
|
|
|
run = get_object_or_404(AnalysisRun, display_id=display_id)
|
|
if run.status != 'failed':
|
|
return JsonResponse({'error': '只能重试失败的分析运行'}, status=400)
|
|
|
|
run.status = 'pending'
|
|
run.error_message = ''
|
|
run.progress_pct = 0
|
|
run.progress_msg = ''
|
|
run.save(update_fields=['status', 'error_message', 'progress_pct', 'progress_msg'])
|
|
|
|
if run.run_type == 'upload' and run.csv_glob:
|
|
upload_dir = Path(run.csv_glob).parent
|
|
if upload_dir.is_dir():
|
|
t = threading.Thread(
|
|
target=_background_process,
|
|
args=(run.id, upload_dir),
|
|
daemon=True,
|
|
)
|
|
t.start()
|
|
else:
|
|
run.error_message = '上传数据目录已不存在，请重新上传'
|
|
run.status = 'failed'
|
|
run.save(update_fields=['status', 'error_message'])
|
|
|
|
return redirect('analysis:run_detail', display_id=run.display_id)
|
|
|