Files
tianxuan/templates/analysis/cluster_overview.html
T
PM-pinou 8fa36b774e v7: 19 issues fixed — SQLite storage, new clustering pipeline, display_id, globe rewrite, Chinese tools
Key changes:
- New: data_loader.py SQLite persistence with drop_sqlite_table
- New: db_utils.py retry_on_lock decorator (3 retries, exponential backoff)
- New: tool_registry.py with 27 MCP tools (filter_and_cluster, compute_scores, etc.)
- New: tls_ref.py for TLS cipher/reference data
- New: import_tlsdb.py management command
- New: scripts/start_server.py for portable runtime
- New: migrations 0003-0007 for SQLite table, display_id, llm fields
- Changed: views.py unified pipeline worker, retry_run, display_id everywhere
- Changed: models.py with display_id auto-assignment, run_type, sqlite_table, llm_thinking, tool_calls_json
- Changed: urls.py added retry_run route
- Changed: session_store.py robust JSON persistence
- Changed: AGENTS.md v7 fix summary added
- Changed: templates — globe rewrite (inertia/polar flip), auto.html (thinking/tool accordions), base.html (toast/config), all pages use display_id
- Changed: run.bat PYTHONUTF8=1
- Deleted: entity_detector.py, entity_aggregator.py (replaced by filter_and_cluster clustering pipeline)
- Test: 92/92 unit tests passing
2026-07-20 13:33:13 +08:00

335 lines
14 KiB
HTML

{% extends 'base.html' %}
{% load static %}
{% block title %}Cluster Overview - Run #{{ run.display_id }}{% endblock %}
{% block content %}
<style>
.feature-chart { width: 100%; display: block; }
#featureChartsContainer canvas { max-height: 400px; }
.chart-wrapper { overflow-x: auto; }
.chart-container { position: relative; }
.canvas-legend {
max-height: 110px; overflow-y: auto;
display: flex; flex-wrap: wrap; gap: 4px 14px;
padding: 6px 10px; font-size: 11px;
border-top: 1px solid #eee; margin-top: 4px;
}
.canvas-legend .legend-item {
display: inline-flex; align-items: center; gap: 4px;
white-space: nowrap;
}
.canvas-legend .legend-swatch {
width: 10px; height: 10px; display: inline-block;
border-radius: 2px; flex-shrink: 0;
}
</style>
<div class="card">
<h2>Cluster Overview — Run #{{ run.display_id }}</h2>
<p>{{ run.total_flows }} flows, {{ run.cluster_count }} clusters</p>
</div>
<div class="card">
<h2>PCA-2D Embedding</h2>
<div class="chart-wrapper">
<div class="chart-container">
<canvas id="scatterChart" class="scatter-canvas"></canvas>
<div id="scatterLegend" class="canvas-legend"></div>
</div>
</div>
</div>
{% if geo_count > 0 %}
<div class="card">
<h2>地理位置分布 <span class="badge badge-info">{{ geo_count }} entities</span></h2>
{% if geo_skipped > 0 %}
<p style="color:#888;font-size:0.85rem;">{{ geo_skipped }} entities excluded (missing lat/lon)</p>
{% endif %}
<div class="chart-wrapper">
<div class="chart-container">
<canvas id="geoChart" class="scatter-canvas"></canvas>
<div id="geoLegend" class="canvas-legend"></div>
</div>
</div>
</div>
{% endif %}
<div class="card" id="clusterSizesCard">
<h2>Cluster Size Distribution</h2>
<div class="chart-wrapper">
<canvas id="clusterSizesChart" class="scatter-canvas"></canvas>
</div>
</div>
<div class="card" id="silhouetteCard">
<h2>Silhouette Score Comparison</h2>
<div class="chart-wrapper">
<canvas id="silhouetteChart" class="scatter-canvas"></canvas>
</div>
</div>
<div class="card" id="topFeaturesCard">
<h2>Top Distinguishing Features per Cluster</h2>
<div id="featureChartsContainer"></div>
</div>
<div class="grid-2">
{% for c in clusters %}
<div class="card">
<h3>Cluster #{{ c.cluster_label }} <span class="badge badge-info">{{ c.size }} entities</span></h3>
<p>Proportion: {{ c.proportion|floatformat:3 }}</p>
<p>Silhouette: {{ c.silhouette_score|floatformat:4|default:"-" }}</p>
<h4 style="margin-top:0.75rem;font-size:0.9rem;">Top Distinguishing Features</h4>
<table>
<thead>
<tr><th>Feature</th><th>Score</th><th>Mean</th></tr>
</thead>
<tbody>
{% for f in c.features.all|slice:":5" %}
<tr>
<td>{{ f.feature_name }}</td>
<td>{{ f.distinguishing_score|floatformat:3|default:"-" }}</td>
<td>{{ f.mean|floatformat:3|default:"-" }}</td>
</tr>
{% empty %}
<tr><td colspan="3">No features</td></tr>
{% endfor %}
</tbody>
</table>
<a href="{% url 'analysis:cluster_detail' run.display_id c.cluster_label %}" class="btn btn-primary" style="margin-top:0.75rem;">Detail</a>
</div>
{% empty %}
<div class="card">
<div class="empty-state"><p>No clusters found.</p></div>
</div>
{% endfor %}
</div>
{% if noise %}
<div class="card">
<h3>Noise (Cluster -1)</h3>
<p>{{ noise.size }} entities classified as noise</p>
</div>
{% endif %}
<script>
// Pure Canvas scatter plot — no Chart.js needed (offline compatible)
const scatterData = {{ scatter_data_json|safe }};
const colors = ['#4361ee','#f72585','#7209b7','#4cc9f0','#f8961e','#43aa8b','#f94144','#577590','#e9c46a','#9b5de5'];
function drawScatter(canvasId, data, xLabel, yLabel) {
const canvas = document.getElementById(canvasId);
if (!canvas || data.length === 0) return;
const ctx = canvas.getContext('2d');
const W = canvas.parentElement.clientWidth || 600;
const H = 400;
canvas.width = W; canvas.height = H;
const pad = { top: 30, bottom: 50, left: 60, right: 30 };
const plotW = W - pad.left - pad.right;
const plotH = H - pad.top - pad.bottom;
// Compute bounds
const xs = data.map(d => d.x); const ys = data.map(d => d.y);
const xMin = Math.min(...xs), xMax = Math.max(...xs);
const yMin = Math.min(...ys), yMax = Math.max(...ys);
const xRange = xMax - xMin || 1; const yRange = yMax - yMin || 1;
const toX = v => pad.left + (v - xMin) / xRange * plotW;
const toY = v => pad.top + plotH - (v - yMin) / yRange * plotH;
// Clear
ctx.clearRect(0, 0, W, H);
// Grid lines
ctx.strokeStyle = '#eee'; ctx.lineWidth = 1;
for (let i = 0; i <= 5; i++) {
const y = pad.top + i * plotH / 5;
ctx.beginPath(); ctx.moveTo(pad.left, y); ctx.lineTo(W - pad.right, y); ctx.stroke();
}
// Axes
ctx.strokeStyle = '#333'; ctx.lineWidth = 2;
ctx.beginPath(); ctx.moveTo(pad.left, pad.top); ctx.lineTo(pad.left, H - pad.bottom); ctx.stroke();
ctx.beginPath(); ctx.moveTo(pad.left, H - pad.bottom); ctx.lineTo(W - pad.right, H - pad.bottom); ctx.stroke();
// Labels
ctx.fillStyle = '#666'; ctx.font = '13px sans-serif'; ctx.textAlign = 'center';
ctx.fillText(xLabel, W / 2, H - 5);
ctx.save(); ctx.translate(15, H / 2); ctx.rotate(-Math.PI / 2); ctx.fillText(yLabel, 0, 0); ctx.restore();
// Points
data.forEach(d => {
const cx = toX(d.x), cy = toY(d.y);
const color = d.label === -1 ? 'rgba(150,150,150,0.5)' : colors[Math.abs(d.label) % colors.length];
const r = d.label === -1 ? 2 : 4;
ctx.beginPath(); ctx.arc(cx, cy, r, 0, Math.PI * 2);
ctx.fillStyle = color; ctx.fill();
});
// Tooltip on hover
canvas.onmousemove = function(e) {
const rect = canvas.getBoundingClientRect();
const mx = e.clientX - rect.left, my = e.clientY - rect.top;
const hit = data.find(d => Math.abs(toX(d.x) - mx) < 6 && Math.abs(toY(d.y) - my) < 6);
if (hit) {
canvas.title = `${hit.entity} (${hit.label === -1 ? 'Noise' : 'Cluster ' + hit.label})`;
}
};
// Legend — HTML overlay (supports overflow-y: auto via .canvas-legend)
const legendEl = document.getElementById(canvasId.replace('Chart', 'Legend'));
if (legendEl) {
legendEl.innerHTML = '';
const uniqueLabels = [...new Set(data.map(d => d.label))];
uniqueLabels.forEach(label => {
const color = label === -1 ? 'rgba(150,150,150,0.8)' : colors[Math.abs(label) % colors.length];
const item = document.createElement('span');
item.className = 'legend-item';
item.innerHTML = '<span class="legend-swatch" style="background:' + color + '"></span>' + (label === -1 ? 'Noise' : 'Cluster ' + label);
legendEl.appendChild(item);
});
}
}
drawScatter('scatterChart', scatterData, 'PCA-1', 'PCA-2');
{% if geo_count > 0 %}
const geoData = {{ geo_data_json|safe }};
drawScatter('geoChart', geoData, 'Longitude', 'Latitude');
{% endif %}
// ── Cluster size horizontal bar chart ──
const clusterSizes = {{ cluster_sizes_json|safe }};
const topFeaturesAll = {{ top_features_json|safe }};
const chartColors = ['#4361ee','#f72585','#7209b7','#4cc9f0','#f8961e','#43aa8b','#f94144','#577590','#e9c46a','#9b5de5'];
if (clusterSizes.length > 0) {
// Cluster size bar chart
(function() {
const canvas = document.getElementById('clusterSizesChart');
if (!canvas) return;
const ctx = canvas.getContext('2d');
var maxClusters = 50;
var shownClusters = clusterSizes.slice(0, maxClusters);
var note = '';
if (clusterSizes.length > maxClusters) {
note = ' (top ' + maxClusters + ' of ' + clusterSizes.length + ' clusters)';
}
var W = canvas.parentElement.clientWidth || 600;
var H = Math.max(200, shownClusters.length * 36 + 60);
canvas.width = W; canvas.height = H;
var pad = { top: 20, bottom: 30, left: 80, right: 30 };
var plotW = W - pad.left - pad.right;
var plotH = H - pad.top - pad.bottom;
var maxSize = Math.max(1, ...shownClusters.map(function(d) { return d.size; }));
ctx.clearRect(0, 0, W, H);
if (note) {
ctx.fillStyle = '#999'; ctx.font = '11px sans-serif'; ctx.textAlign = 'left';
ctx.fillText(note, pad.left, 15);
}
shownClusters.forEach(function(d, i) {
var barH = Math.min(32, plotH / shownClusters.length * 0.7);
var y = pad.top + i * (barH + 4);
var bw = Math.max(2, (d.size / maxSize) * plotW);
ctx.fillStyle = chartColors[i % chartColors.length];
ctx.fillRect(pad.left, y, bw, barH);
ctx.fillStyle = '#333';
ctx.font = '12px sans-serif';
ctx.textAlign = 'right';
ctx.fillText('#' + d.label, pad.left - 5, y + barH / 2 + 4);
ctx.textAlign = 'left';
ctx.fillStyle = '#666';
ctx.fillText(d.size, pad.left + bw + 5, y + barH / 2 + 4);
});
})();
// Silhouette score bar chart
(function() {
const canvas = document.getElementById('silhouetteChart');
if (!canvas) return;
const silData = clusterSizes.filter(d => d.silhouette !== null && d.silhouette !== undefined);
if (silData.length === 0) {
document.getElementById('silhouetteCard').style.display = 'none';
return;
}
const ctx = canvas.getContext('2d');
const W = canvas.parentElement.clientWidth || 600;
const H = Math.max(200, silData.length * 40 + 60);
canvas.width = W; canvas.height = H;
const pad = { top: 20, bottom: 30, left: 80, right: 40 };
const plotW = W - pad.left - pad.right;
const plotH = H - pad.top - pad.bottom;
const maxScore = Math.max(1, ...silData.map(d => Math.abs(d.silhouette)));
ctx.clearRect(0, 0, W, H);
silData.forEach((d, i) => {
const barH = Math.min(40, plotH / silData.length * 0.6);
const y = pad.top + i * (barH + 4);
const bw = (d.silhouette / maxScore) * plotW * 0.5;
const x = pad.left + plotW / 2;
ctx.fillStyle = d.silhouette >= 0 ? '#43aa8b' : '#f94144';
ctx.fillRect(x, y, d.silhouette >= 0 ? bw : -bw, barH);
ctx.fillStyle = '#333';
ctx.font = '12px sans-serif';
ctx.textAlign = 'right';
ctx.fillText('#' + d.label, pad.left - 5, y + barH / 2 + 4);
ctx.fillStyle = '#666';
if (d.silhouette >= 0) {
ctx.textAlign = 'left';
ctx.fillText(d.silhouette.toFixed(4), x + bw + 5, y - 2);
} else {
ctx.textAlign = 'right';
ctx.fillText(d.silhouette.toFixed(4), x - Math.abs(bw) - 5, y + barH + 12);
}
});
// Zero line
ctx.strokeStyle = '#999'; ctx.lineWidth = 1;
ctx.beginPath(); ctx.moveTo(pad.left + plotW / 2, pad.top); ctx.lineTo(pad.left + plotW / 2, H - pad.bottom); ctx.stroke();
ctx.fillStyle = '#999'; ctx.font = '11px sans-serif'; ctx.textAlign = 'center';
ctx.fillText('0', pad.left + plotW / 2, H - 5);
ctx.fillText('+1', W - pad.right, H - 5);
ctx.fillText('-1', pad.left, H - 5);
})();
// Top features per cluster horizontal bar charts
(function() {
const container = document.getElementById('featureChartsContainer');
if (!container) return;
clusterSizes.forEach(d => {
const label = d.label;
const features = topFeaturesAll[String(label)] || [];
if (features.length === 0) return;
const card = document.createElement('div');
card.style.marginBottom = '1.5rem';
card.innerHTML = '<h4 style="margin-bottom:0.5rem;">Cluster #' + label + ' - Top Features</h4><canvas class="feature-chart" data-label="' + label + '" style="width:100%;height:' + Math.max(120, features.length * 28 + 30) + 'px;"></canvas>';
container.appendChild(card);
const canvas = card.querySelector('canvas');
if (!canvas) return;
const ctx = canvas.getContext('2d');
const W2 = canvas.parentElement.clientWidth || 600;
const H2 = Math.max(120, features.length * 28 + 30);
canvas.width = W2; canvas.height = H2;
const pad2 = { top: 10, bottom: 20, left: 100, right: 60 };
const plotW2 = W2 - pad2.left - pad2.right;
const plotH2 = H2 - pad2.top - pad2.bottom;
const maxScore = Math.max(0.01, ...features.map(f => Math.abs(f.score || 0)));
ctx.clearRect(0, 0, W2, H2);
features.forEach((f, i) => {
const barH = Math.min(30, plotH2 / features.length * 0.6);
const y = pad2.top + i * (barH + 3);
const score = f.score || 0;
const bw = (score / maxScore) * plotW2;
ctx.fillStyle = chartColors[label % chartColors.length];
ctx.fillRect(pad2.left, y, bw, barH);
ctx.fillStyle = '#333';
ctx.font = '11px sans-serif';
ctx.textAlign = 'right';
const name = f.feature_name.length > 18 ? f.feature_name.slice(0, 16) + '..' : f.feature_name;
ctx.fillText(name, pad2.left - 5, y + barH / 2 + 4);
ctx.textAlign = 'left';
ctx.fillStyle = '#666';
ctx.fillText(score.toFixed(3), pad2.left + bw + 5, y + barH / 2 + 4);
});
});
})();
}
</script>
{% endblock %}