8fa36b774e
Key changes: - New: data_loader.py SQLite persistence with drop_sqlite_table - New: db_utils.py retry_on_lock decorator (3 retries, exponential backoff) - New: tool_registry.py with 27 MCP tools (filter_and_cluster, compute_scores, etc.) - New: tls_ref.py for TLS cipher/reference data - New: import_tlsdb.py management command - New: scripts/start_server.py for portable runtime - New: migrations 0003-0007 for SQLite table, display_id, llm fields - Changed: views.py unified pipeline worker, retry_run, display_id everywhere - Changed: models.py with display_id auto-assignment, run_type, sqlite_table, llm_thinking, tool_calls_json - Changed: urls.py added retry_run route - Changed: session_store.py robust JSON persistence - Changed: AGENTS.md v7 fix summary added - Changed: templates — globe rewrite (inertia/polar flip), auto.html (thinking/tool accordions), base.html (toast/config), all pages use display_id - Changed: run.bat PYTHONUTF8=1 - Deleted: entity_detector.py, entity_aggregator.py (replaced by filter_and_cluster clustering pipeline) - Test: 92/92 unit tests passing
335 lines
14 KiB
HTML
335 lines
14 KiB
HTML
{% extends 'base.html' %}
|
|
{% load static %}
|
|
{% block title %}Cluster Overview - Run #{{ run.display_id }}{% endblock %}
|
|
{% block content %}
|
|
<style>
|
|
.feature-chart { width: 100%; display: block; }
|
|
#featureChartsContainer canvas { max-height: 400px; }
|
|
.chart-wrapper { overflow-x: auto; }
|
|
.chart-container { position: relative; }
|
|
.canvas-legend {
|
|
max-height: 110px; overflow-y: auto;
|
|
display: flex; flex-wrap: wrap; gap: 4px 14px;
|
|
padding: 6px 10px; font-size: 11px;
|
|
border-top: 1px solid #eee; margin-top: 4px;
|
|
}
|
|
.canvas-legend .legend-item {
|
|
display: inline-flex; align-items: center; gap: 4px;
|
|
white-space: nowrap;
|
|
}
|
|
.canvas-legend .legend-swatch {
|
|
width: 10px; height: 10px; display: inline-block;
|
|
border-radius: 2px; flex-shrink: 0;
|
|
}
|
|
</style>
|
|
<div class="card">
|
|
<h2>Cluster Overview — Run #{{ run.display_id }}</h2>
|
|
<p>{{ run.total_flows }} flows, {{ run.cluster_count }} clusters</p>
|
|
</div>
|
|
|
|
<div class="card">
|
|
<h2>PCA-2D Embedding</h2>
|
|
<div class="chart-wrapper">
|
|
<div class="chart-container">
|
|
<canvas id="scatterChart" class="scatter-canvas"></canvas>
|
|
<div id="scatterLegend" class="canvas-legend"></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
{% if geo_count > 0 %}
|
|
<div class="card">
|
|
<h2>地理位置分布 <span class="badge badge-info">{{ geo_count }} entities</span></h2>
|
|
{% if geo_skipped > 0 %}
|
|
<p style="color:#888;font-size:0.85rem;">{{ geo_skipped }} entities excluded (missing lat/lon)</p>
|
|
{% endif %}
|
|
<div class="chart-wrapper">
|
|
<div class="chart-container">
|
|
<canvas id="geoChart" class="scatter-canvas"></canvas>
|
|
<div id="geoLegend" class="canvas-legend"></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
{% endif %}
|
|
|
|
<div class="card" id="clusterSizesCard">
|
|
<h2>Cluster Size Distribution</h2>
|
|
<div class="chart-wrapper">
|
|
<canvas id="clusterSizesChart" class="scatter-canvas"></canvas>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="card" id="silhouetteCard">
|
|
<h2>Silhouette Score Comparison</h2>
|
|
<div class="chart-wrapper">
|
|
<canvas id="silhouetteChart" class="scatter-canvas"></canvas>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="card" id="topFeaturesCard">
|
|
<h2>Top Distinguishing Features per Cluster</h2>
|
|
<div id="featureChartsContainer"></div>
|
|
</div>
|
|
|
|
<div class="grid-2">
|
|
{% for c in clusters %}
|
|
<div class="card">
|
|
<h3>Cluster #{{ c.cluster_label }} <span class="badge badge-info">{{ c.size }} entities</span></h3>
|
|
<p>Proportion: {{ c.proportion|floatformat:3 }}</p>
|
|
<p>Silhouette: {{ c.silhouette_score|floatformat:4|default:"-" }}</p>
|
|
<h4 style="margin-top:0.75rem;font-size:0.9rem;">Top Distinguishing Features</h4>
|
|
<table>
|
|
<thead>
|
|
<tr><th>Feature</th><th>Score</th><th>Mean</th></tr>
|
|
</thead>
|
|
<tbody>
|
|
{% for f in c.features.all|slice:":5" %}
|
|
<tr>
|
|
<td>{{ f.feature_name }}</td>
|
|
<td>{{ f.distinguishing_score|floatformat:3|default:"-" }}</td>
|
|
<td>{{ f.mean|floatformat:3|default:"-" }}</td>
|
|
</tr>
|
|
{% empty %}
|
|
<tr><td colspan="3">No features</td></tr>
|
|
{% endfor %}
|
|
</tbody>
|
|
</table>
|
|
<a href="{% url 'analysis:cluster_detail' run.display_id c.cluster_label %}" class="btn btn-primary" style="margin-top:0.75rem;">Detail</a>
|
|
</div>
|
|
{% empty %}
|
|
<div class="card">
|
|
<div class="empty-state"><p>No clusters found.</p></div>
|
|
</div>
|
|
{% endfor %}
|
|
</div>
|
|
|
|
{% if noise %}
|
|
<div class="card">
|
|
<h3>Noise (Cluster -1)</h3>
|
|
<p>{{ noise.size }} entities classified as noise</p>
|
|
</div>
|
|
{% endif %}
|
|
|
|
<script>
|
|
// Pure Canvas scatter plot — no Chart.js needed (offline compatible)
|
|
const scatterData = {{ scatter_data_json|safe }};
|
|
const colors = ['#4361ee','#f72585','#7209b7','#4cc9f0','#f8961e','#43aa8b','#f94144','#577590','#e9c46a','#9b5de5'];
|
|
|
|
function drawScatter(canvasId, data, xLabel, yLabel) {
|
|
const canvas = document.getElementById(canvasId);
|
|
if (!canvas || data.length === 0) return;
|
|
const ctx = canvas.getContext('2d');
|
|
const W = canvas.parentElement.clientWidth || 600;
|
|
const H = 400;
|
|
canvas.width = W; canvas.height = H;
|
|
const pad = { top: 30, bottom: 50, left: 60, right: 30 };
|
|
const plotW = W - pad.left - pad.right;
|
|
const plotH = H - pad.top - pad.bottom;
|
|
|
|
// Compute bounds
|
|
const xs = data.map(d => d.x); const ys = data.map(d => d.y);
|
|
const xMin = Math.min(...xs), xMax = Math.max(...xs);
|
|
const yMin = Math.min(...ys), yMax = Math.max(...ys);
|
|
const xRange = xMax - xMin || 1; const yRange = yMax - yMin || 1;
|
|
|
|
const toX = v => pad.left + (v - xMin) / xRange * plotW;
|
|
const toY = v => pad.top + plotH - (v - yMin) / yRange * plotH;
|
|
|
|
// Clear
|
|
ctx.clearRect(0, 0, W, H);
|
|
|
|
// Grid lines
|
|
ctx.strokeStyle = '#eee'; ctx.lineWidth = 1;
|
|
for (let i = 0; i <= 5; i++) {
|
|
const y = pad.top + i * plotH / 5;
|
|
ctx.beginPath(); ctx.moveTo(pad.left, y); ctx.lineTo(W - pad.right, y); ctx.stroke();
|
|
}
|
|
|
|
// Axes
|
|
ctx.strokeStyle = '#333'; ctx.lineWidth = 2;
|
|
ctx.beginPath(); ctx.moveTo(pad.left, pad.top); ctx.lineTo(pad.left, H - pad.bottom); ctx.stroke();
|
|
ctx.beginPath(); ctx.moveTo(pad.left, H - pad.bottom); ctx.lineTo(W - pad.right, H - pad.bottom); ctx.stroke();
|
|
|
|
// Labels
|
|
ctx.fillStyle = '#666'; ctx.font = '13px sans-serif'; ctx.textAlign = 'center';
|
|
ctx.fillText(xLabel, W / 2, H - 5);
|
|
ctx.save(); ctx.translate(15, H / 2); ctx.rotate(-Math.PI / 2); ctx.fillText(yLabel, 0, 0); ctx.restore();
|
|
|
|
// Points
|
|
data.forEach(d => {
|
|
const cx = toX(d.x), cy = toY(d.y);
|
|
const color = d.label === -1 ? 'rgba(150,150,150,0.5)' : colors[Math.abs(d.label) % colors.length];
|
|
const r = d.label === -1 ? 2 : 4;
|
|
ctx.beginPath(); ctx.arc(cx, cy, r, 0, Math.PI * 2);
|
|
ctx.fillStyle = color; ctx.fill();
|
|
});
|
|
|
|
// Tooltip on hover
|
|
canvas.onmousemove = function(e) {
|
|
const rect = canvas.getBoundingClientRect();
|
|
const mx = e.clientX - rect.left, my = e.clientY - rect.top;
|
|
const hit = data.find(d => Math.abs(toX(d.x) - mx) < 6 && Math.abs(toY(d.y) - my) < 6);
|
|
if (hit) {
|
|
canvas.title = `${hit.entity} (${hit.label === -1 ? 'Noise' : 'Cluster ' + hit.label})`;
|
|
}
|
|
};
|
|
|
|
// Legend — HTML overlay (supports overflow-y: auto via .canvas-legend)
|
|
const legendEl = document.getElementById(canvasId.replace('Chart', 'Legend'));
|
|
if (legendEl) {
|
|
legendEl.innerHTML = '';
|
|
const uniqueLabels = [...new Set(data.map(d => d.label))];
|
|
uniqueLabels.forEach(label => {
|
|
const color = label === -1 ? 'rgba(150,150,150,0.8)' : colors[Math.abs(label) % colors.length];
|
|
const item = document.createElement('span');
|
|
item.className = 'legend-item';
|
|
item.innerHTML = '<span class="legend-swatch" style="background:' + color + '"></span>' + (label === -1 ? 'Noise' : 'Cluster ' + label);
|
|
legendEl.appendChild(item);
|
|
});
|
|
}
|
|
}
|
|
|
|
drawScatter('scatterChart', scatterData, 'PCA-1', 'PCA-2');
|
|
|
|
{% if geo_count > 0 %}
|
|
const geoData = {{ geo_data_json|safe }};
|
|
drawScatter('geoChart', geoData, 'Longitude', 'Latitude');
|
|
{% endif %}
|
|
|
|
// ── Cluster size horizontal bar chart ──
|
|
const clusterSizes = {{ cluster_sizes_json|safe }};
|
|
const topFeaturesAll = {{ top_features_json|safe }};
|
|
const chartColors = ['#4361ee','#f72585','#7209b7','#4cc9f0','#f8961e','#43aa8b','#f94144','#577590','#e9c46a','#9b5de5'];
|
|
|
|
if (clusterSizes.length > 0) {
|
|
// Cluster size bar chart
|
|
(function() {
|
|
const canvas = document.getElementById('clusterSizesChart');
|
|
if (!canvas) return;
|
|
const ctx = canvas.getContext('2d');
|
|
var maxClusters = 50;
|
|
var shownClusters = clusterSizes.slice(0, maxClusters);
|
|
var note = '';
|
|
if (clusterSizes.length > maxClusters) {
|
|
note = ' (top ' + maxClusters + ' of ' + clusterSizes.length + ' clusters)';
|
|
}
|
|
var W = canvas.parentElement.clientWidth || 600;
|
|
var H = Math.max(200, shownClusters.length * 36 + 60);
|
|
canvas.width = W; canvas.height = H;
|
|
var pad = { top: 20, bottom: 30, left: 80, right: 30 };
|
|
var plotW = W - pad.left - pad.right;
|
|
var plotH = H - pad.top - pad.bottom;
|
|
var maxSize = Math.max(1, ...shownClusters.map(function(d) { return d.size; }));
|
|
ctx.clearRect(0, 0, W, H);
|
|
if (note) {
|
|
ctx.fillStyle = '#999'; ctx.font = '11px sans-serif'; ctx.textAlign = 'left';
|
|
ctx.fillText(note, pad.left, 15);
|
|
}
|
|
shownClusters.forEach(function(d, i) {
|
|
var barH = Math.min(32, plotH / shownClusters.length * 0.7);
|
|
var y = pad.top + i * (barH + 4);
|
|
var bw = Math.max(2, (d.size / maxSize) * plotW);
|
|
ctx.fillStyle = chartColors[i % chartColors.length];
|
|
ctx.fillRect(pad.left, y, bw, barH);
|
|
ctx.fillStyle = '#333';
|
|
ctx.font = '12px sans-serif';
|
|
ctx.textAlign = 'right';
|
|
ctx.fillText('#' + d.label, pad.left - 5, y + barH / 2 + 4);
|
|
ctx.textAlign = 'left';
|
|
ctx.fillStyle = '#666';
|
|
ctx.fillText(d.size, pad.left + bw + 5, y + barH / 2 + 4);
|
|
});
|
|
})();
|
|
|
|
// Silhouette score bar chart
|
|
(function() {
|
|
const canvas = document.getElementById('silhouetteChart');
|
|
if (!canvas) return;
|
|
const silData = clusterSizes.filter(d => d.silhouette !== null && d.silhouette !== undefined);
|
|
if (silData.length === 0) {
|
|
document.getElementById('silhouetteCard').style.display = 'none';
|
|
return;
|
|
}
|
|
const ctx = canvas.getContext('2d');
|
|
const W = canvas.parentElement.clientWidth || 600;
|
|
const H = Math.max(200, silData.length * 40 + 60);
|
|
canvas.width = W; canvas.height = H;
|
|
const pad = { top: 20, bottom: 30, left: 80, right: 40 };
|
|
const plotW = W - pad.left - pad.right;
|
|
const plotH = H - pad.top - pad.bottom;
|
|
const maxScore = Math.max(1, ...silData.map(d => Math.abs(d.silhouette)));
|
|
ctx.clearRect(0, 0, W, H);
|
|
silData.forEach((d, i) => {
|
|
const barH = Math.min(40, plotH / silData.length * 0.6);
|
|
const y = pad.top + i * (barH + 4);
|
|
const bw = (d.silhouette / maxScore) * plotW * 0.5;
|
|
const x = pad.left + plotW / 2;
|
|
ctx.fillStyle = d.silhouette >= 0 ? '#43aa8b' : '#f94144';
|
|
ctx.fillRect(x, y, d.silhouette >= 0 ? bw : -bw, barH);
|
|
ctx.fillStyle = '#333';
|
|
ctx.font = '12px sans-serif';
|
|
ctx.textAlign = 'right';
|
|
ctx.fillText('#' + d.label, pad.left - 5, y + barH / 2 + 4);
|
|
ctx.fillStyle = '#666';
|
|
if (d.silhouette >= 0) {
|
|
ctx.textAlign = 'left';
|
|
ctx.fillText(d.silhouette.toFixed(4), x + bw + 5, y - 2);
|
|
} else {
|
|
ctx.textAlign = 'right';
|
|
ctx.fillText(d.silhouette.toFixed(4), x - Math.abs(bw) - 5, y + barH + 12);
|
|
}
|
|
});
|
|
// Zero line
|
|
ctx.strokeStyle = '#999'; ctx.lineWidth = 1;
|
|
ctx.beginPath(); ctx.moveTo(pad.left + plotW / 2, pad.top); ctx.lineTo(pad.left + plotW / 2, H - pad.bottom); ctx.stroke();
|
|
ctx.fillStyle = '#999'; ctx.font = '11px sans-serif'; ctx.textAlign = 'center';
|
|
ctx.fillText('0', pad.left + plotW / 2, H - 5);
|
|
ctx.fillText('+1', W - pad.right, H - 5);
|
|
ctx.fillText('-1', pad.left, H - 5);
|
|
})();
|
|
|
|
// Top features per cluster horizontal bar charts
|
|
(function() {
|
|
const container = document.getElementById('featureChartsContainer');
|
|
if (!container) return;
|
|
clusterSizes.forEach(d => {
|
|
const label = d.label;
|
|
const features = topFeaturesAll[String(label)] || [];
|
|
if (features.length === 0) return;
|
|
const card = document.createElement('div');
|
|
card.style.marginBottom = '1.5rem';
|
|
card.innerHTML = '<h4 style="margin-bottom:0.5rem;">Cluster #' + label + ' - Top Features</h4><canvas class="feature-chart" data-label="' + label + '" style="width:100%;height:' + Math.max(120, features.length * 28 + 30) + 'px;"></canvas>';
|
|
container.appendChild(card);
|
|
const canvas = card.querySelector('canvas');
|
|
if (!canvas) return;
|
|
const ctx = canvas.getContext('2d');
|
|
const W2 = canvas.parentElement.clientWidth || 600;
|
|
const H2 = Math.max(120, features.length * 28 + 30);
|
|
canvas.width = W2; canvas.height = H2;
|
|
const pad2 = { top: 10, bottom: 20, left: 100, right: 60 };
|
|
const plotW2 = W2 - pad2.left - pad2.right;
|
|
const plotH2 = H2 - pad2.top - pad2.bottom;
|
|
const maxScore = Math.max(0.01, ...features.map(f => Math.abs(f.score || 0)));
|
|
ctx.clearRect(0, 0, W2, H2);
|
|
features.forEach((f, i) => {
|
|
const barH = Math.min(30, plotH2 / features.length * 0.6);
|
|
const y = pad2.top + i * (barH + 3);
|
|
const score = f.score || 0;
|
|
const bw = (score / maxScore) * plotW2;
|
|
ctx.fillStyle = chartColors[label % chartColors.length];
|
|
ctx.fillRect(pad2.left, y, bw, barH);
|
|
ctx.fillStyle = '#333';
|
|
ctx.font = '11px sans-serif';
|
|
ctx.textAlign = 'right';
|
|
const name = f.feature_name.length > 18 ? f.feature_name.slice(0, 16) + '..' : f.feature_name;
|
|
ctx.fillText(name, pad2.left - 5, y + barH / 2 + 4);
|
|
ctx.textAlign = 'left';
|
|
ctx.fillStyle = '#666';
|
|
ctx.fillText(score.toFixed(3), pad2.left + bw + 5, y + barH / 2 + 4);
|
|
});
|
|
});
|
|
})();
|
|
}
|
|
</script>
|
|
{% endblock %}
|