fix: entity_count fallback + globe SQLite loading

- entity_count: fallback from ClusterResult sizes when PCA is skipped
- globe_view: load from SQLite table (_data_N) instead of csv_glob path
This commit is contained in:
PM-pinou
2026-07-21 13:26:50 +08:00
parent 745b1481cd
commit 9de1d93146
+25 -17
View File
@@ -816,6 +816,13 @@ def _run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorith
except Exception as pca_err:
logger.warning(f'PCA-2D embedding skipped: {pca_err}')
# Fallback: if entity_count not set (PCA skipped), derive from cluster sizes
if run.entity_count is None:
from django.db.models import Sum
total = run.clusters.aggregate(total=Sum('size'))['total'] or 0
run.entity_count = total
run.save(update_fields=['entity_count'])
# ── Done ──
run.status = 'completed'
run.progress_pct = 100
@@ -1358,7 +1365,7 @@ def _extract_flows_from_df(df, MAX_ROWS):
def globe_view(request):
import json
from analysis.geoip import lookup as geo_lookup
from analysis.data_loader import load_csv_directory
from analysis.data_loader import load_csv_directory, load_from_db
from analysis.models import AnalysisRun
from analysis.session_store import SessionStore
@@ -1412,7 +1419,11 @@ def globe_view(request):
for run in selected_runs:
try:
lf, schema, rc, fc, mem = load_csv_directory(run.csv_glob)
lf = None
if run.sqlite_table:
lf = load_from_db(run.sqlite_table)
if lf is None:
lf, schema, rc, fc, mem = load_csv_directory(run.csv_glob)
df = lf.head(MAX_ROWS_PER_RUN).collect()
flows.extend(_extract_flows_from_df(df, MAX_ROWS_PER_RUN))
all_selected_failed = False
@@ -1422,25 +1433,22 @@ def globe_view(request):
continue
if all_selected_failed or not flows:
# Fallback: latest completed run with valid files
# Fallback: latest completed run with valid data (SQLite or CSV)
for latest in AnalysisRun.objects.filter(status='completed').order_by('-id'):
if not latest.csv_glob:
if not latest.csv_glob and not latest.sqlite_table:
continue
glob_path = Path(latest.csv_glob)
try:
if '/*' in latest.csv_glob:
if glob_path.parent.is_dir():
lf, schema, rc, fc, mem = load_csv_directory(latest.csv_glob)
df = lf.head(MAX_ROWS_PER_RUN).collect()
flows.extend(_extract_flows_from_df(df, MAX_ROWS_PER_RUN))
selected_runs = [latest]
break
elif glob_path.exists():
lf = None
if latest.sqlite_table:
lf = load_from_db(latest.sqlite_table)
if lf is None:
if not latest.csv_glob:
continue
lf, schema, rc, fc, mem = load_csv_directory(latest.csv_glob)
df = lf.head(MAX_ROWS_PER_RUN).collect()
flows.extend(_extract_flows_from_df(df, MAX_ROWS_PER_RUN))
selected_runs = [latest]
break
df = lf.head(MAX_ROWS_PER_RUN).collect()
flows.extend(_extract_flows_from_df(df, MAX_ROWS_PER_RUN))
selected_runs = [latest]
break
except Exception:
continue