From 656f0ae2ab690811bc2779c7e038f6ebcec17267 Mon Sep 17 00:00:00 2001
From: PM-pinou <2504420230@qq.com>
Date: Fri, 24 Jul 2026 22:36:30 +0800
Subject: [PATCH] fix(upload): auto-delete uploaded CSV files after SQLite
 ingestion to save C drive space

---
 .server_pid                  |   1 -
 analysis/views/pipeline.py   |   9 ++
 scripts/gen_coverage_test.py | 186 ++++++++++++++++++++++++++++++++++
 server_stderr.txt            | 187 +++++++++++++----------------------
 server_stdout.txt            |   8 +-
 5 files changed, 273 insertions(+), 118 deletions(-)
 delete mode 100644 .server_pid
 create mode 100644 scripts/gen_coverage_test.py

diff --git a/.server_pid b/.server_pid
deleted file mode 100644
index 86d068e..0000000
--- a/.server_pid
+++ /dev/null
@@ -1 +0,0 @@
-11572
\ No newline at end of file
diff --git a/analysis/views/pipeline.py b/analysis/views/pipeline.py
index 861e5a5..114de71 100644
--- a/analysis/views/pipeline.py
+++ b/analysis/views/pipeline.py
@@ -262,6 +262,15 @@ def _background_process(run_id, upload_dir):
         # ── Phase 3: Data ready for filtering/clustering via MCP tools ──
         run.status = 'ready'; run.progress_pct = 60; run.progress_msg = '数据准备就绪'
         run.save(update_fields=['status', 'progress_pct', 'progress_msg'])
+
+        # ── Phase 4: Clean up uploaded CSV files ──
+        try:
+            if upload_dir and upload_dir.is_dir():
+                import shutil
+                shutil.rmtree(upload_dir, ignore_errors=True)
+                logger.info('[BACKGROUND] Cleaned up upload directory: %s', upload_dir)
+        except Exception as clean_err:
+            logger.warning('[BACKGROUND] Upload cleanup failed (non-fatal): %s', clean_err)
     except Exception:
         tb = traceback.format_exc()
         logger.error(tb)
diff --git a/scripts/gen_coverage_test.py b/scripts/gen_coverage_test.py
new file mode 100644
index 0000000..e661882
--- /dev/null
+++ b/scripts/gen_coverage_test.py
@@ -0,0 +1,186 @@
+"""Generate test CSVs with alternating high/low column coverage.
+Output files: 0.csv .. N-1.csv in specified directory.
+Columns that are all-blank in a CSV are dropped from that file.
+Usage:
+    runtime\python\python.exe scripts\gen_coverage_test.py --files 2500 --rows 10000 --output-dir ../test_data
+"""
+import argparse, csv, os, random, sys
+from datetime import datetime, timedelta
+from pathlib import Path
+
+# ── Column spec with coverage profiles ──
+# HIGH = present in 95-100% of files, MEDIUM = 40-60%, LOW = 5-15%
+COLUMNS = [
+    # HIGH coverage — always present
+    (':ips', 1.0), (':ipd', 1.0), (':prs', 1.0), (':prd', 1.0),
+    ('scnt', 1.0), ('dcnt', 1.0), ('tabl', 1.0), ('name', 1.0),
+    ('source-node', 1.0), ('server-ip', 1.0), ('client-ip', 1.0),
+    ('timestamp', 1.0), ('time', 1.0),
+    ('1ipp', 1.0), ('4dbn', 1.0),
+    # MEDIUM-HIGH coverage
+    (':ips.latd', 0.85), (':ips.lond', 0.85),
+    (':ipd.latd', 0.85), (':ipd.lond', 0.85),
+    (':ips.ispn', 0.80), (':ipd.ispn', 0.80),
+    (':ips.orgn', 0.70), (':ipd.orgn', 0.70),
+    (':ips.city', 0.85), (':ipd.city', 0.85),
+    ('4dur', 0.95), ('8seq', 0.95), ('2tmo', 0.90),
+    ('8ack', 0.95), ('8pak', 0.90), ('8byt', 0.85), ('8ppk', 0.80),
+    ('8ses', 0.80), ('8did', 0.85), ('4srs', 0.80),
+    # MEDIUM coverage — present in about half the files
+    ('0rnd', 0.55), ('0rnt', 0.55), ('0ver', 0.60),
+    ('snam', 0.50), ('cnam', 0.35),
+    ('0cph', 0.45), ('0crv', 0.35),
+    ('cnrs', 0.50), ('isrs', 0.50),
+    # LOW coverage — rarely present
+    (':ips.anon', 0.12), (':ipd.anon', 0.12),
+    (':ips.doma', 0.15), (':ipd.doma', 0.15),
+    ('4ksz', 0.20), ('cipher-suite', 0.15),
+    ('ecdhe-named-curve', 0.12),
+    ('crcc', 0.06), ('orga', 0.06), ('orgu', 0.05),
+    ('eiph', 0.02), ('@iph', 0.02),
+]
+
+def generate_row(row_idx: int, base_time: datetime, file_rng: random.Random,
+                 present_cols: set) -> dict:
+    """Generate a single row of TLS flow data."""
+    row = {}
+    src_octets = file_rng.randint(1, 223)
+    src_ip = f'{src_octets}.{file_rng.randint(0,255)}.{file_rng.randint(0,255)}.{file_rng.randint(1,254)}'
+    dst_octets = file_rng.randint(1, 223)
+    dst_ip = f'{dst_octets}.{file_rng.randint(0,255)}.{file_rng.randint(0,255)}.{file_rng.randint(1,254)}'
+    ts_val = base_time.timestamp() + file_rng.randint(0, 86400 * 7) + row_idx * 0.1
+
+    for col, _ in COLUMNS:
+        if col not in present_cols:
+            continue
+        v = ''
+        if col in (':ips', 'server-ip'):
+            v = src_ip
+        elif col in (':ipd', 'client-ip'):
+            v = dst_ip
+        elif col == ':prs':
+            v = str(file_rng.randint(1024, 65535))
+        elif col == ':prd':
+            v = str(file_rng.choice([80, 443, 8080, 8443, 53, 22, 25, 993, 3306, 6379]))
+        elif col in ('scnt', 'dcnt'):
+            v = file_rng.choice(['CN', 'US', 'JP', 'KR', 'GB', 'DE', 'SG', 'HK', 'AU', 'RU'])
+        elif col == 'tabl':
+            v = file_rng.choice(['TlsC', 'TlsS'])
+        elif col == 'name':
+            v = file_rng.choice(['流量_01', '流量_02', '流量_03', '流量_04', '流量_05'])
+        elif col == 'source-node':
+            v = file_rng.choice(['node-a', 'node-b', 'node-c', 'node-d'])
+        elif col in ('4dur', '8ses', '2tmo'):
+            v = f'{file_rng.uniform(1, 300):.2f}' if file_rng.random() < 0.95 else ''
+        elif col in ('8ack', '8pak', '8byt', '8ppk', '8did', '4srs'):
+            v = str(file_rng.randint(50, 100000))
+        elif col == '8seq':
+            v = f'{file_rng.uniform(1, 1000):.2f}' if file_rng.random() < 0.95 else ''
+        elif col in ('timestamp',):
+            v = f'{ts_val:.2f}'
+        elif col == 'time':
+            dt = datetime.fromtimestamp(ts_val)
+            v = dt.strftime('%Y-%m-%d-%H-%M-%S')
+        elif col == '1ipp':
+            v = str(file_rng.choice([6, 17, 1, 58]))
+        elif col == '4dbn':
+            v = str(file_rng.randint(1, 200))
+        elif col in (':ips.latd', ':ipd.latd'):
+            v = f'{file_rng.uniform(-90, 90):.4f}' if file_rng.random() < 0.98 else ''
+        elif col in (':ips.lond', ':ipd.lond'):
+            v = f'{file_rng.uniform(-180, 180):.4f}' if file_rng.random() < 0.98 else ''
+        elif col in (':ips.ispn', ':ipd.ispn'):
+            v = file_rng.choices(['电信', '联通', '移动', 'AWS', 'Azure', 'GCP', 'Cloudflare', ''], weights=[0.25,0.2,0.15,0.1,0.1,0.05,0.05,0.1], k=1)[0]
+        elif col in (':ips.orgn', ':ipd.orgn'):
+            v = file_rng.choices(['阿里巴巴', '腾讯', '华为', 'Amazon', 'Microsoft', 'Google', ''], weights=[0.2,0.15,0.1,0.15,0.1,0.05,0.25], k=1)[0]
+        elif col in (':ips.city', ':ipd.city'):
+            v = file_rng.choice(['北京', '上海', '深圳', '广州', '杭州', '东京', '新加坡', '伦敦', '纽约', '硅谷', '首尔', '悉尼', '法兰克福', ''])
+        elif col in (':ips.anon', ':ipd.anon'):
+            v = file_rng.choices(['', '+'], weights=[0.87, 0.13], k=1)[0]
+        elif col in (':ips.doma', ':ipd.doma'):
+            v = file_rng.choice(['example.com', 'api.example.com', 'cdn.example.net', 'db.internal', ''])
+        elif col in ('cnrs', 'isrs'):
+            v = file_rng.choices(['', '+', ''], weights=[0.5, 0.2, 0.3], k=1)[0]
+        elif col in ('0ver',):
+            v = file_rng.choices(['0303', '0304', '0302', ''], weights=[0.6, 0.3, 0.05, 0.05], k=1)[0]
+        elif col in ('snam',):
+            v = file_rng.choice(['www.example.com', 'api.server.com', 'cdn.cdn.net', 'mail.host.com', 'db.internal.net', ''])
+        elif col in ('cnam',):
+            v = file_rng.choice(['*.example.com', '*.server.com', 'www', ''])
+        elif col in ('0rnd',):
+            v = ''.join(file_rng.choices('0123456789abcdef', k=28)) if file_rng.random() < 0.99 else ''
+        elif col in ('0rnt',):
+            v = f'{file_rng.uniform(1, 2**32):.0f}' if file_rng.random() < 0.99 else ''
+        elif col in ('4ksz',):
+            v = str(file_rng.choice([128, 256, 1024, 2048, 4096]))
+        elif col in ('0cph', '0crv'):
+            v = file_rng.choice(['0e a6 3f 2b', 'ab cd ef 01', '12 34 56 78', '00 00 00 00', ''])
+        elif col in ('cipher-suite',):
+            v = file_rng.choice(['TLS_AES_128_GCM_SHA256', 'TLS_AES_256_GCM_SHA384', 'TLS_CHACHA20_POLY1305_SHA256', ''])
+        elif col in ('ecdhe-named-curve',):
+            v = file_rng.choice(['X25519', 'secp256r1', 'secp384r1', ''])
+        elif col in ('crcc',):
+            v = file_rng.choices(['AB', 'CD', 'EF', ''], weights=[0.4, 0.3, 0.1, 0.2], k=1)[0]
+        elif col in ('orga', 'orgu'):
+            v = file_rng.choice(['ORG_A', 'ORG_B', 'ORG_C', 'ORG_D', ''])
+        elif col in ('eiph', '@iph'):
+            v = file_rng.choices(['', '+', '', ''], weights=[0.02, 0.02, 0.48, 0.48], k=1)[0]
+        row[col] = v
+    return row
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Generate test CSVs with alternating column coverage')
+    parser.add_argument('--files', type=int, default=100, help='Number of CSV files')
+    parser.add_argument('--rows', type=int, default=10000, help='Rows per file')
+    parser.add_argument('--output-dir', type=str, default='../test_data', help='Output directory')
+    parser.add_argument('--seed', type=int, default=42, help='Random seed')
+    args = parser.parse_args()
+
+    out_dir = Path(args.output_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    base_time = datetime(2026, 1, 1, 0, 0, 0)
+    total_rows = 0
+    col_names = [c[0] for c in COLUMNS]
+
+    for file_idx in range(args.files):
+        file_rng = random.Random(args.seed + file_idx * 7)
+        # Decide per-file which LOW/MEDIUM columns are present
+        present = set()
+        for col, cov in COLUMNS:
+            if file_rng.random() < cov:
+                present.add(col)
+
+        # Generate rows, track which columns actually have data
+        non_blank = {c: False for c in present}
+        rows = []
+        for row_i in range(args.rows):
+            row = generate_row(total_rows + row_i, base_time, file_rng, present)
+            rows.append(row)
+            for col in present:
+                if row.get(col, '') and not non_blank[col]:
+                    non_blank[col] = True
+
+        # Determine final column list: only columns that have at least some data
+        final_cols = [c for c in col_names if non_blank.get(c, False)]
+        if not final_cols:
+            final_cols = [c for c in present]  # fallback
+
+        # Write file
+        out_path = out_dir / f'{file_idx}.csv'
+        with open(out_path, 'w', newline='', encoding='utf-8') as f:
+            writer = csv.DictWriter(f, fieldnames=final_cols, extrasaction='ignore')
+            writer.writeheader()
+            for row in rows:
+                writer.writerow(row)
+
+        total_rows += args.rows
+        if (file_idx + 1) % 100 == 0 or file_idx == 0 or file_idx == args.files - 1:
+            print(f'[{file_idx+1}/{args.files}] {total_rows} rows, {len(final_cols)} cols, file={file_idx}.csv', flush=True)
+
+    print(f'\nDone: {args.files} files x {args.rows} rows = {total_rows} total rows')
+    print(f'Output: {out_dir.resolve()}')
+
+
+if __name__ == '__main__':
+    main()
diff --git a/server_stderr.txt b/server_stderr.txt
index cb3dc31..c41144e 100644
--- a/server_stderr.txt
+++ b/server_stderr.txt
@@ -1,117 +1,72 @@
 Restored 1 datasets from disk
-[2026-07-24 14:04:56,246] INFO django: Restored 1 datasets from disk
-[2026-07-24 14:04:56,258] INFO analysis.data_loader._csv: [LOAD] files=1 total_rows_est=200
-[2026-07-24 14:04:56,258] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=0cph dtype=String
-[2026-07-24 14:04:56,258] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=0crv dtype=String
-[2026-07-24 14:04:56,258] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=0rnd dtype=String
-[2026-07-24 14:04:56,258] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=0rnt dtype=String
-[2026-07-24 14:04:56,258] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=0ver dtype=String
-[2026-07-24 14:04:56,258] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=1ipp dtype=Int64
-[2026-07-24 14:04:56,258] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=2tmo dtype=Float64
-[2026-07-24 14:04:56,265] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=4dbn dtype=Int64
-[2026-07-24 14:04:56,265] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=4dur dtype=Float64
-[2026-07-24 14:04:56,265] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=4ksz dtype=Int64
-[2026-07-24 14:04:56,265] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=4srs dtype=Int64
-[2026-07-24 14:04:56,265] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=8ack dtype=Int64
-[2026-07-24 14:04:56,265] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=8byt dtype=Int64
-[2026-07-24 14:04:56,265] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=8dbd dtype=Int64
-[2026-07-24 14:04:56,265] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=8did dtype=Int64
-[2026-07-24 14:04:56,265] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=8pak dtype=Int64
-[2026-07-24 14:04:56,265] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=8ppk dtype=Int64
-[2026-07-24 14:04:56,265] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=8seq dtype=Float64
-[2026-07-24 14:04:56,265] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=8ses dtype=Float64
-[2026-07-24 14:04:56,270] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ipd dtype=String
-[2026-07-24 14:04:56,270] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ipd.anon dtype=String
-[2026-07-24 14:04:56,270] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ipd.city dtype=String
-[2026-07-24 14:04:56,271] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ipd.doma dtype=String
-[2026-07-24 14:04:56,271] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ipd.ispn dtype=String
-[2026-07-24 14:04:56,271] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ipd.latd dtype=String
-[2026-07-24 14:04:56,271] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ipd.lond dtype=String
-[2026-07-24 14:04:56,271] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ipd.orgn dtype=String
-[2026-07-24 14:04:56,271] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ips dtype=String
-[2026-07-24 14:04:56,271] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ips.anon dtype=String
-[2026-07-24 14:04:56,271] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ips.city dtype=String
-[2026-07-24 14:04:56,274] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ips.doma dtype=String
-[2026-07-24 14:04:56,274] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ips.ispn dtype=String
-[2026-07-24 14:04:56,274] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ips.latd dtype=String
-[2026-07-24 14:04:56,274] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ips.lond dtype=String
-[2026-07-24 14:04:56,274] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ips.orgn dtype=String
-[2026-07-24 14:04:56,274] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:prd dtype=Int64
-[2026-07-24 14:04:56,274] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:prs dtype=Int64
-[2026-07-24 14:04:56,276] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=cipher-suite dtype=String
-[2026-07-24 14:04:56,276] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=client-ip dtype=String
-[2026-07-24 14:04:56,276] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=cnam dtype=String
-[2026-07-24 14:04:56,276] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=crcc dtype=String
-[2026-07-24 14:04:56,276] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=dcnt dtype=String
-[2026-07-24 14:04:56,276] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=ecdhe-named-curve dtype=String
-[2026-07-24 14:04:56,276] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=name dtype=String
-[2026-07-24 14:04:56,276] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=orga dtype=String
-[2026-07-24 14:04:56,276] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=orgu dtype=Int64
-[2026-07-24 14:04:56,280] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=scnt dtype=String
-[2026-07-24 14:04:56,280] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=server-ip dtype=String
-[2026-07-24 14:04:56,280] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=snam dtype=String
-[2026-07-24 14:04:56,280] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=source-node dtype=String
-[2026-07-24 14:04:56,280] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=tabl dtype=String
-[2026-07-24 14:04:56,280] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=time dtype=String
-[2026-07-24 14:04:56,280] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=timestamp dtype=Float64
-[2026-07-24 14:04:56,290] INFO analysis.data_loader._csv: [CLEAN] Generic numeric coercion applied: [':ipd.lond', '0cph', ':ips.anon', '0crv', ':ipd.doma', ':ipd.ispn', '0ver', 'scnt', 'cnam', ':ips.ispn', 'tabl', 'cipher-suite', ':ips.doma', 'orga', 'snam', 'time', ':ips.orgn', ':ipd.orgn', '0rnt', 'crcc', 'ecdhe-named-curve', 'dcnt', ':ips.latd', ':ipd.latd', ':ips.city', ':ipd.city', ':ipd.anon', ':ips.lond', '0rnd']
-[24/Jul/2026 14:05:07] "HEAD / HTTP/1.1" 200 0
-[2026-07-24 14:05:22,226] INFO analysis.data_loader._csv: [LOAD] files=1 total_rows_est=200
-[2026-07-24 14:05:22,292] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=0cph dtype=String
-[2026-07-24 14:05:22,293] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=0crv dtype=String
-[2026-07-24 14:05:22,293] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=0rnd dtype=String
-[2026-07-24 14:05:22,293] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=0rnt dtype=String
-[2026-07-24 14:05:22,293] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=0ver dtype=String
-[2026-07-24 14:05:22,293] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=1ipp dtype=Int64
-[2026-07-24 14:05:22,293] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=2tmo dtype=Float64
-[2026-07-24 14:05:22,293] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=4dbn dtype=Int64
-[2026-07-24 14:05:22,293] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=4dur dtype=Float64
-[2026-07-24 14:05:22,293] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=4ksz dtype=Int64
-[2026-07-24 14:05:22,293] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=4srs dtype=Int64
-[2026-07-24 14:05:22,293] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=8ack dtype=Int64
-[2026-07-24 14:05:22,293] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=8byt dtype=Int64
-[2026-07-24 14:05:22,293] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=8dbd dtype=Int64
-[2026-07-24 14:05:22,293] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=8did dtype=Int64
-[2026-07-24 14:05:22,293] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=8pak dtype=Int64
-[2026-07-24 14:05:22,293] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=8ppk dtype=Int64
-[2026-07-24 14:05:22,293] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=8seq dtype=Float64
-[2026-07-24 14:05:22,293] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=8ses dtype=Float64
-[2026-07-24 14:05:22,293] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ipd dtype=String
-[2026-07-24 14:05:22,293] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ipd.anon dtype=String
-[2026-07-24 14:05:22,293] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ipd.city dtype=String
-[2026-07-24 14:05:22,293] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ipd.doma dtype=String
-[2026-07-24 14:05:22,300] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ipd.ispn dtype=String
-[2026-07-24 14:05:22,300] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ipd.latd dtype=String
-[2026-07-24 14:05:22,301] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ipd.lond dtype=String
-[2026-07-24 14:05:22,301] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ipd.orgn dtype=String
-[2026-07-24 14:05:22,301] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ips dtype=String
-[2026-07-24 14:05:22,301] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ips.anon dtype=String
-[2026-07-24 14:05:22,301] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ips.city dtype=String
-[2026-07-24 14:05:22,301] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ips.doma dtype=String
-[2026-07-24 14:05:22,301] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ips.ispn dtype=String
-[2026-07-24 14:05:22,304] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ips.latd dtype=String
-[2026-07-24 14:05:22,304] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ips.lond dtype=String
-[2026-07-24 14:05:22,304] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ips.orgn dtype=String
-[2026-07-24 14:05:22,304] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:prd dtype=Int64
-[2026-07-24 14:05:22,304] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:prs dtype=Int64
-[2026-07-24 14:05:22,304] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=cipher-suite dtype=String
-[2026-07-24 14:05:22,304] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=client-ip dtype=String
-[2026-07-24 14:05:22,304] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=cnam dtype=String
-[2026-07-24 14:05:22,304] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=crcc dtype=String
-[2026-07-24 14:05:22,304] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=dcnt dtype=String
-[2026-07-24 14:05:22,304] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=ecdhe-named-curve dtype=String
-[2026-07-24 14:05:22,304] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=name dtype=String
-[2026-07-24 14:05:22,304] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=orga dtype=String
-[2026-07-24 14:05:22,304] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=orgu dtype=Int64
-[2026-07-24 14:05:22,309] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=scnt dtype=String
-[2026-07-24 14:05:22,310] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=server-ip dtype=String
-[2026-07-24 14:05:22,310] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=snam dtype=String
-[2026-07-24 14:05:22,310] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=source-node dtype=String
-[2026-07-24 14:05:22,310] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=tabl dtype=String
-[2026-07-24 14:05:22,310] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=time dtype=String
-[2026-07-24 14:05:22,310] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=timestamp dtype=Float64
-[2026-07-24 14:05:22,315] INFO analysis.data_loader._csv: [CLEAN] Generic numeric coercion applied: [':ipd.lond', '0cph', ':ips.anon', '0crv', ':ipd.doma', ':ipd.ispn', '0ver', 'scnt', 'cnam', ':ips.ispn', 'tabl', 'cipher-suite', ':ips.doma', 'orga', 'snam', 'time', ':ips.orgn', ':ipd.orgn', '0rnt', 'crcc', 'ecdhe-named-curve', 'dcnt', ':ips.latd', ':ipd.latd', ':ips.city', ':ipd.city', ':ipd.anon', ':ips.lond', '0rnd']
-[2026-07-24 14:05:22,337] INFO analysis.geoip: [GEOIP] loaded 0 ranges (0 skipped)
-[2026-07-24 14:05:22,337] INFO analysis.geoip: [GEOIP] loaded 211 cached entries
-[24/Jul/2026 14:05:22] "GET /clusters/9/ HTTP/1.1" 200 28390
-[24/Jul/2026 14:05:22] "GET /globe/?embed=1&data=upload_9 HTTP/1.1" 200 5545
+[2026-07-24 20:44:23,972] INFO django: Restored 1 datasets from disk
+[2026-07-24 20:44:23,986] INFO analysis.data_loader._csv: [LOAD] files=1 total_rows_est=200
+[2026-07-24 20:44:23,988] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=0cph dtype=String
+[2026-07-24 20:44:23,988] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=0crv dtype=String
+[2026-07-24 20:44:23,988] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=0rnd dtype=String
+[2026-07-24 20:44:23,988] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=0rnt dtype=String
+[2026-07-24 20:44:23,988] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=0ver dtype=String
+[2026-07-24 20:44:23,988] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=1ipp dtype=Int64
+[2026-07-24 20:44:23,988] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=2tmo dtype=Float64
+[2026-07-24 20:44:23,988] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=4dbn dtype=Int64
+[2026-07-24 20:44:23,993] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=4dur dtype=Float64
+[2026-07-24 20:44:23,993] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=4ksz dtype=Int64
+[2026-07-24 20:44:23,993] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=4srs dtype=Int64
+[2026-07-24 20:44:23,993] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=8ack dtype=Int64
+[2026-07-24 20:44:23,993] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=8byt dtype=Int64
+[2026-07-24 20:44:23,993] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=8dbd dtype=Int64
+[2026-07-24 20:44:23,993] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=8did dtype=Int64
+[2026-07-24 20:44:23,993] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=8pak dtype=Int64
+[2026-07-24 20:44:23,993] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=8ppk dtype=Int64
+[2026-07-24 20:44:23,993] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=8seq dtype=Float64
+[2026-07-24 20:44:23,993] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=8ses dtype=Float64
+[2026-07-24 20:44:23,993] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ipd dtype=String
+[2026-07-24 20:44:23,993] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ipd.anon dtype=String
+[2026-07-24 20:44:23,993] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ipd.city dtype=String
+[2026-07-24 20:44:23,993] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ipd.doma dtype=String
+[2026-07-24 20:44:23,993] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ipd.ispn dtype=String
+[2026-07-24 20:44:23,993] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ipd.latd dtype=String
+[2026-07-24 20:44:23,993] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ipd.lond dtype=String
+[2026-07-24 20:44:23,993] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ipd.orgn dtype=String
+[2026-07-24 20:44:23,993] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ips dtype=String
+[2026-07-24 20:44:23,993] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ips.anon dtype=String
+[2026-07-24 20:44:23,993] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ips.city dtype=String
+[2026-07-24 20:44:23,993] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ips.doma dtype=String
+[2026-07-24 20:44:23,993] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ips.ispn dtype=String
+[2026-07-24 20:44:23,993] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ips.latd dtype=String
+[2026-07-24 20:44:23,993] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ips.lond dtype=String
+[2026-07-24 20:44:24,004] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:ips.orgn dtype=String
+[2026-07-24 20:44:24,004] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:prd dtype=Int64
+[2026-07-24 20:44:24,004] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=:prs dtype=Int64
+[2026-07-24 20:44:24,004] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=cipher-suite dtype=String
+[2026-07-24 20:44:24,004] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=client-ip dtype=String
+[2026-07-24 20:44:24,004] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=cnam dtype=String
+[2026-07-24 20:44:24,004] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=crcc dtype=String
+[2026-07-24 20:44:24,004] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=dcnt dtype=String
+[2026-07-24 20:44:24,004] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=ecdhe-named-curve dtype=String
+[2026-07-24 20:44:24,004] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=name dtype=String
+[2026-07-24 20:44:24,004] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=orga dtype=String
+[2026-07-24 20:44:24,004] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=orgu dtype=Int64
+[2026-07-24 20:44:24,004] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=scnt dtype=String
+[2026-07-24 20:44:24,004] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=server-ip dtype=String
+[2026-07-24 20:44:24,004] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=snam dtype=String
+[2026-07-24 20:44:24,004] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=source-node dtype=String
+[2026-07-24 20:44:24,004] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=tabl dtype=String
+[2026-07-24 20:44:24,004] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=time dtype=String
+[2026-07-24 20:44:24,004] INFO analysis.data_loader._csv: [LOAD_SCHEMA] col=timestamp dtype=Float64
+[2026-07-24 20:44:24,004] INFO analysis.data_loader._csv: [CLEAN] Generic numeric coercion applied: [':ipd.lond', '0cph', ':ips.anon', '0crv', ':ipd.doma', ':ipd.ispn', '0ver', 'scnt', 'cnam', ':ips.ispn', 'tabl', 'cipher-suite', ':ips.doma', 'orga', 'snam', 'time', ':ips.orgn', ':ipd.orgn', '0rnt', 'crcc', 'ecdhe-named-curve', 'dcnt', ':ips.latd', ':ipd.latd', ':ips.city', ':ipd.city', ':ipd.anon', ':ips.lond', '0rnd']
+[24/Jul/2026 20:47:43] "GET /upload/ HTTP/1.1" 200 23884
+[24/Jul/2026 20:48:13] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 20:48:43] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 20:49:13] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 20:49:43] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 20:50:13] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 20:50:43] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 20:51:13] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 20:51:43] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 20:52:13] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 20:52:43] "HEAD / HTTP/1.1" 200 0
+[24/Jul/2026 20:53:13] "HEAD / HTTP/1.1" 200 0
+Restored 1 datasets from disk
+[2026-07-24 20:54:11,295] INFO django: Restored 1 datasets from disk
+Error: You don't have permission to access that port.
diff --git a/server_stdout.txt b/server_stdout.txt
index e65e70c..6057b01 100644
--- a/server_stdout.txt
+++ b/server_stdout.txt
@@ -1,8 +1,14 @@
 Performing system checks...
 
 System check identified no issues (0 silenced).
-July 24, 2026 - 14:04:56
+July 24, 2026 - 20:44:23
 Django version 4.2.30, using settings 'tianxuan.settings'
 Starting development server at http://127.0.0.1:8765/
 Quit the server with CTRL-BREAK.
 
+Performing system checks...
+
+System check identified no issues (0 silenced).
+Performing system checks...
+
+System check identified no issues (0 silenced).