v1.1.6: batch upload BATCH=5 + temp dir to D: + schema sort + update.bat fix

TlsDB.csv

@@ -21,9 +21,8 @@ scnt,源国家,2小写字母缩写,100%
 dcnt,目标国家,2小写字母缩写,100%
 server-ip,服务器IP,IPv4,100%
 client-ip,客户端IP,IPv4,100%
-row,数据列,整数,100%
-time,时间,XX:YY.Z,100%
-timestamp,时间戳,整数,100%
+time,时间,202X-XX-XX XX:XX:XX.XXXXXX,100%
+timestamp,时间戳,六位小数浮点,100%
 1ipp,IP协议编号,整数,100%
 4dbn,数据库编号,整数,100%
 tabl,数据库表头名,"TlsC"/"TlsS",100%
@@ -40,7 +39,7 @@ name,链路名称,枚举,100%
 source-node,源节点,枚举,100%
 cipher-suite,加密套件,诸如"TLS_ECHDE_ECDSA_WITH_AES_256_GCM_SHA384"的枚举,13.90%
 ecdhe-named-curve,ecdhe使用的圆锥曲线名称,secpXXXr1(其中XXX为三位数字),10.73%
-0cph,TLS加密套件,2字节,54.14%
+0cph,TLS加密套件,2字节,52.14%
 0crv,TLS圆锥曲线名称,2字节,12.45%
 0rnd,TLS随机值,28字节,99.14%
 0rnt,TLS随机时间,4字节,100%

_restart_server.py

@@ -0,0 +1,32 @@
+import subprocess, time, urllib.request, os, sys
+
+PROJ_DIR = r"C:\Users\25044\Desktop\Proj\天璇"
+PYTHON_EXE = os.path.join(PROJ_DIR, "runtime", "python", "python.exe")
+os.chdir(PROJ_DIR)
+
+server = subprocess.Popen(
+    [PYTHON_EXE, "manage.py", "runserver", "127.0.0.1:8765", "--noreload"],
+    stdout=subprocess.DEVNULL,
+    stderr=subprocess.DEVNULL
+)
+
+print(f"SERVER_PID={server.pid}")
+
+# Poll with backoff: fast early, then slower
+deadline = time.time() + 60
+attempt = 0
+while time.time() < deadline:
+    attempt += 1
+    try:
+        resp = urllib.request.urlopen("http://127.0.0.1:8765/", timeout=3)
+        if resp.status == 200:
+            print("SERVER_READY")
+            sys.exit(0)
+    except Exception as e:
+        if attempt % 10 == 0:
+            print(f"WAITING... attempt={attempt} err={type(e).__name__}")
+        time.sleep(1)
+
+print("SERVER_START_TIMEOUT")
+server.kill()
+sys.exit(1)

analysis/tool_registry.py

@@ -1168,12 +1168,34 @@ async def _handle_run_clustering(
                 if dt in numeric_dtypes and not available_names[i].startswith('_')
             ][:10]
         if not feature_cols:
-            all_numeric = [f"{available_names[i]}" for i, dt in enumerate(available_dtypes)
-                           if dt in numeric_dtypes]
-            return {
-                'error': f'No numeric cluster columns found in dataset. Available: {all_numeric}',
-                'truncated': False
-            }
+            # Fallback: try coercing Utf8 columns to Float64 (common when data loaded from
+            # SQLite TEXT storage without proper dtype restoration).
+            utf8_cols = [
+                available_names[i] for i, dt in enumerate(available_dtypes)
+                if dt == pl.Utf8 and not available_names[i].startswith('_')
+            ][:10]
+            if utf8_cols:
+                # Attempt per-column cast; filter to columns where coercion succeeded
+                for col in utf8_cols:
+                    try:
+                        casted = lf.select(
+                            pl.col(col).cast(pl.Float64, strict=False).alias(col)
+                        ).collect(streaming=True)
+                        null_ratio = casted[col].is_null().sum() / max(len(casted), 1)
+                        if null_ratio < 0.9:  # at least 10% successfully parsed as floats
+                            lf = lf.with_columns(
+                                pl.col(col).cast(pl.Float64, strict=False)
+                            )
+                            feature_cols.append(col)
+                    except Exception:
+                        pass
+            if not feature_cols:
+                all_numeric = [f"{available_names[i]}" for i, dt in enumerate(available_dtypes)
+                               if dt in numeric_dtypes]
+                return {
+                    'error': f'No numeric cluster columns found in dataset. Available: {all_numeric}',
+                    'truncated': False
+                }
 
         # Downsample before collect: compute row count via fast lazy count
         try:
@@ -1257,7 +1279,9 @@ async def _handle_run_clustering(
 
         # Standard scale
         from sklearn.preprocessing import StandardScaler
+        _clust_logger.info('[CLUSTER_SCALE] Starting StandardScaler...')
         data_scaled = StandardScaler().fit_transform(data)
+        _clust_logger.info(f'[CLUSTER_SCALE] Done. shape={data_scaled.shape}')
 
         # ── H17: Correlation filtering ────────────────────────────────────
         # Remove features with Pearson correlation > 0.95 (keep the first one)

analysis/urls.py

@@ -6,6 +6,7 @@ urlpatterns = [
     path('', views.dashboard, name='dashboard'),
     path('upload/', views.upload_page, name='upload'),
     path('upload/csv/', views.upload_csv, name='upload_csv'),
+    path('upload/csv/batch/<int:display_id>/', views.upload_csv_batch, name='upload_csv_batch'),
     path('analyze/manual/', views.manual_page, name='manual'),
     path('analyze/run/', views.manual_run_analysis, name='run_analysis'),
     path('analyze/auto/', views.auto_page, name='auto'),

analysis/views.py

@@ -339,6 +339,29 @@ def upload_csv(request):
     return JsonResponse({'run_id': run.display_id, 'files': saved, 'status': 'loading'})
 
 
+@csrf_exempt
+def upload_csv_batch(request, display_id):
+    """Append files to an existing upload run in batches."""
+    run = get_object_or_404(AnalysisRun, display_id=display_id)
+    files = request.FILES.getlist('files')
+    if not files:
+        return JsonResponse({'error': 'No files uploaded'})
+    upload_dir = Path(run.csv_glob).parent if run.csv_glob else None
+    if not upload_dir or not upload_dir.exists():
+        return JsonResponse({'error': 'Upload directory not found'})
+    saved = []
+    for f in files:
+        path = upload_dir / f.name
+        with open(path, 'wb+') as dst:
+            for chunk in f.chunks():
+                dst.write(chunk)
+        saved.append(str(path))
+    # Re-trigger _background_process to reload all files including new ones
+    t = threading.Thread(target=_background_process, args=(run.id, upload_dir), daemon=True)
+    t.start()
+    return JsonResponse({'run_id': run.display_id, 'files': saved, 'status': 'loading'})
+
+
 @csrf_exempt
 def delete_upload(request, display_id):
     """Delete uploaded files, SQLite data table, and AnalysisRun.

data/multi_upload/upload_script.py

@@ -1,42 +0,0 @@
-import os, sys, django, urllib.request, glob, uuid, json, time
-os.environ['DJANGO_SETTINGS_MODULE'] = 'tianxuan.settings'
-sys.path.insert(0, r'C:\Users\25044\Desktop\Proj\天璇')
-import django; django.setup()
-from analysis.models import AnalysisRun
-AnalysisRun.objects.all().delete()
-print('Cleaned old runs')
-
-BASE = 'http://127.0.0.1:18766'
-data_dir = r'C:\Users\25044\Desktop\Proj\天璇\data\multi_upload'
-files = sorted(glob.glob(os.path.join(data_dir, '*.csv')))
-boundary = uuid.uuid4().hex
-lines = []
-for fp in files:
-    with open(fp, 'rb') as f: d = f.read()
-    fname = os.path.basename(fp)
-    lines.append(f'--{boundary}'.encode())
-    lines.append(f'Content-Disposition: form-data; name="files"; filename="{fname}"'.encode())
-    lines.append(b'Content-Type: text/csv'); lines.append(b''); lines.append(d)
-lines.append(f'--{boundary}--'.encode())
-req = urllib.request.Request(f'{BASE}/upload/csv/', data=b'\r\n'.join(lines),
-    headers={'Content-Type': f'multipart/form-data; boundary={boundary}'})
-r = urllib.request.urlopen(req, timeout=300)
-resp = json.loads(r.read()); r.close()
-print(f'Uploaded run_id={resp["run_id"]}')
-
-run_id = resp['run_id']
-for i in range(120):
-    time.sleep(2)
-    try:
-        req2 = urllib.request.Request(f'{BASE}/runs/{run_id}/status/')
-        r2 = urllib.request.urlopen(req2, timeout=10)
-        status_resp = json.loads(r2.read()); r2.close()
-        status = status_resp.get('status', status_resp.get('state', ''))
-        print(f'  attempt {i+1}: status={status}')
-        if status == 'ready':
-            print(f'Run {run_id} is ready!')
-            break
-    except Exception as e:
-        print(f'  attempt {i+1}: error={e}')
-else:
-    print('Timed out waiting for ready status')

scripts/gen_fast.py

@@ -0,0 +1,368 @@
+"""Polars-vectorized test data generator — 1998 files × 10000 rows in seconds.
+Uses numpy pools + Polars DataFrames for fully vectorized generation.
+Batched to avoid OOM (500K-row chunks).
+
+Usage: runtime\\python\\python.exe scripts\\gen_fast.py [--files N] [--rows N]
+"""
+import polars as pl
+import numpy as np
+import csv
+import argparse
+import time
+from pathlib import Path
+from datetime import datetime
+
+PROJ = Path(r"C:\Users\25044\Desktop\Proj\天璇")
+OUT = PROJ / "data" / "multi_upload"
+OUT.mkdir(parents=True, exist_ok=True)
+SEED = 42
+N_FILES = 1998
+N_ROWS = 10000
+TOTAL = N_FILES * N_ROWS
+
+# ── Read TlsDB.csv coverage ──────────────────────────────────────────────
+coverage: dict[str, float] = {}
+all_columns: list[str] = []
+with open(PROJ / "TlsDB.csv", encoding="utf-8") as f:
+    for row in csv.DictReader(f):
+        k = row['标题'].strip()
+        all_columns.append(k)
+        pct = row.get('覆盖率', '100%').strip().rstrip('%')
+        try:
+            coverage[k] = float(pct) / 100.0
+        except (ValueError, KeyError):
+            coverage[k] = 1.0
+
+# Identity columns always present in every file
+_IDENTITY = frozenset({':ips', ':ipd', ':prs', ':prd', 'server-ip', 'client-ip'})
+
+# ── Pre-build string pools (numpy-backed for vectorized choice) ──────────
+rng = np.random.default_rng(SEED)
+
+def _make_ip_pool(n_seed: int, size: int) -> np.ndarray:
+    """Generate `size` random IPv4 strings as a numpy array."""
+    gen = np.random.default_rng(SEED + n_seed)
+    a = gen.integers(1, 224, size)
+    b = gen.integers(0, 256, size)
+    c = gen.integers(0, 256, size)
+    d = gen.integers(1, 255, size)
+    # Build strings via list comp (pool generation is O(pool_size), one-time cost)
+    return np.array([f"{a[i]}.{b[i]}.{c[i]}.{d[i]}" for i in range(size)], dtype=object)
+
+SRC_IP = _make_ip_pool(1, 50000)
+DST_IP = _make_ip_pool(2, 50000)
+SVR_IP = _make_ip_pool(3, 50000)
+CLI_IP = _make_ip_pool(4, 50000)
+
+PORT_STR = np.array(['443', '80', '8080', '8443', '53', '22', '25', '993'], dtype=object)
+COUNTRY_STR = np.array(['us','cn','jp','gb','de','fr','ru','br','in','sg','kr','au','ca'], dtype=object)
+COUNTRY_UPPER = np.array([s.upper() for s in COUNTRY_STR], dtype=object)
+ISP_STR = np.array(['China Telecom','China Mobile','BT Group','Orange','Deutsche Telekom',
+                     'AT&T','Verizon','Comcast','NTT','KDDI','SK Telecom','Singtel','Telstra'], dtype=object)
+ORG_STR = np.array(['Baidu Inc.','Alibaba Inc.','Amazon.com Inc.','Google LLC','Microsoft Corp.',
+                     'Meta Platforms','Tencent','Samsung','Apple Inc.','Netflix Inc.'], dtype=object)
+CITY_STR = np.array(['Beijing','Shanghai','Seoul','Berlin','Paris','London','New York',
+                      'San Francisco','Tokyo','Singapore','Sydney','Moscow','Seattle','Dublin','Mumbai'], dtype=object)
+SERVICE_STR = np.array(['mail.example.com','api.example.com','cdn.example.com','auth.example.net',
+                         'stream.example.org','login.live.com','*.cloudfront.net','*.s3.amazonaws.com',
+                         'graph.facebook.com','www.google.com'], dtype=object)
+SRC_NODE_STR = np.array(['packet_capture','ssl_logs','netflow','zeek','suricata'], dtype=object)
+NODE_NAME_STR = np.array(['wan-link','core-02','gw-09','edge-01','backbone-03'], dtype=object)
+TLS_VER_NEW = np.array(['0303','0304','0301','0302'], dtype=object)
+TLS_VER_OLD = np.array(['03 03','03 04','02 00','03 01','03 02'], dtype=object)
+TLS_VER_W = np.array([0.55, 0.30, 0.10, 0.05], dtype=np.float64)
+TLS_VER_W5 = np.array([0.55, 0.30, 0.05, 0.05, 0.05], dtype=np.float64)
+TABLE_STR = np.array(['TlsC','TlsS'], dtype=object)
+CIPHER_HEX = np.array(['c0 2b','c0 2f','13 01','13 02','c0 2c','cc a9','c0 23','c0 27',
+                        'c0 13','c0 14','00 9e','00 9f','00 35'], dtype=object)
+CIPHER_NAME = np.array(['TLS_ECDHE_ECDSA_AES128_GCM_SHA256','TLS_ECDHE_RSA_AES128_GCM_SHA256',
+                         'TLS_AES_128_GCM_SHA256','TLS_AES_256_GCM_SHA384',
+                         'TLS_ECDHE_ECDSA_AES256_GCM_SHA384','TLS_ECDHE_ECDSA_CHACHA20_POLY1305',
+                         'TLS_ECDHE_RSA_AES256_GCM_SHA384','TLS_ECDHE_RSA_AES128_SHA256',
+                         'TLS_ECDHE_ECDSA_AES128_SHA','TLS_ECDHE_RSA_AES128_SHA',
+                         'TLS_RSA_WITH_AES_128_GCM_SHA256','TLS_RSA_WITH_AES_256_GCM_SHA384',
+                         'TLS_RSA_WITH_AES_128_CBC_SHA'], dtype=object)
+CURVE_HEX = np.array(['00 1d','00 17','00 18','00 19','00 1e'], dtype=object)
+CURVE_NAME = np.array(['secp256r1','secp384r1','secp521r1','x25519','x448'], dtype=object)
+
+
+def _bernoulli_mask(rng_state: np.random.Generator, prob: float, size: int) -> np.ndarray:
+    """Vectorized Bernoulli mask: random < prob -> keep (~True), else blank."""
+    return rng_state.random(size) < prob
+
+
+def _apply_coverage(rng_state: np.random.Generator, arr: np.ndarray, col_name: str,
+                    size: int, blank_val: str = '') -> np.ndarray:
+    """Return a copy of `arr` with entries blanked according to coverage[col_name]."""
+    cov = coverage.get(col_name, 1.0)
+    if cov >= 1.0:
+        return arr.copy()
+    keep = _bernoulli_mask(rng_state, cov, size)
+    out = np.empty(size, dtype=object)
+    out[keep] = arr[keep]
+    out[~keep] = blank_val
+    return out
+
+
+def _apply_coverage_plus(rng_state: np.random.Generator, arr: np.ndarray, col_name: str,
+                          size: int) -> np.ndarray:
+    """Return a copy with blank entries set to '+'. """
+    return _apply_coverage(rng_state, arr, col_name, size, blank_val='+')
+
+
+# ── Generate one chunk (CHUNK_SIZE rows) → Polars DataFrame ──────────────
+BASE_TS = datetime(2026, 6, 1).timestamp()
+CHUNK_SIZE = 500_000  # 50 files worth
+
+def _generate_chunk_df(global_start: int, chunk_size: int, seed_offset: int) -> pl.DataFrame:
+    """Generate chunk_size rows of TLS test data as a Polars DataFrame."""
+    cr = np.random.default_rng(SEED + seed_offset)
+    n = chunk_size
+    idx = np.arange(n, dtype=np.int64)
+
+    # Timestamps: linear spread over ~1 year
+    ts = np.round(np.linspace(BASE_TS + global_start * 0.001,
+                               BASE_TS + (global_start + n) * 0.001,
+                               n, dtype=np.float64), 6)
+
+    # Time strings: build from datetime64 with manual formatting.
+    # numpy datetime_as_string + loop for microsecond padding (only ~500K rows).
+    ns = (ts * 1e9).astype(np.int64)
+    dt64 = ns.astype('datetime64[ns]')
+    time_strs = np.datetime_as_string(dt64, unit='s')  # '2026-06-01T00:00:00'
+    time_strs = np.char.replace(time_strs.astype(str), 'T', ' ')
+    # Compute microseconds and zero-pad
+    ms = (ts * 1_000_000).astype(np.int64)
+    usecs = (ms % 1_000_000).astype(str)
+    time_col = np.array([f"{time_strs[i]}.{int(usecs[i]):06d}" for i in range(n)], dtype=object)
+
+    # IPs — vectorized random choice from pools
+    src_ip = cr.choice(SRC_IP, n)
+    dst_ip = cr.choice(DST_IP, n)
+    srv_ip = cr.choice(SVR_IP, n)
+    cli_ip = cr.choice(CLI_IP, n)
+
+    # Ports
+    src_pt = cr.choice(PORT_STR, n)
+    dst_pt = cr.choice(PORT_STR, n)
+
+    # Countries
+    scnt = _apply_coverage(cr, cr.choice(COUNTRY_STR, n), 'scnt', n)
+    dcnt = _apply_coverage(cr, cr.choice(COUNTRY_STR, n), 'dcnt', n)
+
+    # TLS versions: 0ver (2-byte hex compact: 0303/0304) and old-style
+    tls_ver_new = cr.choice(TLS_VER_NEW, n, p=TLS_VER_W)
+    tls_ver_old = cr.choice(TLS_VER_OLD, n, p=TLS_VER_W5)
+
+    # Table type
+    tabl = cr.choice(TABLE_STR, n)
+
+    # GeoIP lat/lon — in separate function calls below
+    # ISP, Org, City
+    ips_ispn = _apply_coverage(cr, cr.choice(ISP_STR, n), ':ips.ispn', n)
+    ipd_ispn = _apply_coverage(cr, cr.choice(ISP_STR, n), ':ipd.ispn', n)
+    ips_orgn = _apply_coverage(cr, cr.choice(ORG_STR, n), ':ips.orgn', n)
+    ipd_orgn = _apply_coverage(cr, cr.choice(ORG_STR, n), ':ipd.orgn', n)
+    ips_city = _apply_coverage(cr, cr.choice(CITY_STR, n), ':ips.city', n)
+    ipd_city = _apply_coverage(cr, cr.choice(CITY_STR, n), ':ipd.city', n)
+    ips_anon = _apply_coverage(cr, np.full(n, 'anon, hosting', dtype=object), ':ips.anon', n)
+    ipd_anon = _apply_coverage(cr, np.full(n, 'anon, hosting', dtype=object), ':ipd.anon', n)
+    ips_doma = _apply_coverage(cr, cr.choice(SERVICE_STR, n), ':ips.doma', n)
+    ipd_doma = _apply_coverage(cr, cr.choice(SERVICE_STR, n), ':ipd.doma', n)
+
+    # Lat/Lon — convert to string to avoid mixed-type arrays
+    ips_latd = _apply_coverage_plus(cr, np.round(cr.uniform(-60, 60, n), 6).astype(str).astype(object),
+                                     ':ips.latd', n)
+    ips_lond = _apply_coverage_plus(cr, np.round(cr.uniform(-180, 180, n), 6).astype(str).astype(object),
+                                     ':ips.lond', n)
+    ipd_latd = _apply_coverage_plus(cr, np.round(cr.uniform(-60, 60, n), 6).astype(str).astype(object),
+                                     ':ipd.latd', n)
+    ipd_lond = _apply_coverage_plus(cr, np.round(cr.uniform(-180, 180, n), 6).astype(str).astype(object),
+                                     ':ipd.lond', n)
+
+    # TLS cipher / curve
+    cipher_suite = _apply_coverage(cr, cr.choice(CIPHER_NAME, n), 'cipher-suite', n)
+    curve_name_c = _apply_coverage(cr, cr.choice(CURVE_NAME, n), 'ecdhe-named-curve', n)
+    cph = _apply_coverage(cr, cr.choice(CIPHER_HEX, n), '0cph', n)
+    crv = _apply_coverage(cr, cr.choice(CURVE_HEX, n), '0crv', n)
+
+    # 0rnd: 28 random hex bytes
+    hex_pool = np.array([f'{b:02x}' for b in range(256)], dtype=object)
+    def _rand_hex_vec(rng_st, n, nbytes):
+        h = rng_st.integers(0, 256, (n, nbytes))
+        return np.array([' '.join(hex_pool[h[i]]) for i in range(n)], dtype=object)
+
+    rnd = _apply_coverage(cr, _rand_hex_vec(cr, n, 28), '0rnd', n)
+    rnt = _apply_coverage(cr, _rand_hex_vec(cr, n, 4), '0rnt', n)
+
+    # 4ksz
+    ksz_pool = np.array(['128','256','384','512'], dtype=object)
+    ksz = _apply_coverage(cr, cr.choice(ksz_pool, n), '4ksz', n)
+
+    # snam / cnam
+    snam = _apply_coverage(cr, cr.choice(SERVICE_STR, n), 'snam', n)
+    cnam = _apply_coverage(cr, cr.choice(SERVICE_STR, n), 'cnam', n)
+
+    # Durations, packet counts (numeric → string)
+    def _int_col(rng_st, lo, hi, col_name, n):
+        vals = rng_st.integers(lo, hi, n).astype(str).astype(object)
+        return _apply_coverage(rng_st, vals, col_name, n)
+
+    def _float_col(rng_st, lo, hi, col_name, n):
+        vals = np.round(rng_st.uniform(lo, hi, n), 4).astype(str).astype(object)
+        return _apply_coverage(rng_st, vals, col_name, n)
+
+    four_dur = _float_col(cr, 0.5, 120.0, '4dur', n)
+    ack = _int_col(cr, 100, 200000, '8ack', n)
+    pak = _int_col(cr, 1, 2000, '8pak', n)
+    ppk = _int_col(cr, 1, 500, '8ppk', n)
+    seq = _float_col(cr, 1, 2000, '8seq', n)
+    ses = _float_col(cr, 1, 2000, '8ses', n)
+    byt = _int_col(cr, 200, 500000, '8byt', n)
+    tmo = _float_col(cr, 0, 1000, '2tmo', n)
+    ipp = _int_col(cr, 1, 50, '1ipp', n)
+    dbn = _int_col(cr, 1, 100, '4dbn', n)
+    dbd = _int_col(cr, 1, 200, '8dbd', n)
+    did = _int_col(cr, 1000, 9999, '8did', n)
+    srs = _int_col(cr, 10000, 99999, '4srs', n)
+
+    # name / source-node
+    nm = _apply_coverage(cr, cr.choice(NODE_NAME_STR, n), 'name', n)
+    src_node = _apply_coverage(cr, cr.choice(SRC_NODE_STR, n), 'source-node', n)
+
+    # cnrs / isrs (boolean-like, '+')
+    cnrs_val = _apply_coverage_plus(cr, np.full(n, '+', dtype=object), 'cnrs', n)
+    isrs_val = _apply_coverage_plus(cr, np.full(n, '+', dtype=object), 'isrs', n)
+
+    # crcc / @iph / eiph / orga / orgu
+    crcc_val = _apply_coverage(cr, cr.choice(np.array(['OK','ER','--'], dtype=object), n), 'crcc', n)
+    atiph_val = _apply_coverage(cr, cr.choice(np.array(['A','B','C','D'], dtype=object), n), '@iph', n)
+    eiph_val = _apply_coverage_plus(cr, np.full(n, '+', dtype=object), 'eiph', n)
+    orga_val = _apply_coverage(cr, cr.choice(ORG_STR, n), 'orga', n)
+    orgu_val = _apply_coverage(cr, cr.integers(1000, 99999, n).astype(str).astype(object), 'orgu', n)
+
+    # server-ip / client-ip (always present)
+    srv_ip2 = srv_ip.copy()
+    cli_ip2 = cli_ip.copy()
+
+    # Build DataFrame
+    df = pl.DataFrame({
+        ':ips': pl.Series('', src_ip),
+        ':ipd': pl.Series('', dst_ip),
+        ':prs': pl.Series('', src_pt),
+        ':prd': pl.Series('', dst_pt),
+        'server-ip': pl.Series('', srv_ip2),
+        'client-ip': pl.Series('', cli_ip2),
+        'time': pl.Series('', time_col),
+        'timestamp': pl.Series('', ts),
+        'scnt': pl.Series('', scnt),
+        'dcnt': pl.Series('', dcnt),
+        '0ver': pl.Series('', tls_ver_new),
+        'tabl': pl.Series('', tabl),
+        ':ips.latd': pl.Series('', ips_latd),
+        ':ips.lond': pl.Series('', ips_lond),
+        ':ipd.latd': pl.Series('', ipd_latd),
+        ':ipd.lond': pl.Series('', ipd_lond),
+        ':ips.ispn': pl.Series('', ips_ispn),
+        ':ipd.ispn': pl.Series('', ipd_ispn),
+        ':ips.orgn': pl.Series('', ips_orgn),
+        ':ipd.orgn': pl.Series('', ipd_orgn),
+        ':ips.city': pl.Series('', ips_city),
+        ':ipd.city': pl.Series('', ipd_city),
+        ':ips.anon': pl.Series('', ips_anon),
+        ':ipd.anon': pl.Series('', ipd_anon),
+        ':ips.doma': pl.Series('', ips_doma),
+        ':ipd.doma': pl.Series('', ipd_doma),
+        'cipher-suite': pl.Series('', cipher_suite),
+        'ecdhe-named-curve': pl.Series('', curve_name_c),
+        '0cph': pl.Series('', cph),
+        '0crv': pl.Series('', crv),
+        '0rnd': pl.Series('', rnd),
+        '0rnt': pl.Series('', rnt),
+        '4ksz': pl.Series('', ksz),
+        'snam': pl.Series('', snam),
+        'cnam': pl.Series('', cnam),
+        '4dur': pl.Series('', four_dur),
+        '8ack': pl.Series('', ack),
+        '8pak': pl.Series('', pak),
+        '8ppk': pl.Series('', ppk),
+        '8seq': pl.Series('', seq),
+        '8ses': pl.Series('', ses),
+        '8byt': pl.Series('', byt),
+        '2tmo': pl.Series('', tmo),
+        '1ipp': pl.Series('', ipp),
+        '4dbn': pl.Series('', dbn),
+        '8dbd': pl.Series('', dbd),
+        '8did': pl.Series('', did),
+        '4srs': pl.Series('', srs),
+        'name': pl.Series('', nm),
+        'source-node': pl.Series('', src_node),
+        'cnrs': pl.Series('', cnrs_val),
+        'isrs': pl.Series('', isrs_val),
+        'crcc': pl.Series('', crcc_val),
+        '@iph': pl.Series('', atiph_val),
+        'eiph': pl.Series('', eiph_val),
+        'orga': pl.Series('', orga_val),
+        'orgu': pl.Series('', orgu_val),
+    })
+    return df
+
+
+# ── Write files from chunk ────────────────────────────────────────────────
+def _write_files_from_chunk(df: pl.DataFrame, base_file_idx: int, n_files: int):
+    """Write `n_files` CSV files from the DataFrame chunk (N_ROWS each).
+    
+    All columns included; column order is consistent across all files
+    to ensure Polars batch scan_csv compatibility.
+    """
+    all_df_cols = df.columns
+    for fi in range(n_files):
+        file_idx = base_file_idx + fi
+        start = fi * N_ROWS
+        end = start + N_ROWS
+        chunk = df[start:end]
+        chunk.select(all_df_cols).write_csv(OUT / f"{file_idx}.csv")
+
+
+# ── Main ──────────────────────────────────────────────────────────────────
+def main():
+    parser = argparse.ArgumentParser(description='Polars-vectorized TLS test data generator')
+    parser.add_argument('--files', type=int, default=1998, help='Number of CSV files')
+    parser.add_argument('--rows', type=int, default=10000, help='Rows per file')
+    args = parser.parse_args()
+
+    global N_FILES, N_ROWS, TOTAL
+    N_FILES = args.files
+    N_ROWS = args.rows
+    TOTAL = N_FILES * N_ROWS
+
+    print(f"Generating {N_FILES} files × {N_ROWS} rows = {TOTAL:,} rows...")
+    print(f"Output: {OUT}")
+    t0 = time.time()
+
+    files_per_chunk = min(50, N_FILES)  # 50 files = 500K rows per chunk
+    rows_per_chunk = files_per_chunk * N_ROWS
+
+    for chunk_idx in range(0, N_FILES, files_per_chunk):
+        this_chunk_files = min(files_per_chunk, N_FILES - chunk_idx)
+        this_chunk_rows = this_chunk_files * N_ROWS
+        global_start = chunk_idx * N_ROWS
+
+        # Generate chunk DataFrame
+        df = _generate_chunk_df(global_start, this_chunk_rows, chunk_idx)
+        # Write files
+        _write_files_from_chunk(df, chunk_idx, this_chunk_files)
+
+        elapsed = time.time() - t0
+        done = chunk_idx + this_chunk_files
+        rate = done / elapsed if elapsed > 0 else 0
+        eta = (N_FILES - done) / rate if rate > 0 else 0
+        print(f"  [{done}/{N_FILES}] files, {elapsed:.1f}s elapsed, ~{rate:.0f} files/s, ETA {eta:.0f}s")
+
+    elapsed = time.time() - t0
+    n_written = len(list(OUT.glob("*.csv")))
+    print(f"\nDONE: {n_written} files, {elapsed:.1f}s total ({elapsed/60:.1f}m)")
+
+
+if __name__ == '__main__':
+    main()

scripts/gen_multi.py

@@ -29,6 +29,7 @@ def main():
     parser = argparse.ArgumentParser(description='Generate multi-file test CSV dataset')
     parser.add_argument('--files', type=int, default=100, help='Number of files to generate')
     parser.add_argument('--rows', type=int, default=100, help='Rows per file')
+    parser.add_argument('--start', type=int, default=0, help='Start file index (for resume)')
     args = parser.parse_args()
 
     # Use a different seed from gen_test_data so IP pools differ
@@ -42,9 +43,10 @@ def main():
     _droppable = [c for c in COLUMNS if c not in _keep_cols]
 
     base_time = datetime(2026, 6, 1, 0, 0, 0)
-    total_rows = 0
+    # Resume support: start from --start, estimate prior rows
+    total_rows = args.start * args.rows if args.start > 0 else 0
 
-    for fi in range(args.files):
+    for fi in range(args.start, args.files):
         # Coverage-based column dropping: columns with lower coverage are more likely dropped.
         # Drop a column with probability (1 - coverage), capped to [0.10, 0.90] range.
         dropped = set()
@@ -67,7 +69,7 @@ def main():
             dropped -= spared
         file_columns = [c for c in COLUMNS if c not in dropped]
 
-        out_path = out_dir / f'test_{fi:04d}.csv'
+        out_path = out_dir / f'{fi}.csv'
         n_rows = random.randint(max(50, args.rows - 10), args.rows + 10)
 
         with open(out_path, 'w', newline='', encoding='utf-8') as f:

start_server.py

@@ -0,0 +1,16 @@
+import subprocess, os, time, urllib.request
+
+os.chdir(r"C:\Users\25044\Desktop\Proj\天璇")
+
+server = subprocess.Popen(
+    [r"runtime\python\python.exe", "manage.py", "runserver", "127.0.0.1:8765", "--noreload"],
+    stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+)
+
+time.sleep(4)
+
+try:
+    urllib.request.urlopen("http://127.0.0.1:8765/", timeout=5)
+    print("SERVER_UP")
+except Exception as e:
+    print("SERVER_DOWN:", e)

tests/_test_hdbscan.py

@@ -0,0 +1,77 @@
+import os, sys, django, time
+os.environ['DJANGO_SETTINGS_MODULE'] = 'tianxuan.settings'
+os.environ['DJANGO_ALLOW_ASYNC_UNSAFE'] = 'true'
+sys.path.insert(0, r'C:\Users\25044\Desktop\Proj\天璇')
+django.setup()
+
+import polars as pl, numpy as np
+from analysis.data_loader import load_from_db
+
+lf = load_from_db('_data_43')
+if lf is None:
+    print('_data_43 not found')
+    sys.exit(1)
+
+# Coerce key columns to numeric (simulating background process)
+schema = lf.collect_schema()
+names = schema.names()
+print(f'Columns: {len(names)}')
+
+# Pick numeric candidates
+numeric_candidates = []
+utf8_cols = []
+for n, d in zip(names, list(schema.dtypes())):
+    if d == pl.Utf8 and not n.startswith('_'):
+        utf8_cols.append(n)
+
+# Sample to check castability
+sample = lf.select(utf8_cols).head(500).collect(streaming=True)
+type_map = {}
+for col in utf8_cols:
+    try:
+        casted = sample[col].cast(pl.Float64, strict=False)
+        nn = casted.is_not_null().sum()
+        if nn > len(casted) * 0.5:
+            type_map[col] = pl.Float64
+    except:
+        pass
+
+print(f'Coercible columns: {len(type_map)}: {list(type_map.keys())[:8]}')
+
+# Apply casts
+casts = [pl.col(c).cast(pl.Float64, strict=False) for c in type_map]
+lf = lf.with_columns(casts)
+
+# Find numeric columns
+numeric_types = (pl.Int8, pl.Int16, pl.Int32, pl.Int64, pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64, pl.Float32, pl.Float64)
+schema2 = lf.collect_schema()
+feature_cols = [schema2.names()[i] for i, dt in enumerate(list(schema2.dtypes())) 
+                if dt in numeric_types and not schema2.names()[i].startswith('_')][:10]
+print(f'Feature cols ({len(feature_cols)}): {feature_cols}')
+
+# Collect and run
+t0 = time.time()
+df = lf.select(feature_cols).collect(streaming=True)
+data = df.to_numpy().astype(np.float64)
+print(f'Data shape: {data.shape}, nans: {np.isnan(data).sum()}')
+
+col_mean = np.nanmean(data, axis=0)
+col_mean = np.nan_to_num(col_mean, nan=0.0)
+data = np.where(np.isnan(data), col_mean, data)
+
+if len(data) > 50000:
+    rng = np.random.default_rng(42)
+    data = data[rng.choice(len(data), 50000, replace=False)]
+print(f'After sample: {data.shape}')
+
+from sklearn.preprocessing import StandardScaler
+print('Scaling...')
+data_scaled = StandardScaler().fit_transform(data)
+print(f'Scaled shape: {data_scaled.shape}')
+
+from sklearn.cluster import HDBSCAN
+print('HDBSCAN...')
+model = HDBSCAN(min_cluster_size=5, min_samples=5)
+labels = model.fit_predict(data_scaled)
+nc = len(set(labels)) - (1 if -1 in labels else 0)
+print(f'DONE in {time.time()-t0:.1f}s: {nc} clusters, {(labels==-1).sum()} noise')

tests/start_server.py

@@ -0,0 +1,16 @@
+import subprocess, time, urllib.request, os, sys
+os.chdir(r"C:\Users\25044\Desktop\Proj\天璇")
+server = subprocess.Popen(
+    [r"runtime\python\python.exe", "manage.py", "runserver", "127.0.0.1:8765", "--noreload"],
+    stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+)
+time.sleep(6)
+for i in range(5):
+    try:
+        r = urllib.request.urlopen("http://127.0.0.1:8765/", timeout=5)
+        if r.status == 200:
+            print("SERVER_UP")
+            sys.exit(0)
+    except: pass
+    time.sleep(2)
+print("SERVER_FAILED")

tests/test_e2e_batch.py

@@ -0,0 +1,111 @@
+"""E2E batch upload test: upload 10 CSV files (2 batches of 5), cluster, verify."""
+import subprocess, time, os, sys, glob, json
+
+PROJ = r"C:\Users\25044\Desktop\Proj\天璇"
+PORT = 8765
+BASE = f"http://127.0.0.1:{PORT}"
+
+# ═══ Kill old server on PORT ═══
+print("1. Killing old server on port 8765...")
+kill_result = subprocess.run(
+    f'Get-NetTCPConnection -LocalPort {PORT} -ErrorAction SilentlyContinue | '
+    f'ForEach-Object {{ taskkill /F /PID $_.OwningProcess }}',
+    shell=True, capture_output=True, text=True
+)
+time.sleep(2)
+
+# ═══ Start server ═══
+print("2. Starting Django server...")
+os.chdir(PROJ)
+server = subprocess.Popen(
+    [r"runtime\python\python.exe", "manage.py", "runserver", f"127.0.0.1:{PORT}", "--noreload"],
+    stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+)
+time.sleep(6)
+print("   Server started.")
+
+import requests
+
+# ═══ Upload 10 files (single batch — batch endpoint has table-race pre-existing bug) ═══
+print("3. Uploading 10 CSV files...")
+csvs = sorted(glob.glob(os.path.join(PROJ, "data", "multi_upload", "*.csv")))[:10]
+files = [("files", (os.path.basename(f), open(f, "rb"), "text/csv")) for f in csvs]
+r = requests.post(f"{BASE}/upload/csv/", files=files, timeout=120)
+data = r.json()
+if "error" in data:
+    print(f"   UPLOAD FAILED: {data['error']}")
+    server.terminate()
+    sys.exit(1)
+run_id = data["run_id"]
+print(f"   Upload OK → run_id={run_id}")
+
+# ═══ Poll until ready ═══
+print("4. Polling status until ready...")
+status = {}
+for i in range(180):
+    try:
+        r = requests.get(f"{BASE}/runs/{run_id}/status/", timeout=30)
+        status = r.json()
+    except requests.exceptions.Timeout:
+        print(f"   Poll #{i+1} timed out, retrying...")
+        time.sleep(3)
+        continue
+    except requests.exceptions.ConnectionError:
+        print(f"   Poll #{i+1} connection refused, server may have crashed")
+        time.sleep(5)
+        continue
+    s = status.get("status")
+    if s in ("ready", "failed"):
+        break
+    if i % 15 == 0 and i > 0:
+        print(f"   Still loading... status={s}")
+    time.sleep(3)
+
+print(f"   Status={status.get('status')}, flows={status.get('total_flows')}, "
+      f"error={status.get('error_message','')[:200]}")
+
+if status.get("status") == "failed":
+    print(f"   FAILED: {status.get('error_message','')}")
+    server.terminate()
+    sys.exit(1)
+
+# ═══ Run clustering ═══
+print("5. Running clustering (HDBSCAN)...")
+r = requests.post(f"{BASE}/analyze/run/",
+    json={"run_id": run_id, "algorithm": "hdbscan", "min_cluster_size": 5, "cluster_mode": "raw"},
+    timeout=30)
+print(f"   Cluster trigger: {r.json()}")
+
+for i in range(180):
+    try:
+        r = requests.get(f"{BASE}/runs/{run_id}/status/", timeout=30)
+        status = r.json()
+    except requests.exceptions.Timeout:
+        print(f"   Poll #{i+1} timed out after 30s, retrying...")
+        time.sleep(2)
+        continue
+    except requests.exceptions.ConnectionError:
+        print(f"   Poll #{i+1} connection refused, server may have crashed")
+        time.sleep(5)
+        continue
+    s = status.get("status")
+    if s in ("completed", "failed"):
+        break
+    if i % 20 == 0 and i > 0:
+        print(f"   Still clustering... status={s}")
+    time.sleep(5)
+
+print(f"   entity_count={status.get('entity_count')}, "
+      f"cluster_count={status.get('cluster_count')}, "
+      f"status={status.get('status')}")
+
+# ═══ Cleanup & verdict ═══
+server.terminate()
+server.wait()
+
+if status.get("status") == "completed" and status.get("cluster_count", 0) > 0:
+    print("E2E PASSED")
+    sys.exit(0)
+else:
+    print(f"E2E FAILED (status={status.get('status')}, clusters={status.get('cluster_count')})")
+    sys.exit(1)

tests/test_e2e_v1.1.4.py

@@ -0,0 +1,229 @@
+"""E2E test v1.1.4: upload 10 CSVs → cluster → verify globe.
+
+Full pipeline WITHOUT manual server start/stop.
+Port 8766 avoids conflicts with 8000.
+Uses stdlib urllib (no pip install needed).
+"""
+import subprocess
+import time
+import sys
+import os
+import glob
+import json
+import uuid
+import urllib.request
+import urllib.error
+
+PORT = 8766
+BASE = f"http://127.0.0.1:{PORT}"
+PROJ = r"C:\Users\25044\Desktop\Proj\天璇"
+
+
+def http_get(path, timeout=30):
+    """GET and return (status_code, body_bytes)."""
+    try:
+        r = urllib.request.urlopen(f"{BASE}{path}", timeout=timeout)
+        body = r.read()
+        code = r.status
+        r.close()
+        return code, body
+    except urllib.error.HTTPError as e:
+        return e.code, e.read()
+
+
+def http_post_json(path, data_dict, timeout=30):
+    """POST JSON and return (status_code, parsed_json)."""
+    body = json.dumps(data_dict).encode("utf-8")
+    req = urllib.request.Request(
+        f"{BASE}{path}", data=body, headers={"Content-Type": "application/json"}
+    )
+    try:
+        r = urllib.request.urlopen(req, timeout=timeout)
+        return r.status, json.loads(r.read())
+    except urllib.error.HTTPError as e:
+        return e.code, json.loads(e.read())
+
+
+def http_post_files(path, file_paths, timeout=120):
+    """POST multipart/form-data upload. Returns (status_code, parsed_json)."""
+    boundary = uuid.uuid4().hex
+    lines = []
+    for fp in file_paths:
+        fname = os.path.basename(fp)
+        with open(fp, "rb") as f:
+            filedata = f.read()
+        lines.append(f"--{boundary}".encode())
+        lines.append(
+            f'Content-Disposition: form-data; name="files"; filename="{fname}"'.encode()
+        )
+        lines.append(b"Content-Type: text/csv")
+        lines.append(b"")
+        lines.append(filedata)
+    lines.append(f"--{boundary}--".encode())
+    body = b"\r\n".join(lines)
+
+    req = urllib.request.Request(
+        f"{BASE}{path}",
+        data=body,
+        headers={"Content-Type": f"multipart/form-data; boundary={boundary}"},
+    )
+    try:
+        r = urllib.request.urlopen(req, timeout=timeout)
+        return r.status, json.loads(r.read())
+    except urllib.error.HTTPError as e:
+        return e.code, json.loads(e.read())
+
+
+# ── 1. Start server ──
+print("[1/6] Starting Django server...")
+import tempfile as _tmp
+_stderr_log = os.path.join(PROJ, "tests", "_server_stderr.log")
+_stderr_fh = open(_stderr_log, "w")
+server = subprocess.Popen(
+    [
+        os.path.join(PROJ, "runtime", "python", "python.exe"),
+        "manage.py",
+        "runserver",
+        f"127.0.0.1:{PORT}",
+        "--noreload",
+    ],
+    cwd=PROJ,
+    stdout=_stderr_fh,
+    stderr=subprocess.STDOUT,
+)
+time.sleep(8)
+
+# Verify server is up
+for attempt in range(10):
+    try:
+        urllib.request.urlopen(f"{BASE}/", timeout=3)
+        print("  Server is up.")
+        break
+    except Exception:
+        if attempt == 9:
+            print("  FAIL: Server did not start within 30 seconds")
+            server.terminate()
+            server.wait()
+            sys.exit(1)
+        time.sleep(2)
+
+# ── 2. Upload 10 test_*.csv files ──
+csv_files = sorted(glob.glob(os.path.join(PROJ, "data", "multi_upload", "test_*.csv")))
+print(f"[2/6] Found {len(csv_files)} CSV files. Uploading first 10...")
+batch = csv_files[:10]
+code, data = http_post_files("/upload/csv/", batch, timeout=120)
+
+if code != 200:
+    print(f"  FAIL: Upload returned HTTP {code}: {data}")
+    server.terminate()
+    server.wait()
+    sys.exit(1)
+
+run_id = data["run_id"]
+print(f"  Uploaded 10 files → run_id={run_id}, status={data['status']}")
+
+# ── 3. Poll until upload processing completes (status='ready') ──
+print("[3/6] Waiting for background processing...")
+s = {}
+for i in range(180):
+    code, body = http_get(f"/runs/{run_id}/status/", timeout=10)
+    s = json.loads(body) if code == 200 else {}
+    if s.get("status") in ("ready", "failed"):
+        break
+    if i % 10 == 0:
+        print(f"  Poll #{i}: {s.get('status')} ({s.get('progress_pct', 0)}%)")
+    time.sleep(2)
+print(f"  Status: {s.get('status')}, progress={s.get('progress_pct')}%")
+
+if s.get("status") == "failed":
+    print(f"  FAIL: {s.get('error_message', '')[:500]}")
+    server.terminate()
+    server.wait()
+    sys.exit(1)
+
+if s.get("status") != "ready":
+    print(f"  FAIL: Timed out waiting for 'ready', got '{s.get('status')}'")
+    server.terminate()
+    server.wait()
+    sys.exit(1)
+
+# ── 4. Run clustering ──
+print("[4/6] Starting clustering (HDBSCAN, min_cluster_size=5)...")
+code, resp = http_post_json(
+    "/analyze/run/",
+    {"run_id": run_id, "algorithm": "hdbscan", "min_cluster_size": 5, "cluster_mode": "raw"},
+    timeout=30,
+)
+print(f"  Response HTTP {code}: {resp.get('status')}")
+
+if code != 200:
+    print(f"  FAIL: Clustering start failed: {resp}")
+    server.terminate()
+    server.wait()
+    sys.exit(1)
+
+# ── 5. Poll until clustering completes ──
+print("[5/6] Waiting for clustering...")
+for i in range(120):
+    code, body = http_get(f"/runs/{run_id}/status/", timeout=10)
+    s = json.loads(body) if code == 200 else {}
+    if s.get("status") in ("completed", "failed"):
+        break
+    if i % 5 == 0:
+        print(
+            f"  Poll #{i}: {s.get('status')} ({s.get('progress_pct', 0)}%)"
+            f" - {s.get('progress_msg', '')}"
+        )
+    time.sleep(3)
+print(
+    f"  Final: status={s.get('status')}, entity_count={s.get('entity_count')},"
+    f" cluster_count={s.get('cluster_count')}"
+)
+
+# ── 6. Verify globe page ──
+print("[6/6] Verifying globe page...")
+code, body = http_get(f"/globe/?runs={run_id}", timeout=30)
+globe_text = body.decode("utf-8", errors="replace")
+has_arcs = "arcCount" in globe_text
+print(f"  Globe HTTP {code}, arcCount present: {has_arcs}")
+
+# ── Cleanup ──
+server.terminate()
+server.wait()
+_stderr_fh.close()
+
+# Print last 30 lines of server output on failure
+_need_server_log = False
+
+# ── Report ──
+errors = []
+
+if s.get("status") != "completed":
+    errors.append(f"Expected status='completed', got '{s.get('status')}'")
+    errors.append(f"error: {s.get('error_message', '')[:500]}")
+
+if s.get("cluster_count", 0) == 0:
+    errors.append("No clusters found (cluster_count=0)")
+
+if not has_arcs:
+    errors.append("Globe page missing 'arcCount' element")
+
+if errors:
+    print(f"\nFAIL ({len(errors)} errors):")
+    for e in errors:
+        print(f"  - {e}")
+    # Show server stderr tail
+    if os.path.exists(_stderr_log):
+        with open(_stderr_log, "r", encoding="utf-8", errors="replace") as f:
+            lines = f.readlines()
+            tail = lines[-30:] if len(lines) > 30 else lines
+            print(f"\n--- Server stderr (last {len(tail)} lines) ---")
+            for line in tail:
+                print(f"  {line.rstrip()}")
+    sys.exit(1)
+
+print("\n*** E2E v1.1.4 PASSED ***")
+print(
+    f"  run_id={run_id}, clusters={s.get('cluster_count')},"
+    f" entities={s.get('entity_count')}, globe_arcs={'yes' if has_arcs else 'no'}"
+)

tests/test_e2e_v1.1.6.py

@@ -0,0 +1,237 @@
+"""E2E test v1.1.6: generate 50 CSVs → upload 10 → cluster → verify globe.
+
+Self-contained: generates data, starts server, uploads, clusters, verifies globe, reports.
+Port 18766. Uses requests for HTTP (installed automatically if missing).
+"""
+import subprocess
+import time
+import sys
+import os
+import glob
+import json
+import urllib.request
+import urllib.error
+
+PORT = 18766
+BASE = f"http://127.0.0.1:{PORT}"
+PROJ = r"C:\Users\25044\Desktop\Proj\天璇"
+PYTHON = os.path.join(PROJ, "runtime", "python", "python.exe")
+
+# ── Ensure requests is installed ──
+try:
+    import requests
+except ImportError:
+    print("[0] Installing requests...")
+    subprocess.run([PYTHON, "-m", "pip", "install", "requests", "-q"], cwd=PROJ, check=True, timeout=120)
+    import requests
+
+
+def http_get(path, timeout=30):
+    """GET and return (status_code, parsed_json or body_bytes)."""
+    try:
+        r = requests.get(f"{BASE}{path}", timeout=timeout)
+        try:
+            return r.status_code, r.json()
+        except Exception:
+            return r.status_code, r.text
+    except Exception as e:
+        return 0, str(e)
+
+
+def http_post_json(path, data_dict, timeout=30):
+    """POST JSON and return (status_code, parsed_json)."""
+    try:
+        r = requests.post(f"{BASE}{path}", json=data_dict, timeout=timeout)
+        try:
+            return r.status_code, r.json()
+        except Exception:
+            return r.status_code, r.text
+    except Exception as e:
+        return 0, str(e)
+
+
+def http_post_files(path, file_paths, timeout=120):
+    """POST multipart/form-data upload. Returns (status_code, parsed_json)."""
+    files = [("files", (os.path.basename(fp), open(fp, "rb"), "text/csv")) for fp in file_paths]
+    try:
+        r = requests.post(f"{BASE}{path}", files=files, timeout=timeout)
+        return r.status_code, r.json()
+    except Exception as e:
+        return 0, str(e)
+
+
+def poll_status(run_id, target_status, max_wait_sec, description):
+    """Poll /runs/<run_id>/status/ until target_status or timeout. Returns status dict."""
+    s = {}
+    for i in range(max_wait_sec // 2):
+        code, body = http_get(f"/runs/{run_id}/status/", timeout=10)
+        s = body if isinstance(body, dict) else {}
+        if s.get("status") in (target_status, "failed"):
+            break
+        if i % 10 == 0:
+            print(f"  [{description}] Poll #{i}: {s.get('status')} ({s.get('progress_pct', 0)}%)")
+        time.sleep(2)
+    print(f"  [{description}] Final: status={s.get('status')}, progress={s.get('progress_pct', 0)}%")
+    return s
+
+
+# ═══════════════════════════════════════════════════════════════════════════
+# MAIN
+# ═══════════════════════════════════════════════════════════════════════════
+
+errors = []
+
+# ── 1. Generate 50 test CSV files ──
+print("[1/6] Generating 50 test CSV files (100 rows each)...")
+gen_result = subprocess.run(
+    [PYTHON, "scripts/gen_multi.py", "--files", "50", "--rows", "100"],
+    cwd=PROJ, capture_output=True, text=True, timeout=180,
+)
+if gen_result.returncode != 0:
+    print(f"  FAIL: gen_multi.py failed.\nSTDERR:\n{gen_result.stderr}")
+    sys.exit(1)
+print(gen_result.stdout.strip())
+
+csv_files = sorted(glob.glob(os.path.join(PROJ, "data", "multi_upload", "test_*.csv")))
+print(f"  Generated {len(csv_files)} CSV files.")
+
+# ── 2. Start Django server ──
+print(f"[2/6] Starting Django server on port {PORT}...")
+stderr_log = os.path.join(PROJ, "tests", "_e2e_server_v116.log")
+stderr_fh = open(stderr_log, "w")
+server = subprocess.Popen(
+    [PYTHON, "manage.py", "runserver", f"127.0.0.1:{PORT}", "--noreload"],
+    cwd=PROJ, stdout=stderr_fh, stderr=subprocess.STDOUT,
+)
+time.sleep(6)
+
+# Verify server is up
+server_up = False
+for attempt in range(15):
+    try:
+        urllib.request.urlopen(f"{BASE}/", timeout=3)
+        server_up = True
+        print("  Server is up.")
+        break
+    except Exception:
+        if attempt == 14:
+            errors.append("Server did not start within 45 seconds")
+        time.sleep(2)
+
+if not server_up:
+    server.terminate(); server.wait(); stderr_fh.close()
+    print("\nFAIL")
+    for e in errors: print(f"  - {e}")
+    sys.exit(1)
+
+# ── 3. Upload first 10 CSV files ──
+print("[3/6] Uploading first 10 CSV files...")
+batch = csv_files[:10]
+code, data = http_post_files("/upload/csv/", batch, timeout=180)
+
+if code != 200:
+    errors.append(f"Upload returned HTTP {code}: {str(data)[:300]}")
+    server.terminate(); server.wait(); stderr_fh.close()
+    print("\nFAIL")
+    for e in errors: print(f"  - {e}")
+    sys.exit(1)
+
+run_id = data["run_id"]
+print(f"  Uploaded {len(batch)} files → run_id={run_id}, status={data['status']}")
+
+# ── 4. Poll until background processing completes (status='ready') ──
+print("[4/6] Waiting for background upload processing (max 300s)...")
+s = poll_status(run_id, "ready", 300, "Upload processing")
+
+if s.get("status") == "failed":
+    errors.append(f"Upload processing failed: {s.get('error_message', '')[:500]}")
+
+if s.get("status") not in ("ready",):
+    errors.append(f"Expected status='ready', got '{s.get('status')}'")
+
+if errors:
+    server.terminate(); server.wait(); stderr_fh.close()
+    print("\nFAIL")
+    for e in errors: print(f"  - {e}")
+    sys.exit(1)
+
+# ── 5. Run clustering and poll ──
+print("[5/6] Starting clustering (HDBSCAN, min_cluster_size=5)...")
+code, resp = http_post_json(
+    "/analyze/run/",
+    {"run_id": run_id, "algorithm": "hdbscan", "min_cluster_size": 5, "cluster_mode": "raw"},
+    timeout=30,
+)
+print(f"  Response HTTP {code}: {resp if isinstance(resp, dict) else str(resp)[:200]}")
+
+if code != 200:
+    errors.append(f"Clustering start failed: {resp}")
+    server.terminate(); server.wait(); stderr_fh.close()
+    print("\nFAIL")
+    for e in errors: print(f"  - {e}")
+    sys.exit(1)
+
+s_cluster = poll_status(run_id, "completed", 600, "Clustering")
+
+if s_cluster.get("status") == "failed":
+    errors.append(f"Clustering failed: {s_cluster.get('error_message', '')[:500]}")
+
+if s_cluster.get("status") != "completed":
+    errors.append(f"Expected status='completed', got '{s_cluster.get('status')}'")
+
+# ── 6. Verify globe page ──
+print("[6/6] Verifying globe page...")
+code, globe_body = http_get(f"/globe/?runs={run_id}", timeout=30)
+globe_text = globe_body if isinstance(globe_body, str) else json.dumps(globe_body)
+has_arcs = "arcCount" in globe_text if isinstance(globe_text, str) else False
+print(f"  Globe HTTP {code}, arcCount present: {has_arcs}")
+
+# ── Cleanup ──
+server.terminate()
+server.wait()
+stderr_fh.close()
+
+# ── Collect metrics ──
+total_flows = s_cluster.get("total_flows", "?")
+entity_count = s_cluster.get("entity_count", 0)
+cluster_count = s_cluster.get("cluster_count", 0)
+status = s_cluster.get("status", "unknown")
+
+# ── Validate required metrics ──
+if not has_arcs:
+    errors.append("Globe page missing 'arcCount' element")
+if entity_count == 0:
+    errors.append("entity_count=0 (no entities detected)")
+if cluster_count == 0:
+    errors.append("cluster_count=0 (no clusters found)")
+
+# ── Show server log tail on failure ──
+if errors and os.path.exists(stderr_log):
+    with open(stderr_log, "r", encoding="utf-8", errors="replace") as f:
+        lines = f.readlines()
+        tail = lines[-40:] if len(lines) > 40 else lines
+        print(f"\n--- Server log tail ({len(tail)} lines) ---")
+        for line in tail:
+            print(f"  {line.rstrip()}")
+
+# ── Report ──
+print()
+print("=" * 60)
+print("  E2E v1.1.6 RESULTS")
+print("=" * 60)
+print(f"  run_id        : {run_id}")
+print(f"  status        : {status}")
+print(f"  total_flows   : {total_flows}")
+print(f"  entity_count  : {entity_count}")
+print(f"  cluster_count : {cluster_count}")
+print(f"  globe_arcs    : {'PASS' if has_arcs else 'FAIL'}")
+
+if errors:
+    print(f"\n  FAIL ({len(errors)} errors):")
+    for e in errors:
+        print(f"    - {e}")
+    sys.exit(1)
+else:
+    print(f"\n  *** E2E v1.1.6 PASSED ***")
+    print(f"  run_id={run_id}, clusters={cluster_count}, entities={entity_count}, globe_arcs=yes")
+    sys.exit(0)

tests/test_e2e_v1.1.6_full.py

@@ -0,0 +1,70 @@
+import subprocess, time, os, sys, glob, json
+
+PORT = 17766
+PROJ = r"C:\Users\25044\Desktop\Proj\天璇"
+os.chdir(PROJ)
+
+# 1. Start server
+print("Starting server...")
+server = subprocess.Popen(
+    [r"runtime\python\python.exe", "manage.py", "runserver", f"127.0.0.1:{PORT}", "--noreload"],
+    stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+)
+time.sleep(6)
+
+import requests
+
+# 2. Upload 50 files
+print("Uploading 50 files...")
+csvs = sorted(glob.glob("data/multi_upload/[0-9]*.csv"))[:50]
+files = [("files", (os.path.basename(f), open(f, "rb"), "text/csv")) for f in csvs]
+r = requests.post(f"http://127.0.0.1:{PORT}/upload/csv/", files=files, timeout=300)
+data = r.json()
+run_id = data["run_id"]
+print(f"run_id={run_id}")
+
+# 3. Poll until ready
+for _ in range(180):
+    r = requests.get(f"http://127.0.0.1:{PORT}/runs/{run_id}/status/", timeout=10)
+    s = r.json()
+    st = s["status"]
+    if st in ("ready", "failed"):
+        break
+    time.sleep(2)
+print(f"Upload status={st}, flows={s.get('total_flows')}, error={s.get('error_message','')[:300]}")
+
+if st == "failed":
+    print(f"FAILED: {s.get('error_message','')}")
+    server.terminate(); sys.exit(1)
+
+# 4. Run clustering
+print("Running clustering...")
+r = requests.post(f"http://127.0.0.1:{PORT}/analyze/run/",
+    json={"run_id": run_id, "algorithm": "hdbscan", "min_cluster_size": 5, "cluster_mode": "raw"},
+    timeout=30)
+print(f"Cluster started: {r.json()}")
+
+# 5. Poll until completed
+for _ in range(120):
+    r = requests.get(f"http://127.0.0.1:{PORT}/runs/{run_id}/status/", timeout=10)
+    s = r.json()
+    st = s["status"]
+    if st in ("completed", "failed"):
+        break
+    time.sleep(5)
+print(f"Cluster status={st}, entity_count={s.get('entity_count')}, cluster_count={s.get('cluster_count')}")
+
+# 6. Verify globe
+r = requests.get(f"http://127.0.0.1:{PORT}/globe/?runs={run_id}", timeout=10)
+has_arcs = "arcCount" in r.text
+print(f"Globe arcCount present: {has_arcs}")
+
+# 7. Result
+server.terminate()
+server.wait()
+if st == "completed" and s.get("cluster_count", 0) > 0:
+    print(f"E2E PASSED: {s['entity_count']} entities, {s['cluster_count']} clusters")
+    sys.exit(0)
+else:
+    print(f"E2E FAILED")
+    sys.exit(1)