From 1df4c3c2ebb7a0e88582dafeab63b57f101bb9d0 Mon Sep 17 00:00:00 2001 From: PM-pinou <2504420230@qq.com> Date: Wed, 22 Jul 2026 21:53:25 +0800 Subject: [PATCH] chore: bump VERSION to v2.0.0beta, reimport TlsDB, remove stale test files --- TlsDB.csv | 4 +- VERSION | 2 +- tests/test_e2e_batch.py | 2 +- tests/test_e2e_v1.1.4.py | 229 -------------------------------- tests/test_e2e_v1.1.6.py | 237 ---------------------------------- tests/test_e2e_v1.1.6_full.py | 70 ---------- 6 files changed, 4 insertions(+), 540 deletions(-) delete mode 100644 tests/test_e2e_v1.1.4.py delete mode 100644 tests/test_e2e_v1.1.6.py delete mode 100644 tests/test_e2e_v1.1.6_full.py diff --git a/TlsDB.csv b/TlsDB.csv index 94041d5..fe1d5d7 100644 --- a/TlsDB.csv +++ b/TlsDB.csv @@ -11,8 +11,8 @@ :ipd.orgn,目标IP组织名称,字符串,97.61% :ips.city,源IP所在城市,字符串,99.26% :ipd.city,目标IP所在城市,字符串,98.74% -:ips.anon,源IP匿名状态,"anon, hosting"或无,12.66% -:ipd.anon,目标IP匿名状态,"anon, hosting"或无,15.54% +:ips.anon,源IP匿名状态,"anon, hosting"或"anon, vpn"或"anon, server"或无,12.66% +:ipd.anon,目标IP匿名状态,"anon, hosting"或"anon, vpn"或"anon, server"或无,15.54% :ips.doma,源IP域名,字符串,28.91% :ipd.doma,目标IP域名,字符串,13.48% :prs,源端口,整数,100% diff --git a/VERSION b/VERSION index 46882bc..828b6bf 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v1.1.6 \ No newline at end of file +v2.0.0beta \ No newline at end of file diff --git a/tests/test_e2e_batch.py b/tests/test_e2e_batch.py index 379d7bb..f28fd01 100644 --- a/tests/test_e2e_batch.py +++ b/tests/test_e2e_batch.py @@ -1,7 +1,7 @@ """E2E batch upload test: upload 10 CSV files (2 batches of 5), cluster, verify.""" import subprocess, time, os, sys, glob, json -PROJ = r"C:\Users\25044\Desktop\Proj\天璇" +PROJ = r"E:\hjq\天璇" PORT = 8765 BASE = f"http://127.0.0.1:{PORT}" diff --git a/tests/test_e2e_v1.1.4.py b/tests/test_e2e_v1.1.4.py deleted file mode 100644 index 4b7a356..0000000 --- a/tests/test_e2e_v1.1.4.py +++ /dev/null @@ -1,229 +0,0 @@ -"""E2E test v1.1.4: upload 10 CSVs → cluster → verify globe. - -Full pipeline WITHOUT manual server start/stop. -Port 8766 avoids conflicts with 8000. -Uses stdlib urllib (no pip install needed). -""" -import subprocess -import time -import sys -import os -import glob -import json -import uuid -import urllib.request -import urllib.error - -PORT = 8766 -BASE = f"http://127.0.0.1:{PORT}" -PROJ = r"C:\Users\25044\Desktop\Proj\天璇" - - -def http_get(path, timeout=30): - """GET and return (status_code, body_bytes).""" - try: - r = urllib.request.urlopen(f"{BASE}{path}", timeout=timeout) - body = r.read() - code = r.status - r.close() - return code, body - except urllib.error.HTTPError as e: - return e.code, e.read() - - -def http_post_json(path, data_dict, timeout=30): - """POST JSON and return (status_code, parsed_json).""" - body = json.dumps(data_dict).encode("utf-8") - req = urllib.request.Request( - f"{BASE}{path}", data=body, headers={"Content-Type": "application/json"} - ) - try: - r = urllib.request.urlopen(req, timeout=timeout) - return r.status, json.loads(r.read()) - except urllib.error.HTTPError as e: - return e.code, json.loads(e.read()) - - -def http_post_files(path, file_paths, timeout=120): - """POST multipart/form-data upload. Returns (status_code, parsed_json).""" - boundary = uuid.uuid4().hex - lines = [] - for fp in file_paths: - fname = os.path.basename(fp) - with open(fp, "rb") as f: - filedata = f.read() - lines.append(f"--{boundary}".encode()) - lines.append( - f'Content-Disposition: form-data; name="files"; filename="{fname}"'.encode() - ) - lines.append(b"Content-Type: text/csv") - lines.append(b"") - lines.append(filedata) - lines.append(f"--{boundary}--".encode()) - body = b"\r\n".join(lines) - - req = urllib.request.Request( - f"{BASE}{path}", - data=body, - headers={"Content-Type": f"multipart/form-data; boundary={boundary}"}, - ) - try: - r = urllib.request.urlopen(req, timeout=timeout) - return r.status, json.loads(r.read()) - except urllib.error.HTTPError as e: - return e.code, json.loads(e.read()) - - -# ── 1. Start server ── -print("[1/6] Starting Django server...") -import tempfile as _tmp -_stderr_log = os.path.join(PROJ, "tests", "_server_stderr.log") -_stderr_fh = open(_stderr_log, "w") -server = subprocess.Popen( - [ - os.path.join(PROJ, "runtime", "python", "python.exe"), - "manage.py", - "runserver", - f"127.0.0.1:{PORT}", - "--noreload", - ], - cwd=PROJ, - stdout=_stderr_fh, - stderr=subprocess.STDOUT, -) -time.sleep(8) - -# Verify server is up -for attempt in range(10): - try: - urllib.request.urlopen(f"{BASE}/", timeout=3) - print(" Server is up.") - break - except Exception: - if attempt == 9: - print(" FAIL: Server did not start within 30 seconds") - server.terminate() - server.wait() - sys.exit(1) - time.sleep(2) - -# ── 2. Upload 10 test_*.csv files ── -csv_files = sorted(glob.glob(os.path.join(PROJ, "data", "multi_upload", "test_*.csv"))) -print(f"[2/6] Found {len(csv_files)} CSV files. Uploading first 10...") -batch = csv_files[:10] -code, data = http_post_files("/upload/csv/", batch, timeout=120) - -if code != 200: - print(f" FAIL: Upload returned HTTP {code}: {data}") - server.terminate() - server.wait() - sys.exit(1) - -run_id = data["run_id"] -print(f" Uploaded 10 files → run_id={run_id}, status={data['status']}") - -# ── 3. Poll until upload processing completes (status='ready') ── -print("[3/6] Waiting for background processing...") -s = {} -for i in range(180): - code, body = http_get(f"/runs/{run_id}/status/", timeout=10) - s = json.loads(body) if code == 200 else {} - if s.get("status") in ("ready", "failed"): - break - if i % 10 == 0: - print(f" Poll #{i}: {s.get('status')} ({s.get('progress_pct', 0)}%)") - time.sleep(2) -print(f" Status: {s.get('status')}, progress={s.get('progress_pct')}%") - -if s.get("status") == "failed": - print(f" FAIL: {s.get('error_message', '')[:500]}") - server.terminate() - server.wait() - sys.exit(1) - -if s.get("status") != "ready": - print(f" FAIL: Timed out waiting for 'ready', got '{s.get('status')}'") - server.terminate() - server.wait() - sys.exit(1) - -# ── 4. Run clustering ── -print("[4/6] Starting clustering (HDBSCAN, min_cluster_size=5)...") -code, resp = http_post_json( - "/analyze/run/", - {"run_id": run_id, "algorithm": "hdbscan", "min_cluster_size": 5, "cluster_mode": "raw"}, - timeout=30, -) -print(f" Response HTTP {code}: {resp.get('status')}") - -if code != 200: - print(f" FAIL: Clustering start failed: {resp}") - server.terminate() - server.wait() - sys.exit(1) - -# ── 5. Poll until clustering completes ── -print("[5/6] Waiting for clustering...") -for i in range(120): - code, body = http_get(f"/runs/{run_id}/status/", timeout=10) - s = json.loads(body) if code == 200 else {} - if s.get("status") in ("completed", "failed"): - break - if i % 5 == 0: - print( - f" Poll #{i}: {s.get('status')} ({s.get('progress_pct', 0)}%)" - f" - {s.get('progress_msg', '')}" - ) - time.sleep(3) -print( - f" Final: status={s.get('status')}, entity_count={s.get('entity_count')}," - f" cluster_count={s.get('cluster_count')}" -) - -# ── 6. Verify globe page ── -print("[6/6] Verifying globe page...") -code, body = http_get(f"/globe/?runs={run_id}", timeout=30) -globe_text = body.decode("utf-8", errors="replace") -has_arcs = "arcCount" in globe_text -print(f" Globe HTTP {code}, arcCount present: {has_arcs}") - -# ── Cleanup ── -server.terminate() -server.wait() -_stderr_fh.close() - -# Print last 30 lines of server output on failure -_need_server_log = False - -# ── Report ── -errors = [] - -if s.get("status") != "completed": - errors.append(f"Expected status='completed', got '{s.get('status')}'") - errors.append(f"error: {s.get('error_message', '')[:500]}") - -if s.get("cluster_count", 0) == 0: - errors.append("No clusters found (cluster_count=0)") - -if not has_arcs: - errors.append("Globe page missing 'arcCount' element") - -if errors: - print(f"\nFAIL ({len(errors)} errors):") - for e in errors: - print(f" - {e}") - # Show server stderr tail - if os.path.exists(_stderr_log): - with open(_stderr_log, "r", encoding="utf-8", errors="replace") as f: - lines = f.readlines() - tail = lines[-30:] if len(lines) > 30 else lines - print(f"\n--- Server stderr (last {len(tail)} lines) ---") - for line in tail: - print(f" {line.rstrip()}") - sys.exit(1) - -print("\n*** E2E v1.1.4 PASSED ***") -print( - f" run_id={run_id}, clusters={s.get('cluster_count')}," - f" entities={s.get('entity_count')}, globe_arcs={'yes' if has_arcs else 'no'}" -) diff --git a/tests/test_e2e_v1.1.6.py b/tests/test_e2e_v1.1.6.py deleted file mode 100644 index 3283206..0000000 --- a/tests/test_e2e_v1.1.6.py +++ /dev/null @@ -1,237 +0,0 @@ -"""E2E test v1.1.6: generate 50 CSVs → upload 10 → cluster → verify globe. - -Self-contained: generates data, starts server, uploads, clusters, verifies globe, reports. -Port 18766. Uses requests for HTTP (installed automatically if missing). -""" -import subprocess -import time -import sys -import os -import glob -import json -import urllib.request -import urllib.error - -PORT = 18766 -BASE = f"http://127.0.0.1:{PORT}" -PROJ = r"C:\Users\25044\Desktop\Proj\天璇" -PYTHON = os.path.join(PROJ, "runtime", "python", "python.exe") - -# ── Ensure requests is installed ── -try: - import requests -except ImportError: - print("[0] Installing requests...") - subprocess.run([PYTHON, "-m", "pip", "install", "requests", "-q"], cwd=PROJ, check=True, timeout=120) - import requests - - -def http_get(path, timeout=30): - """GET and return (status_code, parsed_json or body_bytes).""" - try: - r = requests.get(f"{BASE}{path}", timeout=timeout) - try: - return r.status_code, r.json() - except Exception: - return r.status_code, r.text - except Exception as e: - return 0, str(e) - - -def http_post_json(path, data_dict, timeout=30): - """POST JSON and return (status_code, parsed_json).""" - try: - r = requests.post(f"{BASE}{path}", json=data_dict, timeout=timeout) - try: - return r.status_code, r.json() - except Exception: - return r.status_code, r.text - except Exception as e: - return 0, str(e) - - -def http_post_files(path, file_paths, timeout=120): - """POST multipart/form-data upload. Returns (status_code, parsed_json).""" - files = [("files", (os.path.basename(fp), open(fp, "rb"), "text/csv")) for fp in file_paths] - try: - r = requests.post(f"{BASE}{path}", files=files, timeout=timeout) - return r.status_code, r.json() - except Exception as e: - return 0, str(e) - - -def poll_status(run_id, target_status, max_wait_sec, description): - """Poll /runs//status/ until target_status or timeout. Returns status dict.""" - s = {} - for i in range(max_wait_sec // 2): - code, body = http_get(f"/runs/{run_id}/status/", timeout=10) - s = body if isinstance(body, dict) else {} - if s.get("status") in (target_status, "failed"): - break - if i % 10 == 0: - print(f" [{description}] Poll #{i}: {s.get('status')} ({s.get('progress_pct', 0)}%)") - time.sleep(2) - print(f" [{description}] Final: status={s.get('status')}, progress={s.get('progress_pct', 0)}%") - return s - - -# ═══════════════════════════════════════════════════════════════════════════ -# MAIN -# ═══════════════════════════════════════════════════════════════════════════ - -errors = [] - -# ── 1. Generate 50 test CSV files ── -print("[1/6] Generating 50 test CSV files (100 rows each)...") -gen_result = subprocess.run( - [PYTHON, "scripts/gen_multi.py", "--files", "50", "--rows", "100"], - cwd=PROJ, capture_output=True, text=True, timeout=180, -) -if gen_result.returncode != 0: - print(f" FAIL: gen_multi.py failed.\nSTDERR:\n{gen_result.stderr}") - sys.exit(1) -print(gen_result.stdout.strip()) - -csv_files = sorted(glob.glob(os.path.join(PROJ, "data", "multi_upload", "test_*.csv"))) -print(f" Generated {len(csv_files)} CSV files.") - -# ── 2. Start Django server ── -print(f"[2/6] Starting Django server on port {PORT}...") -stderr_log = os.path.join(PROJ, "tests", "_e2e_server_v116.log") -stderr_fh = open(stderr_log, "w") -server = subprocess.Popen( - [PYTHON, "manage.py", "runserver", f"127.0.0.1:{PORT}", "--noreload"], - cwd=PROJ, stdout=stderr_fh, stderr=subprocess.STDOUT, -) -time.sleep(6) - -# Verify server is up -server_up = False -for attempt in range(15): - try: - urllib.request.urlopen(f"{BASE}/", timeout=3) - server_up = True - print(" Server is up.") - break - except Exception: - if attempt == 14: - errors.append("Server did not start within 45 seconds") - time.sleep(2) - -if not server_up: - server.terminate(); server.wait(); stderr_fh.close() - print("\nFAIL") - for e in errors: print(f" - {e}") - sys.exit(1) - -# ── 3. Upload first 10 CSV files ── -print("[3/6] Uploading first 10 CSV files...") -batch = csv_files[:10] -code, data = http_post_files("/upload/csv/", batch, timeout=180) - -if code != 200: - errors.append(f"Upload returned HTTP {code}: {str(data)[:300]}") - server.terminate(); server.wait(); stderr_fh.close() - print("\nFAIL") - for e in errors: print(f" - {e}") - sys.exit(1) - -run_id = data["run_id"] -print(f" Uploaded {len(batch)} files → run_id={run_id}, status={data['status']}") - -# ── 4. Poll until background processing completes (status='ready') ── -print("[4/6] Waiting for background upload processing (max 300s)...") -s = poll_status(run_id, "ready", 300, "Upload processing") - -if s.get("status") == "failed": - errors.append(f"Upload processing failed: {s.get('error_message', '')[:500]}") - -if s.get("status") not in ("ready",): - errors.append(f"Expected status='ready', got '{s.get('status')}'") - -if errors: - server.terminate(); server.wait(); stderr_fh.close() - print("\nFAIL") - for e in errors: print(f" - {e}") - sys.exit(1) - -# ── 5. Run clustering and poll ── -print("[5/6] Starting clustering (HDBSCAN, min_cluster_size=5)...") -code, resp = http_post_json( - "/analyze/run/", - {"run_id": run_id, "algorithm": "hdbscan", "min_cluster_size": 5, "cluster_mode": "raw"}, - timeout=30, -) -print(f" Response HTTP {code}: {resp if isinstance(resp, dict) else str(resp)[:200]}") - -if code != 200: - errors.append(f"Clustering start failed: {resp}") - server.terminate(); server.wait(); stderr_fh.close() - print("\nFAIL") - for e in errors: print(f" - {e}") - sys.exit(1) - -s_cluster = poll_status(run_id, "completed", 600, "Clustering") - -if s_cluster.get("status") == "failed": - errors.append(f"Clustering failed: {s_cluster.get('error_message', '')[:500]}") - -if s_cluster.get("status") != "completed": - errors.append(f"Expected status='completed', got '{s_cluster.get('status')}'") - -# ── 6. Verify globe page ── -print("[6/6] Verifying globe page...") -code, globe_body = http_get(f"/globe/?runs={run_id}", timeout=30) -globe_text = globe_body if isinstance(globe_body, str) else json.dumps(globe_body) -has_arcs = "arcCount" in globe_text if isinstance(globe_text, str) else False -print(f" Globe HTTP {code}, arcCount present: {has_arcs}") - -# ── Cleanup ── -server.terminate() -server.wait() -stderr_fh.close() - -# ── Collect metrics ── -total_flows = s_cluster.get("total_flows", "?") -entity_count = s_cluster.get("entity_count", 0) -cluster_count = s_cluster.get("cluster_count", 0) -status = s_cluster.get("status", "unknown") - -# ── Validate required metrics ── -if not has_arcs: - errors.append("Globe page missing 'arcCount' element") -if entity_count == 0: - errors.append("entity_count=0 (no entities detected)") -if cluster_count == 0: - errors.append("cluster_count=0 (no clusters found)") - -# ── Show server log tail on failure ── -if errors and os.path.exists(stderr_log): - with open(stderr_log, "r", encoding="utf-8", errors="replace") as f: - lines = f.readlines() - tail = lines[-40:] if len(lines) > 40 else lines - print(f"\n--- Server log tail ({len(tail)} lines) ---") - for line in tail: - print(f" {line.rstrip()}") - -# ── Report ── -print() -print("=" * 60) -print(" E2E v1.1.6 RESULTS") -print("=" * 60) -print(f" run_id : {run_id}") -print(f" status : {status}") -print(f" total_flows : {total_flows}") -print(f" entity_count : {entity_count}") -print(f" cluster_count : {cluster_count}") -print(f" globe_arcs : {'PASS' if has_arcs else 'FAIL'}") - -if errors: - print(f"\n FAIL ({len(errors)} errors):") - for e in errors: - print(f" - {e}") - sys.exit(1) -else: - print(f"\n *** E2E v1.1.6 PASSED ***") - print(f" run_id={run_id}, clusters={cluster_count}, entities={entity_count}, globe_arcs=yes") - sys.exit(0) diff --git a/tests/test_e2e_v1.1.6_full.py b/tests/test_e2e_v1.1.6_full.py deleted file mode 100644 index 7319451..0000000 --- a/tests/test_e2e_v1.1.6_full.py +++ /dev/null @@ -1,70 +0,0 @@ -import subprocess, time, os, sys, glob, json - -PORT = 17766 -PROJ = r"C:\Users\25044\Desktop\Proj\天璇" -os.chdir(PROJ) - -# 1. Start server -print("Starting server...") -server = subprocess.Popen( - [r"runtime\python\python.exe", "manage.py", "runserver", f"127.0.0.1:{PORT}", "--noreload"], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL -) -time.sleep(6) - -import requests - -# 2. Upload 50 files -print("Uploading 50 files...") -csvs = sorted(glob.glob("data/multi_upload/[0-9]*.csv"))[:50] -files = [("files", (os.path.basename(f), open(f, "rb"), "text/csv")) for f in csvs] -r = requests.post(f"http://127.0.0.1:{PORT}/upload/csv/", files=files, timeout=300) -data = r.json() -run_id = data["run_id"] -print(f"run_id={run_id}") - -# 3. Poll until ready -for _ in range(180): - r = requests.get(f"http://127.0.0.1:{PORT}/runs/{run_id}/status/", timeout=10) - s = r.json() - st = s["status"] - if st in ("ready", "failed"): - break - time.sleep(2) -print(f"Upload status={st}, flows={s.get('total_flows')}, error={s.get('error_message','')[:300]}") - -if st == "failed": - print(f"FAILED: {s.get('error_message','')}") - server.terminate(); sys.exit(1) - -# 4. Run clustering -print("Running clustering...") -r = requests.post(f"http://127.0.0.1:{PORT}/analyze/run/", - json={"run_id": run_id, "algorithm": "hdbscan", "min_cluster_size": 5, "cluster_mode": "raw"}, - timeout=30) -print(f"Cluster started: {r.json()}") - -# 5. Poll until completed -for _ in range(120): - r = requests.get(f"http://127.0.0.1:{PORT}/runs/{run_id}/status/", timeout=10) - s = r.json() - st = s["status"] - if st in ("completed", "failed"): - break - time.sleep(5) -print(f"Cluster status={st}, entity_count={s.get('entity_count')}, cluster_count={s.get('cluster_count')}") - -# 6. Verify globe -r = requests.get(f"http://127.0.0.1:{PORT}/globe/?runs={run_id}", timeout=10) -has_arcs = "arcCount" in r.text -print(f"Globe arcCount present: {has_arcs}") - -# 7. Result -server.terminate() -server.wait() -if st == "completed" and s.get("cluster_count", 0) > 0: - print(f"E2E PASSED: {s['entity_count']} entities, {s['cluster_count']} clusters") - sys.exit(0) -else: - print(f"E2E FAILED") - sys.exit(1)