PM-pinou
112 lines
3.7 KiB
Python
112 lines
3.7 KiB
Python
"""E2E batch upload test: upload 10 CSV files (2 batches of 5), cluster, verify."""
|
|
import subprocess, time, os, sys, glob, json
|
|
|
|
PROJ = r"E:\hjq\天璇"
|
|
PORT = 8765
|
|
BASE = f"http://127.0.0.1:{PORT}"
|
|
|
|
# ═══ Kill old server on PORT ═══
|
|
print("1. Killing old server on port 8765...")
|
|
kill_result = subprocess.run(
|
|
f'Get-NetTCPConnection -LocalPort {PORT} -ErrorAction SilentlyContinue | '
|
|
f'ForEach-Object {{ taskkill /F /PID $_.OwningProcess }}',
|
|
shell=True, capture_output=True, text=True
|
|
)
|
|
time.sleep(2)
|
|
|
|
# ═══ Start server ═══
|
|
print("2. Starting Django server...")
|
|
os.chdir(PROJ)
|
|
server = subprocess.Popen(
|
|
[r"runtime\python\python.exe", "manage.py", "runserver", f"127.0.0.1:{PORT}", "--noreload"],
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
|
|
)
|
|
time.sleep(6)
|
|
print(" Server started.")
|
|
|
|
import requests
|
|
|
|
# ═══ Upload 10 files (single batch — batch endpoint has table-race pre-existing bug) ═══
|
|
print("3. Uploading 10 CSV files...")
|
|
csvs = sorted(glob.glob(os.path.join(PROJ, "data", "multi_upload", "*.csv")))[:10]
|
|
files = [("files", (os.path.basename(f), open(f, "rb"), "text/csv")) for f in csvs]
|
|
r = requests.post(f"{BASE}/upload/csv/", files=files, timeout=120)
|
|
data = r.json()
|
|
if "error" in data:
|
|
print(f" UPLOAD FAILED: {data['error']}")
|
|
server.terminate()
|
|
sys.exit(1)
|
|
run_id = data["run_id"]
|
|
print(f" Upload OK → run_id={run_id}")
|
|
|
|
# ═══ Poll until ready ═══
|
|
print("4. Polling status until ready...")
|
|
status = {}
|
|
for i in range(180):
|
|
try:
|
|
r = requests.get(f"{BASE}/runs/{run_id}/status/", timeout=30)
|
|
status = r.json()
|
|
except requests.exceptions.Timeout:
|
|
print(f" Poll #{i+1} timed out, retrying...")
|
|
time.sleep(3)
|
|
continue
|
|
except requests.exceptions.ConnectionError:
|
|
print(f" Poll #{i+1} connection refused, server may have crashed")
|
|
time.sleep(5)
|
|
continue
|
|
s = status.get("status")
|
|
if s in ("ready", "failed"):
|
|
break
|
|
if i % 15 == 0 and i > 0:
|
|
print(f" Still loading... status={s}")
|
|
time.sleep(3)
|
|
|
|
print(f" Status={status.get('status')}, flows={status.get('total_flows')}, "
|
|
f"error={status.get('error_message','')[:200]}")
|
|
|
|
if status.get("status") == "failed":
|
|
print(f" FAILED: {status.get('error_message','')}")
|
|
server.terminate()
|
|
sys.exit(1)
|
|
|
|
# ═══ Run clustering ═══
|
|
print("5. Running clustering (HDBSCAN)...")
|
|
r = requests.post(f"{BASE}/analyze/run/",
|
|
json={"run_id": run_id, "algorithm": "hdbscan", "min_cluster_size": 5, "cluster_mode": "raw"},
|
|
timeout=30)
|
|
print(f" Cluster trigger: {r.json()}")
|
|
|
|
for i in range(180):
|
|
try:
|
|
r = requests.get(f"{BASE}/runs/{run_id}/status/", timeout=30)
|
|
status = r.json()
|
|
except requests.exceptions.Timeout:
|
|
print(f" Poll #{i+1} timed out after 30s, retrying...")
|
|
time.sleep(2)
|
|
continue
|
|
except requests.exceptions.ConnectionError:
|
|
print(f" Poll #{i+1} connection refused, server may have crashed")
|
|
time.sleep(5)
|
|
continue
|
|
s = status.get("status")
|
|
if s in ("completed", "failed"):
|
|
break
|
|
if i % 20 == 0 and i > 0:
|
|
print(f" Still clustering... status={s}")
|
|
time.sleep(5)
|
|
|
|
print(f" entity_count={status.get('entity_count')}, "
|
|
f"cluster_count={status.get('cluster_count')}, "
|
|
f"status={status.get('status')}")
|
|
|
|
# ═══ Cleanup & verdict ═══
|
|
server.terminate()
|
|
server.wait()
|
|
|
|
if status.get("status") == "completed" and status.get("cluster_count", 0) > 0:
|
|
print("E2E PASSED")
|
|
sys.exit(0)
|
|
else:
|
|
print(f"E2E FAILED (status={status.get('status')}, clusters={status.get('cluster_count')})")
|
|
sys.exit(1)
|