HJQ/tianxuan
2
1
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

fix: classify cnrs/isrs as BOOL_ENUM for proper boolean cleaning (blank→False, +→True)

Browse Source
This commit is contained in:
PM-pinou
2026-07-22 23:39:37 +08:00
parent 38e8985718
commit 7d6d29475d
11 changed files with 779 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_e2e_full.py
+87 -57
View File
@@ -1,18 +1,20 @@
"""
Full-stack end-to-end test: real DeepSeek LLM auto-analysis workflow.
Full-stack end-to-end test: upload → process → analyze workflow.
1. Start Django server on port 18766, monitor stderr
2. Upload ALL CSV files from data/multi_upload/ (0.csv..1997.csv) via POST /upload/csv/
2. Upload ALL CSV files from data/multi_upload/ via POST /upload/csv/
3. Poll /runs/{id}/status/ until 'ready' or 'failed'
4. POST /analyze/llm/ to run LLM auto-analysis (real DeepSeek LLM)
5. Poll /runs/{id}/status/ until 'completed' or 'failed' (LLM may take 2-5 min)
5. Poll /runs/{id}/status/ until 'completed' or 'failed'
6. Check status is 'completed', not 'failed'
7. GET /globe/ → 200
8. POST /runs/{id}/delete/ → 200
9. Stop server
10. FAIL on any error → sys.exit(1)
Usage: runtime/python/python.exe tests/test_e2e_full.py
Usage:
runtime/python/python.exe tests/test_e2e_full.py # Full test with LLM
runtime/python/python.exe tests/test_e2e_full.py --skip-llm # Upload+process only, skip LLM
"""
import subprocess
import time
@@ -21,6 +23,7 @@ import sys
import threading
import os
import uuid
import argparse
import urllib.request
import urllib.error
import glob
@@ -107,10 +110,15 @@ def http_post_raw(path, data=None):
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--skip-llm', action='store_true',
help='Skip LLM analysis steps (4-6), only test upload+processing')
args = parser.parse_args()
proc = start_server()
print("[1/8] Server started")
# ── Upload ALL 1998 CSV files in a single request ──
# ── Upload ALL CSV files in a single request ──
data_dir = os.path.join(os.path.dirname(__file__), '..', 'data', 'multi_upload')
csv_files = sorted(glob.glob(os.path.join(data_dir, '*.csv')))
print(f"[2/8] Uploading {len(csv_files)} CSV files (single multipart request)...")
@@ -137,8 +145,8 @@ def main():
resp = json.loads(r.read())
r.close()
except urllib.error.HTTPError as e:
body = e.read().decode('utf-8', errors='replace')
fail(f"Upload failed: HTTP {e.code} - {body[:300]}")
err_body = e.read().decode('utf-8', errors='replace')
fail(f"Upload failed: HTTP {e.code} - {err_body[:300]}")
proc.terminate()
proc.wait()
sys.exit(1)
@@ -150,16 +158,20 @@ def main():
run_id = resp.get('run_id')
check(run_id is not None, "No run_id in upload response")
print(f" Uploaded {len(csv_files)} files, Run ID: {run_id}")
total_files = resp.get('total_files', len(csv_files))
total_rows = resp.get('total_rows', '?')
print(f" Uploaded {total_files} files ({total_rows} rows), Run ID: {run_id}")
# ── Poll status until 'ready' ──
print("[3/8] Waiting for processing (background: load -> profile -> ready)...")
ready = False
final_status_info = {}
for i in range(180):
code, body = http_get(f'/runs/{run_id}/status/')
code, resp_body = http_get(f'/runs/{run_id}/status/')
check(code == 200, f"Status poll -> {code}")
s = json.loads(body)
s = json.loads(resp_body)
status = s['status']
final_status_info = s
if status == 'failed':
err = s.get('error_message', '')[:200]
fail(f"Run failed: {err}")
@@ -178,64 +190,82 @@ def main():
proc.wait()
sys.exit(1)
# ── POST /analyze/llm/ to run real LLM auto-analysis ──
print("[4/8] Starting LLM auto-analysis (real DeepSeek LLM, may take 2-5 min)...")
code, resp = http_post_json('/analyze/llm/', {'run_id': run_id})
check(code == 200, f"LLM start -> HTTP {code}")
llm_status = resp.get('status')
print(f" LLM started: {llm_status}")
# ── Report upload stats ──
run_total_files = final_status_info.get('total_files', total_files)
run_total_rows = final_status_info.get('total_rows', total_rows)
print(f"\n === UPLOAD RESULTS ===")
print(f" Status: {final_status_info.get('status')}")
print(f" Progress: {final_status_info.get('progress_pct', 0)}%")
print(f" Total files: {run_total_files}")
print(f" Total rows: {run_total_rows}")
csv_glob_info = final_status_info.get('csv_glob', '')
if csv_glob_info:
print(f" CSV glob: {csv_glob_info}")
print(f" ====================\n")
# ── Poll status until 'completed' or 'failed' (LLM phase) ──
print("[5/8] Waiting for LLM analysis to complete...")
completed = False
for i in range(120): # 120 × 3s = 360s = 6 min max
code, body = http_get(f'/runs/{run_id}/status/')
check(code == 200, f"LLM status poll -> {code}")
s = json.loads(body)
status = s['status']
pct = s.get('progress_pct', 0)
msg = s.get('progress_msg', '')
if status == 'failed':
err = s.get('error_message', '') or s.get('progress_msg', '') or '(no details)'
fail(f"LLM analysis failed: {err[:300]}")
break
if status == 'completed':
completed = True
print(f" LLM completed ({pct}%)")
# Print last 300 chars of run_log for interest
run_log = s.get('run_log', '')
if run_log:
# Strip non-ASCII chars for console-safe printing
log_snippet = run_log[-200:].encode('ascii', errors='replace').decode('ascii')
print(f" Last log: {log_snippet}")
break
if i % 5 == 0 or i == 10:
print(f" Poll #{i}: {status} ({pct}%) - {msg}")
time.sleep(3)
if args.skip_llm:
print("[4/8] SKIP: LLM analysis (--skip-llm)")
print("[5/8] SKIP: LLM polling (--skip-llm)")
print("[6/8] Verifying upload status (--skip-llm mode)...")
check(status == 'ready', f"Run status '{status}' after upload, expected 'ready'")
print(f" Upload status: {status} [OK]")
else:
# ── POST /analyze/llm/ to run real LLM auto-analysis ──
print("[4/8] Starting LLM auto-analysis (real DeepSeek LLM, may take 2-5 min)...")
code, resp = http_post_json('/analyze/llm/', {'run_id': run_id})
check(code == 200, f"LLM start -> HTTP {code}")
llm_status = resp.get('status')
print(f" LLM started: {llm_status}")
check(completed, "LLM analysis never completed")
if not completed:
proc.terminate()
proc.wait()
sys.exit(1)
# ── Poll status until 'completed' or 'failed' (LLM phase) ──
print("[5/8] Waiting for LLM analysis to complete...")
completed = False
for i in range(120): # 120 × 3s = 360s = 6 min max
code, resp_body = http_get(f'/runs/{run_id}/status/')
check(code == 200, f"LLM status poll -> {code}")
s = json.loads(resp_body)
status = s['status']
pct = s.get('progress_pct', 0)
msg = s.get('progress_msg', '')
if status == 'failed':
err = s.get('error_message', '') or s.get('progress_msg', '') or '(no details)'
fail(f"LLM analysis failed: {err[:300]}")
break
if status == 'completed':
completed = True
print(f" LLM completed ({pct}%)")
run_log = s.get('run_log', '')
if run_log:
log_snippet = run_log[-200:].encode('ascii', errors='replace').decode('ascii')
print(f" Last log: {log_snippet}")
break
if i % 5 == 0 or i == 10:
print(f" Poll #{i}: {status} ({pct}%) - {msg}")
time.sleep(3)
# ── Check status is 'completed', not 'failed' ──
print("[6/8] Verifying final status...")
code, body = http_get(f'/runs/{run_id}/status/')
s = json.loads(body) if code == 200 else {}
final_status = s.get('status', 'unknown')
check(final_status == 'completed', f"Final status is '{final_status}', expected 'completed'")
print(f" Final status: {final_status} [OK]")
check(completed, "LLM analysis never completed")
if not completed:
proc.terminate()
proc.wait()
sys.exit(1)
# ── Check status is 'completed', not 'failed' ──
print("[6/8] Verifying final status...")
code, resp_body = http_get(f'/runs/{run_id}/status/')
s = json.loads(resp_body) if code == 200 else {}
final_status = s.get('status', 'unknown')
check(final_status == 'completed', f"Final status is '{final_status}', expected 'completed'")
print(f" Final status: {final_status} [OK]")
# ── GET /globe/ → 200 ──
print("[7/8] Checking globe page...")
code, body = http_get('/globe/')
code, _ = http_get('/globe/')
check(code == 200, f"GET /globe/ -> {code}")
print(f" Globe -> HTTP {code} [OK]")
# ── POST /runs/{run_id}/delete/ → 200 ──
print("[8/8] Deleting run...")
code, body = http_post_raw(f'/runs/{run_id}/delete/')
code, _ = http_post_raw(f'/runs/{run_id}/delete/')
check(code == 200, f"DELETE -> HTTP {code}")
print(f" Delete -> HTTP {code} [OK]")