fix: defer background processing until all upload batches complete (race condition fix)

Add finalize_upload endpoint that starts _background_process only after
all file batches have been saved to disk. Fixes race where glob(*.csv)
captured only first batch's files.

- upload_csv: no longer starts thread, sets status='pending'
- finalize_upload: validates dir, starts processing thread
- upload.html: calls finalize after batch upload loop completes
This commit is contained in:
PM-pinou
2026-07-22 22:34:54 +08:00
parent 2c8d1e90ae
commit 29484ef033
3 changed files with 41 additions and 5 deletions
+1
View File
@@ -20,6 +20,7 @@ urlpatterns = [
path('config/', views.config_view, name='config'),
path('api/llm/test/', views.llm_test, name='llm_test'),
path('runs/<int:display_id>/retry/', views.retry_run, name='retry_run'),
path('runs/<int:display_id>/finalize/', views.finalize_upload, name='finalize_upload'),
path('runs/<int:display_id>/delete/', views.delete_upload, name='delete_upload'),
path('logs/', views.log_viewer, name='log_viewer'),
path('globe/', views.globe_view, name='globe'),
+22 -5
View File
@@ -331,15 +331,14 @@ def upload_csv(request):
for chunk in f.chunks():
dst.write(chunk)
saved.append(str(path))
# Auto-process in background
# Create run with status='pending' — background processing deferred to finalize endpoint
run = AnalysisRun.objects.create(
csv_glob=str(upload_dir / '*.csv'),
status='loading',
status='pending',
run_type='upload',
)
t = threading.Thread(target=_background_process, args=(run.id, upload_dir), daemon=True)
t.start()
return JsonResponse({'run_id': run.display_id, 'files': saved, 'status': 'loading'})
# Deferred: _background_process will be started by finalize_upload endpoint
return JsonResponse({'run_id': run.display_id, 'files': saved, 'status': 'pending'})
@csrf_exempt
@@ -367,6 +366,24 @@ def upload_csv_batch(request, display_id):
return JsonResponse({'run_id': run.display_id, 'files': len(saved), 'status': 'ok'})
@csrf_exempt
def finalize_upload(request, display_id):
"""Finalize batch upload: validate upload_dir, set status='loading', start background processing."""
if request.method != 'POST':
return JsonResponse({'error': 'POST required'}, status=405)
run = get_object_or_404(AnalysisRun, display_id=display_id)
if run.status != 'pending':
return JsonResponse({'error': f'Run status is {run.status}, expected pending'}, status=400)
upload_dir = Path(run.csv_glob).parent if run.csv_glob else None
if not upload_dir or not upload_dir.exists():
return JsonResponse({'error': 'Upload directory not found'}, status=400)
run.status = 'loading'
run.save(update_fields=['status'])
t = threading.Thread(target=_background_process, args=(run.id, upload_dir), daemon=True)
t.start()
return JsonResponse({'run_id': run.display_id, 'status': 'loading'})
@csrf_exempt
def delete_upload(request, display_id):
"""Delete uploaded files, SQLite data table, and AnalysisRun.
+18
View File
@@ -126,6 +126,24 @@ document.getElementById('uploadForm').addEventListener('submit', async (e) => {
return;
}
}
// All batches uploaded — finalize to trigger background processing
document.getElementById('progressText').textContent = '正在初始化后台处理...';
try {
const finResp = await fetch(`/runs/${runId}/finalize/`, {
method: 'POST',
headers: { 'X-CSRFToken': csrfToken }
});
const finData = await finResp.json();
if (finData.error) {
document.getElementById('progressText').textContent = '初始化失败: ' + finData.error;
submitBtn.disabled = false;
return;
}
} catch (err) {
document.getElementById('progressText').textContent = '初始化失败: ' + err.message;
submitBtn.disabled = false;
return;
}
// Upload complete — poll for background processing
document.getElementById('progressText').textContent = '上传完成,后台处理中...';
const poll = setInterval(async () => {