fix: defer background processing until all upload batches complete (race condition fix)
Add finalize_upload endpoint that starts _background_process only after all file batches have been saved to disk. Fixes race where glob(*.csv) captured only first batch's files. - upload_csv: no longer starts thread, sets status='pending' - finalize_upload: validates dir, starts processing thread - upload.html: calls finalize after batch upload loop completes
This commit is contained in:
@@ -20,6 +20,7 @@ urlpatterns = [
|
||||
path('config/', views.config_view, name='config'),
|
||||
path('api/llm/test/', views.llm_test, name='llm_test'),
|
||||
path('runs/<int:display_id>/retry/', views.retry_run, name='retry_run'),
|
||||
path('runs/<int:display_id>/finalize/', views.finalize_upload, name='finalize_upload'),
|
||||
path('runs/<int:display_id>/delete/', views.delete_upload, name='delete_upload'),
|
||||
path('logs/', views.log_viewer, name='log_viewer'),
|
||||
path('globe/', views.globe_view, name='globe'),
|
||||
|
||||
+22
-5
@@ -331,15 +331,14 @@ def upload_csv(request):
|
||||
for chunk in f.chunks():
|
||||
dst.write(chunk)
|
||||
saved.append(str(path))
|
||||
# Auto-process in background
|
||||
# Create run with status='pending' — background processing deferred to finalize endpoint
|
||||
run = AnalysisRun.objects.create(
|
||||
csv_glob=str(upload_dir / '*.csv'),
|
||||
status='loading',
|
||||
status='pending',
|
||||
run_type='upload',
|
||||
)
|
||||
t = threading.Thread(target=_background_process, args=(run.id, upload_dir), daemon=True)
|
||||
t.start()
|
||||
return JsonResponse({'run_id': run.display_id, 'files': saved, 'status': 'loading'})
|
||||
# Deferred: _background_process will be started by finalize_upload endpoint
|
||||
return JsonResponse({'run_id': run.display_id, 'files': saved, 'status': 'pending'})
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
@@ -367,6 +366,24 @@ def upload_csv_batch(request, display_id):
|
||||
return JsonResponse({'run_id': run.display_id, 'files': len(saved), 'status': 'ok'})
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
def finalize_upload(request, display_id):
|
||||
"""Finalize batch upload: validate upload_dir, set status='loading', start background processing."""
|
||||
if request.method != 'POST':
|
||||
return JsonResponse({'error': 'POST required'}, status=405)
|
||||
run = get_object_or_404(AnalysisRun, display_id=display_id)
|
||||
if run.status != 'pending':
|
||||
return JsonResponse({'error': f'Run status is {run.status}, expected pending'}, status=400)
|
||||
upload_dir = Path(run.csv_glob).parent if run.csv_glob else None
|
||||
if not upload_dir or not upload_dir.exists():
|
||||
return JsonResponse({'error': 'Upload directory not found'}, status=400)
|
||||
run.status = 'loading'
|
||||
run.save(update_fields=['status'])
|
||||
t = threading.Thread(target=_background_process, args=(run.id, upload_dir), daemon=True)
|
||||
t.start()
|
||||
return JsonResponse({'run_id': run.display_id, 'status': 'loading'})
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
def delete_upload(request, display_id):
|
||||
"""Delete uploaded files, SQLite data table, and AnalysisRun.
|
||||
|
||||
@@ -126,6 +126,24 @@ document.getElementById('uploadForm').addEventListener('submit', async (e) => {
|
||||
return;
|
||||
}
|
||||
}
|
||||
// All batches uploaded — finalize to trigger background processing
|
||||
document.getElementById('progressText').textContent = '正在初始化后台处理...';
|
||||
try {
|
||||
const finResp = await fetch(`/runs/${runId}/finalize/`, {
|
||||
method: 'POST',
|
||||
headers: { 'X-CSRFToken': csrfToken }
|
||||
});
|
||||
const finData = await finResp.json();
|
||||
if (finData.error) {
|
||||
document.getElementById('progressText').textContent = '初始化失败: ' + finData.error;
|
||||
submitBtn.disabled = false;
|
||||
return;
|
||||
}
|
||||
} catch (err) {
|
||||
document.getElementById('progressText').textContent = '初始化失败: ' + err.message;
|
||||
submitBtn.disabled = false;
|
||||
return;
|
||||
}
|
||||
// Upload complete — poll for background processing
|
||||
document.getElementById('progressText').textContent = '上传完成,后台处理中...';
|
||||
const poll = setInterval(async () => {
|
||||
|
||||
Reference in New Issue
Block a user