PM-pinou
8fa36b774e
Key changes: - New: data_loader.py SQLite persistence with drop_sqlite_table - New: db_utils.py retry_on_lock decorator (3 retries, exponential backoff) - New: tool_registry.py with 27 MCP tools (filter_and_cluster, compute_scores, etc.) - New: tls_ref.py for TLS cipher/reference data - New: import_tlsdb.py management command - New: scripts/start_server.py for portable runtime - New: migrations 0003-0007 for SQLite table, display_id, llm fields - Changed: views.py unified pipeline worker, retry_run, display_id everywhere - Changed: models.py with display_id auto-assignment, run_type, sqlite_table, llm_thinking, tool_calls_json - Changed: urls.py added retry_run route - Changed: session_store.py robust JSON persistence - Changed: AGENTS.md v7 fix summary added - Changed: templates — globe rewrite (inertia/polar flip), auto.html (thinking/tool accordions), base.html (toast/config), all pages use display_id - Changed: run.bat PYTHONUTF8=1 - Deleted: entity_detector.py, entity_aggregator.py (replaced by filter_and_cluster clustering pipeline) - Test: 92/92 unit tests passing
114 lines
4.4 KiB
Python
114 lines
4.4 KiB
Python
"""Django management command: ``python manage.py import_tlsdb``
|
|
|
|
Reads ``TlsDB.csv`` from the project root and populates the
|
|
``tls_ref_data`` SQLite table, replacing the CSV as the authoritative
|
|
source of TLS field-code reference metadata.
|
|
|
|
Usage:
|
|
python manage.py import_tlsdb
|
|
python manage.py import_tlsdb --csv /path/to/TlsDB.csv
|
|
python manage.py import_tlsdb --clear # re-import from scratch
|
|
"""
|
|
|
|
import csv
|
|
import os
|
|
from pathlib import Path
|
|
|
|
from django.conf import settings
|
|
from django.core.management.base import BaseCommand, CommandError
|
|
from django.db import connection, transaction
|
|
|
|
|
|
class Command(BaseCommand):
|
|
"""Import ``TlsDB.csv`` field definitions into ``tls_ref_data``."""
|
|
|
|
help = 'Import TlsDB.csv reference data into the tls_ref_data SQLite table'
|
|
|
|
def add_arguments(self, parser):
|
|
parser.add_argument(
|
|
'--csv', type=str, default=None,
|
|
help='Path to TlsDB.csv (default: project-root/TlsDB.csv)',
|
|
)
|
|
parser.add_argument(
|
|
'--clear', action='store_true', default=False,
|
|
help='Clear existing data before importing (default: upsert)',
|
|
)
|
|
|
|
def _resolve_csv_path(self, provided: str | None) -> Path:
|
|
"""Locate the TlsDB.csv file.
|
|
|
|
Priority:
|
|
1. Explicit ``--csv`` path.
|
|
2. Project root (where ``manage.py`` lives).
|
|
"""
|
|
if provided:
|
|
path = Path(provided).resolve()
|
|
else:
|
|
# manage.py lives in the project root — use that as anchor
|
|
manage_dir = Path(__file__).resolve().parent.parent.parent.parent
|
|
path = manage_dir / 'TlsDB.csv'
|
|
|
|
if not path.exists():
|
|
raise CommandError(f'TlsDB.csv not found at: {path}')
|
|
return path
|
|
|
|
def handle(self, *args, **options):
|
|
csv_path = self._resolve_csv_path(options['csv'])
|
|
clear = options['clear']
|
|
|
|
# ── Parse CSV ────────────────────────────────────────────────────────
|
|
rows: list[dict[str, str]] = []
|
|
with open(csv_path, 'r', encoding='utf-8-sig') as f:
|
|
reader = csv.DictReader(f)
|
|
# Expected columns: 标题,含义,数据类型
|
|
for line_no, row in enumerate(reader, start=2):
|
|
field_code = (row.get('标题') or '').strip()
|
|
meaning = (row.get('含义') or '').strip()
|
|
data_type = (row.get('数据类型') or '').strip()
|
|
if not field_code:
|
|
self.stderr.write(self.style.WARNING(
|
|
f' [SKIP] Line {line_no}: empty field_code, skipping'
|
|
))
|
|
continue
|
|
rows.append({
|
|
'field_code': field_code,
|
|
'meaning': meaning,
|
|
'data_type': data_type,
|
|
})
|
|
|
|
if not rows:
|
|
self.stderr.write(self.style.ERROR('No valid rows found in TlsDB.csv'))
|
|
return
|
|
|
|
# ── Insert into tls_ref_data ─────────────────────────────────────────
|
|
with connection.cursor() as cursor:
|
|
with transaction.atomic():
|
|
if clear:
|
|
cursor.execute('DELETE FROM tls_ref_data')
|
|
self.stdout.write(self.style.WARNING(f' Cleared existing data ({len(rows)} rows to insert)'))
|
|
|
|
inserted = 0
|
|
updated = 0
|
|
for r in rows:
|
|
cursor.execute(
|
|
"""
|
|
INSERT INTO tls_ref_data (field_code, meaning, data_type)
|
|
VALUES (%s, %s, %s)
|
|
ON CONFLICT(field_code) DO UPDATE SET
|
|
meaning = EXCLUDED.meaning,
|
|
data_type = EXCLUDED.data_type
|
|
""",
|
|
[r['field_code'], r['meaning'], r['data_type']],
|
|
)
|
|
if cursor.rowcount == 1:
|
|
inserted += 1
|
|
else:
|
|
updated += 1
|
|
|
|
self.stdout.write(self.style.SUCCESS(
|
|
f'\n[OK] Imported {len(rows)} TLS field definitions from: {csv_path}'
|
|
))
|
|
self.stdout.write(self.style.SUCCESS(
|
|
f' New: {inserted} | Updated: {updated} | Total: {len(rows)}'
|
|
))
|