Compare commits
14 Commits
908bfcaa32
...
beta
| Author | SHA1 | Date | |
|---|---|---|---|
 PM-pinou
|
8477291f53 | ||
|
PM-pinou
|
464642288e | ||
|
PM-pinou
|
a75c5a7a0e | ||
|
PM-pinou
|
10c9c46a84 | ||
|
PM-pinou
|
92e1305c07 | ||
|
PM-pinou
|
656f0ae2ab | ||
|
PM-pinou
|
806cfc0a64 | ||
|
PM-pinou
|
e54499fa78 | ||
|
PM-pinou
|
9f06e40a41 | ||
|
PM-pinou
|
6cbbc24f14 | ||
|
PM-pinou
|
c54917791e | ||
|
PM-pinou
|
3fd3896aba | ||
|
PM-pinou
|
66f68a2062 | ||
|
PM-pinou
|
b768d02987 |
@@ -52,3 +52,7 @@ data/geoip_chunks/
|
||||
|
||||
# Playwright MCP sessions
|
||||
.playwright-mcp/
|
||||
|
||||
tmp_uploads/
|
||||
*.bat
|
||||
*.py
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
11572
|
||||
30672
|
||||
+14
-742
@@ -1,746 +1,18 @@
|
||||
"""CSV loading with per-file schema validation, BOM detection, and multi-file merge.
|
||||
"""Thin re-export wrapper — implementation moved to analysis/data_loader/ package.
|
||||
|
||||
Supports multiple encodings and delimiter-separated files. The main entry
|
||||
point is :func:`load_csv_directory`, which globs files, detects BOM per file,
|
||||
validates schema consistency, and returns a merged :class:`pl.LazyFrame`.
|
||||
When both ``analysis/data_loader.py`` and ``analysis/data_loader/`` exist,
|
||||
Python prefers the package directory, so this file is effectively dead code.
|
||||
Kept for git history continuity — all real code lives in the package.
|
||||
|
||||
If you need to restore the monolithic version, see git history.
|
||||
"""
|
||||
import glob
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import polars as pl
|
||||
import yaml
|
||||
|
||||
from analysis.types import classify_schema, TYPE_REGISTRY
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
SUPPORTED_ENCODINGS = ['utf-8', 'utf-8-sig', 'utf-16le', 'utf-16be', 'latin-1']
|
||||
|
||||
|
||||
# ── numeric coercion helper ──────────────────────────────────────────────
|
||||
|
||||
def _coerce_to_float(series: pl.Series) -> pl.Series:
|
||||
"""Convert string series to Float64.
|
||||
|
||||
Every entry that doesn't parse as a valid float becomes null.
|
||||
Handles blank strings, standalone '+'/'-', 'N/A', 'null', 'None', etc.
|
||||
"""
|
||||
series = series.cast(pl.Utf8).str.strip_chars()
|
||||
artifacts = ('', '+', '-', '.', '..', 'N/A', 'n/a', 'NA', 'null', 'None', 'NULL', '?', '---', '--', '/')
|
||||
is_artifact = series.is_in(pl.Series(artifacts).implode())
|
||||
result = []
|
||||
for v, bad in zip(series.to_list(), is_artifact.to_list()):
|
||||
if v is None or bad:
|
||||
result.append(None)
|
||||
else:
|
||||
# Check for "+"/"-" prefix artifacts (e.g., "+abc", "-xyz")
|
||||
# where the value isn't a genuine numeric string. Values like
|
||||
# "+5" or "-3.14" are still accepted as valid floats below.
|
||||
if len(v) > 1 and v[0] in '+-':
|
||||
try:
|
||||
float(v[1:]) # validate suffix minus prefix
|
||||
except ValueError:
|
||||
result.append(None)
|
||||
continue
|
||||
try:
|
||||
result.append(float(v))
|
||||
except (ValueError, TypeError):
|
||||
result.append(None)
|
||||
return pl.Series(result, dtype=pl.Float64)
|
||||
|
||||
|
||||
# ── BOM detection ───────────────────────────────────────────────────────
|
||||
|
||||
def detect_bom(path: str) -> str:
|
||||
"""Read the first 4 bytes of *path* and return the detected encoding.
|
||||
|
||||
Returns ``'utf-8'`` when no BOM is found.
|
||||
|
||||
Note: Python 3 ``open()`` fully supports Unicode/Chinese paths on all
|
||||
platforms (no encoding dance needed).
|
||||
"""
|
||||
with open(path, 'rb') as f:
|
||||
header = f.read(4)
|
||||
|
||||
if header[:3] == b'\xef\xbb\xbf':
|
||||
return 'utf-8-sig'
|
||||
if header[:2] == b'\xff\xfe':
|
||||
return 'utf-16le'
|
||||
if header[:2] == b'\xfe\xff':
|
||||
return 'utf-16be'
|
||||
return 'utf-8'
|
||||
|
||||
|
||||
# ── schema utilities ────────────────────────────────────────────────────
|
||||
|
||||
def collect_schema(
|
||||
path: str,
|
||||
encoding: str = 'utf-8',
|
||||
delimiter: str = ',',
|
||||
sample_rows: int = 10000,
|
||||
) -> dict:
|
||||
"""Return {column_name: dtype_string} for a single CSV file.
|
||||
|
||||
Uses :func:`pl.scan_csv` with ``infer_schema_length=0`` to skip type
|
||||
inference. All columns are forced to ``Utf8`` for cross-file compatibility,
|
||||
avoiding type-inference failures when one file's column is ``Int64`` and
|
||||
another's is ``String``.
|
||||
"""
|
||||
lf = pl.scan_csv(
|
||||
str(path), # str() ensures Unicode/Chinese paths resolve correctly
|
||||
encoding=encoding,
|
||||
separator=delimiter,
|
||||
infer_schema_length=0,
|
||||
from analysis.data_loader._csv import load_csv_directory
|
||||
from analysis.data_loader._sqlite import (
|
||||
load_from_db,
|
||||
load_from_db_lazy,
|
||||
save_to_db,
|
||||
drop_sqlite_table,
|
||||
)
|
||||
schema = lf.collect_schema()
|
||||
result = {}
|
||||
for name, dtype in zip(schema.names(), schema.dtypes()):
|
||||
result[name] = 'Utf8' # force Utf8 for cross-file compatibility
|
||||
return result
|
||||
|
||||
|
||||
def _merge_schema_entries(
|
||||
schemas: list[dict],
|
||||
) -> dict:
|
||||
"""Merge multiple schema dicts, raising on type conflicts.
|
||||
|
||||
Returns a single {name: dtype} dict. If a column appears in multiple
|
||||
files with different dtypes the *last* non-null dtype wins (with a
|
||||
warning embedded in the result).
|
||||
"""
|
||||
merged: dict[str, str] = {}
|
||||
conflicts: list[str] = []
|
||||
for s in schemas:
|
||||
for name, dtype in s.items():
|
||||
if name in merged and merged[name] != dtype:
|
||||
conflicts.append(name)
|
||||
merged[name] = dtype # last wins
|
||||
return merged, conflicts
|
||||
|
||||
|
||||
def validate_schemas(schemas: list[dict], strict: bool = True) -> None:
|
||||
"""Compare schemas across files and raise or warn on column mismatch.
|
||||
|
||||
Checks that every file has the same set of column names. Type
|
||||
differences across files are **not** considered an error (Polars can
|
||||
cast), but differences in *column sets* are.
|
||||
|
||||
Args:
|
||||
schemas: List of ``{column_name: dtype_string}`` dicts, one per file.
|
||||
strict: If ``True`` (default), raise :class:`ValueError` on any
|
||||
column mismatch. If ``False``, log a warning instead.
|
||||
"""
|
||||
if not schemas:
|
||||
return
|
||||
|
||||
col_sets = [set(s.keys()) for s in schemas]
|
||||
reference = col_sets[0]
|
||||
|
||||
for i, cs in enumerate(col_sets[1:], start=1):
|
||||
only_in_ref = reference - cs
|
||||
only_in_current = cs - reference
|
||||
if only_in_ref or only_in_current:
|
||||
msg_parts = []
|
||||
if only_in_ref:
|
||||
msg_parts.append(f"File #{i} missing columns: {sorted(only_in_ref)}")
|
||||
if only_in_current:
|
||||
msg_parts.append(f"File #{i} has extra columns: {sorted(only_in_current)}")
|
||||
full_msg = f"Schema mismatch: {'; '.join(msg_parts)}"
|
||||
if strict:
|
||||
raise ValueError(full_msg)
|
||||
else:
|
||||
logger.warning(full_msg)
|
||||
|
||||
|
||||
# ── dtype inference from schema ─────────────────────────────────────────
|
||||
|
||||
def detect_schema_dtypes(
|
||||
lf: pl.LazyFrame,
|
||||
sample_rows: int = 10000,
|
||||
) -> dict:
|
||||
"""Infer Polars dtype strings from a LazyFrame's schema.
|
||||
|
||||
The schema is read without collecting the full data (the scan scans
|
||||
a small sample).
|
||||
"""
|
||||
schema = lf.collect_schema()
|
||||
return {name: str(dtype) for name, dtype in zip(schema.names(), schema.dtypes())}
|
||||
|
||||
|
||||
# ── config loading ──────────────────────────────────────────────────────
|
||||
|
||||
def load_config(config_path: Optional[str]) -> dict:
|
||||
"""Load a YAML config file; returns an empty dict if *config_path* is None."""
|
||||
if config_path is None:
|
||||
return {}
|
||||
path = Path(config_path)
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Config file not found: {config_path}")
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
return yaml.safe_load(f) or {}
|
||||
|
||||
|
||||
# ── SQLite persistence ──────────────────────────────────────────────────
|
||||
|
||||
def _dtype_to_sqlite(dtype: pl.DataType) -> str:
|
||||
"""Map a Polars dtype to a SQLite column type affinity."""
|
||||
if dtype in (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
|
||||
pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64):
|
||||
return 'INTEGER'
|
||||
if dtype in (pl.Float32, pl.Float64):
|
||||
return 'REAL'
|
||||
if dtype == pl.Boolean:
|
||||
return 'INTEGER'
|
||||
return 'TEXT' # String, Date, Datetime, etc. all map to TEXT
|
||||
|
||||
|
||||
def save_to_db(run_id: int, lf: pl.LazyFrame, mode: str = 'replace') -> str:
|
||||
"""Write a LazyFrame to a SQLite table ``_data_{run_id}``.
|
||||
|
||||
Collects the LazyFrame (streaming if possible), creates or replaces the
|
||||
table, bulk-inserts rows in chunks, and returns the table name.
|
||||
|
||||
Args:
|
||||
run_id: Analysis run ID (used as ``_data_{run_id}``).
|
||||
lf: LazyFrame to persist.
|
||||
mode: ``'replace'`` (default) → drop+create+insert.
|
||||
``'append'`` → skip DDL, only INSERT (table must exist).
|
||||
|
||||
This is a *persistence copy* — the original CSV files remain untouched.
|
||||
"""
|
||||
from django.conf import settings
|
||||
|
||||
table_name = f"_data_{run_id}"
|
||||
db_path = settings.DATABASES['default']['NAME']
|
||||
|
||||
df = lf.collect(streaming=True)
|
||||
conn = sqlite3.connect(db_path)
|
||||
|
||||
try:
|
||||
if mode == 'replace':
|
||||
# Drop existing table for retry/idempotency safety
|
||||
conn.execute(f'DROP TABLE IF EXISTS "{table_name}"')
|
||||
|
||||
# Build CREATE TABLE from Polars schema
|
||||
col_defs = [f'"{name}" {_dtype_to_sqlite(dtype)}'
|
||||
for name, dtype in zip(df.columns, df.dtypes)]
|
||||
conn.execute(f'CREATE TABLE "{table_name}" ({", ".join(col_defs)})')
|
||||
|
||||
# Bulk insert in chunks
|
||||
col_names = [f'"{c}"' for c in df.columns]
|
||||
placeholders = ','.join(['?' for _ in df.columns])
|
||||
insert_sql = f'INSERT INTO "{table_name}" ({", ".join(col_names)}) VALUES ({placeholders})'
|
||||
|
||||
total_rows = len(df)
|
||||
chunk_size = 10000
|
||||
for i in range(0, total_rows, chunk_size):
|
||||
chunk = df.slice(i, chunk_size)
|
||||
conn.executemany(insert_sql, chunk.rows())
|
||||
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
logger.info('[SAVE_TO_DB] table=%s rows=%d mode=%s', table_name, total_rows, mode)
|
||||
return table_name
|
||||
|
||||
|
||||
_STR_TO_DTYPE = {
|
||||
'Float64': pl.Float64, 'Float32': pl.Float32,
|
||||
'Int64': pl.Int64, 'Int32': pl.Int32, 'Int16': pl.Int16, 'Int8': pl.Int8,
|
||||
'UInt64': pl.UInt64, 'UInt32': pl.UInt32, 'UInt16': pl.UInt16, 'UInt8': pl.UInt8,
|
||||
'Utf8': pl.Utf8, 'String': pl.Utf8, 'Categorical': pl.Categorical,
|
||||
'Boolean': pl.Boolean, 'Bool': pl.Boolean,
|
||||
'Date': pl.Date, 'Datetime': pl.Datetime, 'Time': pl.Time,
|
||||
}
|
||||
|
||||
|
||||
def _str_to_polars_dtype(s: str) -> pl.DataType:
|
||||
"""Convert a Polars dtype string (e.g. ``'Float64'``, ``'Int32'``) to a ``pl.DataType``.
|
||||
|
||||
Strips trailing metadata like ``Datetime(time_unit='us', time_zone=None)``.
|
||||
"""
|
||||
base = s.split('(')[0].strip()
|
||||
return _STR_TO_DTYPE.get(base, pl.Utf8)
|
||||
|
||||
|
||||
def _rows_to_csv_df(rows: list[sqlite3.Row]) -> pl.DataFrame:
|
||||
"""Convert a batch of sqlite3.Row objects to a DataFrame via CSV fallback.
|
||||
|
||||
Loads all columns as Utf8 to handle mixed-type SQLite columns.
|
||||
"""
|
||||
import csv as _csv_mod
|
||||
|
||||
buf = io.StringIO()
|
||||
writer = _csv_mod.writer(buf)
|
||||
writer.writerow(rows[0].keys())
|
||||
for r in rows:
|
||||
writer.writerow(str(v) if v is not None else '' for v in r)
|
||||
buf.seek(0)
|
||||
return pl.read_csv(buf, infer_schema_length=0)
|
||||
|
||||
|
||||
def load_from_db(
|
||||
table_name: str,
|
||||
schema_overrides: dict[str, pl.DataType | str] | None = None,
|
||||
) -> Optional[pl.LazyFrame]:
|
||||
"""Load a SQLite table into a LazyFrame.
|
||||
|
||||
Args:
|
||||
table_name: SQLite table name.
|
||||
schema_overrides: Optional dict of ``{column_name: dtype}`` to cast
|
||||
columns after loading. Values can be ``pl.DataType`` instances
|
||||
(e.g. ``pl.Float64``) or dtype strings (e.g. ``'Float64'``).
|
||||
Use when SQLite ``TEXT`` columns need to be restored to
|
||||
``Float64`` / ``Int64`` etc.
|
||||
|
||||
Returns ``None`` if the table does not exist.
|
||||
"""
|
||||
from django.conf import settings
|
||||
|
||||
db_path = settings.DATABASES['default']['NAME']
|
||||
conn = sqlite3.connect(db_path)
|
||||
|
||||
try:
|
||||
# Check table exists
|
||||
cursor = conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name=?",
|
||||
(table_name,),
|
||||
)
|
||||
if cursor.fetchone() is None:
|
||||
return None
|
||||
|
||||
conn.row_factory = sqlite3.Row
|
||||
cursor = conn.execute(f'SELECT * FROM "{table_name}"')
|
||||
|
||||
# Peek first row to detect if table is empty
|
||||
first_row = cursor.fetchone()
|
||||
if first_row is None:
|
||||
return pl.DataFrame().lazy()
|
||||
|
||||
batch_rows = [first_row]
|
||||
dfs = []
|
||||
_use_csv_fallback = False
|
||||
|
||||
while True:
|
||||
chunk = cursor.fetchmany(10000)
|
||||
if not chunk:
|
||||
# Flush remaining rows
|
||||
if batch_rows:
|
||||
if _use_csv_fallback:
|
||||
dfs.append(_rows_to_csv_df(batch_rows))
|
||||
else:
|
||||
try:
|
||||
dfs.append(pl.from_dicts([dict(r) for r in batch_rows]))
|
||||
except Exception:
|
||||
_use_csv_fallback = True
|
||||
dfs.append(_rows_to_csv_df(batch_rows))
|
||||
break
|
||||
|
||||
batch_rows.extend(chunk)
|
||||
if len(batch_rows) >= 10000:
|
||||
if _use_csv_fallback:
|
||||
dfs.append(_rows_to_csv_df(batch_rows))
|
||||
else:
|
||||
try:
|
||||
dfs.append(pl.from_dicts([dict(r) for r in batch_rows]))
|
||||
except Exception:
|
||||
_use_csv_fallback = True
|
||||
# Re-process current batch with CSV fallback
|
||||
dfs.append(_rows_to_csv_df(batch_rows))
|
||||
batch_rows = []
|
||||
|
||||
# Concat all batches
|
||||
if len(dfs) == 1:
|
||||
df = dfs[0]
|
||||
else:
|
||||
df = pl.concat(dfs, how='vertical')
|
||||
|
||||
# Apply type overrides to restore original column types
|
||||
if schema_overrides:
|
||||
casts = {}
|
||||
for col, dtype in schema_overrides.items():
|
||||
if col not in df.columns:
|
||||
continue
|
||||
dtype_obj = dtype if isinstance(dtype, pl.DataType) else _str_to_polars_dtype(dtype)
|
||||
if df[col].dtype != dtype_obj:
|
||||
try:
|
||||
casts[col] = pl.col(col).cast(dtype_obj)
|
||||
except Exception:
|
||||
# If a cast fails (e.g. "A" → Int64), keep the column as-is
|
||||
pass
|
||||
if casts:
|
||||
df = df.with_columns(list(casts.values()))
|
||||
|
||||
return df.lazy()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def load_from_db_lazy(
|
||||
table_name: str,
|
||||
schema_overrides: dict[str, pl.DataType | str] | None = None,
|
||||
) -> Optional[pl.LazyFrame]:
|
||||
"""Load a SQLite table into a LazyFrame using Polars native DB reader.
|
||||
|
||||
Uses ``pl.read_database`` for efficient bulk reading — an order of
|
||||
magnitude faster than the manual batch-processing path in
|
||||
:func:`load_from_db`. Returns a :class:`pl.LazyFrame` so downstream
|
||||
operations remain lazy (query-plan, not materialised).
|
||||
|
||||
Args:
|
||||
table_name: SQLite table name.
|
||||
schema_overrides: Optional ``{column_name: dtype}`` dict to cast
|
||||
columns after loading. Values can be ``pl.DataType`` instances
|
||||
(e.g. ``pl.Float64``) or dtype strings (e.g. ``'Float64'``).
|
||||
|
||||
Returns:
|
||||
``None`` if the table does not exist or is empty.
|
||||
"""
|
||||
from django.conf import settings
|
||||
|
||||
db_path = settings.DATABASES['default']['NAME']
|
||||
conn = sqlite3.connect(db_path)
|
||||
|
||||
try:
|
||||
# Check table existence (fast metadata query)
|
||||
cursor = conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name=?",
|
||||
(table_name,),
|
||||
)
|
||||
if cursor.fetchone() is None:
|
||||
return None
|
||||
|
||||
# Bulk-read with Polars native engine — leverages the DBAPI2
|
||||
# connection directly for efficient C-level data transfer.
|
||||
query = f'SELECT * FROM "{table_name}"'
|
||||
df = pl.read_database(connection=conn, query=query)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
if df.height == 0:
|
||||
return None
|
||||
|
||||
# Apply type overrides to restore original column types
|
||||
# (SQLite stores everything as TEXT/INTEGER/REAL/BLOB;
|
||||
# we need to cast back to Float64 / Int64 / etc.)
|
||||
if schema_overrides:
|
||||
casts = {}
|
||||
for col, dtype in schema_overrides.items():
|
||||
if col not in df.columns:
|
||||
continue
|
||||
dtype_obj = (
|
||||
dtype if isinstance(dtype, pl.DataType)
|
||||
else _str_to_polars_dtype(dtype)
|
||||
)
|
||||
if df[col].dtype != dtype_obj:
|
||||
try:
|
||||
casts[col] = pl.col(col).cast(dtype_obj)
|
||||
except Exception:
|
||||
pass # uncastable column — keep as-is
|
||||
if casts:
|
||||
df = df.with_columns(list(casts.values()))
|
||||
|
||||
rows = df.height
|
||||
cols = df.width
|
||||
logger.info(
|
||||
'[LOAD_FROM_DB_LAZY] table=%s rows=%d cols=%d',
|
||||
table_name, rows, cols,
|
||||
)
|
||||
return df.lazy()
|
||||
|
||||
|
||||
def drop_sqlite_table(table_name: str) -> None:
|
||||
"""Drop a SQLite data table silently if it exists."""
|
||||
from django.conf import settings
|
||||
|
||||
db_path = settings.DATABASES['default']['NAME']
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
conn.execute(f'DROP TABLE IF EXISTS "{table_name}"')
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ── helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
def _file_count(paths: list[str]) -> int:
|
||||
return len(paths)
|
||||
|
||||
|
||||
def _resolve_encoding(encoding: str, file_encoding: str) -> str:
|
||||
"""BOM-detected encoding overrides user-supplied encoding.
|
||||
|
||||
Polars >= 1.0 requires ``'utf8'`` (not ``'utf-8'``).
|
||||
"""
|
||||
resolved = file_encoding if file_encoding != 'utf-8' else encoding
|
||||
return resolved.replace('-', '') if resolved in ('utf-8', 'utf-8-sig') else resolved
|
||||
|
||||
|
||||
# ── main entry ──────────────────────────────────────────────────────────
|
||||
|
||||
def load_csv_directory(
|
||||
glob_pattern: str,
|
||||
encoding: str = 'utf-8',
|
||||
delimiter: str = ',',
|
||||
config_path: Optional[str] = None,
|
||||
schema_override: Optional[dict] = None,
|
||||
sample_rows: int = 10000,
|
||||
schema_strict: bool = False,
|
||||
recursive: bool = False,
|
||||
head: Optional[int] = None,
|
||||
) -> tuple[pl.LazyFrame, dict, int, int, float]:
|
||||
"""Load, validate, and merge CSV files matching *glob_pattern*.
|
||||
|
||||
Args:
|
||||
glob_pattern: Glob pattern for CSV files (e.g. ``data/*.csv``).
|
||||
encoding: Fallback encoding when no BOM is detected.
|
||||
delimiter: Column delimiter character.
|
||||
config_path: Optional path to a YAML config (merged into metadata).
|
||||
schema_override: If given, force these {column: dtype} overrides.
|
||||
sample_rows: Rows to scan for dtype inference per file.
|
||||
schema_strict: When True, raise ValueError on column mismatch across
|
||||
files. When False (default), log warnings and merge with
|
||||
``diagonal_relaxed`` (missing columns get null).
|
||||
recursive: When True, use ``glob.glob(..., recursive=True)`` to
|
||||
support ``**/*.csv`` patterns.
|
||||
head: Optional row limit for preview scenarios. When set, applies
|
||||
``lf.head(head)`` before returning.
|
||||
|
||||
Returns:
|
||||
A tuple of ``(lazyframe, schema_dict, total_row_count, file_count,
|
||||
memory_estimate_mb)``.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If no files match the glob pattern.
|
||||
ValueError: If schemas are inconsistent across files and
|
||||
*schema_strict* is True.
|
||||
"""
|
||||
# Normalise path separators for cross-platform Unicode support
|
||||
glob_pattern = os.path.normpath(glob_pattern)
|
||||
|
||||
# Resolve file list
|
||||
paths = sorted(glob.glob(glob_pattern, recursive=recursive))
|
||||
if not paths and not recursive:
|
||||
# Fallback: try recursive pattern
|
||||
paths = sorted(glob.glob(glob_pattern, recursive=True))
|
||||
if not paths:
|
||||
raise FileNotFoundError(f"No files match glob pattern: {glob_pattern}")
|
||||
|
||||
# Load config (user-provided or fall back to project default)
|
||||
config = load_config(config_path)
|
||||
if not config:
|
||||
default_path = Path(__file__).parent.parent / 'config' / 'config.yaml'
|
||||
if default_path.exists():
|
||||
config = load_config(str(default_path))
|
||||
|
||||
# Collect per-file schema
|
||||
schemas: list[dict] = []
|
||||
lazyframes: list[pl.LazyFrame] = []
|
||||
for path in paths:
|
||||
file_encoding = detect_bom(path)
|
||||
actual_encoding = _resolve_encoding(encoding, file_encoding)
|
||||
|
||||
lf = pl.scan_csv(
|
||||
str(path), # str() ensures Unicode/Chinese paths resolve correctly
|
||||
encoding=actual_encoding,
|
||||
separator=delimiter,
|
||||
infer_schema_length=max(sample_rows, 1000),
|
||||
)
|
||||
schema = detect_schema_dtypes(lf, sample_rows=sample_rows)
|
||||
|
||||
# Apply schema override for this file
|
||||
if schema_override:
|
||||
for col, dtype in schema_override.items():
|
||||
if col in schema:
|
||||
schema[col] = dtype
|
||||
|
||||
schemas.append(schema)
|
||||
lazyframes.append(lf)
|
||||
|
||||
if schema_strict:
|
||||
# Raise on column mismatch
|
||||
validate_schemas(schemas)
|
||||
else:
|
||||
# Lenient mode: log warnings for column mismatches
|
||||
col_sets = [set(s.keys()) for s in schemas]
|
||||
reference = col_sets[0]
|
||||
for i, cs in enumerate(col_sets[1:], start=1):
|
||||
only_in_ref = reference - cs
|
||||
only_in_current = cs - reference
|
||||
if only_in_ref:
|
||||
logger.warning(
|
||||
"File #%d missing columns: %s (will be null)",
|
||||
i + 1, sorted(only_in_ref),
|
||||
)
|
||||
if only_in_current:
|
||||
logger.warning(
|
||||
"File #%d has extra columns: %s (will be null in other files)",
|
||||
i + 1, sorted(only_in_current),
|
||||
)
|
||||
|
||||
# Merge schemas (unify column ordering — first file wins order)
|
||||
merged_schema, _ = _merge_schema_entries(schemas)
|
||||
|
||||
# Concatenate all LazyFrames (union vertically)
|
||||
merged_lf: pl.LazyFrame = pl.concat(lazyframes, how='diagonal_relaxed')
|
||||
|
||||
# Log schema and size estimate
|
||||
total_row_count_est = len(paths) * (lazyframes[0].collect(streaming=True).height if lazyframes else 0)
|
||||
logger.info(f'[LOAD] files={len(paths)} total_rows_est={total_row_count_est}')
|
||||
for col_name, dtype in sorted(merged_schema.items()):
|
||||
logger.info(f'[LOAD_SCHEMA] col={col_name} dtype={dtype}')
|
||||
|
||||
# Apply schema override on merged frame (cast columns)
|
||||
if schema_override:
|
||||
casts = {}
|
||||
for col, dtype_str in schema_override.items():
|
||||
if col in merged_schema:
|
||||
try:
|
||||
polars_dtype = getattr(pl, dtype_str.split('(')[0], None)
|
||||
if polars_dtype:
|
||||
casts[col] = pl.col(col).cast(polars_dtype)
|
||||
except Exception:
|
||||
pass # skip uncastable overrides silently
|
||||
if casts:
|
||||
merged_lf = merged_lf.with_columns(list(casts.values()))
|
||||
|
||||
# ── Cleaning: drop outdated columns ──────────────────────────────────
|
||||
geoip_cfg = config.get('data', {}).get('geoip', {})
|
||||
drop_cols = geoip_cfg.get(
|
||||
'drop_columns',
|
||||
[], # Empty by default — lat/lon columns are now cleaned, not dropped
|
||||
)
|
||||
existing_drop = [c for c in drop_cols if c in merged_schema]
|
||||
if existing_drop:
|
||||
merged_lf = merged_lf.drop(existing_drop)
|
||||
merged_schema = {k: v for k, v in merged_schema.items() if k not in existing_drop}
|
||||
logger.info('[CLEAN] dropped columns: %s', existing_drop)
|
||||
|
||||
# ── Cleaning: BOOL normalisation ─────────────────────────────────────
|
||||
try:
|
||||
type_map = classify_schema(
|
||||
merged_lf.head(1000),
|
||||
config_overrides=config.get('columns', {}),
|
||||
)
|
||||
except Exception:
|
||||
type_map = {}
|
||||
|
||||
bool_cols = [
|
||||
c for c, t in type_map.items()
|
||||
if t.name == "布尔" and c in merged_schema
|
||||
]
|
||||
if bool_cols:
|
||||
schema_dtypes = dict(zip(
|
||||
merged_lf.collect_schema().names(),
|
||||
merged_lf.collect_schema().dtypes(),
|
||||
))
|
||||
bool_exprs = []
|
||||
cleaned_names: list[str] = []
|
||||
for c in bool_cols:
|
||||
if schema_dtypes.get(c) not in (pl.Utf8, pl.String):
|
||||
continue
|
||||
c_str = pl.col(c).cast(pl.Utf8).str.strip_chars()
|
||||
cleaned = (
|
||||
pl.when(c_str == "")
|
||||
.then(pl.lit("False"))
|
||||
.when(c_str.str.to_lowercase() == "+")
|
||||
.then(pl.lit("True"))
|
||||
.when(c_str.str.to_lowercase() == "-")
|
||||
.then(pl.lit("False"))
|
||||
.otherwise(c_str)
|
||||
)
|
||||
bool_exprs.append(cleaned.alias(c))
|
||||
cleaned_names.append(c)
|
||||
if bool_exprs:
|
||||
merged_lf = merged_lf.with_columns(bool_exprs)
|
||||
for c in cleaned_names:
|
||||
merged_schema[c] = 'Utf8'
|
||||
logger.info('[CLEAN] BOOL columns normalised: %s', cleaned_names)
|
||||
|
||||
# ── Generic numeric coercion ────────────────────────────────────────
|
||||
# For remaining string columns (not IPv4/BYTES/ENUM/BOOL etc.),
|
||||
# try to coerce to Float64.
|
||||
_SKIP_NUM_COERCE = {"IPv4", "字节", "枚举", "布尔"}
|
||||
_SKIP_NAME_PATTERNS = ('mac', 'mac_', '_mac', 'addr', 'address')
|
||||
schema_dtypes_gn = dict(zip(
|
||||
merged_lf.collect_schema().names(),
|
||||
merged_lf.collect_schema().dtypes(),
|
||||
))
|
||||
generic_numeric_cols = []
|
||||
for c, t in type_map.items():
|
||||
if t.name in _SKIP_NUM_COERCE:
|
||||
continue
|
||||
if c not in merged_schema:
|
||||
continue
|
||||
dtype = schema_dtypes_gn.get(c)
|
||||
if dtype not in (pl.Utf8, pl.String):
|
||||
continue
|
||||
cl = c.lower().replace('-', '_').replace(' ', '_')
|
||||
if any(p in cl for p in _SKIP_NAME_PATTERNS):
|
||||
continue
|
||||
generic_numeric_cols.append(c)
|
||||
|
||||
if generic_numeric_cols:
|
||||
num_exprs = []
|
||||
for c in generic_numeric_cols:
|
||||
num_exprs.append(
|
||||
pl.col(c).map_batches(_coerce_to_float, return_dtype=pl.Float64).alias(c)
|
||||
)
|
||||
if num_exprs:
|
||||
merged_lf = merged_lf.with_columns(num_exprs)
|
||||
for c in generic_numeric_cols:
|
||||
merged_schema[c] = 'Float64'
|
||||
logger.info('[CLEAN] Generic numeric coercion applied: %s', generic_numeric_cols)
|
||||
|
||||
# ── Cleaning: BYTES (hex) preprocessing ─────────────────────────────
|
||||
hex_cols = [
|
||||
c for c, t in type_map.items()
|
||||
if t.name == "字节" and c in merged_schema
|
||||
]
|
||||
if hex_cols:
|
||||
hex_exprs = []
|
||||
for c in hex_cols:
|
||||
hex_exprs.append(
|
||||
pl.col(c).str.split(" ").list.len().alias(f"{c}_hex_pairs_count")
|
||||
)
|
||||
merged_lf = merged_lf.with_columns(hex_exprs)
|
||||
for c in hex_cols:
|
||||
merged_schema[f"{c}_hex_pairs_count"] = 'UInt32'
|
||||
logger.info('[CLEAN] BYTES columns added hex_pairs_count: %s', hex_cols)
|
||||
|
||||
# Row count estimate (fast path: use first file row count × files)
|
||||
# For accurate count we'd need to collect, but that defeats lazy.
|
||||
# We estimate from the first file's scanned schema info.
|
||||
first_rows = lazyframes[0].collect(streaming=True).height if lazyframes else 0
|
||||
total_row_count = first_rows * len(paths)
|
||||
|
||||
# Memory estimate
|
||||
bytes_per_row = 0
|
||||
for col_name, dtype_str in merged_schema.items():
|
||||
numeric_part = dtype_str.split('(')[0].lower()
|
||||
size_map = {
|
||||
'int8': 1, 'int16': 2, 'int32': 4, 'int64': 8,
|
||||
'uint8': 1, 'uint16': 2, 'uint32': 4, 'uint64': 8,
|
||||
'float32': 4, 'float64': 8,
|
||||
'boolean': 1, 'bool': 1,
|
||||
'utf8': 50, 'string': 50, 'str': 50,
|
||||
'categorical': 8, 'cat': 8,
|
||||
'date': 4, 'datetime': 8, 'time': 8, 'duration': 8,
|
||||
}
|
||||
bytes_per_row += size_map.get(numeric_part, 8)
|
||||
memory_mb = (bytes_per_row * total_row_count) / (1024 * 1024)
|
||||
|
||||
# Apply head limit for preview (keeps LazyFrame lazy — rows are limited on collect)
|
||||
if head is not None:
|
||||
merged_lf = merged_lf.head(head)
|
||||
total_row_count = min(total_row_count, head)
|
||||
|
||||
return merged_lf, merged_schema, total_row_count, len(paths), memory_mb
|
||||
from analysis.data_loader._clean import _coerce_to_float
|
||||
from analysis.data_loader._sqlite import _dtype_to_sqlite
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
"""CSV data loading, schema handling, cleaning, and SQLite persistence.
|
||||
|
||||
Package layout:
|
||||
_csv.py — CSV scanning, BOM/encoding detection, :func:`load_csv_directory`
|
||||
_schema.py — Schema merging and cross-file validation
|
||||
_clean.py — Data cleaning helpers (:func:`_coerce_to_float`)
|
||||
_sqlite.py — SQLite persistence (save/load/drop tables)
|
||||
"""
|
||||
|
||||
from ._csv import (
|
||||
SUPPORTED_ENCODINGS,
|
||||
detect_bom,
|
||||
collect_schema,
|
||||
detect_schema_dtypes,
|
||||
load_config,
|
||||
_file_count,
|
||||
_resolve_encoding,
|
||||
load_csv_directory,
|
||||
)
|
||||
from ._schema import _merge_schema_entries, validate_schemas
|
||||
from ._clean import _coerce_to_float
|
||||
from ._sqlite import (
|
||||
_dtype_to_sqlite,
|
||||
_str_to_polars_dtype,
|
||||
_rows_to_csv_df,
|
||||
save_to_db,
|
||||
load_from_db,
|
||||
load_from_db_lazy,
|
||||
drop_sqlite_table,
|
||||
)
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Data cleaning helpers: numeric coercion for string columns."""
|
||||
import polars as pl
|
||||
|
||||
|
||||
def _coerce_to_float(series: pl.Series) -> pl.Series:
|
||||
"""Convert string series to Float64.
|
||||
|
||||
Every entry that doesn't parse as a valid float becomes null.
|
||||
Handles blank strings, standalone '+'/'-', 'N/A', 'null', 'None', etc.
|
||||
"""
|
||||
series = series.cast(pl.Utf8).str.strip_chars()
|
||||
artifacts = ('', '+', '-', '.', '..', 'N/A', 'n/a', 'NA', 'null', 'None', 'NULL', '?', '---', '--', '/')
|
||||
is_artifact = series.is_in(pl.Series(artifacts).implode())
|
||||
result = []
|
||||
for v, bad in zip(series.to_list(), is_artifact.to_list()):
|
||||
if v is None or bad:
|
||||
result.append(None)
|
||||
else:
|
||||
# Check for "+"/"-" prefix artifacts (e.g., "+abc", "-xyz")
|
||||
# where the value isn't a genuine numeric string. Values like
|
||||
# "+5" or "-3.14" are still accepted as valid floats below.
|
||||
if len(v) > 1 and v[0] in '+-':
|
||||
try:
|
||||
float(v[1:]) # validate suffix minus prefix
|
||||
except ValueError:
|
||||
result.append(None)
|
||||
continue
|
||||
try:
|
||||
result.append(float(v))
|
||||
except (ValueError, TypeError):
|
||||
result.append(None)
|
||||
return pl.Series(result, dtype=pl.Float64)
|
||||
@@ -0,0 +1,375 @@
|
||||
"""CSV loading: glob, BOM detection, encoding, schema collection, multi-file merge.
|
||||
|
||||
Main entry: :func:`load_csv_directory`.
|
||||
"""
|
||||
import glob
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import polars as pl
|
||||
import yaml
|
||||
|
||||
from analysis.types import classify_schema, TYPE_REGISTRY
|
||||
from ._clean import _coerce_to_float
|
||||
from ._schema import _merge_schema_entries, validate_schemas
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SUPPORTED_ENCODINGS = ['utf-8', 'utf-8-sig', 'utf-16le', 'utf-16be', 'latin-1']
|
||||
|
||||
|
||||
# ── BOM detection ───────────────────────────────────────────────────────
|
||||
|
||||
def detect_bom(path: str) -> str:
|
||||
"""Read the first 4 bytes of *path* and return the detected encoding.
|
||||
|
||||
Returns ``'utf-8'`` when no BOM is found.
|
||||
|
||||
Note: Python 3 ``open()`` fully supports Unicode/Chinese paths on all
|
||||
platforms (no encoding dance needed).
|
||||
"""
|
||||
with open(path, 'rb') as f:
|
||||
header = f.read(4)
|
||||
|
||||
if header[:3] == b'\xef\xbb\xbf':
|
||||
return 'utf-8-sig'
|
||||
if header[:2] == b'\xff\xfe':
|
||||
return 'utf-16le'
|
||||
if header[:2] == b'\xfe\xff':
|
||||
return 'utf-16be'
|
||||
return 'utf-8'
|
||||
|
||||
|
||||
# ── schema utilities ────────────────────────────────────────────────────
|
||||
|
||||
def collect_schema(
|
||||
path: str,
|
||||
encoding: str = 'utf-8',
|
||||
delimiter: str = ',',
|
||||
sample_rows: int = 10000,
|
||||
) -> dict:
|
||||
"""Return {column_name: dtype_string} for a single CSV file.
|
||||
|
||||
Uses :func:`pl.scan_csv` with ``infer_schema_length=0`` to skip type
|
||||
inference. All columns are forced to ``Utf8`` for cross-file compatibility,
|
||||
avoiding type-inference failures when one file's column is ``Int64`` and
|
||||
another's is ``String``.
|
||||
"""
|
||||
lf = pl.scan_csv(
|
||||
str(path), # str() ensures Unicode/Chinese paths resolve correctly
|
||||
encoding=encoding,
|
||||
separator=delimiter,
|
||||
infer_schema_length=0,
|
||||
)
|
||||
schema = lf.collect_schema()
|
||||
result = {}
|
||||
for name, dtype in zip(schema.names(), schema.dtypes()):
|
||||
result[name] = 'Utf8' # force Utf8 for cross-file compatibility
|
||||
return result
|
||||
|
||||
|
||||
# ── dtype inference from schema ─────────────────────────────────────────
|
||||
|
||||
def detect_schema_dtypes(
|
||||
lf: pl.LazyFrame,
|
||||
sample_rows: int = 10000,
|
||||
) -> dict:
|
||||
"""Infer Polars dtype strings from a LazyFrame's schema.
|
||||
|
||||
The schema is read without collecting the full data (the scan scans
|
||||
a small sample).
|
||||
"""
|
||||
schema = lf.collect_schema()
|
||||
return {name: str(dtype) for name, dtype in zip(schema.names(), schema.dtypes())}
|
||||
|
||||
|
||||
# ── config loading ──────────────────────────────────────────────────────
|
||||
|
||||
def load_config(config_path: Optional[str]) -> dict:
|
||||
"""Load a YAML config file; returns an empty dict if *config_path* is None."""
|
||||
if config_path is None:
|
||||
return {}
|
||||
path = Path(config_path)
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Config file not found: {config_path}")
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
return yaml.safe_load(f) or {}
|
||||
|
||||
|
||||
# ── helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
def _file_count(paths: list[str]) -> int:
|
||||
return len(paths)
|
||||
|
||||
|
||||
def _resolve_encoding(encoding: str, file_encoding: str) -> str:
|
||||
"""BOM-detected encoding overrides user-supplied encoding.
|
||||
|
||||
Polars >= 1.0 requires ``'utf8'`` (not ``'utf-8'``).
|
||||
"""
|
||||
resolved = file_encoding if file_encoding != 'utf-8' else encoding
|
||||
return resolved.replace('-', '') if resolved in ('utf-8', 'utf-8-sig') else resolved
|
||||
|
||||
|
||||
# ── main entry ──────────────────────────────────────────────────────────
|
||||
|
||||
def load_csv_directory(
|
||||
glob_pattern: str,
|
||||
encoding: str = 'utf-8',
|
||||
delimiter: str = ',',
|
||||
config_path: Optional[str] = None,
|
||||
schema_override: Optional[dict] = None,
|
||||
sample_rows: int = 10000,
|
||||
schema_strict: bool = False,
|
||||
recursive: bool = False,
|
||||
head: Optional[int] = None,
|
||||
) -> tuple[pl.LazyFrame, dict, int, int, float]:
|
||||
"""Load, validate, and merge CSV files matching *glob_pattern*.
|
||||
|
||||
Args:
|
||||
glob_pattern: Glob pattern for CSV files (e.g. ``data/*.csv``).
|
||||
encoding: Fallback encoding when no BOM is detected.
|
||||
delimiter: Column delimiter character.
|
||||
config_path: Optional path to a YAML config (merged into metadata).
|
||||
schema_override: If given, force these {column: dtype} overrides.
|
||||
sample_rows: Rows to scan for dtype inference per file.
|
||||
schema_strict: When True, raise ValueError on column mismatch across
|
||||
files. When False (default), log warnings and merge with
|
||||
``diagonal_relaxed`` (missing columns get null).
|
||||
recursive: When True, use ``glob.glob(..., recursive=True)`` to
|
||||
support ``**/*.csv`` patterns.
|
||||
head: Optional row limit for preview scenarios. When set, applies
|
||||
``lf.head(head)`` before returning.
|
||||
|
||||
Returns:
|
||||
A tuple of ``(lazyframe, schema_dict, total_row_count, file_count,
|
||||
memory_estimate_mb)``.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If no files match the glob pattern.
|
||||
ValueError: If schemas are inconsistent across files and
|
||||
*schema_strict* is True.
|
||||
"""
|
||||
# Normalise path separators for cross-platform Unicode support
|
||||
glob_pattern = os.path.normpath(glob_pattern)
|
||||
|
||||
# Resolve file list
|
||||
paths = sorted(glob.glob(glob_pattern, recursive=recursive))
|
||||
if not paths and not recursive:
|
||||
# Fallback: try recursive pattern
|
||||
paths = sorted(glob.glob(glob_pattern, recursive=True))
|
||||
if not paths:
|
||||
raise FileNotFoundError(f"No files match glob pattern: {glob_pattern}")
|
||||
|
||||
# Load config (user-provided or fall back to project default)
|
||||
config = load_config(config_path)
|
||||
if not config:
|
||||
default_path = Path(__file__).parent.parent.parent / 'config' / 'config.yaml'
|
||||
if default_path.exists():
|
||||
config = load_config(str(default_path))
|
||||
|
||||
# Collect per-file schema
|
||||
schemas: list[dict] = []
|
||||
lazyframes: list[pl.LazyFrame] = []
|
||||
for path in paths:
|
||||
file_encoding = detect_bom(path)
|
||||
actual_encoding = _resolve_encoding(encoding, file_encoding)
|
||||
|
||||
lf = pl.scan_csv(
|
||||
str(path), # str() ensures Unicode/Chinese paths resolve correctly
|
||||
encoding=actual_encoding,
|
||||
separator=delimiter,
|
||||
infer_schema_length=max(sample_rows, 1000),
|
||||
)
|
||||
schema = detect_schema_dtypes(lf, sample_rows=sample_rows)
|
||||
|
||||
# Apply schema override for this file
|
||||
if schema_override:
|
||||
for col, dtype in schema_override.items():
|
||||
if col in schema:
|
||||
schema[col] = dtype
|
||||
|
||||
schemas.append(schema)
|
||||
lazyframes.append(lf)
|
||||
|
||||
if schema_strict:
|
||||
# Raise on column mismatch
|
||||
validate_schemas(schemas)
|
||||
else:
|
||||
# Lenient mode: log warnings for column mismatches
|
||||
col_sets = [set(s.keys()) for s in schemas]
|
||||
reference = col_sets[0]
|
||||
for i, cs in enumerate(col_sets[1:], start=1):
|
||||
only_in_ref = reference - cs
|
||||
only_in_current = cs - reference
|
||||
if only_in_ref:
|
||||
logger.warning(
|
||||
"File #%d missing columns: %s (will be null)",
|
||||
i + 1, sorted(only_in_ref),
|
||||
)
|
||||
if only_in_current:
|
||||
logger.warning(
|
||||
"File #%d has extra columns: %s (will be null in other files)",
|
||||
i + 1, sorted(only_in_current),
|
||||
)
|
||||
|
||||
# Merge schemas (unify column ordering — first file wins order)
|
||||
merged_schema, _ = _merge_schema_entries(schemas)
|
||||
|
||||
# Concatenate all LazyFrames (union vertically)
|
||||
merged_lf: pl.LazyFrame = pl.concat(lazyframes, how='diagonal_relaxed')
|
||||
|
||||
# Log schema and size estimate
|
||||
total_row_count_est = len(paths) * (lazyframes[0].collect(streaming=True).height if lazyframes else 0)
|
||||
logger.info(f'[LOAD] files={len(paths)} total_rows_est={total_row_count_est}')
|
||||
for col_name, dtype in sorted(merged_schema.items()):
|
||||
logger.info(f'[LOAD_SCHEMA] col={col_name} dtype={dtype}')
|
||||
|
||||
# Apply schema override on merged frame (cast columns)
|
||||
if schema_override:
|
||||
casts = {}
|
||||
for col, dtype_str in schema_override.items():
|
||||
if col in merged_schema:
|
||||
try:
|
||||
polars_dtype = getattr(pl, dtype_str.split('(')[0], None)
|
||||
if polars_dtype:
|
||||
casts[col] = pl.col(col).cast(polars_dtype)
|
||||
except Exception:
|
||||
pass # skip uncastable overrides silently
|
||||
if casts:
|
||||
merged_lf = merged_lf.with_columns(list(casts.values()))
|
||||
|
||||
# ── Cleaning: drop outdated columns ──────────────────────────────────
|
||||
geoip_cfg = config.get('data', {}).get('geoip', {})
|
||||
drop_cols = geoip_cfg.get(
|
||||
'drop_columns',
|
||||
[], # Empty by default — lat/lon columns are now cleaned, not dropped
|
||||
)
|
||||
existing_drop = [c for c in drop_cols if c in merged_schema]
|
||||
if existing_drop:
|
||||
merged_lf = merged_lf.drop(existing_drop)
|
||||
merged_schema = {k: v for k, v in merged_schema.items() if k not in existing_drop}
|
||||
logger.info('[CLEAN] dropped columns: %s', existing_drop)
|
||||
|
||||
# ── Cleaning: BOOL normalisation ─────────────────────────────────────
|
||||
try:
|
||||
type_map = classify_schema(
|
||||
merged_lf.head(1000),
|
||||
config_overrides=config.get('columns', {}),
|
||||
)
|
||||
except Exception:
|
||||
type_map = {}
|
||||
|
||||
bool_cols = [
|
||||
c for c, t in type_map.items()
|
||||
if t.name == "布尔" and c in merged_schema
|
||||
]
|
||||
if bool_cols:
|
||||
schema_dtypes = dict(zip(
|
||||
merged_lf.collect_schema().names(),
|
||||
merged_lf.collect_schema().dtypes(),
|
||||
))
|
||||
bool_exprs = []
|
||||
cleaned_names: list[str] = []
|
||||
for c in bool_cols:
|
||||
if schema_dtypes.get(c) not in (pl.Utf8, pl.String):
|
||||
continue
|
||||
c_str = pl.col(c).cast(pl.Utf8).str.strip_chars()
|
||||
cleaned = (
|
||||
pl.when(c_str == "")
|
||||
.then(pl.lit("False"))
|
||||
.when(c_str.str.to_lowercase() == "+")
|
||||
.then(pl.lit("True"))
|
||||
.when(c_str.str.to_lowercase() == "-")
|
||||
.then(pl.lit("False"))
|
||||
.otherwise(c_str)
|
||||
)
|
||||
bool_exprs.append(cleaned.alias(c))
|
||||
cleaned_names.append(c)
|
||||
if bool_exprs:
|
||||
merged_lf = merged_lf.with_columns(bool_exprs)
|
||||
for c in cleaned_names:
|
||||
merged_schema[c] = 'Utf8'
|
||||
logger.info('[CLEAN] BOOL columns normalised: %s', cleaned_names)
|
||||
|
||||
# ── Generic numeric coercion ────────────────────────────────────────
|
||||
# For remaining string columns (not IPv4/BYTES/ENUM/BOOL etc.),
|
||||
# try to coerce to Float64.
|
||||
_SKIP_NUM_COERCE = {"IPv4", "字节", "枚举", "布尔"}
|
||||
_SKIP_NAME_PATTERNS = ('mac', 'mac_', '_mac', 'addr', 'address')
|
||||
schema_dtypes_gn = dict(zip(
|
||||
merged_lf.collect_schema().names(),
|
||||
merged_lf.collect_schema().dtypes(),
|
||||
))
|
||||
generic_numeric_cols = []
|
||||
for c, t in type_map.items():
|
||||
if t.name in _SKIP_NUM_COERCE:
|
||||
continue
|
||||
if c not in merged_schema:
|
||||
continue
|
||||
dtype = schema_dtypes_gn.get(c)
|
||||
if dtype not in (pl.Utf8, pl.String):
|
||||
continue
|
||||
cl = c.lower().replace('-', '_').replace(' ', '_')
|
||||
if any(p in cl for p in _SKIP_NAME_PATTERNS):
|
||||
continue
|
||||
generic_numeric_cols.append(c)
|
||||
|
||||
if generic_numeric_cols:
|
||||
num_exprs = []
|
||||
for c in generic_numeric_cols:
|
||||
num_exprs.append(
|
||||
pl.col(c).map_batches(_coerce_to_float, return_dtype=pl.Float64).alias(c)
|
||||
)
|
||||
if num_exprs:
|
||||
merged_lf = merged_lf.with_columns(num_exprs)
|
||||
for c in generic_numeric_cols:
|
||||
merged_schema[c] = 'Float64'
|
||||
logger.info('[CLEAN] Generic numeric coercion applied: %s', generic_numeric_cols)
|
||||
|
||||
# ── Cleaning: BYTES (hex) preprocessing ─────────────────────────────
|
||||
hex_cols = [
|
||||
c for c, t in type_map.items()
|
||||
if t.name == "字节" and c in merged_schema
|
||||
]
|
||||
if hex_cols:
|
||||
hex_exprs = []
|
||||
for c in hex_cols:
|
||||
hex_exprs.append(
|
||||
pl.col(c).str.split(" ").list.len().alias(f"{c}_hex_pairs_count")
|
||||
)
|
||||
merged_lf = merged_lf.with_columns(hex_exprs)
|
||||
for c in hex_cols:
|
||||
merged_schema[f"{c}_hex_pairs_count"] = 'UInt32'
|
||||
logger.info('[CLEAN] BYTES columns added hex_pairs_count: %s', hex_cols)
|
||||
|
||||
# Row count estimate (fast path: use first file row count × files)
|
||||
# For accurate count we'd need to collect, but that defeats lazy.
|
||||
# We estimate from the first file's scanned schema info.
|
||||
first_rows = lazyframes[0].collect(streaming=True).height if lazyframes else 0
|
||||
total_row_count = first_rows * len(paths)
|
||||
|
||||
# Memory estimate
|
||||
bytes_per_row = 0
|
||||
for col_name, dtype_str in merged_schema.items():
|
||||
numeric_part = dtype_str.split('(')[0].lower()
|
||||
size_map = {
|
||||
'int8': 1, 'int16': 2, 'int32': 4, 'int64': 8,
|
||||
'uint8': 1, 'uint16': 2, 'uint32': 4, 'uint64': 8,
|
||||
'float32': 4, 'float64': 8,
|
||||
'boolean': 1, 'bool': 1,
|
||||
'utf8': 50, 'string': 50, 'str': 50,
|
||||
'categorical': 8, 'cat': 8,
|
||||
'date': 4, 'datetime': 8, 'time': 8, 'duration': 8,
|
||||
}
|
||||
bytes_per_row += size_map.get(numeric_part, 8)
|
||||
memory_mb = (bytes_per_row * total_row_count) / (1024 * 1024)
|
||||
|
||||
# Apply head limit for preview (keeps LazyFrame lazy — rows are limited on collect)
|
||||
if head is not None:
|
||||
merged_lf = merged_lf.head(head)
|
||||
total_row_count = min(total_row_count, head)
|
||||
|
||||
return merged_lf, merged_schema, total_row_count, len(paths), memory_mb
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Schema detection, validation, and merging across CSV files."""
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _merge_schema_entries(
|
||||
schemas: list[dict],
|
||||
) -> tuple[dict[str, str], list[str]]:
|
||||
"""Merge multiple schema dicts, raising on type conflicts.
|
||||
|
||||
Returns a single {name: dtype} dict. If a column appears in multiple
|
||||
files with different dtypes the *last* non-null dtype wins (with a
|
||||
warning embedded in the result).
|
||||
"""
|
||||
merged: dict[str, str] = {}
|
||||
conflicts: list[str] = []
|
||||
for s in schemas:
|
||||
for name, dtype in s.items():
|
||||
if name in merged and merged[name] != dtype:
|
||||
conflicts.append(name)
|
||||
merged[name] = dtype # last wins
|
||||
return merged, conflicts
|
||||
|
||||
|
||||
def validate_schemas(schemas: list[dict], strict: bool = True) -> None:
|
||||
"""Compare schemas across files and raise or warn on column mismatch.
|
||||
|
||||
Checks that every file has the same set of column names. Type
|
||||
differences across files are **not** considered an error (Polars can
|
||||
cast), but differences in *column sets* are.
|
||||
|
||||
Args:
|
||||
schemas: List of ``{column_name: dtype_string}`` dicts, one per file.
|
||||
strict: If ``True`` (default), raise :class:`ValueError` on any
|
||||
column mismatch. If ``False``, log a warning instead.
|
||||
"""
|
||||
if not schemas:
|
||||
return
|
||||
|
||||
col_sets = [set(s.keys()) for s in schemas]
|
||||
reference = col_sets[0]
|
||||
|
||||
for i, cs in enumerate(col_sets[1:], start=1):
|
||||
only_in_ref = reference - cs
|
||||
only_in_current = cs - reference
|
||||
if only_in_ref or only_in_current:
|
||||
msg_parts = []
|
||||
if only_in_ref:
|
||||
msg_parts.append(f"File #{i} missing columns: {sorted(only_in_ref)}")
|
||||
if only_in_current:
|
||||
msg_parts.append(f"File #{i} has extra columns: {sorted(only_in_current)}")
|
||||
full_msg = f"Schema mismatch: {'; '.join(msg_parts)}"
|
||||
if strict:
|
||||
raise ValueError(full_msg)
|
||||
else:
|
||||
logger.warning(full_msg)
|
||||
@@ -0,0 +1,292 @@
|
||||
"""SQLite persistence layer for LazyFrame data."""
|
||||
import csv as _csv_mod
|
||||
import io
|
||||
import logging
|
||||
import sqlite3
|
||||
from typing import Optional
|
||||
|
||||
import polars as pl
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── dtype mapping ─────────────────────────────────────────────────────────
|
||||
|
||||
_STR_TO_DTYPE = {
|
||||
'Float64': pl.Float64, 'Float32': pl.Float32,
|
||||
'Int64': pl.Int64, 'Int32': pl.Int32, 'Int16': pl.Int16, 'Int8': pl.Int8,
|
||||
'UInt64': pl.UInt64, 'UInt32': pl.UInt32, 'UInt16': pl.UInt16, 'UInt8': pl.UInt8,
|
||||
'Utf8': pl.Utf8, 'String': pl.Utf8, 'Categorical': pl.Categorical,
|
||||
'Boolean': pl.Boolean, 'Bool': pl.Boolean,
|
||||
'Date': pl.Date, 'Datetime': pl.Datetime, 'Time': pl.Time,
|
||||
}
|
||||
|
||||
|
||||
def _dtype_to_sqlite(dtype: pl.DataType) -> str:
|
||||
"""Map a Polars dtype to a SQLite column type affinity."""
|
||||
if dtype in (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
|
||||
pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64):
|
||||
return 'INTEGER'
|
||||
if dtype in (pl.Float32, pl.Float64):
|
||||
return 'REAL'
|
||||
if dtype == pl.Boolean:
|
||||
return 'INTEGER'
|
||||
return 'TEXT' # String, Date, Datetime, etc. all map to TEXT
|
||||
|
||||
|
||||
def _str_to_polars_dtype(s: str) -> pl.DataType:
|
||||
"""Convert a Polars dtype string (e.g. ``'Float64'``, ``'Int32'``) to a ``pl.DataType``.
|
||||
|
||||
Strips trailing metadata like ``Datetime(time_unit='us', time_zone=None)``.
|
||||
"""
|
||||
base = s.split('(')[0].strip()
|
||||
return _STR_TO_DTYPE.get(base, pl.Utf8)
|
||||
|
||||
|
||||
def _rows_to_csv_df(rows: list[sqlite3.Row]) -> pl.DataFrame:
|
||||
"""Convert a batch of sqlite3.Row objects to a DataFrame via CSV fallback.
|
||||
|
||||
Loads all columns as Utf8 to handle mixed-type SQLite columns.
|
||||
"""
|
||||
buf = io.StringIO()
|
||||
writer = _csv_mod.writer(buf)
|
||||
writer.writerow(rows[0].keys())
|
||||
for r in rows:
|
||||
writer.writerow(str(v) if v is not None else '' for v in r)
|
||||
buf.seek(0)
|
||||
return pl.read_csv(buf, infer_schema_length=0)
|
||||
|
||||
|
||||
# ── public API ───────────────────────────────────────────────────────────
|
||||
|
||||
def save_to_db(run_id: int, lf: pl.LazyFrame, mode: str = 'replace') -> str:
|
||||
"""Write a LazyFrame to a SQLite table ``_data_{run_id}``.
|
||||
|
||||
Collects the LazyFrame (streaming if possible), creates or replaces the
|
||||
table, bulk-inserts rows in chunks, and returns the table name.
|
||||
|
||||
Args:
|
||||
run_id: Analysis run ID (used as ``_data_{run_id}``).
|
||||
lf: LazyFrame to persist.
|
||||
mode: ``'replace'`` (default) → drop+create+insert.
|
||||
``'append'`` → skip DDL, only INSERT (table must exist).
|
||||
|
||||
This is a *persistence copy* — the original CSV files remain untouched.
|
||||
"""
|
||||
from django.conf import settings
|
||||
|
||||
table_name = f"_data_{run_id}"
|
||||
db_path = settings.DATABASES['default']['NAME']
|
||||
|
||||
df = lf.collect(streaming=True)
|
||||
conn = sqlite3.connect(db_path)
|
||||
|
||||
try:
|
||||
if mode == 'replace':
|
||||
# Drop existing table for retry/idempotency safety
|
||||
conn.execute(f'DROP TABLE IF EXISTS "{table_name}"')
|
||||
|
||||
# Build CREATE TABLE from Polars schema
|
||||
col_defs = [f'"{name}" {_dtype_to_sqlite(dtype)}'
|
||||
for name, dtype in zip(df.columns, df.dtypes)]
|
||||
conn.execute(f'CREATE TABLE "{table_name}" ({", ".join(col_defs)})')
|
||||
|
||||
# Bulk insert in chunks
|
||||
col_names = [f'"{c}"' for c in df.columns]
|
||||
placeholders = ','.join(['?' for _ in df.columns])
|
||||
insert_sql = f'INSERT INTO "{table_name}" ({", ".join(col_names)}) VALUES ({placeholders})'
|
||||
|
||||
total_rows = len(df)
|
||||
chunk_size = 10000
|
||||
for i in range(0, total_rows, chunk_size):
|
||||
chunk = df.slice(i, chunk_size)
|
||||
conn.executemany(insert_sql, chunk.rows())
|
||||
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
logger.info('[SAVE_TO_DB] table=%s rows=%d mode=%s', table_name, total_rows, mode)
|
||||
return table_name
|
||||
|
||||
|
||||
def load_from_db(
|
||||
table_name: str,
|
||||
schema_overrides: dict[str, pl.DataType | str] | None = None,
|
||||
) -> Optional[pl.LazyFrame]:
|
||||
"""Load a SQLite table into a LazyFrame.
|
||||
|
||||
Args:
|
||||
table_name: SQLite table name.
|
||||
schema_overrides: Optional dict of ``{column_name: dtype}`` to cast
|
||||
columns after loading. Values can be ``pl.DataType`` instances
|
||||
(e.g. ``pl.Float64``) or dtype strings (e.g. ``'Float64'``).
|
||||
Use when SQLite ``TEXT`` columns need to be restored to
|
||||
``Float64`` / ``Int64`` etc.
|
||||
|
||||
Returns ``None`` if the table does not exist.
|
||||
"""
|
||||
from django.conf import settings
|
||||
|
||||
db_path = settings.DATABASES['default']['NAME']
|
||||
conn = sqlite3.connect(db_path)
|
||||
|
||||
try:
|
||||
# Check table exists
|
||||
cursor = conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name=?",
|
||||
(table_name,),
|
||||
)
|
||||
if cursor.fetchone() is None:
|
||||
return None
|
||||
|
||||
conn.row_factory = sqlite3.Row
|
||||
cursor = conn.execute(f'SELECT * FROM "{table_name}"')
|
||||
|
||||
# Peek first row to detect if table is empty
|
||||
first_row = cursor.fetchone()
|
||||
if first_row is None:
|
||||
return pl.DataFrame().lazy()
|
||||
|
||||
batch_rows = [first_row]
|
||||
dfs = []
|
||||
_use_csv_fallback = False
|
||||
|
||||
while True:
|
||||
chunk = cursor.fetchmany(10000)
|
||||
if not chunk:
|
||||
# Flush remaining rows
|
||||
if batch_rows:
|
||||
if _use_csv_fallback:
|
||||
dfs.append(_rows_to_csv_df(batch_rows))
|
||||
else:
|
||||
try:
|
||||
dfs.append(pl.from_dicts([dict(r) for r in batch_rows]))
|
||||
except Exception:
|
||||
_use_csv_fallback = True
|
||||
dfs.append(_rows_to_csv_df(batch_rows))
|
||||
break
|
||||
|
||||
batch_rows.extend(chunk)
|
||||
if len(batch_rows) >= 10000:
|
||||
if _use_csv_fallback:
|
||||
dfs.append(_rows_to_csv_df(batch_rows))
|
||||
else:
|
||||
try:
|
||||
dfs.append(pl.from_dicts([dict(r) for r in batch_rows]))
|
||||
except Exception:
|
||||
_use_csv_fallback = True
|
||||
# Re-process current batch with CSV fallback
|
||||
dfs.append(_rows_to_csv_df(batch_rows))
|
||||
batch_rows = []
|
||||
|
||||
# Concat all batches
|
||||
if len(dfs) == 1:
|
||||
df = dfs[0]
|
||||
else:
|
||||
df = pl.concat(dfs, how='vertical')
|
||||
|
||||
# Apply type overrides to restore original column types
|
||||
if schema_overrides:
|
||||
casts = {}
|
||||
for col, dtype in schema_overrides.items():
|
||||
if col not in df.columns:
|
||||
continue
|
||||
dtype_obj = dtype if isinstance(dtype, pl.DataType) else _str_to_polars_dtype(dtype)
|
||||
if df[col].dtype != dtype_obj:
|
||||
try:
|
||||
casts[col] = pl.col(col).cast(dtype_obj)
|
||||
except Exception:
|
||||
# If a cast fails (e.g. "A" → Int64), keep the column as-is
|
||||
pass
|
||||
if casts:
|
||||
df = df.with_columns(list(casts.values()))
|
||||
|
||||
return df.lazy()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def load_from_db_lazy(
|
||||
table_name: str,
|
||||
schema_overrides: dict[str, pl.DataType | str] | None = None,
|
||||
) -> Optional[pl.LazyFrame]:
|
||||
"""Load a SQLite table into a LazyFrame using Polars native DB reader.
|
||||
|
||||
Uses ``pl.read_database`` for efficient bulk reading — an order of
|
||||
magnitude faster than the manual batch-processing path in
|
||||
:func:`load_from_db`. Returns a :class:`pl.LazyFrame` so downstream
|
||||
operations remain lazy (query-plan, not materialised).
|
||||
|
||||
Args:
|
||||
table_name: SQLite table name.
|
||||
schema_overrides: Optional ``{column_name: dtype}`` dict to cast
|
||||
columns after loading. Values can be ``pl.DataType`` instances
|
||||
(e.g. ``pl.Float64``) or dtype strings (e.g. ``'Float64'``).
|
||||
|
||||
Returns:
|
||||
``None`` if the table does not exist or is empty.
|
||||
"""
|
||||
from django.conf import settings
|
||||
|
||||
db_path = settings.DATABASES['default']['NAME']
|
||||
conn = sqlite3.connect(db_path)
|
||||
|
||||
try:
|
||||
# Check table existence (fast metadata query)
|
||||
cursor = conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name=?",
|
||||
(table_name,),
|
||||
)
|
||||
if cursor.fetchone() is None:
|
||||
return None
|
||||
|
||||
# Bulk-read with Polars native engine — leverages the DBAPI2
|
||||
# connection directly for efficient C-level data transfer.
|
||||
query = f'SELECT * FROM "{table_name}"'
|
||||
df = pl.read_database(connection=conn, query=query)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
if df.height == 0:
|
||||
return None
|
||||
|
||||
# Apply type overrides to restore original column types
|
||||
# (SQLite stores everything as TEXT/INTEGER/REAL/BLOB;
|
||||
# we need to cast back to Float64 / Int64 / etc.)
|
||||
if schema_overrides:
|
||||
casts = {}
|
||||
for col, dtype in schema_overrides.items():
|
||||
if col not in df.columns:
|
||||
continue
|
||||
dtype_obj = (
|
||||
dtype if isinstance(dtype, pl.DataType)
|
||||
else _str_to_polars_dtype(dtype)
|
||||
)
|
||||
if df[col].dtype != dtype_obj:
|
||||
try:
|
||||
casts[col] = pl.col(col).cast(dtype_obj)
|
||||
except Exception:
|
||||
pass # uncastable column — keep as-is
|
||||
if casts:
|
||||
df = df.with_columns(list(casts.values()))
|
||||
|
||||
rows = df.height
|
||||
cols = df.width
|
||||
logger.info(
|
||||
'[LOAD_FROM_DB_LAZY] table=%s rows=%d cols=%d',
|
||||
table_name, rows, cols,
|
||||
)
|
||||
return df.lazy()
|
||||
|
||||
|
||||
def drop_sqlite_table(table_name: str) -> None:
|
||||
"""Drop a SQLite data table silently if it exists."""
|
||||
from django.conf import settings
|
||||
|
||||
db_path = settings.DATABASES['default']['NAME']
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
conn.execute(f'DROP TABLE IF EXISTS "{table_name}"')
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,37 @@
|
||||
"""Distance computation package for TianXuan analysis pipeline.
|
||||
|
||||
Provides distance functions for various data types (IP, string, boolean,
|
||||
bytes, timestamp), plus normalisation, SVD denoising, UMAP reduction,
|
||||
and per-cluster SVD feature extraction.
|
||||
|
||||
Public API:
|
||||
ip_distance — subnet-mask + Haversine geographic distance for IP pairs
|
||||
string_distance — Levenshtein / enum-like distance for string columns
|
||||
bool_distance — 0/1 match-to-mode distance for boolean columns
|
||||
bytes_distance — Hamming (popcount) distance for hex byte columns
|
||||
timestamp_fft_distance — FFT phase distance for timestamp columns
|
||||
normalize_features — StandardScaler normalisation (μ=0, σ=1)
|
||||
svd_denoise — TruncatedSVD noise removal (returns noise_profile)
|
||||
umap_reduce — UMAP dimensionality reduction (3D→2D fallback)
|
||||
cluster_svd_extract — Per-cluster SVD distinguishing feature extraction
|
||||
"""
|
||||
|
||||
from analysis.distance._geo import ip_distance
|
||||
from analysis.distance._text import bool_distance, string_distance
|
||||
from analysis.distance._numeric import bytes_distance, timestamp_fft_distance
|
||||
from analysis.distance._normalize import normalize_features
|
||||
from analysis.distance._svd import svd_denoise
|
||||
from analysis.distance._umap import umap_reduce
|
||||
from analysis.distance._cluster_svd import cluster_svd_extract
|
||||
|
||||
__all__ = [
|
||||
"ip_distance",
|
||||
"string_distance",
|
||||
"bool_distance",
|
||||
"bytes_distance",
|
||||
"timestamp_fft_distance",
|
||||
"normalize_features",
|
||||
"svd_denoise",
|
||||
"umap_reduce",
|
||||
"cluster_svd_extract",
|
||||
]
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Per-cluster SVD feature extraction."""
|
||||
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
from sklearn.decomposition import TruncatedSVD
|
||||
|
||||
|
||||
def cluster_svd_extract(
|
||||
lf: pl.LazyFrame,
|
||||
labels: list[int],
|
||||
feature_cols: list[str],
|
||||
) -> dict[int, dict]:
|
||||
"""Per-cluster SVD-based distinguishing feature extraction.
|
||||
|
||||
For each cluster label:
|
||||
1. Extracts rows belonging to that cluster.
|
||||
2. Fits ``TruncatedSVD(n_components=min(N, len(feature_cols)))``.
|
||||
3. The absolute loadings on the first SVD component serve as
|
||||
*feature_strength* — higher values mean more distinguishing.
|
||||
|
||||
Returns a mapping ``{cluster_label: {...}}`` where each value is::
|
||||
|
||||
{
|
||||
'features': [...], # top-10 feature names by strength
|
||||
'explained_variance_ratio': [...], # per-component ratios
|
||||
'feature_strength': [...], # per-feature loading magnitude
|
||||
}
|
||||
|
||||
Parameters
|
||||
----------
|
||||
lf:
|
||||
LazyFrame with rows aligned to *labels*.
|
||||
labels:
|
||||
Cluster labels (same length as *lf* row count, ``-1`` = noise).
|
||||
feature_cols:
|
||||
Numeric column names to analyse.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict[int, dict]
|
||||
Per-cluster SVD extraction results.
|
||||
"""
|
||||
available = [c for c in feature_cols if c in lf.collect_schema().names()]
|
||||
if not available:
|
||||
return {}
|
||||
|
||||
df = lf.select(available).collect(streaming=True)
|
||||
mat = df.to_numpy().astype(np.float64)
|
||||
mat = np.nan_to_num(mat, nan=0.0, posinf=0.0, neginf=0.0)
|
||||
|
||||
if len(labels) != len(mat):
|
||||
labels = labels[:len(mat)]
|
||||
|
||||
unique_labels = sorted(set(labels))
|
||||
result: dict[int, dict] = {}
|
||||
|
||||
for label in unique_labels:
|
||||
mask = np.array(labels) == label
|
||||
n = int(mask.sum())
|
||||
if n < 2:
|
||||
continue
|
||||
|
||||
cluster_mat = mat[mask]
|
||||
n_comp = min(n, len(available))
|
||||
svd = TruncatedSVD(n_components=n_comp, random_state=42)
|
||||
svd.fit(cluster_mat)
|
||||
|
||||
# First component loadings → feature strength
|
||||
if n_comp > 0 and svd.components_.shape[0] > 0:
|
||||
feature_strength = np.abs(svd.components_[0]).astype(np.float64)
|
||||
else:
|
||||
feature_strength = np.zeros(len(available), dtype=np.float64)
|
||||
|
||||
total_strength = float(feature_strength.sum())
|
||||
if total_strength > 1e-10:
|
||||
feature_strength = feature_strength / total_strength
|
||||
|
||||
top_indices = np.argsort(-feature_strength)
|
||||
top_k = min(10, len(available))
|
||||
|
||||
result[int(label)] = {
|
||||
'features': [available[i] for i in top_indices[:top_k]],
|
||||
'explained_variance_ratio': svd.explained_variance_ratio_.tolist(),
|
||||
'feature_strength': feature_strength.tolist(),
|
||||
}
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,170 @@
|
||||
"""IP distance functions for Polars LazyFrames.
|
||||
|
||||
Provides subnet-mask distance and Haversine geographic distance
|
||||
between pairs of IP columns. All distances are normalised to [0, 1];
|
||||
invalid / non-geolocatable IPs produce ``NaN``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Optional
|
||||
|
||||
import polars as pl
|
||||
|
||||
from analysis.geoip import ip_to_int, lookup
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Earth constant
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_HALF_EARTH_CIRCUMFERENCE_KM = 20_037.508 # equatorial / 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Core distance helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _subnet_mask_distance(ip1: Optional[str], ip2: Optional[str], mask: int) -> float:
|
||||
"""Normalised subnet-mask distance between two IP strings.
|
||||
|
||||
Computes ``abs(int(ip1)/2**mask - int(ip2)/2**mask) / 2**(32-mask)``.
|
||||
Returns ``NaN`` when either IP is ``None`` or unparseable.
|
||||
"""
|
||||
if ip1 is None or ip2 is None:
|
||||
return float("nan")
|
||||
if not isinstance(ip1, str):
|
||||
ip1 = str(ip1)
|
||||
if not isinstance(ip2, str):
|
||||
ip2 = str(ip2)
|
||||
|
||||
int1 = ip_to_int(ip1)
|
||||
int2 = ip_to_int(ip2)
|
||||
if int1 is None or int2 is None:
|
||||
return float("nan")
|
||||
|
||||
divisor = 2.0 ** mask
|
||||
max_val = 2.0 ** (32 - mask)
|
||||
return abs(int1 / divisor - int2 / divisor) / max_val
|
||||
|
||||
|
||||
def _haversine_km(lat1: float, lon1: float, lat2: float, lon2: float) -> float:
|
||||
"""Great-circle distance (km) via the Haversine formula."""
|
||||
dlat = math.radians(lat2 - lat1)
|
||||
dlon = math.radians(lon2 - lon1)
|
||||
a = (
|
||||
math.sin(dlat / 2) ** 2
|
||||
+ math.cos(math.radians(lat1))
|
||||
* math.cos(math.radians(lat2))
|
||||
* math.sin(dlon / 2) ** 2
|
||||
)
|
||||
return 6371.0 * 2.0 * math.asin(math.sqrt(a))
|
||||
|
||||
|
||||
def _geo_distance(ip1: Optional[str], ip2: Optional[str]) -> float:
|
||||
"""Normalised [0, 1] geographic distance between two IP strings.
|
||||
|
||||
Uses ``analysis.geoip.lookup()`` to obtain lat/lon, then Haversine.
|
||||
Returns ``NaN`` when either IP cannot be geolocated.
|
||||
"""
|
||||
if ip1 is None or ip2 is None:
|
||||
return float("nan")
|
||||
if not isinstance(ip1, str):
|
||||
ip1 = str(ip1)
|
||||
if not isinstance(ip2, str):
|
||||
ip2 = str(ip2)
|
||||
|
||||
geo1 = lookup(ip1)
|
||||
geo2 = lookup(ip2)
|
||||
if geo1 is None or geo2 is None:
|
||||
return float("nan")
|
||||
|
||||
km = _haversine_km(geo1["lat"], geo1["lon"], geo2["lat"], geo2["lon"])
|
||||
return km / _HALF_EARTH_CIRCUMFERENCE_KM
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def ip_distance(
|
||||
lf: pl.LazyFrame,
|
||||
ip_columns: list[str],
|
||||
subnet_masks: Optional[list[int]] = None,
|
||||
) -> pl.LazyFrame:
|
||||
"""Append IP distance columns to a LazyFrame.
|
||||
|
||||
For each **consecutive pair** of *ip_columns*
|
||||
(``ip_columns[0]`` vs ``ip_columns[1]``,
|
||||
``ip_columns[2]`` vs ``ip_columns[3]``, …) computes:
|
||||
|
||||
* one subnet-mask distance per entry in *subnet_masks*
|
||||
* one Haversine geographic distance
|
||||
|
||||
All distances are normalised to [0, 1]. Invalid IPs or IPs not
|
||||
found in the GeoIP database produce ``NaN``.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
lf:
|
||||
LazyFrame containing the IP columns.
|
||||
ip_columns:
|
||||
List of IP column names (used in consecutive pairs).
|
||||
subnet_masks:
|
||||
Subnet masks for network distance (default ``[24, 28]``).
|
||||
|
||||
Returns
|
||||
-------
|
||||
pl.LazyFrame
|
||||
Input frame with ``_ip_dist_0``, ``_ip_dist_1``, …
|
||||
columns appended.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> lf = pl.LazyFrame({
|
||||
... "src_ip": ["8.8.8.8", "1.1.1.1"],
|
||||
... "dst_ip": ["8.8.4.4", "1.0.0.1"],
|
||||
... })
|
||||
>>> result = ip_distance(lf, ["src_ip", "dst_ip"], subnet_masks=[24])
|
||||
>>> result.collect().columns
|
||||
['src_ip', 'dst_ip', '_ip_dist_0', '_ip_dist_1']
|
||||
"""
|
||||
if subnet_masks is None:
|
||||
subnet_masks = [24, 28]
|
||||
|
||||
result = lf
|
||||
dist_idx = 0
|
||||
|
||||
# Walk consecutive pairs
|
||||
for i in range(0, len(ip_columns) - 1, 2):
|
||||
col_a = ip_columns[i]
|
||||
col_b = ip_columns[i + 1]
|
||||
|
||||
# -- subnet-mask distances --
|
||||
for mask in subnet_masks:
|
||||
name = f"_ip_dist_{dist_idx}"
|
||||
result = result.with_columns(
|
||||
pl.struct([col_a, col_b])
|
||||
.map_elements(
|
||||
lambda s, _mask=mask, _ca=col_a, _cb=col_b: (
|
||||
_subnet_mask_distance(s[_ca], s[_cb], _mask)
|
||||
),
|
||||
return_dtype=pl.Float64,
|
||||
)
|
||||
.alias(name)
|
||||
)
|
||||
dist_idx += 1
|
||||
|
||||
# -- geographic distance --
|
||||
name = f"_ip_dist_{dist_idx}"
|
||||
result = result.with_columns(
|
||||
pl.struct([col_a, col_b])
|
||||
.map_elements(
|
||||
lambda s, _ca=col_a, _cb=col_b: _geo_distance(s[_ca], s[_cb]),
|
||||
return_dtype=pl.Float64,
|
||||
)
|
||||
.alias(name)
|
||||
)
|
||||
dist_idx += 1
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,52 @@
|
||||
"""StandardScaler normalisation (step 2 of preprocessing pipeline)."""
|
||||
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
|
||||
|
||||
def normalize_features(lf: pl.LazyFrame, feature_cols: list[str]) -> pl.LazyFrame:
|
||||
"""Apply StandardScaler normalisation to numeric feature columns.
|
||||
|
||||
Collects *feature_cols* into a numpy array, replaces ``NaN`` with
|
||||
zero, centres to μ=0 σ=1 via ``sklearn.preprocessing.StandardScaler``,
|
||||
then returns the result as *lf* extended with ``_norm_*`` prefixed
|
||||
columns.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
lf:
|
||||
LazyFrame containing *feature_cols*.
|
||||
feature_cols:
|
||||
Numeric column names to normalise. Columns not present in
|
||||
*lf* are silently skipped; non-numeric columns raise an error.
|
||||
|
||||
Returns
|
||||
-------
|
||||
pl.LazyFrame
|
||||
Input frame with additional ``_norm_{col}`` columns.
|
||||
|
||||
Memory
|
||||
------
|
||||
Peak memory ≈ collected numpy matrix + scaler workspace
|
||||
(well under the 800 MB budget for typical inputs).
|
||||
"""
|
||||
available = [c for c in feature_cols if c in lf.collect_schema().names()]
|
||||
if not available:
|
||||
return lf
|
||||
|
||||
df = lf.select(available).collect(streaming=True)
|
||||
mat = df.to_numpy().astype(np.float64)
|
||||
mat = np.nan_to_num(mat, nan=0.0)
|
||||
|
||||
scaler = StandardScaler()
|
||||
scaled = scaler.fit_transform(mat)
|
||||
|
||||
# Build new columns with _norm_ prefix
|
||||
df_result = lf.collect(streaming=True)
|
||||
for i, col in enumerate(available):
|
||||
df_result = df_result.with_columns(
|
||||
pl.Series(f'_norm_{col}', scaled[:, i])
|
||||
)
|
||||
|
||||
return df_result.lazy()
|
||||
@@ -0,0 +1,278 @@
|
||||
"""Bytes/hex distance (Hamming/popcount) and timestamp FFT phase distance."""
|
||||
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bytes Hamming distance
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_HEX_POPCOUNT: dict[str, int] = {
|
||||
'0': 0, '1': 1, '2': 1, '3': 2,
|
||||
'4': 1, '5': 2, '6': 2, '7': 3,
|
||||
'8': 1, '9': 2,
|
||||
'a': 2, 'b': 3, 'c': 2, 'd': 3, 'e': 3, 'f': 4,
|
||||
'A': 2, 'B': 3, 'C': 2, 'D': 3, 'E': 3, 'F': 4,
|
||||
}
|
||||
|
||||
|
||||
def _hex_popcount_normalized(val: str | None) -> float:
|
||||
"""Return the proportion of 1-bits in a hex byte string (0.0–1.0).
|
||||
|
||||
This is the Hamming distance from the all-zero bit string,
|
||||
normalised by total bits. Returns ``NaN`` for null or
|
||||
non-hex input.
|
||||
"""
|
||||
if val is None:
|
||||
return float("nan")
|
||||
s = str(val).replace(' ', '').replace('\t', '').replace('\n', '')
|
||||
if not s:
|
||||
return 0.0
|
||||
|
||||
total_ones = 0
|
||||
for ch in s:
|
||||
ones = _HEX_POPCOUNT.get(ch)
|
||||
if ones is None:
|
||||
return float("nan") # non-hex character
|
||||
total_ones += ones
|
||||
|
||||
total_bits = len(s) * 4
|
||||
return total_ones / total_bits
|
||||
|
||||
|
||||
def bytes_distance(
|
||||
lf: pl.LazyFrame,
|
||||
byte_columns: list[str],
|
||||
) -> pl.LazyFrame:
|
||||
"""Append normalised Hamming-distance columns for hex byte columns.
|
||||
|
||||
Each byte column is converted from hex to binary, and the
|
||||
proportion of 1-bits (Hamming distance from the all-zero
|
||||
bit vector) is computed as a ``_bytes_dist_N`` Float64 column
|
||||
normalised to [0, 1]. Non-hex / null values produce ``NaN``.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
lf:
|
||||
LazyFrame containing the byte columns.
|
||||
byte_columns:
|
||||
Column names whose values are hex byte strings
|
||||
(with optional whitespace separators, e.g. ``"A3 F2"``).
|
||||
|
||||
Returns
|
||||
-------
|
||||
pl.LazyFrame
|
||||
Input frame with ``_bytes_dist_0``, ``_bytes_dist_1``, …
|
||||
appended. Columns not present in *lf* are silently skipped.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> df = pl.DataFrame({"payload": ["A3F2", "FFFF", None]})
|
||||
>>> result = bytes_distance(df.lazy(), ["payload"])
|
||||
>>> result.collect()
|
||||
shape: (3, 2)
|
||||
┌─────────┬────────────────┐
|
||||
│ payload ┆ _bytes_dist_0 │
|
||||
│ --- ┆ --- │
|
||||
│ str ┆ f64 │
|
||||
╞═════════╪════════════════╡
|
||||
│ A3F2 ┆ 0.625 │
|
||||
│ FFFF ┆ 1.0 │
|
||||
│ null ┆ NaN │
|
||||
└─────────┴────────────────┘
|
||||
"""
|
||||
existing = set(lf.collect_schema().names())
|
||||
result = lf
|
||||
|
||||
for i, col_name in enumerate(byte_columns):
|
||||
if col_name not in existing:
|
||||
continue
|
||||
result = result.with_columns(
|
||||
pl.col(col_name)
|
||||
.cast(pl.Utf8)
|
||||
.map_elements(_hex_popcount_normalized, return_dtype=pl.Float64)
|
||||
.alias(f'_bytes_dist_{i}')
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Timestamp FFT phase distance
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# ── threshold: minimum fraction of repeated intervals to use FFT path ──
|
||||
_FFT_PERIODICITY_THRESHOLD = 0.05
|
||||
# ── minimum samples before FFT is attempted ──
|
||||
_FFT_MIN_SAMPLES = 4
|
||||
|
||||
|
||||
def _parse_timestamp_to_epoch(series: pl.Series) -> pl.Series:
|
||||
"""Convert a timestamp series to Unix epoch seconds (float64).
|
||||
|
||||
Handles int/float (assumed already epoch), pl.Datetime/pl.Date
|
||||
(ns→seconds), and str/pl.Utf8 (strptime→ns→seconds, with direct
|
||||
float-cast fallback on parse failure).
|
||||
"""
|
||||
dtype = series.dtype
|
||||
if dtype in (pl.Utf8, pl.String):
|
||||
try:
|
||||
parsed = series.str.strptime(pl.Datetime, format=None, strict=False)
|
||||
except Exception:
|
||||
return series.cast(pl.Float64)
|
||||
return parsed.cast(pl.Int64).cast(pl.Float64) / 1_000_000_000.0
|
||||
elif dtype in (pl.Datetime, pl.Date):
|
||||
return series.cast(pl.Int64).cast(pl.Float64) / 1_000_000_000.0
|
||||
else:
|
||||
return series.cast(pl.Float64)
|
||||
|
||||
|
||||
def _compute_fft_distances(values: np.ndarray) -> np.ndarray:
|
||||
"""Compute per-row phase distances for a 1-D array.
|
||||
|
||||
Sorts *values*, checks for periodicity, then either:
|
||||
* FFT path: ``np.fft.fft()`` → ``np.angle()``, mapped back to
|
||||
original (unsorted) row order via rank lookup.
|
||||
* Statistical fallback: interval z-scores mapped to row order.
|
||||
|
||||
Returns an ``(n,)`` float64 array, normalised to zero-mean
|
||||
unit-variance.
|
||||
"""
|
||||
n = len(values)
|
||||
if n < _FFT_MIN_SAMPLES:
|
||||
return np.full(n, np.nan, dtype=np.float64)
|
||||
|
||||
# Sort once; record rank→position mapping for unsorting later
|
||||
sorted_order = np.argsort(values)
|
||||
ranks = np.empty(n, dtype=np.int64)
|
||||
ranks[sorted_order] = np.arange(n)
|
||||
sorted_vals = values[sorted_order]
|
||||
|
||||
# ── periodicity check ──────────────────────────────────────────
|
||||
intervals = np.diff(sorted_vals)
|
||||
unique_int, counts = np.unique(
|
||||
np.round(intervals, 6), return_counts=True
|
||||
)
|
||||
repeat_ratio = float(np.sum(counts[counts > 1])) / max(len(intervals), 1)
|
||||
|
||||
if repeat_ratio >= _FFT_PERIODICITY_THRESHOLD:
|
||||
# ── FFT phase path ─────────────────────────────────────────
|
||||
centered = sorted_vals - np.mean(sorted_vals)
|
||||
fft_result = np.fft.fft(centered)
|
||||
phases = np.angle(fft_result)
|
||||
dist_values = phases[ranks]
|
||||
else:
|
||||
# ── statistical fallback ───────────────────────────────────
|
||||
mean_int = float(np.mean(intervals))
|
||||
std_int = float(np.std(intervals))
|
||||
if std_int > 1e-10:
|
||||
z_scores = (intervals - mean_int) / std_int
|
||||
else:
|
||||
z_scores = np.zeros_like(intervals)
|
||||
|
||||
# Row at sorted position r gets the z-score of interval r-1
|
||||
# (preceding interval); row at position 0 reuses last interval.
|
||||
dist_values = np.empty(n, dtype=np.float64)
|
||||
for r in range(n):
|
||||
if r > 0:
|
||||
dist_values[ranks[r]] = z_scores[r - 1]
|
||||
else:
|
||||
dist_values[ranks[0]] = (
|
||||
z_scores[-1] if n > 1 else 0.0
|
||||
)
|
||||
|
||||
# Normalise to zero-mean unit-variance
|
||||
d_std = float(np.std(dist_values))
|
||||
d_mean = float(np.mean(dist_values))
|
||||
if d_std > 1e-10:
|
||||
dist_values = (dist_values - d_mean) / d_std
|
||||
else:
|
||||
dist_values = dist_values - d_mean
|
||||
|
||||
return dist_values
|
||||
|
||||
|
||||
def timestamp_fft_distance(
|
||||
lf: pl.LazyFrame, ts_columns: list[str]
|
||||
) -> pl.LazyFrame:
|
||||
"""Append FFT-based phase-distance columns for timestamp columns.
|
||||
|
||||
For each column in *ts_columns* that exists in the schema:
|
||||
|
||||
1. Parse values to Unix epoch float seconds (handles int, float,
|
||||
datetime strings, and ``pl.Datetime``).
|
||||
2. Collect the column to numpy, sort, then run ``np.fft.fft``
|
||||
followed by ``np.angle`` to extract per-row phase values.
|
||||
3. If fewer than 5 % of timestamp intervals repeat (weak
|
||||
periodicity), fall back to statistical features derived
|
||||
from interval mean, standard deviation, and z-scores.
|
||||
|
||||
Columns are appended as ``_ts_dist_0``, ``_ts_dist_1``, … in the
|
||||
order *ts_columns* are supplied. Rows whose timestamp is null
|
||||
receive ``NaN``.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
lf : pl.LazyFrame
|
||||
The input LazyFrame with at least one timestamp column.
|
||||
ts_columns : list[str]
|
||||
Column names to process.
|
||||
|
||||
Returns
|
||||
-------
|
||||
pl.LazyFrame
|
||||
A new LazyFrame with ``_ts_dist_N`` columns appended.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> import polars as pl
|
||||
>>> from analysis.distance import timestamp_fft_distance
|
||||
>>> df = pl.DataFrame({
|
||||
... 'ts': [0, 60, 120, 180, 240, 300],
|
||||
... })
|
||||
>>> result = timestamp_fft_distance(df.lazy(), ['ts'])
|
||||
>>> result.collect().columns
|
||||
['ts', '_ts_dist_0']
|
||||
"""
|
||||
schema = lf.collect_schema()
|
||||
available = [c for c in ts_columns if c in schema.names()]
|
||||
if not available:
|
||||
return lf
|
||||
|
||||
# Collect all needed columns once
|
||||
df = lf.select(available).collect(streaming=True)
|
||||
n_rows = len(df)
|
||||
|
||||
dist_arrays: list[np.ndarray] = []
|
||||
|
||||
for col in available:
|
||||
series = df[col]
|
||||
ts_numeric = _parse_timestamp_to_epoch(series)
|
||||
|
||||
# Identify valid (non-null) rows
|
||||
valid = ~ts_numeric.is_null()
|
||||
valid_indices = np.where(valid.to_numpy())[0]
|
||||
values = ts_numeric.filter(valid).to_numpy().astype(np.float64)
|
||||
|
||||
if len(values) < _FFT_MIN_SAMPLES:
|
||||
continue
|
||||
|
||||
phase_values = _compute_fft_distances(values)
|
||||
|
||||
# Build full-length array; invalid rows stay NaN
|
||||
full_dist = np.full(n_rows, np.nan, dtype=np.float64)
|
||||
full_dist[valid_indices] = phase_values
|
||||
dist_arrays.append(full_dist)
|
||||
|
||||
if not dist_arrays:
|
||||
return lf
|
||||
|
||||
# Re-materialise the original frame and bolt on distance columns
|
||||
result_df = lf.collect(streaming=True)
|
||||
for i, arr in enumerate(dist_arrays):
|
||||
result_df = result_df.with_columns(
|
||||
pl.Series(f'_ts_dist_{i}', arr)
|
||||
)
|
||||
|
||||
return result_df.lazy()
|
||||
@@ -0,0 +1,116 @@
|
||||
"""SVD noise removal (step 3 of preprocessing pipeline)."""
|
||||
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
|
||||
|
||||
def svd_denoise(
|
||||
lf: pl.LazyFrame,
|
||||
feature_cols: list[str],
|
||||
variance_threshold: float = 0.95,
|
||||
) -> tuple[pl.LazyFrame, dict]:
|
||||
"""Remove noise from numeric features via TruncatedSVD.
|
||||
|
||||
1. Fits TruncatedSVD on *feature_cols*.
|
||||
2. Keeps *k* components where cumulative explained variance
|
||||
``trace(S[:k]) / trace(S) ≥ variance_threshold`` (default 0.95).
|
||||
3. Inverse-transforms to reconstruct the denoised matrix.
|
||||
4. **Replaces** the original *feature_cols* with denoised values.
|
||||
|
||||
The noise components (removed by SVD) are returned as a dict for
|
||||
display, NOT as new feature columns.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
lf:
|
||||
LazyFrame containing *feature_cols*.
|
||||
feature_cols:
|
||||
Numeric column names to denoise. These columns are replaced
|
||||
in-place by their denoised reconstructions.
|
||||
variance_threshold:
|
||||
Fraction of total variance to preserve (0.0–1.0).
|
||||
Default 0.95 keeps 95 % of signal and treats 5 % as noise.
|
||||
|
||||
Returns
|
||||
-------
|
||||
tuple[pl.LazyFrame, dict]
|
||||
``(denoised_lf, noise_profile)`` where *noise_profile* contains:
|
||||
- ``kept_components`` : int — number of components kept
|
||||
- ``total_components`` : int — max possible components
|
||||
- ``kept_variance`` : float — fraction of variance kept
|
||||
- ``noise_components`` : list[dict] — per-component noise info
|
||||
(component index, explained variance, top-5 contributing features)
|
||||
"""
|
||||
noise_profile = {
|
||||
'kept_components': 0,
|
||||
'total_components': 0,
|
||||
'kept_variance': 0.0,
|
||||
'noise_components': [],
|
||||
}
|
||||
|
||||
available = [c for c in feature_cols if c in lf.collect_schema().names()]
|
||||
if not available:
|
||||
return lf, noise_profile
|
||||
|
||||
df = lf.select(available).collect(streaming=True)
|
||||
mat = df.to_numpy().astype(np.float64)
|
||||
mat = np.nan_to_num(mat, nan=0.0)
|
||||
|
||||
n, d = mat.shape
|
||||
max_components = min(n, d - 1)
|
||||
if max_components < 1:
|
||||
return lf, noise_profile
|
||||
|
||||
from sklearn.decomposition import TruncatedSVD
|
||||
svd = TruncatedSVD(n_components=max_components, random_state=42)
|
||||
_ = svd.fit_transform(mat)
|
||||
|
||||
total_variance = float(np.sum(svd.explained_variance_))
|
||||
if total_variance <= 1e-10:
|
||||
return lf, noise_profile
|
||||
|
||||
cumsum = np.cumsum(svd.explained_variance_)
|
||||
for k in range(1, max_components + 1):
|
||||
if cumsum[k - 1] / total_variance >= variance_threshold:
|
||||
break
|
||||
else:
|
||||
k = max_components
|
||||
|
||||
# Re-fit with chosen k
|
||||
svd_k = TruncatedSVD(n_components=k, random_state=42)
|
||||
reduced = svd_k.fit_transform(mat)
|
||||
reconstructed = svd_k.inverse_transform(reduced)
|
||||
|
||||
# --- Build noise profile (removed components) ---
|
||||
# The "noise" is the variance in components k..max_components
|
||||
kept_var_ratio = cumsum[k - 1] / total_variance if k > 0 else 0.0
|
||||
noise_profile = {
|
||||
'kept_components': k,
|
||||
'total_components': max_components,
|
||||
'kept_variance': round(float(kept_var_ratio), 4),
|
||||
}
|
||||
|
||||
# Per-component noise info for removed components (k and beyond)
|
||||
noise_info = []
|
||||
for comp_idx in range(k, min(k + 10, max_components)):
|
||||
loading = np.abs(svd.components_[comp_idx])
|
||||
top5_idx = np.argsort(loading)[-5:][::-1]
|
||||
top5 = [
|
||||
{'feature': available[i], 'strength': round(float(loading[i]), 4)}
|
||||
for i in top5_idx
|
||||
]
|
||||
noise_info.append({
|
||||
'component': int(comp_idx + 1),
|
||||
'explained_var_ratio': round(float(svd.explained_variance_ratio_[comp_idx]) * 100, 3),
|
||||
'top_features': top5,
|
||||
})
|
||||
noise_profile['noise_components'] = noise_info
|
||||
|
||||
# --- Replace original columns with denoised values ---
|
||||
df_result = lf.collect(streaming=True)
|
||||
for i, col in enumerate(available):
|
||||
df_result = df_result.with_columns(
|
||||
pl.Series(col, reconstructed[:, i])
|
||||
)
|
||||
|
||||
return df_result.lazy(), noise_profile
|
||||
@@ -0,0 +1,164 @@
|
||||
"""String distance, enum detection, and boolean distance functions."""
|
||||
|
||||
import polars as pl
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Enum-like detection helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _is_enum_like(s: pl.Series) -> bool:
|
||||
"""Check if ≥50% of rows have values appearing ≥2 times."""
|
||||
total = len(s)
|
||||
if total == 0:
|
||||
return False
|
||||
non_null = s.drop_nulls()
|
||||
if len(non_null) == 0:
|
||||
return False
|
||||
vc = non_null.value_counts()
|
||||
if 'count' not in vc.columns:
|
||||
return False
|
||||
duplicated = vc.filter(pl.col('count') >= 2)
|
||||
if len(duplicated) == 0:
|
||||
return False
|
||||
duplicated_count: int = duplicated['count'].sum() # type: ignore[union-attr]
|
||||
return duplicated_count >= 0.5 * total
|
||||
|
||||
|
||||
def _levenshtein_normalized(s1: str | None, s2: str) -> float:
|
||||
"""Normalised Levenshtein distance between *s1* and *s2*.
|
||||
|
||||
Empty or ``None`` values return 1.0 (maximum distance).
|
||||
"""
|
||||
if s1 is None or s1 == '':
|
||||
return 1.0
|
||||
if s2 is None or s2 == '':
|
||||
return 1.0
|
||||
s1_str = str(s1)
|
||||
s2_str = str(s2)
|
||||
max_len = max(len(s1_str), len(s2_str))
|
||||
if max_len == 0:
|
||||
return 0.0
|
||||
import Levenshtein
|
||||
return Levenshtein.distance(s1_str, s2_str) / max_len
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# String distance
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def string_distance(lf: pl.LazyFrame, str_columns: list[str]) -> pl.LazyFrame:
|
||||
"""Compute string distance columns.
|
||||
|
||||
For each column in *str_columns*:
|
||||
|
||||
* If **≥50%** of rows have values appearing ≥2 times, the column is
|
||||
treated as an enum: 0 when the value equals the mode, 1 otherwise.
|
||||
* Otherwise, the normalised Levenshtein distance to the mode is
|
||||
computed for each row.
|
||||
|
||||
Empty or ``None`` values always produce the maximum distance (1.0).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
lf:
|
||||
Input LazyFrame.
|
||||
str_columns:
|
||||
List of string column names to compute distances for.
|
||||
|
||||
Returns
|
||||
-------
|
||||
pl.LazyFrame
|
||||
Input LazyFrame extended with ``_str_dist_0``, ``_str_dist_1``,
|
||||
... columns (one per entry in *str_columns*).
|
||||
"""
|
||||
df = lf.collect(streaming=True)
|
||||
dist_exprs: list[pl.Expr] = []
|
||||
|
||||
for idx, col in enumerate(str_columns):
|
||||
if col not in df.columns:
|
||||
continue
|
||||
|
||||
series = df[col].cast(pl.Utf8)
|
||||
enum_like = _is_enum_like(series)
|
||||
|
||||
non_null = series.drop_nulls()
|
||||
if len(non_null) > 0:
|
||||
mode_val = non_null.value_counts().sort('count', descending=True).row(0)[0]
|
||||
mode_str = str(mode_val)
|
||||
else:
|
||||
mode_str = ''
|
||||
|
||||
col_name = f'_str_dist_{idx}'
|
||||
|
||||
if enum_like:
|
||||
dist_exprs.append(
|
||||
pl.when(pl.col(col).is_null())
|
||||
.then(pl.lit(1.0))
|
||||
.when(pl.col(col).cast(pl.Utf8) == pl.lit(mode_str))
|
||||
.then(pl.lit(0.0))
|
||||
.otherwise(pl.lit(1.0))
|
||||
.alias(col_name)
|
||||
)
|
||||
else:
|
||||
# Capture in closure for map_elements
|
||||
_mode = mode_str
|
||||
dist_exprs.append(
|
||||
pl.col(col).map_elements(
|
||||
lambda v: _levenshtein_normalized(v, _mode),
|
||||
return_dtype=pl.Float64,
|
||||
).alias(col_name)
|
||||
)
|
||||
|
||||
return lf.with_columns(dist_exprs)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Boolean distance
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def bool_distance(lf: pl.LazyFrame, bool_columns: list[str]) -> pl.LazyFrame:
|
||||
"""Compute boolean distance columns.
|
||||
|
||||
For each column in *bool_columns*, each row gets:
|
||||
0 if its value matches the column mode, 1 otherwise.
|
||||
``None`` values always produce 1.0 (maximum distance).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
lf:
|
||||
Input LazyFrame.
|
||||
bool_columns:
|
||||
List of boolean column names to compute distances for.
|
||||
|
||||
Returns
|
||||
-------
|
||||
pl.LazyFrame
|
||||
Input LazyFrame extended with ``_bool_dist_0``, ``_bool_dist_1``,
|
||||
... columns (one per entry in *bool_columns*).
|
||||
"""
|
||||
df = lf.collect(streaming=True)
|
||||
dist_exprs: list[pl.Expr] = []
|
||||
|
||||
for idx, col in enumerate(bool_columns):
|
||||
if col not in df.columns:
|
||||
continue
|
||||
|
||||
series = df[col]
|
||||
non_null = series.drop_nulls()
|
||||
col_name = f'_bool_dist_{idx}'
|
||||
|
||||
if len(non_null) == 0:
|
||||
dist_exprs.append(pl.lit(0.0).alias(col_name))
|
||||
else:
|
||||
mode_val = non_null.value_counts().sort('count', descending=True).row(0)[0]
|
||||
dist_exprs.append(
|
||||
pl.when(pl.col(col).is_null())
|
||||
.then(pl.lit(1.0))
|
||||
.when(pl.col(col) == pl.lit(mode_val))
|
||||
.then(pl.lit(0.0))
|
||||
.otherwise(pl.lit(1.0))
|
||||
.alias(col_name)
|
||||
)
|
||||
|
||||
return lf.with_columns(dist_exprs)
|
||||
@@ -0,0 +1,122 @@
|
||||
"""UMAP dimensionality reduction (3D → 2D fallback)."""
|
||||
|
||||
import logging
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
import umap
|
||||
|
||||
|
||||
def umap_reduce(
|
||||
lf: pl.LazyFrame,
|
||||
feature_cols: list[str],
|
||||
n_components: int = 3,
|
||||
) -> tuple[pl.LazyFrame, dict]:
|
||||
"""UMAP dimensionality reduction with automatic 3D→2D fallback.
|
||||
|
||||
Trains on a ≤10K-row sample (when *lf* exceeds that limit), then
|
||||
batch-transforms the remaining rows in 1K-row batches.
|
||||
|
||||
If ``n_components=3`` fails (e.g. too few rows or numerical issues),
|
||||
falls back to ``n_components=2`` automatically.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
lf:
|
||||
LazyFrame containing the numeric feature columns.
|
||||
feature_cols:
|
||||
Numeric column names to use for UMAP projection. Columns not
|
||||
present in *lf* are silently skipped.
|
||||
n_components:
|
||||
Target dimensionality (2 or 3). Default 3 with fallback to 2.
|
||||
|
||||
Returns
|
||||
-------
|
||||
tuple[pl.LazyFrame, dict]
|
||||
- LazyFrame: input data extended with ``_umap_x``, ``_umap_y``
|
||||
(and ``_umap_z`` when ``n_components ≥ 3``).
|
||||
- dict: metadata with keys ``n_components`` and ``reducer``.
|
||||
"""
|
||||
logger = logging.getLogger(__name__)
|
||||
MAX_TRAIN = 10000
|
||||
BATCH_SIZE = 1000
|
||||
|
||||
available = [c for c in feature_cols if c in lf.collect_schema().names()]
|
||||
if not available:
|
||||
return lf, {'n_components': 0, 'reducer': 'umap', 'error': 'no feature columns available'}
|
||||
|
||||
df = lf.collect(streaming=True)
|
||||
n_total = len(df)
|
||||
|
||||
if n_total < 2:
|
||||
return lf, {'n_components': 0, 'reducer': 'umap', 'error': 'too few rows'}
|
||||
|
||||
mat = df.select(available).to_numpy().astype(np.float64)
|
||||
mat = np.nan_to_num(mat, nan=0.0, posinf=0.0, neginf=0.0)
|
||||
|
||||
scaler = StandardScaler()
|
||||
if n_total > MAX_TRAIN:
|
||||
rng = np.random.RandomState(42)
|
||||
idx_sample = rng.choice(n_total, size=MAX_TRAIN, replace=False)
|
||||
mat_sample = mat[idx_sample]
|
||||
scaler.fit(mat_sample)
|
||||
mat_sample_scaled = np.nan_to_num(scaler.transform(mat_sample), nan=0.0)
|
||||
else:
|
||||
scaler.fit(mat)
|
||||
mat_sample_scaled = np.nan_to_num(scaler.transform(mat), nan=0.0)
|
||||
|
||||
# ── Train UMAP; fallback 3→2 on failure ──
|
||||
reducer = None
|
||||
coords = None
|
||||
effective_n = n_components
|
||||
|
||||
# Try requested n_components first, then fall back to 2 (no duplicates)
|
||||
attempts = sorted(set((n_components, 2)), reverse=True)
|
||||
for attempt_n in attempts:
|
||||
if attempt_n != n_components:
|
||||
logger.warning(f'UMAP {n_components}D failed, falling back to 2D')
|
||||
effective_n = attempt_n
|
||||
try:
|
||||
reducer = umap.UMAP(
|
||||
n_components=effective_n, random_state=42,
|
||||
n_neighbors=15, min_dist=0.1, metric='euclidean',
|
||||
)
|
||||
if n_total > MAX_TRAIN:
|
||||
reducer.fit(mat_sample_scaled)
|
||||
else:
|
||||
coords = reducer.fit_transform(mat_sample_scaled)
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt_n == 2:
|
||||
raise
|
||||
logger.warning(f'UMAP {attempt_n}D failed: {e}')
|
||||
|
||||
# ── Batch transform if dataset was larger than training sample ──
|
||||
if n_total > MAX_TRAIN and reducer is not None:
|
||||
coords_list: list[np.ndarray] = []
|
||||
for start in range(0, n_total, BATCH_SIZE):
|
||||
end = min(start + BATCH_SIZE, n_total)
|
||||
mat_batch = scaler.transform(mat[start:end])
|
||||
mat_batch = np.nan_to_num(mat_batch, nan=0.0, posinf=0.0, neginf=0.0)
|
||||
coords_list.append(reducer.transform(mat_batch))
|
||||
coords = np.vstack(coords_list)
|
||||
|
||||
if coords is None:
|
||||
return lf, {'n_components': 0, 'reducer': 'umap', 'error': 'UMAP produced no output'}
|
||||
|
||||
# ── Build result LazyFrame with coordinate columns ──
|
||||
coord_series = [
|
||||
pl.Series('_umap_x', coords[:, 0]),
|
||||
pl.Series('_umap_y', coords[:, 1]),
|
||||
]
|
||||
if effective_n >= 3 and coords.shape[1] >= 3:
|
||||
coord_series.append(pl.Series('_umap_z', coords[:, 2]))
|
||||
|
||||
df_result = df.with_columns(coord_series)
|
||||
|
||||
metadata: dict = {
|
||||
'n_components': effective_n,
|
||||
'reducer': 'umap',
|
||||
}
|
||||
|
||||
return df_result.lazy(), metadata
|
||||
@@ -67,9 +67,9 @@ class Command(BaseCommand):
|
||||
_self.stdout.write(f' → 聚类特征: {feature_cols}')
|
||||
run.total_flows = row_count
|
||||
run.save(update_fields=['total_flows'])
|
||||
from analysis.views import _run_clustering_pipeline
|
||||
_run_clustering_pipeline(
|
||||
run=run, store=store, ds_id=ds_id,
|
||||
from analysis.services.clustering import run_clustering_pipeline
|
||||
run_clustering_pipeline(
|
||||
run=run, store=store, entity_ds_id=ds_id,
|
||||
feature_columns=feature_cols,
|
||||
algorithm=algo, min_cluster_size=5,
|
||||
run_umap=True,
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
"""Analysis services package — pure business logic, no request/response/rendering."""
|
||||
from .clustering import run_clustering_pipeline, compute_umap_embedding, save_entity_profiles
|
||||
@@ -0,0 +1,361 @@
|
||||
"""Clustering service layer: pure business logic for the clustering pipeline.
|
||||
|
||||
Extracted from analysis/views/clustering.py — no request/response, no rendering.
|
||||
"""
|
||||
import asyncio
|
||||
import traceback
|
||||
import logging
|
||||
import warnings
|
||||
|
||||
import polars as pl
|
||||
import numpy as np
|
||||
|
||||
from analysis.constants import RANDOM_SEED, UMAP_BATCH_SIZE, UMAP_TRAIN_SAMPLE, NUMERIC_DTYPES
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorithm,
|
||||
min_cluster_size, run_umap=True, head=None):
|
||||
"""Pure business logic — no request/response, no rendering.
|
||||
|
||||
Unified clustering pipeline: clustering → feature extraction → UMAP → ORM save.
|
||||
|
||||
Args:
|
||||
run: AnalysisRun ORM object.
|
||||
store: SessionStore instance.
|
||||
entity_ds_id: dataset ID in SessionStore pointing to entity-aggregated data.
|
||||
feature_columns: list of feature column names (None = auto-detect).
|
||||
algorithm: 'hdbscan' or 'kmeans'.
|
||||
min_cluster_size: int for HDBSCAN.
|
||||
run_umap: bool, whether to compute and save UMAP-2D embeddings.
|
||||
head: optional int, downsample to at most this many rows (min 100).
|
||||
"""
|
||||
warnings.filterwarnings('ignore', category=RuntimeWarning, module='sklearn')
|
||||
|
||||
try:
|
||||
from analysis.tool_registry import _handle_run_clustering, _handle_extract_features
|
||||
|
||||
entry = store.get_dataset(entity_ds_id)
|
||||
if entry is None:
|
||||
run.status = 'failed'
|
||||
run.error_message = 'Entity dataset not found in session store'
|
||||
run.save(update_fields=['status', 'error_message'])
|
||||
return
|
||||
|
||||
schema = entry.get('schema', {})
|
||||
|
||||
# ── Downsampling ──
|
||||
if head is not None and head > 0:
|
||||
lf = entry['lazyframe']
|
||||
try:
|
||||
row_count = lf.select(pl.len()).collect(streaming=True).item()
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
row_count = 0
|
||||
|
||||
if row_count > head:
|
||||
head = max(head, 100) # ensure minimum rows for clustering
|
||||
run.progress_msg = f'正在采样 {head}/{row_count} 行...'
|
||||
run.save(update_fields=['progress_msg'])
|
||||
lf = lf.head(head)
|
||||
store.store_dataset(
|
||||
entity_ds_id, lf,
|
||||
schema=schema,
|
||||
metadata=entry.get('metadata', {}),
|
||||
)
|
||||
entry = store.get_dataset(entity_ds_id) # refresh
|
||||
row_count = head
|
||||
|
||||
# Resolve feature columns — use actual LazyFrame dtypes (not stored schema dict
|
||||
# which may be stale after Utf8 coercion in _background_process).
|
||||
numeric_types = (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
|
||||
pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
|
||||
pl.Float32, pl.Float64)
|
||||
|
||||
lf = entry['lazyframe']
|
||||
live_schema = lf.collect_schema()
|
||||
live_names = live_schema.names()
|
||||
live_dtypes = list(live_schema.dtypes())
|
||||
|
||||
if feature_columns:
|
||||
feature_cols = [
|
||||
c for c in feature_columns
|
||||
if c in live_names and live_dtypes[live_names.index(c)] in numeric_types
|
||||
]
|
||||
else:
|
||||
feature_cols = [
|
||||
live_names[i] for i, dt in enumerate(live_dtypes)
|
||||
if dt in numeric_types and not live_names[i].startswith('_')
|
||||
][:10]
|
||||
|
||||
# ── Step 1: Clustering ────────────────────────────────────────────────
|
||||
# If no numeric columns detected, pass empty list to let
|
||||
# _handle_run_clustering apply its Utf8→Float64 coercion fallback.
|
||||
run.status = 'clustering'
|
||||
run.progress_pct = 70
|
||||
run.progress_msg = '正在进行聚类分析...'
|
||||
run.save(update_fields=['status', 'progress_pct', 'progress_msg'])
|
||||
|
||||
result = asyncio.run(_handle_run_clustering(
|
||||
dataset_id=entity_ds_id,
|
||||
cluster_columns=feature_cols,
|
||||
algorithm=algorithm,
|
||||
params={'min_cluster_size': min_cluster_size},
|
||||
random_state=RANDOM_SEED,
|
||||
))
|
||||
|
||||
if 'error' in result:
|
||||
err = result['error']
|
||||
if 'numeric' in err.lower():
|
||||
available = [
|
||||
f"{k}({v})" for k, v in schema.items()
|
||||
if v.split('(')[0].strip() in numeric_types and not k.startswith('_')
|
||||
]
|
||||
err += f"。可用数值列: {available}"
|
||||
raise Exception(err)
|
||||
|
||||
cluster_id = result.get('cluster_result_id', '')
|
||||
run.cluster_count = result.get('n_clusters', 0)
|
||||
run.save(update_fields=['cluster_count'])
|
||||
|
||||
# ── Step 2: Feature extraction ──────────────────────────────────────
|
||||
run.status = 'extracting'
|
||||
run.progress_pct = 90
|
||||
run.progress_msg = '正在提取聚类特征...'
|
||||
run.save(update_fields=['status', 'progress_pct', 'progress_msg'])
|
||||
|
||||
feat_result = asyncio.run(_handle_extract_features(
|
||||
dataset_id=entity_ds_id,
|
||||
cluster_result_id=cluster_id,
|
||||
top_k=10, method='zscore', save_to_db=True,
|
||||
run_id=run.id,
|
||||
))
|
||||
|
||||
if 'error' in feat_result:
|
||||
raise Exception(feat_result['error'])
|
||||
|
||||
# ── Step 2b: Cluster SVD feature extraction ──────────────────────
|
||||
run.progress_msg = '正在进行聚类SVD特征提取...'
|
||||
run.save(update_fields=['progress_msg'])
|
||||
try:
|
||||
from analysis.distance import cluster_svd_extract
|
||||
lf = entry['lazyframe']
|
||||
cluster_entry = store.get_cluster_result(cluster_id)
|
||||
labels_list = cluster_entry.get('labels', []) if cluster_entry else []
|
||||
|
||||
if labels_list and feature_cols:
|
||||
svd_results = cluster_svd_extract(lf, labels_list, feature_cols)
|
||||
|
||||
# Persist SVD features to ClusterFeature (method='cluster_svd')
|
||||
from analysis.models import ClusterFeature as CF
|
||||
from analysis.models import ClusterResult as CR
|
||||
for label, svd_info in svd_results.items():
|
||||
cr = CR.objects.filter(run=run, cluster_label=label).first()
|
||||
if cr is None:
|
||||
continue
|
||||
# Delete old zscore entries for this cluster, replace with SVD
|
||||
CF.objects.filter(cluster=cr, distinguishing_method='zscore').delete()
|
||||
for feat_name in svd_info.get('features', [])[:10]:
|
||||
idx = svd_info['features'].index(feat_name) if feat_name in svd_info['features'] else -1
|
||||
strength = (svd_info.get('feature_strength', [])[idx]
|
||||
if idx >= 0 and idx < len(svd_info.get('feature_strength', []))
|
||||
else None)
|
||||
CF.objects.get_or_create(
|
||||
cluster=cr,
|
||||
feature_name=feat_name,
|
||||
defaults={
|
||||
'mean': None, 'std': None, 'median': None,
|
||||
'p25': None, 'p75': None, 'missing_rate': None,
|
||||
'distinguishing_score': float(strength) if strength is not None else None,
|
||||
'distinguishing_method': 'cluster_svd',
|
||||
},
|
||||
)
|
||||
except Exception as svd_err:
|
||||
logger.warning(f'Cluster SVD extraction skipped: {svd_err}')
|
||||
|
||||
# ── Step 3: UMAP-3D embedding (2D fallback) ───────────────────────
|
||||
if run_umap:
|
||||
try:
|
||||
lf = entry['lazyframe']
|
||||
# UMAP: sample max 10K for training, batch-transform rest in 1K batches
|
||||
MAX_UMAP_TRAIN = UMAP_TRAIN_SAMPLE
|
||||
BATCH_SIZE = UMAP_BATCH_SIZE
|
||||
try:
|
||||
n_total = lf.select(pl.len()).collect(streaming=True).item()
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
n_total = 0
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
import umap
|
||||
|
||||
live_schema = lf.collect_schema()
|
||||
num_cols = [name for name, dt in zip(live_schema.names(), live_schema.dtypes())
|
||||
if dt in (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
|
||||
pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
|
||||
pl.Float32, pl.Float64) and not name.startswith('_')]
|
||||
|
||||
if len(num_cols) >= 2:
|
||||
if n_total > MAX_UMAP_TRAIN:
|
||||
run.progress_msg = f'UMAP 采样 {MAX_UMAP_TRAIN}/{n_total} 实体训练...'
|
||||
run.save(update_fields=['progress_msg'])
|
||||
df_sample = lf.sample(n=MAX_UMAP_TRAIN, seed=RANDOM_SEED).collect(streaming=True)
|
||||
df_umap = lf.collect(streaming=True)
|
||||
else:
|
||||
df_umap = lf.collect(streaming=True)
|
||||
|
||||
coords, umap_components = compute_umap_embedding(
|
||||
df_umap, num_cols, n_total,
|
||||
max_train=MAX_UMAP_TRAIN,
|
||||
batch_size=BATCH_SIZE,
|
||||
seed=RANDOM_SEED,
|
||||
)
|
||||
|
||||
cluster_entry = store.get_cluster_result(cluster_id)
|
||||
labels_list = cluster_entry.get('labels', []) if cluster_entry else []
|
||||
|
||||
save_entity_profiles(run, df_umap, labels_list, coords, umap_components)
|
||||
except Exception as umap_err:
|
||||
logger.warning(f'UMAP embedding skipped: {umap_err}')
|
||||
|
||||
# Fallback: if entity_count not set (UMAP skipped), derive from cluster sizes
|
||||
if run.entity_count is None:
|
||||
from django.db.models import Sum
|
||||
total = run.clusters.aggregate(total=Sum('size'))['total'] or 0
|
||||
run.entity_count = total
|
||||
run.save(update_fields=['entity_count'])
|
||||
|
||||
# ── Done ──
|
||||
run.status = 'completed'
|
||||
run.progress_pct = 100
|
||||
run.progress_msg = '分析完成'
|
||||
run.save(update_fields=['status', 'progress_pct', 'progress_msg', 'entity_count'])
|
||||
|
||||
except Exception:
|
||||
tb = traceback.format_exc()
|
||||
logger.error(tb)
|
||||
try:
|
||||
run.status = 'failed'
|
||||
run.error_message = tb
|
||||
run.progress_msg = f'失败: {tb}'
|
||||
run.run_log += f'\n[ERROR] {tb}'
|
||||
run.save(update_fields=['status', 'error_message', 'progress_msg', 'run_log'])
|
||||
except Exception as e:
|
||||
logger.warning('save failed after pipeline error: %s', e)
|
||||
|
||||
|
||||
def compute_umap_embedding(df, num_cols, n_total, max_train, batch_size, seed):
|
||||
"""UMAP 3D→2D fallback, returns (coords, umap_components).
|
||||
|
||||
Args:
|
||||
df: Collected Polars DataFrame with entity-level data.
|
||||
num_cols: list of numeric column names to use for UMAP.
|
||||
n_total: total row count.
|
||||
max_train: max rows for UMAP training sample.
|
||||
batch_size: batch size for transform when dataset exceeds max_train.
|
||||
seed: random seed.
|
||||
|
||||
Returns:
|
||||
(coords, umap_components): numpy array of shape (n_total, umap_components),
|
||||
and umap_components is 2 or 3.
|
||||
"""
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
import umap
|
||||
|
||||
if len(df) <= 2 or len(num_cols) < 2:
|
||||
return None, 0
|
||||
|
||||
if n_total > max_train:
|
||||
df_sample = df.sample(n=max_train, seed=seed)
|
||||
else:
|
||||
df_sample = df
|
||||
|
||||
umap_components = 3 # try 3D first
|
||||
coords = None
|
||||
|
||||
for attempt_n in (3, 2):
|
||||
try:
|
||||
if n_total > max_train:
|
||||
# Train UMAP on sample, batch-transform full dataset
|
||||
mat_sample = df_sample.select(num_cols).to_numpy()
|
||||
scaler = StandardScaler().fit(mat_sample)
|
||||
mat_sample_scaled = np.nan_to_num(
|
||||
scaler.transform(mat_sample), nan=0.0)
|
||||
reducer = umap.UMAP(n_components=attempt_n,
|
||||
random_state=seed,
|
||||
n_neighbors=15, min_dist=0.1,
|
||||
metric='euclidean')
|
||||
reducer.fit(mat_sample_scaled)
|
||||
# Batch transform in batch_size-row batches
|
||||
coords_list = []
|
||||
for start in range(0, len(df), batch_size):
|
||||
end = min(start + batch_size, len(df))
|
||||
mat_batch = scaler.transform(
|
||||
df[start:end].select(num_cols).to_numpy())
|
||||
mat_batch = np.nan_to_num(mat_batch, nan=0.0)
|
||||
coords_list.append(reducer.transform(mat_batch))
|
||||
coords = np.vstack(coords_list)
|
||||
else:
|
||||
mat = np.nan_to_num(StandardScaler().fit_transform(
|
||||
df.select(num_cols).to_numpy()), nan=0.0)
|
||||
reducer = umap.UMAP(n_components=attempt_n,
|
||||
random_state=seed,
|
||||
n_neighbors=15, min_dist=0.1,
|
||||
metric='euclidean')
|
||||
coords = reducer.fit_transform(mat)
|
||||
umap_components = attempt_n
|
||||
break
|
||||
except Exception as e_umap:
|
||||
if attempt_n == 2:
|
||||
raise
|
||||
logger.warning(f'UMAP 3D failed ({e_umap}), falling back to 2D')
|
||||
|
||||
if coords is None:
|
||||
raise Exception('UMAP produced no output')
|
||||
|
||||
return coords, umap_components
|
||||
|
||||
|
||||
def save_entity_profiles(run, df_umap, labels_list, coords, umap_components):
|
||||
"""Persist per-row cluster labels + UMAP coords to EntityProfile.
|
||||
|
||||
Args:
|
||||
run: AnalysisRun ORM object.
|
||||
df_umap: Collected Polars DataFrame (used for entity column detection + row iteration).
|
||||
labels_list: list of cluster labels per row.
|
||||
coords: numpy array of UMAP coordinates (must not be None).
|
||||
umap_components: 2 or 3.
|
||||
"""
|
||||
from analysis.models import EntityProfile as EP
|
||||
from analysis.models import ClusterResult as CR
|
||||
|
||||
# Determine entity column (first non-numeric, non-underscore column)
|
||||
df_num_cols = [c for c, dt in zip(df_umap.columns, df_umap.dtypes)
|
||||
if dt in NUMERIC_DTYPES]
|
||||
non_num = [c for c in df_umap.columns
|
||||
if c not in df_num_cols and not c.startswith('_')]
|
||||
ent_col = non_num[0] if non_num else df_umap.columns[0]
|
||||
|
||||
profiles = []
|
||||
cr_cache = {}
|
||||
for i, row in enumerate(df_umap.iter_rows(named=True)):
|
||||
ev = str(row.get(ent_col, '')) or 'entity'
|
||||
ev = f'{ev}_{i}'
|
||||
lbl = labels_list[i] if i < len(labels_list) else -1
|
||||
cache_key = f'{run.id}_{lbl}'
|
||||
if cache_key not in cr_cache:
|
||||
cr_cache[cache_key] = CR.objects.filter(
|
||||
run=run, cluster_label=lbl).first()
|
||||
has_z = umap_components >= 3 and coords.shape[1] >= 3
|
||||
profiles.append(EP(
|
||||
entity_value=ev, run=run, cluster_label=lbl,
|
||||
cluster=cr_cache[cache_key],
|
||||
embedding_x=float(coords[i, 0]) if i < len(coords) else None,
|
||||
embedding_y=float(coords[i, 1]) if i < len(coords) else None,
|
||||
embedding_z=float(coords[i, 2]) if has_z
|
||||
and i < len(coords) else None,
|
||||
))
|
||||
if profiles:
|
||||
EP.objects.bulk_create(profiles, ignore_conflicts=True, batch_size=1000)
|
||||
run.entity_count = len(profiles)
|
||||
run.save(update_fields=['entity_count'])
|
||||
File diff suppressed because it is too large
Load Diff
@@ -12,7 +12,7 @@ from django.views.decorators.csrf import csrf_exempt
|
||||
from config import get_config
|
||||
from analysis.models import AnalysisRun
|
||||
from .pipeline import _run_pipeline_worker
|
||||
from .clustering import _run_clustering_pipeline
|
||||
from analysis.services.clustering import run_clustering_pipeline
|
||||
from .helpers import _PLANS_DIR, _add_to_auto_index
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -271,7 +271,7 @@ def run_llm_analysis_view(request):
|
||||
break
|
||||
|
||||
if entity_ds_id:
|
||||
_run_clustering_pipeline(
|
||||
run_clustering_pipeline(
|
||||
run=run,
|
||||
store=store,
|
||||
ds_id=entity_ds_id,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Cluster views: overview, detail, globe flows, and the clustering pipeline."""
|
||||
"""Cluster views: overview, detail, globe flows, and the clustering pipeline."""
|
||||
import asyncio
|
||||
import json
|
||||
import traceback
|
||||
@@ -222,306 +222,7 @@ def cluster_detail(request, display_id, cluster_label):
|
||||
@profile
|
||||
def _run_clustering_pipeline(run, store, ds_id, feature_columns, algorithm,
|
||||
min_cluster_size, run_umap=True, head=None):
|
||||
"""Clustering pipeline on raw rows: clustering → feature extraction → UMAP → ORM save.
|
||||
|
||||
Operates directly on raw data rows (no entity aggregation).
|
||||
Per-row results stored in EntityProfile with row index as identifier.
|
||||
|
||||
Args:
|
||||
run: AnalysisRun ORM object.
|
||||
store: SessionStore instance.
|
||||
ds_id: dataset ID in SessionStore pointing to raw data.
|
||||
feature_columns: list of feature column names (None = auto-detect all numeric).
|
||||
algorithm: clustering algorithm.
|
||||
min_cluster_size: int for HDBSCAN.
|
||||
run_umap: bool, whether to compute and save UMAP-2D embeddings.
|
||||
head: optional int, downsample to at most this many rows (min 100).
|
||||
"""
|
||||
import warnings
|
||||
warnings.filterwarnings('ignore', category=RuntimeWarning, module='sklearn')
|
||||
|
||||
try:
|
||||
from analysis.tool_registry import _handle_run_clustering, _handle_extract_features
|
||||
import polars as pl
|
||||
|
||||
entry = store.get_dataset(ds_id)
|
||||
if entry is None:
|
||||
run.status = 'failed'
|
||||
run.error_message = '数据集在会话中不存在'
|
||||
run.save(update_fields=['status', 'error_message'])
|
||||
return
|
||||
|
||||
schema = entry.get('schema', {})
|
||||
|
||||
# ── Downsampling ──
|
||||
if head is not None and head > 0:
|
||||
lf = entry['lazyframe']
|
||||
try:
|
||||
row_count = lf.select(pl.len()).collect(streaming=True).item()
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
row_count = 0
|
||||
|
||||
if row_count > head:
|
||||
head = max(head, 100) # ensure minimum rows for clustering
|
||||
run.progress_msg = f'正在采样 {head}/{row_count} 行...'
|
||||
run.save(update_fields=['progress_msg'])
|
||||
lf = lf.head(head)
|
||||
store.store_dataset(
|
||||
ds_id, lf,
|
||||
schema=schema,
|
||||
metadata=entry.get('metadata', {}),
|
||||
)
|
||||
entry = store.get_dataset(ds_id)
|
||||
row_count = head
|
||||
|
||||
# Resolve feature columns — use actual LazyFrame dtypes (not stored schema dict
|
||||
# which may be stale after Utf8 coercion in _background_process).
|
||||
numeric_types = (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
|
||||
pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
|
||||
pl.Float32, pl.Float64)
|
||||
|
||||
lf = entry['lazyframe']
|
||||
live_schema = lf.collect_schema()
|
||||
live_names = live_schema.names()
|
||||
live_dtypes = list(live_schema.dtypes())
|
||||
|
||||
if feature_columns:
|
||||
feature_cols = [
|
||||
c for c in feature_columns
|
||||
if c in live_names and live_dtypes[live_names.index(c)] in numeric_types
|
||||
]
|
||||
else:
|
||||
feature_cols = [
|
||||
live_names[i] for i, dt in enumerate(live_dtypes)
|
||||
if dt in numeric_types and not live_names[i].startswith('_')
|
||||
]
|
||||
|
||||
# ── Step 1: Clustering ────────────────────────────────────────────────
|
||||
# If no numeric columns detected, pass empty list to let
|
||||
# _handle_run_clustering apply its Utf8→Float64 coercion fallback.
|
||||
run.status = 'clustering'
|
||||
run.progress_pct = 70
|
||||
run.progress_msg = '正在进行聚类分析...'
|
||||
run.save(update_fields=['status', 'progress_pct', 'progress_msg'])
|
||||
|
||||
result = asyncio.run(_handle_run_clustering(
|
||||
dataset_id=ds_id,
|
||||
cluster_columns=feature_cols,
|
||||
algorithm=algorithm,
|
||||
params={'min_cluster_size': min_cluster_size},
|
||||
random_state=RANDOM_SEED,
|
||||
))
|
||||
|
||||
if 'error' in result:
|
||||
err = result['error']
|
||||
if 'numeric' in err.lower():
|
||||
available = [
|
||||
f"{k}({v})" for k, v in schema.items()
|
||||
if v.split('(')[0].strip() in numeric_types and not k.startswith('_')
|
||||
]
|
||||
err += f"。可用数值列: {available}"
|
||||
raise Exception(err)
|
||||
|
||||
cluster_id = result.get('cluster_result_id', '')
|
||||
run.cluster_count = result.get('n_clusters', 0)
|
||||
run.save(update_fields=['cluster_count'])
|
||||
|
||||
# ── Step 2: Feature extraction ──────────────────────────────────────
|
||||
run.status = 'extracting'
|
||||
run.progress_pct = 90
|
||||
run.progress_msg = '正在提取聚类特征...'
|
||||
run.save(update_fields=['status', 'progress_pct', 'progress_msg'])
|
||||
|
||||
feat_result = asyncio.run(_handle_extract_features(
|
||||
dataset_id=ds_id,
|
||||
cluster_result_id=cluster_id,
|
||||
top_k=10, method='zscore', save_to_db=True,
|
||||
run_id=run.id,
|
||||
))
|
||||
|
||||
if 'error' in feat_result:
|
||||
raise Exception(feat_result['error'])
|
||||
|
||||
# ── Step 2b: Cluster SVD feature extraction ──────────────────────
|
||||
run.progress_msg = '正在进行聚类SVD特征提取...'
|
||||
run.save(update_fields=['progress_msg'])
|
||||
try:
|
||||
from analysis.distance import cluster_svd_extract
|
||||
lf = entry['lazyframe']
|
||||
cluster_entry = store.get_cluster_result(cluster_id)
|
||||
labels_list = cluster_entry.get('labels', []) if cluster_entry else []
|
||||
|
||||
if labels_list and feature_cols:
|
||||
svd_results = cluster_svd_extract(lf, labels_list, feature_cols)
|
||||
|
||||
# Persist SVD features to ClusterFeature (method='cluster_svd')
|
||||
from analysis.models import ClusterFeature as CF
|
||||
from analysis.models import ClusterResult as CR
|
||||
for label, svd_info in svd_results.items():
|
||||
cr = CR.objects.filter(run=run, cluster_label=label).first()
|
||||
if cr is None:
|
||||
continue
|
||||
# Delete old zscore entries for this cluster, replace with SVD
|
||||
CF.objects.filter(cluster=cr, distinguishing_method='zscore').delete()
|
||||
for feat_name in svd_info.get('features', [])[:10]:
|
||||
idx = svd_info['features'].index(feat_name) if feat_name in svd_info['features'] else -1
|
||||
strength = (svd_info.get('feature_strength', [])[idx]
|
||||
if idx >= 0 and idx < len(svd_info.get('feature_strength', []))
|
||||
else None)
|
||||
CF.objects.get_or_create(
|
||||
cluster=cr,
|
||||
feature_name=feat_name,
|
||||
defaults={
|
||||
'mean': None, 'std': None, 'median': None,
|
||||
'p25': None, 'p75': None, 'missing_rate': None,
|
||||
'distinguishing_score': float(strength) if strength is not None else None,
|
||||
'distinguishing_method': 'cluster_svd',
|
||||
},
|
||||
)
|
||||
except Exception as svd_err:
|
||||
logger.warning(f'Cluster SVD extraction skipped: {svd_err}')
|
||||
|
||||
# ── Step 2c: Save per-row cluster labels to EntityProfile ──
|
||||
# Always persist row→cluster mapping so cluster_detail can show data rows
|
||||
# even when UMAP is skipped or fails.
|
||||
try:
|
||||
cluster_entry_save = store.get_cluster_result(cluster_id)
|
||||
labels_list_save = cluster_entry_save.get('labels', []) if cluster_entry_save else []
|
||||
if labels_list_save:
|
||||
from analysis.models import EntityProfile as EP_save
|
||||
from analysis.models import ClusterResult as CR_save
|
||||
# Delete stale EP records for this run first
|
||||
EP_save.objects.filter(run=run).delete()
|
||||
cr_cache_save = {}
|
||||
row_profiles = []
|
||||
for r_idx in range(len(labels_list_save)):
|
||||
lbl = labels_list_save[r_idx]
|
||||
ck = f'{run.id}_{lbl}'
|
||||
if ck not in cr_cache_save:
|
||||
cr_cache_save[ck] = CR_save.objects.filter(run=run, cluster_label=lbl).first()
|
||||
row_profiles.append(EP_save(
|
||||
entity_value=f'row_{r_idx}', run=run, cluster_label=lbl,
|
||||
cluster=cr_cache_save[ck],
|
||||
embedding_x=None, embedding_y=None, embedding_z=None,
|
||||
))
|
||||
if row_profiles:
|
||||
EP_save.objects.bulk_create(row_profiles, ignore_conflicts=True, batch_size=1000)
|
||||
run.entity_count = len(row_profiles)
|
||||
run.save(update_fields=['entity_count'])
|
||||
except Exception as save_err:
|
||||
logger.warning(f'Row labels save skipped: {save_err}')
|
||||
|
||||
# ── Step 3: UMAP-3D embedding (2D fallback) ───────────────────────
|
||||
if run_umap:
|
||||
try:
|
||||
lf = entry['lazyframe']
|
||||
# UMAP: sample max 10K for training, batch-transform rest in 1K batches
|
||||
MAX_UMAP_TRAIN = UMAP_TRAIN_SAMPLE
|
||||
BATCH_SIZE = UMAP_BATCH_SIZE
|
||||
try:
|
||||
n_total = lf.select(pl.len()).collect(streaming=True).item()
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
n_total = 0
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
import umap
|
||||
import numpy as np
|
||||
|
||||
live_schema = lf.collect_schema()
|
||||
num_cols = [name for name, dt in zip(live_schema.names(), live_schema.dtypes())
|
||||
if dt in (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
|
||||
pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
|
||||
pl.Float32, pl.Float64) and not name.startswith('_')]
|
||||
|
||||
if len(num_cols) >= 2:
|
||||
if n_total > MAX_UMAP_TRAIN:
|
||||
run.progress_msg = f'UMAP 采样 {MAX_UMAP_TRAIN}/{n_total} 行训练...'
|
||||
run.save(update_fields=['progress_msg'])
|
||||
# LazyFrame.sample() is unavailable in Polars 1.42 — collect then sample
|
||||
df_sample = df_umap.sample(n=MAX_UMAP_TRAIN, seed=RANDOM_SEED)
|
||||
df_sample = df_umap
|
||||
|
||||
umap_components = 3 # try 3D first
|
||||
coords = None
|
||||
reducer = None
|
||||
|
||||
if len(df_umap) > 2:
|
||||
for attempt_n in (3, 2):
|
||||
try:
|
||||
if n_total > MAX_UMAP_TRAIN:
|
||||
# Train UMAP on 10K sample, batch-transform full dataset
|
||||
mat_sample = df_sample.select(num_cols).to_numpy()
|
||||
scaler = StandardScaler().fit(mat_sample)
|
||||
mat_sample_scaled = np.nan_to_num(
|
||||
scaler.transform(mat_sample), nan=0.0)
|
||||
reducer = umap.UMAP(n_components=attempt_n,
|
||||
random_state=RANDOM_SEED,
|
||||
n_neighbors=15, min_dist=0.1,
|
||||
metric='euclidean')
|
||||
reducer.fit(mat_sample_scaled)
|
||||
# Batch transform in 1K-row batches
|
||||
coords_list = []
|
||||
for start in range(0, len(df_umap), BATCH_SIZE):
|
||||
end = min(start + BATCH_SIZE, len(df_umap))
|
||||
mat_batch = scaler.transform(
|
||||
df_umap[start:end].select(num_cols).to_numpy())
|
||||
mat_batch = np.nan_to_num(mat_batch, nan=0.0)
|
||||
coords_list.append(reducer.transform(mat_batch))
|
||||
coords = np.vstack(coords_list)
|
||||
else:
|
||||
mat = np.nan_to_num(StandardScaler().fit_transform(
|
||||
df_umap.select(num_cols).to_numpy()), nan=0.0)
|
||||
reducer = umap.UMAP(n_components=attempt_n,
|
||||
random_state=RANDOM_SEED,
|
||||
n_neighbors=15, min_dist=0.1,
|
||||
metric='euclidean')
|
||||
coords = reducer.fit_transform(mat)
|
||||
umap_components = attempt_n
|
||||
break
|
||||
except Exception as e_umap:
|
||||
if attempt_n == 2:
|
||||
raise
|
||||
logger.warning(f'UMAP 3D failed ({e_umap}), falling back to 2D')
|
||||
|
||||
if coords is None:
|
||||
raise Exception('UMAP produced no output')
|
||||
|
||||
cluster_entry = store.get_cluster_result(cluster_id)
|
||||
labels_list = cluster_entry.get('labels', []) if cluster_entry else []
|
||||
|
||||
from analysis.models import EntityProfile as EP
|
||||
|
||||
# Update existing EP records with UMAP coordinates
|
||||
for i in range(len(df_umap)):
|
||||
ev = f'row_{i}'
|
||||
has_z = umap_components >= 3 and coords.shape[1] >= 3
|
||||
update_fields = {
|
||||
'embedding_x': float(coords[i, 0]) if i < len(coords) else None,
|
||||
'embedding_y': float(coords[i, 1]) if i < len(coords) else None,
|
||||
}
|
||||
if has_z:
|
||||
update_fields['embedding_z'] = float(coords[i, 2]) if i < len(coords) else None
|
||||
EP.objects.filter(run=run, entity_value=ev).update(**update_fields)
|
||||
run.entity_count = len(df_umap)
|
||||
run.save(update_fields=['entity_count'])
|
||||
except Exception as umap_err:
|
||||
logger.warning(f'UMAP embedding skipped: {umap_err}')
|
||||
|
||||
# ── Done ──
|
||||
run.status = 'completed'
|
||||
run.progress_pct = 100
|
||||
run.progress_msg = '分析完成'
|
||||
run.save(update_fields=['status', 'progress_pct', 'progress_msg', 'entity_count'])
|
||||
|
||||
except Exception:
|
||||
tb = traceback.format_exc()
|
||||
logger.error(tb)
|
||||
try:
|
||||
run.status = 'failed'
|
||||
run.error_message = tb
|
||||
run.progress_msg = f'失败: {tb}'
|
||||
run.run_log += f'\n[ERROR] {tb}'
|
||||
run.save(update_fields=['status', 'error_message', 'progress_msg', 'run_log'])
|
||||
except Exception as e:
|
||||
logger.warning('save failed after pipeline error: %s', e)
|
||||
"""Clustering pipeline on raw rows. Delegates to service layer."""
|
||||
from analysis.services.clustering import run_clustering_pipeline
|
||||
run_clustering_pipeline(run, store, ds_id, feature_columns, algorithm,
|
||||
min_cluster_size, run_umap=run_umap, head=head)
|
||||
|
||||
@@ -10,7 +10,7 @@ from django.views.decorators.csrf import csrf_exempt
|
||||
|
||||
from analysis.models import AnalysisRun
|
||||
from .pipeline import _run_pipeline_worker
|
||||
from .clustering import _run_clustering_pipeline
|
||||
from analysis.services.clustering import run_clustering_pipeline
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -23,9 +23,9 @@ def manual_page(request):
|
||||
|
||||
# Get all tools metadata
|
||||
tools = get_tools_meta()
|
||||
core_names = {'profile_data', 'run_clustering', 'extract_features',
|
||||
'filter_data', 'preprocess_data',
|
||||
'detect_anomalies', 'visualize_anomalies'}
|
||||
core_names = {'profile_data', 'build_entity_profiles', 'compute_scores',
|
||||
'run_clustering', 'extract_features', 'detect_anomalies',
|
||||
'visualize_anomalies'}
|
||||
diag_names = {'validate_data', 'explore_distributions', 'find_outliers',
|
||||
'diagnose_clustering', 'compare_datasets', 'export_debug_sample',
|
||||
'repair_schema'}
|
||||
@@ -164,8 +164,8 @@ def manual_run_analysis(request):
|
||||
ds_id = upload_ds_id if store.get_dataset(upload_ds_id) else f'upload_{pk}'
|
||||
|
||||
# Use the unified clustering pipeline (clustering → extraction → UMAP)
|
||||
_run_clustering_pipeline(
|
||||
run=run, store=store, ds_id=ds_id,
|
||||
run_clustering_pipeline(
|
||||
run=run, store=store, entity_ds_id=ds_id,
|
||||
feature_columns=feature_columns,
|
||||
algorithm=algorithm, min_cluster_size=min_cluster_size,
|
||||
run_umap=True, head=head,
|
||||
|
||||
@@ -262,6 +262,15 @@ def _background_process(run_id, upload_dir):
|
||||
# ── Phase 3: Data ready for filtering/clustering via MCP tools ──
|
||||
run.status = 'ready'; run.progress_pct = 60; run.progress_msg = '数据准备就绪'
|
||||
run.save(update_fields=['status', 'progress_pct', 'progress_msg'])
|
||||
|
||||
# ── Phase 4: Clean up uploaded CSV files ──
|
||||
try:
|
||||
if upload_dir and upload_dir.is_dir():
|
||||
import shutil
|
||||
shutil.rmtree(upload_dir, ignore_errors=True)
|
||||
logger.info('[BACKGROUND] Cleaned up upload directory: %s', upload_dir)
|
||||
except Exception as clean_err:
|
||||
logger.warning('[BACKGROUND] Upload cleanup failed (non-fatal): %s', clean_err)
|
||||
except Exception:
|
||||
tb = traceback.format_exc()
|
||||
logger.error(tb)
|
||||
|
||||
@@ -43,20 +43,64 @@ def upload_csv(request):
|
||||
upload_dir = base / ts
|
||||
upload_dir.mkdir(parents=True, exist_ok=True)
|
||||
saved = []
|
||||
run_ids = []
|
||||
for f in files:
|
||||
path = upload_dir / f.name
|
||||
with open(path, 'wb+') as dst:
|
||||
# Each file gets its own run and its own processing thread
|
||||
fpath = upload_dir / f.name
|
||||
with open(fpath, 'wb+') as dst:
|
||||
for chunk in f.chunks():
|
||||
dst.write(chunk)
|
||||
saved.append(str(path))
|
||||
# Create run with status='pending' — background processing deferred to finalize endpoint
|
||||
fsize = fpath.stat().st_size
|
||||
saved.append(str(fpath))
|
||||
|
||||
# Create a run for this single file
|
||||
run = AnalysisRun.objects.create(
|
||||
csv_glob=str(upload_dir / '*.csv'),
|
||||
status='pending',
|
||||
csv_glob=str(fpath),
|
||||
status='loading',
|
||||
run_type='upload',
|
||||
)
|
||||
# Deferred: _background_process will be started by finalize_upload endpoint
|
||||
return JsonResponse({'run_id': run.display_id, 'files': saved, 'status': 'pending'})
|
||||
run_ids.append(run.display_id)
|
||||
|
||||
# Start processing thread immediately (per-file, not batched)
|
||||
t = threading.Thread(
|
||||
target=_process_single_file,
|
||||
args=(run.id, str(fpath)),
|
||||
daemon=True,
|
||||
)
|
||||
t.start()
|
||||
|
||||
return JsonResponse({'run_ids': run_ids, 'files': saved, 'status': 'processing'})
|
||||
|
||||
|
||||
def _process_single_file(run_id, filepath):
|
||||
"""Load a single CSV into SQLite, then delete it. Runs in background thread inside Django."""
|
||||
from analysis.data_loader._sqlite import save_to_db, drop_sqlite_table
|
||||
from analysis.data_loader._csv import load_csv_directory
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
try:
|
||||
run = AnalysisRun.objects.get(id=run_id)
|
||||
lf, schema, row_count, file_count, mem = load_csv_directory(filepath)
|
||||
table_name = save_to_db(run.id, lf)
|
||||
run.sqlite_table = table_name
|
||||
run.total_flows = row_count
|
||||
run.status = 'ready'
|
||||
run.save(update_fields=['sqlite_table', 'total_flows', 'status'])
|
||||
|
||||
# Delete CSV file immediately
|
||||
Path(filepath).unlink(missing_ok=True)
|
||||
logger.info('[UPLOAD] File %s → SQLite table %s (%d rows), CSV deleted',
|
||||
filepath, table_name, row_count)
|
||||
except Exception as exc:
|
||||
logger.error('[UPLOAD] Failed to process %s: %s', filepath, exc)
|
||||
try:
|
||||
run = AnalysisRun.objects.get(id=run_id)
|
||||
run.status = 'failed'
|
||||
run.error_message = str(exc)
|
||||
run.save(update_fields=['status', 'error_message'])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -99,103 +99,3 @@
|
||||
24.350375,1780243200.000097,89,22,8.59.133.46,c0 13,"anon, hosting",,,China Telecom,63,35,507.4165,03 03,55.39,8472,no,,BT Group,TlsC,,8.59.133.46,89840,*.s3.amazonaws.com,3298,,suricata,52020,,210.235.179.79,2026-06-01 00:00:00.000097,Google LLC,,457.5143,404,Netflix Inc.,21 af 73 9e,,12345,,,br,wan-link,1733,25.656376,25.900402,Shanghai,Seattle,"anon, hosting",1328.8239,37.15.46.192,-69.601417,4f 41 4b 95 c4 83 74 0e 6c 9b 4b 06 84 ba cf f4 57 b0 4f a0 bf c1 f8 56 f6 59 f7 c2
|
||||
-57.188778,1780243200.000098,91,12345,58.32.173.6,13 01,"anon, hosting",,,Comcast,170,26,762.6952,03 03,87.31,5829,us,,Orange,TlsC,,58.32.173.6,13103,,69005,,ssl_logs,223178,,64.41.228.136,2026-06-01 00:00:00.000098,Alibaba Inc.,,668.1106,190,Amazon.com Inc.,62 ca 57 50,,443,,,br,edge-01,1754,17.433557,3.862803,San Francisco,Singapore,"anon, hosting",1490.6827,74.251.102.32,-84.888035,39 5d f9 03 a3 f0 fd 53 5b 21 dd 9b 30 e2 af 5d 63 37 76 6d 9d 9c a3 f1 34 99 6b 4f
|
||||
-129.431676,1780243200.000099,80,465,91.107.136.180,c0 27,"anon, hosting",,,KDDI,5,22,73.0258,03 04,10.98,6281,gb,,Orange,TlsS,,91.107.136.180,43928,,160016,,zeek,79999,,41.89.185.131,2026-06-01 00:00:00.000099,Netflix Inc.,,1635.6938,393,Amazon.com Inc.,fb 90 aa 22,,21,,,no,backbone-03,391,6.800932,22.748415,Seoul,Tokyo,"anon, hosting",1778.8222,47.139.158.236,114.231597,a0 b8 e4 6d 5e a4 ef 74 eb b9 aa 65 84 82 0b 47 3d d2 da ca f9 18 1e bf 08 df 10 a7
|
||||
-136.186649,1780243200.0001,42,443,165.204.169.241,13 02,"anon, hosting",,,AT&T,7,11,365.068,03 03,7.81,8708,hk,,BT Group,TlsC,,165.204.169.241,28919,,194047,,zeek,425040,api.example.com,74.164.146.166,2026-06-01 00:00:00.000100,Alibaba Inc.,,345.6676,422,Alibaba Inc.,8c a5 17 48,,143,,,fr,edge-01,762,-38.988386,26.942201,San Francisco,New York,"anon, hosting",1930.6351,39.228.247.89,148.781956,b7 a2 d4 61 6b 2e b8 f4 82 39 e9 08 03 c2 d7 cc 33 b5 b8 9e 21 de 99 75 2b c7 f1 08
|
||||
-61.694727,1780243200.000101,66,53,193.126.141.197,c0 2f,"anon, hosting",,,Orange,175,20,295.7134,03 03,80.33,1273,il,,NTT,TlsS,,193.126.141.197,56391,,136171,,packet_capture,63260,,21.164.75.17,2026-06-01 00:00:00.000101,Apple Inc.,40320,1087.0943,329,Amazon.com Inc.,3c 8f ac ac,,465,,,gb,edge-01,1602,-32.853192,19.124323,Seoul,Dublin,"anon, hosting",1950.2007,113.17.186.188,-113.534368,88 43 a7 d8 bf 4a 11 c5 18 65 5d 07 49 11 79 52 cb a5 05 f6 f7 b0 60 cf e7 6d e8 86
|
||||
-90.513611,1780243200.000102,100,3389,59.148.40.219,13 01,"anon, hosting",,,Telstra,125,1,251.2491,03 04,79.96,7157,fr,,KDDI,TlsC,,139.7.47.238,97498,,113471,,zeek,220215,,210.235.179.79,2026-06-01 00:00:00.000102,Netflix Inc.,,1448.3807,395,Netflix Inc.,56 f9 95 73,,443,,,fi,edge-01,1307,-40.866839,10.969969,New York,Beijing,"anon, hosting",504.0899,79.13.242.68,-29.15666,47 cd 3f 4d 69 e3 48 8d a7 05 3c 9f 34 f1 76 e0 98 3f e8 4e 24 ae 8b 14 c3 82 39 ee
|
||||
91.194896,1780243200.000103,32,110,46.138.247.120,cc a9,"anon, hosting",,,Verizon,157,15,992.6773,03 04,46.54,8191,nl,,KDDI,TlsC,,46.138.247.120,82250,,193112,,zeek,287503,auth.example.net,56.76.216.29,2026-06-01 00:00:00.000103,Meta Platforms,,165.341,315,Microsoft Corp.,5b cb 26 65,,22,,,nl,core-02,1818,43.617948,28.884572,London,Mumbai,"anon, hosting",885.6296,113.22.210.94,53.383288,2d aa b4 74 a2 9c 33 2e cf a0 fe 5f c3 67 87 f8 b3 ac e0 b9 51 56 a5 08 39 61 9e 97
|
||||
132.327773,1780243200.000104,22,12345,64.62.233.35,c0 14,"anon, hosting",00 1d,*.s3.amazonaws.com,BT Group,136,34,339.7202,03 03,17.05,6355,br,,SK Telecom,TlsS,,64.62.233.35,13624,,25229,,packet_capture,339532,,58.58.236.243,2026-06-01 00:00:00.000104,Meta Platforms,,182.3782,451,Amazon.com Inc.,fd 3d a9 2d,,53,,,no,wan-link,364,7.079525,-18.287628,Singapore,San Francisco,"anon, hosting",402.014,84.226.59.175,-159.283524,8d 81 0e fb dc 44 ca 6e e7 9e 9b a9 77 d0 49 a1 03 1b b7 3d 24 20 d3 52 63 7f 5b 5c
|
||||
-112.197533,1780243200.000105,49,465,179.237.25.143,00 35,"anon, hosting",,,Verizon,177,46,949.9643,03 04,22.99,5758,fi,,Orange,TlsS,,214.0.155.74,41508,,13845,Netflix Inc.,zeek,112656,,63.104.203.235,2026-06-01 00:00:00.000105,Amazon.com Inc.,,1698.5924,379,Netflix Inc.,15 60 2d 9d,,443,,,sg,wan-link,1402,-16.116779,38.11026,Mumbai,Sydney,"anon, hosting",1362.5856,194.70.49.232,-93.797512,f5 32 2e a4 c9 66 90 a4 45 e5 65 95 da ce de c1 2a fb 0c fb 35 a4 f7 c7 9c fe 93 1c
|
||||
155.022955,1780243200.000106,42,465,184.101.38.152,00 35,"anon, hosting",,,NTT,37,4,143.3635,03 03,18.8,4828,ru,,SK Telecom,TlsC,,184.101.38.152,10448,,16785,,suricata,151093,graph.facebook.com,55.32.212.105,2026-06-01 00:00:00.000106,,,51.594,337,Tencent,a2 16 07 4e,,12345,,,kr,wan-link,1285,53.220375,21.708672,Singapore,Mumbai,"anon, hosting",1687.0012,215.90.18.215,-87.660301,85 37 3d ca 67 2c fe 66 41 12 f0 f5 37 39 75 13 c6 5e 2a 22 7b df bd e0 49 e2 15 60
|
||||
162.854691,1780243200.000107,80,465,51.214.112.115,13 02,"anon, hosting",,,KDDI,150,6,869.4637,03 03,29.86,8317,cn,,Verizon,TlsS,,51.214.112.115,32966,*.s3.amazonaws.com,127681,,ssl_logs,4744,,84.226.59.175,2026-06-01 00:00:00.000107,Apple Inc.,,1751.9572,272,Samsung,69 c5 58 8b,,3389,,,hk,gw-09,1911,46.034615,-5.722583,Paris,Mumbai,"anon, hosting",1574.1376,150.72.179.80,-14.190646,44 69 7d 6f 61 63 ba ed d1 bf 9d 6d c9 48 f0 37 e7 99 d3 1a 66 6f ac 50 c4 dd 5c e2
|
||||
-52.303734,1780243200.000108,12,25,8.126.102.215,00 9e,"anon, hosting",,,China Mobile,3,9,895.8191,03 04,49.16,6589,kr,,AT&T,TlsC,,8.126.102.215,90931,auth.example.net,90864,,netflow,167245,auth.example.net,26.225.51.135,2026-06-01 00:00:00.000108,Baidu Inc.,,1826.8013,145,Google LLC,f0 c2 c6 12,,110,,,nl,core-02,1132,-24.918064,34.389271,Mumbai,Shanghai,"anon, hosting",170.5058,206.81.15.37,105.401472,d5 56 ba 81 62 8f 0c fa a3 55 1a 7a 9c 86 e5 dd ac 14 8b ea 29 aa 10 42 05 09 9a c2
|
||||
-37.91953,1780243200.000109,76,443,103.30.84.98,c0 13,"anon, hosting",,,Orange,56,25,873.3618,03 03,49.46,7410,us,,NTT,TlsC,,59.148.40.219,45506,stream.example.org,85131,,zeek,220189,,102.28.3.53,2026-06-01 00:00:00.000109,Samsung,,1798.2801,207,Amazon.com Inc.,f5 bd ff 99,,993,,,kr,edge-01,1734,41.833957,-32.674135,Sydney,Shanghai,"anon, hosting",285.0237,93.9.185.80,92.263047,92 6f da 80 b0 7b 03 80 90 52 21 1f 20 fc b0 1e dc 28 e4 06 2d a3 b9 21 86 a3 e6 b7
|
||||
144.662392,1780243200.00011,19,110,60.51.194.72,cc a9,"anon, hosting",,,China Telecom,98,10,10.914,03 03,97.38,8276,gb,,KDDI,TlsS,,218.37.233.107,24122,,156753,,zeek,367540,,67.104.160.62,2026-06-01 00:00:00.000110,Meta Platforms,,1047.4809,377,Meta Platforms,01 a7 b6 f7,,25,,,nl,wan-link,606,49.217535,16.064571,Dublin,Tokyo,"anon, hosting",497.2047,103.218.55.126,144.919467,a9 63 41 a4 75 79 7f fa f0 09 a0 30 c4 4b a4 29 fb ec 88 22 e6 c7 62 8d 49 ba 1e f2
|
||||
17.315107,1780243200.000111,27,53,165.234.73.68,00 35,"anon, hosting",,,Deutsche Telekom,146,42,451.6498,03 03,75.51,5892,kr,,Orange,TlsS,TLS_ECDHE_RSA_AES128_GCM_SHA256,165.234.73.68,98369,,69737,,packet_capture,62678,,211.124.16.147,2026-06-01 00:00:00.000111,Baidu Inc.,82169,1215.6294,428,Google LLC,fb 03 b2 29,,993,,,nl,core-02,1047,-10.03765,23.350393,Dublin,San Francisco,"anon, hosting",654.7891,188.53.170.35,104.302331,85 74 7a 1c 0c 1a 73 f5 15 15 70 77 4e 46 a3 a5 c6 ca 09 94 fc 0c d4 22 14 66 12 6a
|
||||
97.359727,1780243200.000112,82,443,189.52.44.152,c0 2f,"anon, hosting",,,Verizon,45,31,598.0491,03 02,54.03,7096,in,,Verizon,TlsC,,189.52.44.152,55539,,170541,,netflow,150417,auth.example.net,16.45.118.137,2026-06-01 00:00:00.000112,Microsoft Corp.,,1406.1909,223,Meta Platforms,16 20 8f 90,,22,,,hk,gw-09,1154,-50.86806,-57.312656,Sydney,Seattle,"anon, hosting",1431.4231,117.191.30.159,-29.704669,68 3d 0f f3 59 07 39 2c 78 e9 8b 8a 0b f5 20 4e 6e 21 cb c1 0f 13 fe dd ee d3 59 06
|
||||
-37.814213,1780243200.000113,12,443,54.220.164.120,13 01,"anon, hosting",,,Verizon,186,44,587.1247,03 04,76.41,2540,jp,,Comcast,TlsS,,54.220.164.120,54221,,178126,,zeek,51583,*.s3.amazonaws.com,31.94.98.56,2026-06-01 00:00:00.000113,Amazon.com Inc.,,1401.5624,180,Baidu Inc.,3c d8 b7 ce,,80,,,no,wan-link,65,27.822564,5.814085,San Francisco,Seoul,"anon, hosting",768.1268,103.225.197.87,106.888675,d0 36 73 e8 4f 60 00 37 fd 49 24 9c 7b cf 9b 6b 31 61 0c 08 0f 97 3d 6a 5f 24 22 81
|
||||
+,1780243200.000114,23,50000,175.36.87.137,13 01,"anon, hosting",,,Orange,163,21,303.3441,03 04,14.34,2161,ru,,China Mobile,TlsC,,175.36.87.137,26640,*.cloudfront.net,53170,,ssl_logs,307208,,117.252.68.218,2026-06-01 00:00:00.000114,Tencent,,120.928,201,Meta Platforms,2c 5f 64 e4,,50000,,,us,edge-01,273,-19.191551,41.46461,Dublin,Beijing,"anon, hosting",226.3924,34.46.151.84,65.891929,33 54 47 43 0c e0 fe bb dc 0a bc 36 d6 f2 11 8e df d8 22 21 2c 86 b7 f3 cd 7c 46 3d
|
||||
-57.066252,1780243200.000115,33,50000,85.238.212.16,c0 2f,"anon, hosting",,www.google.com,AT&T,117,5,341.3994,03 03,35.33,5539,kr,,Comcast,TlsS,,85.238.212.16,65114,*.cloudfront.net,156896,,suricata,298094,,149.135.19.28,2026-06-01 00:00:00.000115,Amazon.com Inc.,,994.4144,455,Baidu Inc.,47 97 61 b9,,3389,384,,au,wan-link,1701,-26.299983,58.401469,Shanghai,Tokyo,"anon, hosting",969.2023,202.34.176.62,54.06007,f0 f9 c5 46 f0 ac 2e fa af 20 80 89 8a 93 74 a0 a5 7a 61 b2 9c 62 ab d2 8e 87 d9 60
|
||||
170.670964,1780243200.000116,58,465,155.107.175.53,00 35,"anon, hosting",,,Telstra,23,5,563.3091,03 03,84.02,6705,us,,NTT,TlsS,,155.107.175.53,58050,stream.example.org,106007,,netflow,12927,,95.75.220.33,2026-06-01 00:00:00.000116,,,582.0669,141,Meta Platforms,fb df af a4,,53,,,se,wan-link,644,-40.752506,13.695872,Berlin,Mumbai,"anon, hosting",1201.2708,35.215.191.229,25.646499,d0 71 7b eb 54 58 70 8f 36 01 7a 9d de c5 e4 6d 76 33 29 d1 e3 e2 4c c4 c2 0b 8b 69
|
||||
19.237916,1780243200.000117,100,3389,50.97.229.36,13 02,"anon, hosting",,,Comcast,38,25,115.5362,03 03,69.51,3291,se,,Verizon,TlsS,,193.126.141.197,49580,,106320,,netflow,192876,,46.201.65.187,2026-06-01 00:00:00.000117,Google LLC,,999.9537,86,Tencent,93 eb fa ca,,993,,,ua,gw-09,1073,33.977231,33.487118,Berlin,Mumbai,"anon, hosting",414.5791,120.62.136.116,12.223197,5f 7d 04 a6 57 8b 0a 1d c9 45 55 18 f0 4f 43 7b bc 8e ec 7d e8 59 38 7f 52 15 22 c1
|
||||
-77.426219,1780243200.000118,32,8443,114.226.109.131,00 9e,"anon, hosting",00 18,,China Mobile,6,43,814.3804,03 03,111.02,5459,gb,*.s3.amazonaws.com,Telstra,TlsS,TLS_ECDHE_ECDSA_AES128_SHA,114.226.109.131,76694,,21063,,suricata,287470,,221.24.169.70,2026-06-01 00:00:00.000118,Apple Inc.,,744.8607,316,Google LLC,82 6b 00 21,,443,,,se,backbone-03,1663,35.821341,14.612721,Berlin,Seoul,"anon, hosting",1913.091,45.40.195.159,-14.29525,e0 f9 dd 12 22 4e c1 a9 e1 88 49 48 64 2f c4 8d c9 70 15 16 9b 06 44 63 d6 63 d5 4d
|
||||
35.273579,1780243200.000119,91,110,18.169.160.170,c0 23,"anon, hosting",00 17,,Verizon,21,26,293.6533,03 02,77.39,2928,jp,"*.['example', 'org']",Orange,TlsS,,18.169.160.170,68840,api.example.com,25297,,netflow,499167,*.cloudfront.net,65.26.246.224,2026-06-01 00:00:00.000119,Apple Inc.,,677.9766,17,Microsoft Corp.,fb 37 88 95,,25,,,in,core-02,459,-30.811695,-19.666646,Berlin,Sydney,"anon, hosting",945.848,211.33.205.126,109.630624,2a 04 4c db fc 2f 4d 1a 01 f1 5a 10 44 95 bb b6 2c 14 54 9e a9 33 13 2a 14 00 62 57
|
||||
-29.56673,1780243200.00012,37,993,177.124.52.179,13 01,"anon, hosting",,,China Telecom,61,41,952.5973,03 02,27.89,7373,br,,,TlsS,,177.124.52.179,53068,,117868,,zeek,8768,,3.25.254.214,2026-06-01 00:00:00.000120,Microsoft Corp.,,1606.7643,491,Microsoft Corp.,72 e6 87 99,,443,,,au,backbone-03,5,54.847513,26.494127,New York,New York,"anon, hosting",1799.0883,76.11.202.71,146.630483,be ac 01 35 f7 67 6c 4b aa 8c c8 64 de 14 e8 94 e9 16 91 aa 09 d0 20 f2 d5 ed 76 a1
|
||||
-173.438083,1780243200.000121,18,443,140.154.4.171,00 35,"anon, hosting",,,Deutsche Telekom,98,20,835.2267,03 04,10.68,5598,hk,,Verizon,TlsS,,140.154.4.171,64005,api.example.com,155983,,packet_capture,425308,login.live.com,166.183.31.102,2026-06-01 00:00:00.000121,Alibaba Inc.,,1107.6165,70,Apple Inc.,60 01 b5 08,,443,,,il,gw-09,1570,36.792228,-29.96689,Dublin,Shanghai,"anon, hosting",1523.4064,186.152.250.222,16.309604,2c 20 53 4a af 52 63 f5 b3 0f ae c7 dd 35 ce 6c 55 61 f7 22 74 fd ea c6 a4 bc 65 15
|
||||
-164.691055,1780243200.000122,14,8443,10.240.113.52,cc a9,"anon, hosting",00 1d,,Verizon,115,14,72.1993,03 03,71.09,3019,ca,,SK Telecom,TlsS,,64.62.233.35,52613,,181320,,ssl_logs,390731,api.example.com,133.145.40.65,2026-06-01 00:00:00.000122,Google LLC,,1342.3765,298,Microsoft Corp.,d2 93 89 7f,,3389,,,cn,wan-link,378,53.363526,46.968285,Paris,San Francisco,"anon, hosting",81.3722,64.72.49.13,-178.556598,51 7e 29 ab 56 5e 35 08 f0 6f 40 56 7a d0 02 8b 60 fc 79 1e dc 6c c6 62 95 b0 45 cf
|
||||
-140.463216,1780243200.000123,76,22,109.210.239.222,13 01,"anon, hosting",,api.example.com,Telstra,72,29,680.9608,03 03,59.43,8370,ua,,NTT,TlsC,,109.210.239.222,22252,,147539,,ssl_logs,387447,,221.24.169.70,2026-06-01 00:00:00.000123,Samsung,,822.0599,459,Apple Inc.,b6 31 c2 f1,ER,995,,,ca,core-02,157,53.821841,52.077151,Seattle,Beijing,"anon, hosting",934.6987,217.51.19.170,3.491176,8b 9e ce 0a 20 36 7b 90 81 7d 5d fe 01 4c 0a a7 d6 ae 00 f6 63 5f 37 9c d9 80 8b 66
|
||||
-168.663276,1780243200.000124,7,22,174.207.84.216,13 01,"anon, hosting",,,Comcast,156,42,113.1649,03 03,2.89,2215,fr,,BT Group,TlsC,,174.207.84.216,82388,,98541,,suricata,279930,cdn.example.com,74.159.180.27,2026-06-01 00:00:00.000124,Google LLC,95776,1593.407,448,Google LLC,6e e5 d9 58,ER,443,,,de,wan-link,953,-52.646981,14.237546,Shanghai,Singapore,"anon, hosting",108.5306,18.120.187.93,-18.191876,99 dd 4e 20 51 f2 c1 03 57 bf 11 59 d7 37 2b 50 6c 25 4e d4 37 de 75 55 88 a8 e7 50
|
||||
-138.543279,1780243200.000125,33,8080,168.255.202.227,c0 13,"anon, hosting",,,Orange,32,2,460.0237,03 03,6.36,9671,kr,,Comcast,TlsC,,160.51.37.138,44289,,44726,,netflow,449003,,104.235.124.254,2026-06-01 00:00:00.000125,Tencent,,1250.4088,283,Google LLC,8c 4c cf c1,,443,,,fr,core-02,769,-42.02697,-14.514593,Shanghai,Berlin,"anon, hosting",416.6251,45.40.195.159,-52.798744,85 51 2b 4b f4 93 fb 27 4d 59 93 15 a0 c2 f0 14 de ef 0d 16 19 01 85 7d c1 b0 ae 2d
|
||||
-47.142859,1780243200.000126,50,3389,11.157.186.231,00 9e,"anon, hosting",,,China Mobile,129,27,323.0202,03 03,72.18,9592,br,,AT&T,TlsC,,169.98.204.85,11398,www.google.com,189171,,suricata,70825,login.live.com,51.168.117.98,2026-06-01 00:00:00.000126,Netflix Inc.,,482.9515,311,Baidu Inc.,b5 63 83 a1,,8443,,,kr,wan-link,254,-37.609309,26.534895,New York,Sydney,"anon, hosting",1357.8871,102.7.135.138,103.917246,0b 2b f0 c1 9d d2 8f 94 0b 73 37 bc 7b 5b 8d a9 44 ee 1f ae 13 94 1d 95 7a 68 6e 6a
|
||||
-102.389184,1780243200.000127,71,3389,110.69.236.47,c0 27,"anon, hosting",,,SK Telecom,136,10,424.4736,03 03,112.87,4179,jp,,China Telecom,TlsC,,110.69.236.47,41119,,46108,,netflow,328602,,100.60.134.199,2026-06-01 00:00:00.000127,Alibaba Inc.,,860.1571,183,Meta Platforms,1c 68 b8 c7,,443,,,se,backbone-03,521,-36.635289,-41.078627,Moscow,Dublin,"anon, hosting",873.0447,77.54.171.73,113.380711,8c ab ac d6 27 2f 82 12 96 42 d0 15 bb 0a ac 79 fc ca db de 17 31 3b 41 da 09 15 62
|
||||
24.754033,1780243200.000128,87,3389,8.199.172.172,00 35,"anon, hosting",,,KDDI,38,25,783.6008,03 04,17.52,9659,se,,NTT,TlsS,,8.199.172.172,54151,,160009,,ssl_logs,346711,,149.135.19.28,2026-06-01 00:00:00.000128,Microsoft Corp.,,813.0556,435,Microsoft Corp.,b9 7c 46 08,,143,,,fi,edge-01,62,34.023691,16.413241,Beijing,Dublin,"anon, hosting",1089.4563,99.16.78.174,28.54065,9e 63 59 61 37 5e 72 96 48 dc ce 1c d0 0b f2 4f cb 3c ea d5 0f e2 59 6e 9e 74 70 a6
|
||||
-109.476174,1780243200.000129,17,80,2.180.152.193,c0 2f,"anon, hosting",,,BT Group,140,11,678.0327,03 04,114.53,6651,kr,,China Telecom,TlsC,,2.180.152.193,34954,*.s3.amazonaws.com,187511,,zeek,471594,login.live.com,83.250.9.29,2026-06-01 00:00:00.000129,Meta Platforms,,934.9184,493,Tencent,22 86 0b cb,,443,384,,ca,edge-01,415,-11.065074,15.544468,Beijing,Shanghai,"anon, hosting",929.0458,117.174.80.95,-116.211885,49 19 bc 70 8f 8a 08 fd e1 e2 90 c8 53 1d e5 47 ce d2 42 46 b0 09 2f fb f2 c4 18 52
|
||||
92.836244,1780243200.00013,21,443,108.194.89.158,c0 23,"anon, hosting",,,SK Telecom,10,3,968.9165,03 03,64.92,1555,de,,,TlsS,,108.194.89.158,40120,,27599,,suricata,171847,login.live.com,194.226.218.213,2026-06-01 00:00:00.000130,Netflix Inc.,,1302.9521,376,Netflix Inc.,57 5c 72 b2,,443,256,,jp,core-02,830,34.127345,-7.44828,Sydney,Shanghai,"anon, hosting",146.4338,66.104.169.166,67.240525,7c 3a 5a a3 d1 0e ad 75 e6 0b fc db ce 3c e8 f4 1f ce fe 78 cf 4b 07 ba 01 c0 49 2b
|
||||
2.089979,1780243200.000131,76,143,100.214.112.125,c0 27,"anon, hosting",,,China Telecom,178,47,186.6018,03 04,101.69,9679,kr,,Verizon,TlsS,,100.214.112.125,99972,stream.example.org,58347,,netflow,356993,*.s3.amazonaws.com,83.96.122.147,2026-06-01 00:00:00.000131,Samsung,22602,1869.1381,462,Tencent,c9 eb c9 64,,22,384,,fi,core-02,399,-42.00995,49.838193,Tokyo,Singapore,"anon, hosting",329.6786,87.67.192.40,44.51085,a3 64 fc 13 a2 00 cc 5f 79 bc 1a 89 0f b8 c7 c3 a9 e9 13 0f 3f 55 8f 5a af bf 89 80
|
||||
82.41569,1780243200.000132,89,443,207.101.75.7,c0 23,"anon, hosting",,,BT Group,86,21,421.9646,03 03,90.64,1102,ru,,,TlsS,,207.101.75.7,15619,,73379,,netflow,178214,,167.226.142.47,2026-06-01 00:00:00.000132,Alibaba Inc.,,675.2334,362,Meta Platforms,b2 cd 53 46,,25,,,br,core-02,344,-20.78786,-47.133333,Beijing,San Francisco,"anon, hosting",1338.9602,1.111.37.12,-62.004188,f3 31 79 da e5 a2 17 08 88 4c 37 91 2c 12 b5 5a e9 c4 3e f2 1e 43 f4 03 8b b8 42 78
|
||||
-117.765392,1780243200.000133,73,21,108.250.54.111,00 9f,"anon, hosting",,,Singtel,48,42,718.2891,03 03,4.41,1479,gb,,,TlsS,,108.250.54.111,32615,,23754,,ssl_logs,288230,stream.example.org,223.119.168.199,2026-06-01 00:00:00.000133,Meta Platforms,,1799.4033,380,Netflix Inc.,f5 ed a4 85,,3389,,,us,core-02,1589,55.042991,36.646171,Paris,Tokyo,"anon, hosting",1459.383,2.210.6.5,9.297959,61 20 40 66 49 03 9c 2f 1e c1 04 2e 7d d4 86 fa 51 68 ed b0 0c 54 f1 ce 2c e9 5a 5c
|
||||
-112.753899,1780243200.000134,25,443,200.248.122.71,00 35,"anon, hosting",00 1e,*.cloudfront.net,Singtel,51,50,149.2154,03 04,97.85,7808,no,,China Telecom,TlsS,,204.20.183.54,20832,api.example.com,150531,,ssl_logs,49257,cdn.example.com,188.53.170.35,2026-06-01 00:00:00.000134,Google LLC,,1908.2343,428,Samsung,b6 5e 79 c9,,53,128,secp256r1,sg,wan-link,943,38.984285,-7.34448,Seattle,Beijing,"anon, hosting",1141.4617,153.150.232.130,129.821125,7e ef fe 39 6d fb a7 46 7c 42 4c 0e 32 44 ba cb 1b 60 58 13 16 87 fa ee e0 b2 6b be
|
||||
-147.563834,1780243200.000135,14,80,8.199.172.172,c0 13,"anon, hosting",,,KDDI,67,38,85.3736,03 03,3.56,3598,in,,China Telecom,TlsC,TLS_ECDHE_ECDSA_AES128_GCM_SHA256,8.199.172.172,99712,,14207,,ssl_logs,377079,stream.example.org,199.178.179.187,2026-06-01 00:00:00.000135,,15491,990.2193,246,Netflix Inc.,b7 a7 bb c6,,995,256,,no,edge-01,742,-42.044187,-38.86354,Sydney,New York,"anon, hosting",1198.5345,52.138.139.36,102.600364,47 ac 04 a7 44 4c 92 c0 41 08 63 bb b3 c2 21 00 6d b0 41 fe 08 31 75 e8 3c 48 8d 61
|
||||
-27.4301,1780243200.000136,18,443,59.148.40.219,00 35,"anon, hosting",00 1e,,China Telecom,122,28,439.4762,03 01,51.82,1943,jp,,China Mobile,TlsS,TLS_ECDHE_RSA_AES256_GCM_SHA384,59.148.40.219,62819,graph.facebook.com,7881,,packet_capture,424803,,122.77.31.116,2026-06-01 00:00:00.000136,Samsung,23560,1607.5032,102,Alibaba Inc.,59 3a 64 0c,,8443,512,secp384r1,cn,edge-01,1150,-26.859601,-1.321895,Beijing,Sydney,"anon, hosting",1214.3184,102.253.95.240,159.555466,bb e5 36 4b 0f 37 af 1f ed 2b 64 15 d1 e7 34 7e 1b ec 75 aa 28 18 93 b8 8d fb 99 66
|
||||
66.656057,1780243200.000137,69,80,2.180.152.193,c0 23,"anon, hosting",,,AT&T,151,39,750.9257,03 04,19.42,9326,no,,Telstra,TlsS,,2.180.152.193,19254,graph.facebook.com,7290,,packet_capture,193859,,101.13.42.165,2026-06-01 00:00:00.000137,Google LLC,,1803.9708,49,Samsung,47 0d 6d ef,,443,,,in,core-02,1353,17.76323,10.838059,London,Beijing,"anon, hosting",1946.5058,80.203.245.135,-108.481038,95 34 d2 c3 32 4c 6e f8 92 19 f8 f5 88 7f 77 59 a5 df 5d 6b 94 3a e4 07 22 87 54 dc
|
||||
-52.329152,1780243200.000138,80,443,83.250.9.29,c0 13,"anon, hosting",,graph.facebook.com,,23,17,880.3343,03 04,99.15,4513,br,,KDDI,TlsS,,83.250.9.29,24804,,142344,,suricata,119712,www.google.com,149.135.19.28,2026-06-01 00:00:00.000138,Google LLC,,1557.5519,294,Alibaba Inc.,9f 4d 21 e3,,50000,384,,de,edge-01,1,19.750011,-54.823837,San Francisco,Mumbai,"anon, hosting",800.861,147.213.95.171,-161.990865,d8 39 b5 da 16 fd 7f 5a f9 91 72 aa c1 b5 36 eb cb 5d 95 17 d7 90 c0 c4 7f 87 db 0d
|
||||
16.783619,1780243200.000139,1,80,17.108.161.55,c0 23,"anon, hosting",,,Deutsche Telekom,153,46,314.388,03 04,46.09,5933,fr,,Verizon,TlsC,TLS_ECDHE_RSA_AES128_GCM_SHA256,17.108.161.55,49152,,92180,,netflow,207233,,205.136.42.187,2026-06-01 00:00:00.000139,Apple Inc.,80771,454.9114,350,Tencent,19 38 3f 45,,25,256,,in,wan-link,972,24.023353,8.788149,Beijing,New York,"anon, hosting",1505.66,85.48.197.73,34.03699,01 cd ec 4a 31 8b 28 e2 8a d3 35 73 a7 5e c9 c0 87 d8 7f ca e1 9f c3 9f 2e a7 ae 85
|
||||
-160.668335,1780243200.00014,97,22,172.191.224.231,cc a9,"anon, hosting",,,Deutsche Telekom,119,17,164.6239,03 04,113.04,3765,ca,,Orange,TlsC,,120.128.5.175,44698,api.example.com,3670,,zeek,383164,api.example.com,38.1.141.220,2026-06-01 00:00:00.000140,Microsoft Corp.,,526.9813,368,Tencent,07 37 f2 ca,,110,,,ca,edge-01,560,-19.91975,-32.531746,Sydney,Sydney,"anon, hosting",1624.166,93.50.188.140,38.966452,5f 3c 59 4d 96 5c e7 db f4 f2 9f cd 44 a1 22 c7 0e 35 28 5a 8f 33 2f be 80 c6 1e 03
|
||||
-155.782242,1780243200.000141,98,993,64.38.226.207,c0 13,"anon, hosting",,,China Mobile,16,35,512.4923,03 04,54.32,6152,ua,,Orange,TlsS,,64.38.226.207,27412,*.s3.amazonaws.com,28189,,zeek,239533,,43.157.147.113,2026-06-01 00:00:00.000141,Netflix Inc.,,1339.3667,467,Tencent,b8 85 86 da,,3389,,,de,backbone-03,1406,-58.486129,36.143534,Seoul,Mumbai,"anon, hosting",1191.9171,41.255.239.128,15.444715,82 a8 63 10 c5 75 a6 4a 88 75 a9 75 9c 06 12 35 23 ca e1 f2 92 22 d9 b0 58 8e 82 82
|
||||
-43.415128,1780243200.000142,76,80,169.243.84.68,c0 2f,"anon, hosting",,,Singtel,159,6,299.0908,03 03,52.79,9565,no,,BT Group,TlsC,TLS_ECDHE_ECDSA_AES128_GCM_SHA256,169.243.84.68,58140,www.google.com,5223,,zeek,206238,,73.198.209.199,2026-06-01 00:00:00.000142,Netflix Inc.,,780.4782,498,Tencent,65 08 ea ce,,443,,secp384r1,se,gw-09,95,-52.436309,-4.642904,Seattle,Berlin,"anon, hosting",148.5777,121.22.198.252,7.927761,f6 c5 3e 05 2a 97 60 be c4 2f 93 1f 78 94 a3 7c 2b 18 f7 8d d7 46 63 a1 9e d1 a1 32
|
||||
-151.682652,1780243200.000143,28,53,3.57.38.242,00 35,"anon, hosting",,api.example.com,Comcast,72,41,938.0969,02 00,81.88,8700,gb,,China Telecom,TlsS,,3.57.38.242,75942,login.live.com,11993,,netflow,372605,www.google.com,74.80.37.114,2026-06-01 00:00:00.000143,Apple Inc.,,283.9303,2,Microsoft Corp.,f4 ab 26 e6,,443,,,us,core-02,1597,44.721812,-23.672919,Berlin,New York,"anon, hosting",734.9706,223.251.52.224,-43.149447,bc 47 25 ec b8 ab a0 3c eb f9 e9 94 2e 46 f9 97 c5 5e 26 31 4a 8f 2f ce 46 df 88 5a
|
||||
33.713437,1780243200.000144,49,25,36.126.134.192,cc a9,"anon, hosting",,login.live.com,AT&T,162,17,219.0294,03 03,91.79,5881,de,,Singtel,TlsS,,36.126.134.192,28074,,88546,,suricata,101784,,96.220.74.63,2026-06-01 00:00:00.000144,Meta Platforms,,546.744,31,Meta Platforms,b4 ce 71 5d,--,8080,384,secp256r1,au,backbone-03,311,-54.737982,37.159439,Seoul,Seoul,"anon, hosting",78.7967,181.185.39.235,-13.544836,2c 9f a7 96 74 48 ef d3 6d d3 b0 43 83 8e f1 3b d3 2d 48 f2 67 9f 2a 6d 24 12 84 aa
|
||||
151.170164,1780243200.000145,84,443,11.157.186.231,c0 23,"anon, hosting",,,China Mobile,158,40,214.513,03 03,117.92,3855,gb,,China Telecom,TlsS,,11.157.186.231,57077,graph.facebook.com,177712,,netflow,491818,,35.37.231.45,2026-06-01 00:00:00.000145,,,1380.3687,370,Netflix Inc.,61 42 ed 19,,443,,,sg,wan-link,1987,14.517901,-22.184755,Beijing,New York,"anon, hosting",7.0054,49.185.255.161,150.975812,00 2a 45 5d e0 a0 b6 1e 05 50 d5 19 28 a3 38 ac 6d 5d b1 84 e1 ad 10 e0 90 16 fc 33
|
||||
-2.437103,1780243200.000146,33,3389,221.243.230.67,c0 27,"anon, hosting",,,Verizon,119,16,46.4262,03 03,109.5,3065,no,,China Telecom,TlsS,,221.243.230.67,68316,,147575,,ssl_logs,163612,login.live.com,217.51.19.170,2026-06-01 00:00:00.000146,Amazon.com Inc.,,372.95,439,Netflix Inc.,3b 2d c4 c9,,3389,,,nl,core-02,1113,-40.718387,-58.673497,Seoul,Mumbai,"anon, hosting",1474.1563,58.1.11.251,-54.742014,c8 5e 92 36 eb 82 5e 1f 5f 4c 70 71 9d a9 30 23 07 10 14 d7 17 b6 7d 02 d3 81 7c 07
|
||||
17.096003,1780243200.000147,1,993,80.55.13.238,00 35,"anon, hosting",,,SK Telecom,15,50,372.9307,03 04,76.27,4781,jp,,BT Group,TlsC,,80.55.13.238,67667,,72713,,netflow,459376,,201.232.146.109,2026-06-01 00:00:00.000147,,,1434.1621,437,Alibaba Inc.,38 21 d7 b8,,50000,,,ca,backbone-03,1426,36.9955,-34.086333,Mumbai,Moscow,"anon, hosting",1701.1225,112.169.48.230,37.394599,f5 58 8d a9 9f 1b 31 46 56 f4 53 00 1b 5a 6d 96 41 75 45 4f 1e 81 2d 8e 8e f9 8e 6c
|
||||
4.372196,1780243200.000148,37,21,19.30.117.18,cc a9,"anon, hosting",,,Singtel,48,15,570.8936,03 03,88.33,1943,ua,,Orange,TlsC,,19.30.117.18,40913,,155844,,ssl_logs,44573,,142.150.185.148,2026-06-01 00:00:00.000148,Apple Inc.,,295.8903,319,Baidu Inc.,d2 46 af a4,,8443,256,,us,backbone-03,1266,24.051145,-49.90648,Moscow,Dublin,"anon, hosting",622.9587,217.38.24.196,154.750159,92 82 99 12 e1 dc c8 64 9c 60 73 d5 b0 04 9a c0 55 e8 61 42 56 c5 44 de 3c 2d 97 4b
|
||||
12.775114,1780243200.000149,85,50000,55.32.212.105,c0 23,"anon, hosting",,,BT Group,46,6,167.0122,03 04,68.21,9281,br,,Orange,TlsS,,72.35.142.90,44047,,120135,,suricata,431139,auth.example.net,77.54.171.73,2026-06-01 00:00:00.000149,Baidu Inc.,,1068.7817,463,Tencent,87 64 7e 50,,443,,,ca,backbone-03,1969,-50.510562,-15.607309,Mumbai,Seoul,"anon, hosting",244.8419,30.207.10.248,-56.427283,
|
||||
-26.847855,1780243200.00015,46,25,60.51.194.72,00 9e,"anon, hosting",,,Comcast,142,49,497.8342,03 03,42.81,3996,il,,Deutsche Telekom,TlsC,,60.51.194.72,74617,,111046,,packet_capture,98471,,11.89.240.133,2026-06-01 00:00:00.000150,Samsung,,184.1844,18,Amazon.com Inc.,e2 74 61 59,,110,,,cn,core-02,1039,-4.426236,-36.615894,Berlin,Seoul,"anon, hosting",1952.2702,4.145.240.123,124.805476,3f 6f cb 2c fd 54 11 65 3e 65 c1 b1 b3 4e c8 5a 0d 5f f8 7e b0 41 bf b4 6d 78 4b 56
|
||||
-2.96053,1780243200.000151,40,25,3.57.38.242,00 9e,"anon, hosting",,,China Mobile,52,28,137.0879,03 03,2.52,8045,fr,,NTT,TlsC,TLS_RSA_WITH_AES_128_CBC_SHA,3.57.38.242,91304,,24040,,zeek,421427,,8.126.102.215,2026-06-01 00:00:00.000151,Samsung,,902.2677,109,Google LLC,22 bc 2c db,,443,384,,nl,wan-link,770,3.788717,22.040905,Singapore,Seoul,"anon, hosting",1609.9022,98.183.151.180,-82.239832,a1 75 9c 3e a5 4c 06 24 70 c5 59 cd 1d 92 cc 75 f2 bb c5 20 a0 8d f3 17 b9 a3 80 25
|
||||
-136.421502,1780243200.000152,27,995,184.101.38.152,c0 2c,"anon, hosting",,,Verizon,61,47,694.7215,03 03,117.02,6144,jp,,Deutsche Telekom,TlsC,TLS_AES_256_GCM_SHA384,184.101.38.152,78749,,161700,,packet_capture,168546,,98.183.151.180,2026-06-01 00:00:00.000152,Meta Platforms,,1131.2999,57,Microsoft Corp.,11 83 a9 10,,3389,,,fi,core-02,952,-38.19732,-54.518317,Paris,Seattle,"anon, hosting",1139.0502,10.70.170.206,-135.111883,f8 c1 8b 90 e9 91 f5 03 19 00 47 d7 ed ca 9c 9e de fe cd 98 e5 e6 9e 2d d6 67 c5 b1
|
||||
156.152711,1780243200.000153,11,443,103.30.84.98,c0 23,"anon, hosting",,,Orange,139,5,573.6641,03 03,47.52,1787,de,,China Telecom,TlsS,TLS_AES_128_GCM_SHA256,103.30.84.98,48879,,185430,,ssl_logs,422862,,36.79.39.76,2026-06-01 00:00:00.000153,Microsoft Corp.,,1679.3262,173,Tencent,4b 72 1b 0e,,443,384,,gb,gw-09,390,17.105429,9.916428,Shanghai,Berlin,"anon, hosting",1111.1991,46.154.7.182,137.641264,e4 42 1f 58 3a f5 16 f5 a7 9a b2 62 92 d3 bd ac f1 43 40 13 7d 05 15 9a 8e dd 9b c3
|
||||
-121.186972,1780243200.000154,75,12345,213.215.129.214,c0 2c,"anon, hosting",,,Deutsche Telekom,198,14,487.6362,03 02,97.75,8441,in,,Singtel,TlsS,,213.215.129.214,31418,,196286,,suricata,403682,cdn.example.com,211.250.251.118,2026-06-01 00:00:00.000154,Apple Inc.,,1883.1827,108,Alibaba Inc.,3e db e9 ba,,50000,,,kr,core-02,570,35.272706,53.669902,San Francisco,Moscow,"anon, hosting",462.4299,81.129.165.30,120.497466,c8 ad 5c e5 ac 51 65 66 44 4e 77 04 45 08 44 a9 cd 37 6e 68 05 75 0b 70 65 81 d1 ef
|
||||
-126.924167,1780243200.000155,40,993,140.154.4.171,00 35,"anon, hosting",,,China Mobile,133,35,740.0379,03 02,3.06,6416,gb,,NTT,TlsC,,140.154.4.171,70899,,39330,Amazon.com Inc.,netflow,101050,,8.59.133.46,2026-06-01 00:00:00.000155,Meta Platforms,,1622.0068,348,Samsung,38 fc 34 58,,8080,,,sg,core-02,770,25.513385,26.964851,Singapore,Seoul,"anon, hosting",1292.4519,215.255.8.162,149.348916,27 5f 33 42 2b 2c 18 3d ea 15 78 b1 d4 19 25 e8 19 f8 95 12 73 4d b4 35 80 b5 5b a2
|
||||
-101.388131,1780243200.000156,64,465,28.79.139.73,13 01,"anon, hosting",,,Singtel,135,49,389.4634,03 04,85.49,6139,fi,,China Mobile,TlsC,,28.79.139.73,60818,,114248,,zeek,291846,api.example.com,60.135.101.109,2026-06-01 00:00:00.000156,Microsoft Corp.,,17.8638,206,Alibaba Inc.,2a 5a 4d 30,,8080,,,gb,wan-link,1098,50.887266,32.232592,Moscow,Seattle,"anon, hosting",220.3608,32.188.223.228,163.818088,cd 77 df 7d ed 4e d1 58 97 5c 15 dc c1 26 ab 84 0d ad 31 63 68 36 72 14 16 8b 72 50
|
||||
-101.03754,1780243200.000157,25,12345,9.161.205.69,c0 2c,"anon, hosting",,,KDDI,130,30,767.9636,03 02,107.21,4295,gb,,BT Group,TlsC,,9.161.205.69,89793,*.cloudfront.net,53537,,netflow,425568,,200.208.79.237,2026-06-01 00:00:00.000157,Alibaba Inc.,,673.1088,383,Meta Platforms,dd 53 ea df,,443,256,,au,core-02,991,-36.100502,-57.611713,Seattle,Moscow,"anon, hosting",1262.5808,65.26.246.224,84.728106,56 01 1f 02 b2 64 52 ac 63 ca 60 31 01 a4 17 a7 5b 8a 7d 78 00 b5 52 d5 57 1f 1e e6
|
||||
77.828572,1780243200.000158,78,8443,198.10.194.123,c0 14,"anon, hosting",,,Verizon,164,38,343.3627,03 04,68.73,7463,hk,,Comcast,TlsC,,198.10.194.123,52816,,19831,,suricata,272298,,60.241.12.159,2026-06-01 00:00:00.000158,Netflix Inc.,,605.2357,411,Baidu Inc.,e0 ca f8 0b,,21,,,il,edge-01,397,-0.302572,-58.003643,Seoul,Dublin,"anon, hosting",485.8212,47.109.174.243,84.392726,cc c4 57 b5 30 46 a5 e5 68 9d 53 af a6 72 8b de 04 e3 98 45 0b e3 89 f3 49 d2 d8 9b
|
||||
-34.813058,1780243200.000159,38,110,157.194.172.8,00 35,"anon, hosting",,www.google.com,China Mobile,8,12,932.4389,03 02,48.4,6034,au,,Verizon,TlsC,,157.194.172.8,60960,,43310,,netflow,250645,mail.example.com,14.129.244.30,2026-06-01 00:00:00.000159,Google LLC,,234.5415,404,Meta Platforms,97 9b 81 43,,443,,,ru,wan-link,80,21.558845,31.172675,Dublin,Seoul,"anon, hosting",1222.9524,58.58.236.243,79.273386,68 08 b7 c6 95 31 85 62 e0 96 dd e5 1a f5 09 3c f1 52 d4 11 0f 39 68 da be 3e 18 76
|
||||
123.011334,1780243200.00016,72,21,148.97.130.12,cc a9,"anon, hosting",00 18,,Telstra,81,31,427.124,03 03,58.86,6135,br,,Orange,TlsC,,148.97.130.12,68349,,58360,,ssl_logs,315231,graph.facebook.com,100.214.112.125,2026-06-01 00:00:00.000160,Baidu Inc.,,138.2267,43,Netflix Inc.,ec 3c b6 99,,993,512,,se,backbone-03,1086,-38.286643,-39.562658,London,San Francisco,"anon, hosting",1651.1711,2.40.9.66,60.693619,3b a7 fa bc 2b e5 22 1c ac 1a 18 b9 9d 2f 84 e5 e8 9d 89 f1 75 bb 6d b6 48 98 e0 97
|
||||
11.265017,1780243200.000161,36,22,3.57.38.242,c0 2b,"anon, hosting",,,KDDI,87,35,502.7775,03 04,12.32,1371,sg,,China Mobile,TlsC,,3.57.38.242,90897,,54130,,suricata,15763,,179.151.65.50,2026-06-01 00:00:00.000161,Meta Platforms,,1964.1031,268,Tencent,99 60 e7 29,--,21,,,cn,wan-link,1558,6.49243,9.292733,Singapore,Dublin,"anon, hosting",365.2682,149.78.87.169,-7.311398,53 30 2b d1 94 93 57 96 c7 8d 98 ea bd e0 57 bd fb a8 b2 40 38 1a 5b 49 24 e1 ec 25
|
||||
11.202764,1780243200.000162,44,8080,109.93.142.119,c0 27,"anon, hosting",,,Comcast,76,30,239.9842,03 04,29.11,2346,us,,,TlsC,,109.93.142.119,43408,,48842,,zeek,439720,login.live.com,136.216.108.238,2026-06-01 00:00:00.000162,Meta Platforms,,492.9526,52,Google LLC,aa a4 dc 2e,,443,,,ru,core-02,1618,26.820833,-52.911503,Dublin,Mumbai,"anon, hosting",1227.2752,84.44.205.172,-164.785487,3e 15 91 a7 1f 45 99 6c ec 69 cd 23 af fe 69 c2 cd 47 e1 28 6f b2 cd 7d 6b b4 7b 1e
|
||||
70.262109,1780243200.000163,30,22,103.30.84.98,cc a9,"anon, hosting",,,Singtel,157,9,186.2897,03 03,54.22,6834,hk,,Singtel,TlsS,,103.30.84.98,27557,,24818,,packet_capture,219945,www.google.com,93.157.122.15,2026-06-01 00:00:00.000163,Netflix Inc.,,1884.2171,319,Microsoft Corp.,21 fc 69 f0,,22,,,jp,gw-09,18,35.674533,-11.744167,Seattle,Tokyo,"anon, hosting",1025.7895,222.14.41.12,-29.379929,01 58 58 8e e8 8a a3 b3 39 79 a8 e8 45 fb 3f e8 88 64 0a d9 3d 36 c4 8a 11 67 9c 6a
|
||||
-70.027646,1780243200.000164,9,8080,139.102.159.103,00 9f,"anon, hosting",,,BT Group,2,46,253.8858,03 03,21.01,6019,ua,,BT Group,TlsS,,139.102.159.103,17050,,53628,,zeek,13281,,33.140.166.98,2026-06-01 00:00:00.000164,Tencent,,1769.3144,381,Meta Platforms,7c 45 aa 06,,110,512,secp384r1,in,edge-01,780,4.028328,-55.746476,Mumbai,Seattle,"anon, hosting",991.7329,89.153.217.177,43.868508,19 f3 06 4c cc e5 56 1c 4d 11 10 ab 79 c8 d4 45 23 b8 64 d1 0f 05 ad ee f3 a0 0f 1e
|
||||
26.990087,1780243200.000165,22,12345,207.100.186.111,13 01,"anon, hosting",,,Verizon,187,34,157.7203,03 02,116.95,3431,fi,,China Telecom,TlsS,TLS_ECDHE_ECDSA_AES256_GCM_SHA384,207.100.186.111,57091,,191560,,ssl_logs,498579,,179.249.79.49,2026-06-01 00:00:00.000165,Microsoft Corp.,,557.9264,167,Tencent,42 2b df 99,,8080,,,in,gw-09,1708,-46.566389,-47.307756,Singapore,Beijing,"anon, hosting",874.6555,222.46.196.119,-166.660063,2e 57 dc cf 92 d3 82 f5 db d7 40 6d ed c2 1e 04 89 9f e9 84 0c 76 e0 2d 38 10 fb 95
|
||||
147.506751,1780243200.000166,19,443,19.30.117.18,c0 23,"anon, hosting",,,Orange,89,9,688.7183,03 04,33.16,2147,se,,Verizon,TlsC,,19.30.117.18,52567,,115867,,netflow,282830,,53.189.160.106,2026-06-01 00:00:00.000166,Samsung,,1762.5879,282,Microsoft Corp.,7a 8e d3 d2,,12345,,,no,backbone-03,110,41.117427,42.548379,Singapore,Seattle,"anon, hosting",1620.4165,185.250.94.235,-150.407821,14 19 0b 8d 70 7f de 3c 31 7d e4 2a 31 2b a2 2f bd 84 a8 48 9e 4c 09 60 85 06 23 b9
|
||||
-107.81862,1780243200.000167,10,80,53.215.199.232,c0 13,"anon, hosting",,,KDDI,160,46,509.3879,03 03,115.35,7697,br,,China Mobile,TlsS,,53.215.199.232,67933,cdn.example.com,107535,,zeek,237735,*.s3.amazonaws.com,145.57.31.213,2026-06-01 00:00:00.000167,Google LLC,,1359.0985,388,Baidu Inc.,aa 79 03 bb,,53,,,au,gw-09,335,-49.232271,48.885043,Mumbai,Tokyo,"anon, hosting",1336.6888,171.4.238.231,136.432331,da 23 b3 2d 40 0f 87 bf 05 fe db de a6 72 84 9f c3 17 9c 5a 49 e1 d6 70 99 c4 0c 36
|
||||
+,1780243200.000168,64,993,139.7.47.238,c0 23,"anon, hosting",,,,151,1,807.5877,03 03,104.74,7798,nl,,AT&T,TlsC,TLS_RSA_WITH_AES_128_GCM_SHA256,139.7.47.238,23757,,15423,,zeek,26151,,35.135.59.228,2026-06-01 00:00:00.000168,Baidu Inc.,,893.9969,104,Microsoft Corp.,0f 3c cd 1a,,110,512,,br,gw-09,128,20.469674,32.893208,Sydney,Sydney,"anon, hosting",1246.1724,93.188.239.206,40.48528,db 77 9d 73 ac 93 5b 21 8d d6 0b cf cf 84 09 b1 32 33 16 5d 2e 52 bd 9f 64 8e 2a 5f
|
||||
141.75262,1780243200.000169,28,12345,164.57.12.190,c0 2f,"anon, hosting",,auth.example.net,Deutsche Telekom,12,4,943.3219,03 01,80.13,9563,fi,,Orange,TlsC,,164.57.12.190,30518,,116094,,ssl_logs,392665,,42.92.84.185,2026-06-01 00:00:00.000169,Microsoft Corp.,,492.5697,318,Google LLC,5f 53 3e c8,,465,,,cn,edge-01,731,13.868335,-41.444553,Paris,Moscow,"anon, hosting",1828.867,65.26.246.224,171.341266,c4 80 f6 fb 3d 65 f8 ad 74 14 e5 d6 36 03 5d 92 07 bb b4 ee 10 07 56 83 96 63 45 6d
|
||||
30.674809,1780243200.00017,59,25,64.38.226.207,c0 2b,"anon, hosting",,,Comcast,99,2,306.8956,03 03,99.32,3830,ru,,,TlsC,,64.38.226.207,61698,,94905,,packet_capture,51424,*.cloudfront.net,221.50.25.167,2026-06-01 00:00:00.000170,Microsoft Corp.,,1282.7935,247,Samsung,de bd 8f d8,,443,512,,in,edge-01,540,59.235012,38.431547,New York,Seoul,"anon, hosting",1329.4944,25.120.69.100,-3.389769,7e ac a4 38 46 27 e5 fc a8 32 44 77 be fe 32 ae b9 84 4f c0 17 ec 52 74 c4 09 4e 97
|
||||
120.327798,1780243200.000171,75,443,60.196.78.181,13 01,"anon, hosting",,,China Mobile,46,36,837.1497,03 04,56.04,3952,br,,Verizon,TlsC,,198.10.194.123,84532,,128575,,packet_capture,59979,,165.174.57.76,2026-06-01 00:00:00.000171,Apple Inc.,,115.3551,55,Alibaba Inc.,42 e9 69 5c,,21,256,,il,wan-link,248,-18.377624,2.979156,Mumbai,Beijing,"anon, hosting",1233.9594,9.118.147.181,+,2a a7 e6 db 83 04 a0 62 a4 d2 e6 1d f2 ed ff ad b0 82 51 df 27 0e 7b 84 4b 5c f4 6b
|
||||
-158.088393,1780243200.000172,5,12345,109.210.239.222,13 02,"anon, hosting",,,SK Telecom,45,5,46.4575,03 03,0.83,6709,de,,Telstra,TlsS,,109.210.239.222,42409,mail.example.com,95359,,ssl_logs,442931,,184.218.85.177,2026-06-01 00:00:00.000172,Meta Platforms,,21.4489,87,Baidu Inc.,9b b6 69 fa,,8443,384,,no,gw-09,1734,28.971953,-20.988433,Mumbai,London,"anon, hosting",207.4622,98.117.126.118,92.24355,a6 37 70 73 41 ec 50 fc 5c 2c 51 3c c6 71 b2 18 98 db 1e 40 dc d6 e7 ca a9 57 e8 55
|
||||
-121.790773,1780243200.000173,46,443,221.50.25.167,c0 14,"anon, hosting",,,Comcast,85,43,4.5095,03 03,18.56,6206,au,,China Telecom,TlsS,,221.50.25.167,37509,,165134,,netflow,402041,api.example.com,50.151.116.93,2026-06-01 00:00:00.000173,Netflix Inc.,,1402.7378,460,Amazon.com Inc.,e4 26 75 33,,993,,,gb,gw-09,516,55.910169,-27.57326,Singapore,London,"anon, hosting",852.0212,198.252.237.194,-82.992317,c9 a0 44 1c 3c 49 7c fe cd 65 26 f6 eb 9e be 8e 51 21 07 c4 5d 89 2f 68 e5 af 32 43
|
||||
77.908193,1780243200.000174,7,143,72.141.5.133,c0 14,"anon, hosting",,,Orange,93,3,170.2409,03 03,77.71,4032,no,,China Mobile,TlsC,,98.49.183.217,47360,,44540,,netflow,343673,,27.175.43.130,2026-06-01 00:00:00.000174,Google LLC,,1147.222,45,Amazon.com Inc.,36 5d b4 cf,,50000,,,fr,edge-01,504,14.101346,-19.164803,Paris,Shanghai,"anon, hosting",1576.334,65.76.170.21,-159.422473,aa b7 62 57 a4 df f4 45 cf 58 09 88 cb f6 40 56 b9 9c 6e a3 bf 29 c8 56 51 58 5d b3
|
||||
57.252522,1780243200.000175,65,143,72.35.142.90,c0 2b,"anon, hosting",,,Deutsche Telekom,147,18,445.2963,03 03,12.46,1176,in,,NTT,TlsC,,72.35.142.90,48459,,5283,,netflow,150686,,28.152.101.40,2026-06-01 00:00:00.000175,Amazon.com Inc.,,378.3825,318,Tencent,6a 80 95 5c,,465,,,fr,edge-01,1712,-27.434772,39.139483,Paris,Beijing,"anon, hosting",366.3215,188.163.164.61,-147.552581,65 d2 3a 06 84 90 2f de 99 46 66 32 c7 b2 8d 93 3b 64 eb 5b ed f0 c6 b9 f4 d7 10 d8
|
||||
-15.466886,1780243200.000176,28,8080,25.49.220.91,00 9e,"anon, hosting",,*.s3.amazonaws.com,NTT,85,37,410.3126,03 04,25.75,1764,br,,KDDI,TlsS,,25.49.220.91,36902,,169081,,suricata,385496,auth.example.net,179.248.62.8,2026-06-01 00:00:00.000176,Tencent,,1583.1255,208,Meta Platforms,78 ba d2 db,,443,128,,au,wan-link,433,54.742206,-26.903116,Sydney,Beijing,"anon, hosting",1096.4393,132.75.223.24,8.118103,e7 b3 d4 83 d4 ef 43 c6 77 ce 22 bc 98 2c be 11 1e 4c e7 f3 55 38 05 9d ab 14 e8 c5
|
||||
-136.800708,1780243200.000177,97,993,105.250.123.177,c0 14,"anon, hosting",,,Comcast,14,25,501.9979,03 04,61.76,5138,no,,China Telecom,TlsS,,105.250.123.177,86142,graph.facebook.com,172771,,packet_capture,58734,,164.247.249.5,2026-06-01 00:00:00.000177,Alibaba Inc.,,1387.7819,76,,20 ae e0 61,,143,,,us,gw-09,1663,7.998042,-40.83141,New York,Beijing,"anon, hosting",151.614,83.128.13.24,13.142803,68 77 ed 2e ea 03 d8 9d 86 57 cf 95 50 0f 69 78 2c 79 a7 9a 09 39 90 b7 d6 e6 0d c2
|
||||
-139.560022,1780243200.000178,55,12345,207.81.216.88,c0 14,"anon, hosting",,,NTT,193,16,131.8592,03 02,27.13,9729,gb,,Singtel,TlsS,,207.81.216.88,58131,,2447,,netflow,399206,,175.36.87.137,2026-06-01 00:00:00.000178,Alibaba Inc.,,1632.9084,232,Netflix Inc.,97 49 98 53,,443,384,,il,gw-09,1863,4.780428,11.080358,San Francisco,London,"anon, hosting",500.6013,222.81.167.240,-27.726354,57 f1 cf c8 43 07 10 44 08 7a 3d e9 1d 59 d7 63 87 8f 89 6f a6 c3 5e 81 ab f8 03 29
|
||||
155.309838,1780243200.000179,56,465,167.35.20.194,13 01,"anon, hosting",,,NTT,172,32,204.1252,03 03,100.89,3617,cn,,SK Telecom,TlsS,,167.35.20.194,97441,,17752,,suricata,157758,,84.87.235.213,2026-06-01 00:00:00.000179,Meta Platforms,,68.4777,339,Baidu Inc.,ce 05 0f 02,,993,,,br,core-02,625,-9.965654,-5.555412,,New York,"anon, hosting",1871.6315,4.108.97.247,-153.838275,c1 4f 7e 6e a4 f4 cc ed fd e6 5f be de 06 9a 3e 7e 82 58 19 80 7e 30 a7 66 61 2b 4f
|
||||
158.175545,1780243200.00018,89,443,21.240.9.192,c0 23,"anon, hosting",,,NTT,169,49,413.5389,03 01,93.68,9818,hk,,NTT,TlsC,,21.240.9.192,20283,,6829,,suricata,362917,cdn.example.com,145.57.31.213,2026-06-01 00:00:00.000180,Meta Platforms,,451.7598,444,Alibaba Inc.,eb 14 61 98,,53,,,fi,edge-01,1964,41.948893,-30.250942,Seattle,Mumbai,"anon, hosting",1846.087,221.21.223.5,-177.578077,73 25 63 f6 a1 cf 31 06 1a fe fa fd 64 56 25 fe b3 1f 73 d6 2c 5b 17 4b aa 24 65 f7
|
||||
-38.283752,1780243200.000181,85,8443,221.50.25.167,00 35,"anon, hosting",,auth.example.net,Comcast,153,35,378.5723,03 04,72.14,1529,se,,Comcast,TlsS,,193.37.4.118,69614,,141880,,packet_capture,76357,cdn.example.com,31.213.115.199,2026-06-01 00:00:00.000181,Microsoft Corp.,,1731.4398,454,Microsoft Corp.,1b dd f9 15,,3389,,,br,backbone-03,1685,4.755318,10.627962,Moscow,Seattle,"anon, hosting",400.1846,66.104.169.166,65.150144,7a be cc 8e 0a e3 06 4d c8 a4 50 ad 2e c3 90 df 69 68 79 af d5 43 66 1c 61 ef 99 86
|
||||
146.597747,1780243200.000182,43,993,106.12.91.189,13 02,"anon, hosting",,*.cloudfront.net,,24,41,321.6807,03 03,69.15,1266,ru,,Deutsche Telekom,TlsC,TLS_ECDHE_ECDSA_AES128_GCM_SHA256,106.12.91.189,25830,*.cloudfront.net,102971,,suricata,490118,,155.107.175.53,2026-06-01 00:00:00.000182,Amazon.com Inc.,54228,1726.2174,221,Netflix Inc.,0b 86 ad 50,,21,,,sg,wan-link,1277,-58.026734,37.554542,London,San Francisco,"anon, hosting",1834.5457,82.242.195.99,-45.370693,18 a9 1a c9 79 15 ca ca 7a de 04 bd 16 42 2c db 88 04 3d 05 6e 16 3c 54 1e 79 0e 49
|
||||
-146.199812,1780243200.000183,25,110,151.117.112.2,00 9e,"anon, hosting",,,AT&T,37,1,542.0609,03 02,80.63,8848,il,,NTT,TlsC,,151.117.112.2,99471,,143476,,packet_capture,420306,auth.example.net,73.232.36.176,2026-06-01 00:00:00.000183,Amazon.com Inc.,,1941.1678,26,Samsung,99 cf 6d db,OK,8080,,,nl,backbone-03,767,16.177006,-0.353949,Dublin,Seoul,"anon, hosting",1388.4359,102.28.3.53,-147.540192,1f bf 42 5b 82 9a 26 6b 7d a7 9d e2 87 1c c0 87 bb 6d 31 a3 ef 1b a3 eb 3f 28 cb de
|
||||
-60.696207,1780243200.000184,48,993,100.214.112.125,c0 27,"anon, hosting",,,Telstra,1,18,53.6457,03 02,45.73,8040,il,www.google.com,AT&T,TlsC,,100.214.112.125,11807,www.google.com,161151,,netflow,371427,,67.228.109.157,2026-06-01 00:00:00.000184,Tencent,,576.0402,156,Microsoft Corp.,d4 11 cb cd,,53,512,x448,br,edge-01,1278,46.734959,20.720142,Moscow,Singapore,"anon, hosting",76.1495,74.117.180.57,-176.726595,af 40 c7 1d e2 a5 df 23 67 71 3f 0f 01 c9 91 7d 6d 4f 5e dd 4d 41 fc e6 ef ef 6f 68
|
||||
141.547786,1780243200.000185,26,25,72.79.110.246,13 02,"anon, hosting",,,SK Telecom,31,25,269.3395,03 04,77.27,9896,sg,,AT&T,TlsC,TLS_AES_128_GCM_SHA256,72.79.110.246,28442,cdn.example.com,122662,,packet_capture,360227,stream.example.org,164.29.38.172,2026-06-01 00:00:00.000185,Microsoft Corp.,,542.7662,498,Apple Inc.,f7 1d a8 94,,443,,x25519,us,backbone-03,1178,-44.185431,10.857312,London,Seattle,"anon, hosting",1359.2168,74.251.102.32,-99.924576,26 35 a8 93 ad 74 80 08 02 25 5d e6 df 53 46 62 9b fe 99 58 52 ae cb ec fb 20 48 41
|
||||
10.542121,1780243200.000186,71,25,21.214.161.240,00 35,"anon, hosting",00 1e,,NTT,169,41,14.1607,03 03,43.83,1017,jp,,China Mobile,TlsC,,177.76.18.214,90870,,138152,,packet_capture,267292,*.s3.amazonaws.com,171.157.23.57,2026-06-01 00:00:00.000186,Amazon.com Inc.,,1507.2685,238,Microsoft Corp.,dd 21 f3 f8,,22,512,,gb,backbone-03,651,-42.753564,-35.794022,Dublin,Berlin,"anon, hosting",1382.9119,102.28.3.53,125.290832,77 a2 f0 15 20 e7 0a 7d d1 63 17 98 89 4e 6c d2 1f 58 17 aa 18 72 0f 8e 23 6c 89 06
|
||||
169.586656,1780243200.000187,57,443,83.250.9.29,c0 23,"anon, hosting",,,China Mobile,81,43,627.6874,03 04,117.69,9598,il,,China Mobile,TlsS,,83.250.9.29,81387,graph.facebook.com,59868,,suricata,419343,,207.100.186.111,2026-06-01 00:00:00.000187,Netflix Inc.,,1593.3555,19,Netflix Inc.,0b d7 4e 78,,443,256,,gb,edge-01,1695,-40.899179,-33.626028,Tokyo,Seattle,"anon, hosting",633.4005,34.187.139.213,52.813022,c0 8f 54 eb d7 f6 95 6b 32 e3 62 01 c4 ce 5d 6d 2c be 2f 2e ef 4c 96 4b c3 b9 ae 9f
|
||||
53.588162,1780243200.000188,9,443,109.112.90.206,c0 14,"anon, hosting",,,SK Telecom,124,18,516.5783,03 03,39.85,2677,sg,,Comcast,TlsS,,139.102.159.103,39729,cdn.example.com,178710,Microsoft Corp.,suricata,460091,,50.97.229.36,2026-06-01 00:00:00.000188,Apple Inc.,,254.8298,464,Baidu Inc.,e4 06 f9 e2,,443,,,kr,wan-link,652,1.289227,37.010576,San Francisco,San Francisco,"anon, hosting",957.4039,117.74.196.49,-131.826564,12 3c 18 e8 43 70 f6 4b 49 c6 76 72 8c df 9b e8 17 7c a1 16 d7 fb a7 eb d3 73 61 ad
|
||||
-48.221466,1780243200.000189,86,3389,210.235.179.79,cc a9,"anon, hosting",,,BT Group,179,36,664.4044,03 03,78.21,5180,us,,Telstra,TlsC,,210.235.179.79,36659,,127778,,packet_capture,87014,graph.facebook.com,190.22.205.113,2026-06-01 00:00:00.000189,Microsoft Corp.,,128.2204,313,Google LLC,99 95 c4 f1,,21,512,,kr,gw-09,1909,44.232485,53.388395,Seattle,Seattle,"anon, hosting",27.2427,123.192.160.245,-52.175795,4b e0 77 35 c9 6a 13 45 2d a9 bc 9b 53 74 74 59 99 2e c0 be 23 b1 82 7e dd c7 0c ee
|
||||
-104.847562,1780243200.00019,35,25,175.36.87.137,c0 2b,"anon, hosting",,,AT&T,175,36,26.3258,03 02,77.32,7542,ca,,Orange,TlsS,,175.36.87.137,33667,,166222,,netflow,156768,auth.example.net,76.156.228.155,2026-06-01 00:00:00.000190,Alibaba Inc.,,1370.9329,328,Samsung,c0 1a 38 c4,--,465,,,ru,wan-link,1194,10.605972,47.957325,Dublin,London,"anon, hosting",1761.7165,136.176.217.191,-165.737581,d7 f9 b7 08 30 f2 e7 24 a7 4c 9b 8d ce 3d 8c 96 0d 54 6c 7d b3 20 ea 9b ca b8 de 9d
|
||||
-97.617615,1780243200.000191,76,443,132.171.47.210,c0 13,"anon, hosting",,,Deutsche Telekom,171,48,892.8365,03 04,89.04,7102,nl,,China Mobile,TlsC,,132.171.47.210,95854,auth.example.net,40313,,ssl_logs,451043,stream.example.org,162.122.81.80,2026-06-01 00:00:00.000191,Google LLC,,512.933,53,Alibaba Inc.,63 1b 5e dd,,80,,,il,core-02,838,-49.883072,-52.20925,Sydney,Seoul,"anon, hosting",1133.2132,161.20.158.202,-146.844284,07 f8 27 e9 31 17 cf a3 d8 9a 41 a9 e8 1b d0 01 49 db 65 7d 24 be 86 5b 64 06 ce 11
|
||||
145.767865,1780243200.000192,90,993,211.116.114.7,13 01,"anon, hosting",,,KDDI,51,38,754.9782,03 03,67.56,9962,br,,Verizon,TlsC,,211.116.114.7,52464,,183524,,zeek,460225,login.live.com,17.108.161.55,2026-06-01 00:00:00.000192,Alibaba Inc.,12584,1103.3371,451,Apple Inc.,cf bd 51 93,,443,,,no,gw-09,261,30.042504,-3.173587,Singapore,Sydney,"anon, hosting",500.4335,99.120.228.193,49.196664,b8 5a f1 76 b6 cf 2f ab 5a c4 ce bd aa 21 88 f9 c7 50 bd 7e 1f d5 d5 99 35 88 f2 bd
|
||||
-173.124365,1780243200.000193,28,50000,51.214.112.115,00 35,"anon, hosting",,,Deutsche Telekom,171,39,776.0707,03 03,3.56,8897,gb,graph.facebook.com,NTT,TlsS,,51.214.112.115,36885,,144463,,packet_capture,14638,,198.212.174.240,2026-06-01 00:00:00.000193,,,1911.9092,57,Tencent,86 28 38 5c,,995,,,ca,backbone-03,674,37.742189,48.460315,Berlin,Paris,"anon, hosting",700.4588,56.76.216.29,165.213308,6a 3c 43 3e 92 cb 64 98 e2 cb 5e a5 8e 49 54 c6 97 b0 cf 69 61 37 eb 47 99 05 6e ca
|
||||
-158.431061,1780243200.000194,31,80,194.24.56.40,c0 2f,"anon, hosting",00 17,,AT&T,86,19,122.1142,03 03,59.16,1842,kr,,AT&T,TlsC,,194.24.56.40,96960,,112481,,packet_capture,344982,,202.160.223.156,2026-06-01 00:00:00.000194,Samsung,8648,75.9809,404,Alibaba Inc.,eb 99 a2 45,,443,,secp521r1,fi,core-02,147,-23.71639,-41.287639,Sydney,San Francisco,"anon, hosting",604.0293,130.37.158.119,-85.992072,f4 31 f8 33 69 c3 1e 28 a5 f6 32 2c fd 3b 68 45 32 9e 08 45 9b ed 02 52 2d 94 2f 3d
|
||||
-132.023148,1780243200.000195,41,21,61.142.248.55,00 35,"anon, hosting",00 18,,SK Telecom,200,24,605.3395,03 03,2.07,3385,hk,,China Mobile,TlsC,,36.126.134.192,94296,,85753,,netflow,290783,cdn.example.com,177.124.52.179,2026-06-01 00:00:00.000195,Samsung,,595.0233,311,Baidu Inc.,d0 56 62 38,,25,256,,nl,wan-link,1078,26.929316,-13.234047,San Francisco,Beijing,"anon, hosting",185.4466,35.37.231.45,-15.953494,c3 04 f3 0b 15 2a 96 8c 38 71 02 c1 d7 5c 85 14 e3 34 02 01 d2 d1 1e 8b 33 f8 45 48
|
||||
-171.873785,1780243200.000196,100,3389,17.108.161.55,00 9e,"anon, hosting",,,Deutsche Telekom,47,31,20.4957,03 03,107.29,1149,ua,,China Telecom,TlsS,,17.108.161.55,98514,,83552,,suricata,187745,,141.103.76.43,2026-06-01 00:00:00.000196,Amazon.com Inc.,,1120.7174,115,Meta Platforms,db 07 c1 22,,22,,,se,backbone-03,849,1.192727,13.229179,Dublin,Shanghai,"anon, hosting",1797.9377,211.250.251.118,19.967592,0c 74 ed 3b 4a 31 71 e3 80 7a ce 07 a3 a9 a5 b0 fc be 9c 0a 8e 5f 00 d6 dc 19 98 b7
|
||||
-23.369992,1780243200.000197,64,3389,140.154.4.171,c0 13,"anon, hosting",,,,190,28,708.7332,03 03,11.88,6064,nl,,Telstra,TlsC,,140.154.4.171,68916,,27950,,netflow,338956,,47.109.174.243,2026-06-01 00:00:00.000197,Amazon.com Inc.,,979.3286,192,Alibaba Inc.,6e 3e 76 50,,443,,,de,edge-01,1317,-26.455903,-43.221426,Sydney,San Francisco,"anon, hosting",1207.8019,138.4.55.111,148.193432,2e e2 50 32 f8 97 50 47 43 0d 9a 2c d0 ed 5a 3d b7 7b 5a cf c3 6b f3 02 ca 71 40 b5
|
||||
53.281077,1780243200.000198,26,21,8.126.102.215,c0 27,"anon, hosting",,,Verizon,128,14,851.4567,03 03,92.86,2172,au,,SK Telecom,TlsC,,8.126.102.215,54672,,197321,,suricata,345551,cdn.example.com,14.99.185.142,2026-06-01 00:00:00.000198,Amazon.com Inc.,,1394.8855,448,Tencent,fe 2a 2a 92,,50000,384,secp256r1,au,edge-01,1860,9.059837,38.564027,New York,Seattle,"anon, hosting",520.0465,87.67.192.40,19.358755,1e 4f 43 7d 23 3a c8 f4 e2 0c c5 f4 da 68 ab 7d 45 4d 65 f8 3f b6 4f 24 46 c1 91 bc
|
||||
28.294006,1780243200.000199,52,443,165.204.169.241,c0 2c,"anon, hosting",,,BT Group,114,47,243.0155,03 04,35.31,4229,au,,China Telecom,TlsC,,165.204.169.241,38797,,120412,,netflow,496961,,223.73.205.18,2026-06-01 00:00:00.000199,Amazon.com Inc.,,585.9173,272,Baidu Inc.,fc ec 03 92,,3389,,,de,wan-link,472,15.056915,55.028867,Moscow,Paris,"anon, hosting",1681.911,147.213.95.171,-66.635489,aa 38 49 1b d7 40 e5 d5 fc 5b 76 a3 0b e1 6e 83 89 46 25 43 38 9a df be 02 5f 4b 64
|
||||
|
||||
|
@@ -0,0 +1,81 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-hans">
|
||||
<head><meta charset="UTF-8"><title>簇详情 — 预览</title>
|
||||
<style>
|
||||
*{margin:0;padding:0;box-sizing:border-box}
|
||||
body{font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;background:#f5f7fa;color:#1a1a2e;padding:1rem 2rem}
|
||||
.card{background:#fff;border-radius:8px;padding:1.5rem;margin-bottom:1rem;box-shadow:0 1px 3px rgba(0,0,0,.1)}
|
||||
a{color:#4361ee;text-decoration:none;font-size:.9rem}
|
||||
h2{font-size:1.2rem}.badge{display:inline-block;padding:.15rem .5rem;border-radius:12px;font-size:.75rem;font-weight:600}
|
||||
.badge-warning{background:#fff3cd;color:#856404}
|
||||
.feat-table{width:100%;border-collapse:collapse;font-size:.8rem;margin-top:.5rem}
|
||||
.feat-table th{background:#f5f5f5;padding:.3rem .5rem;text-align:left;font-weight:600;border-bottom:2px solid #ddd;white-space:nowrap}
|
||||
.feat-table td{padding:.25rem .5rem;border-bottom:1px solid #eee}
|
||||
.feat-table td code{font-size:.72rem}
|
||||
.feat-p{color:#2e7d32}.feat-n{color:#c62828}
|
||||
.grid-2{display:grid;grid-template-columns:1fr 1fr;gap:1rem;margin-top:.5rem}
|
||||
.svd-card{background:#f8f9fa;border:1px solid #e0e0e0;border-radius:8px;padding:.8rem}
|
||||
.svd-card h4{margin:0 0 .3rem;font-size:.9rem}
|
||||
.data-table{width:100%;border-collapse:collapse;font-size:.72rem;margin-top:.5rem}
|
||||
.data-table th{background:#1a1a2e;color:#e0e0e0;padding:.25rem .4rem;text-align:left;font-weight:600;white-space:nowrap;font-size:.7rem}
|
||||
.data-table td{padding:.2rem .4rem;border-bottom:1px solid #eee;font-size:.7rem}
|
||||
.data-table td code{font-size:.68rem}
|
||||
.data-table tr:nth-child(even){background:#f8f9fa}
|
||||
.scroll-x{overflow-x:auto}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="card">
|
||||
<a href="#">← 聚类概览</a>
|
||||
<div style="display:flex;justify-content:space-between;align-items:center;margin-top:.5rem">
|
||||
<div><h2>簇 #0 <span class="badge badge-warning" style="display:none">噪声</span></h2>
|
||||
<p style="color:#666;font-size:.9rem">847,293 条流 — 66.0% 占总 — 轮廓系数: 0.7100</p></div>
|
||||
</div>
|
||||
<div style="background:#f8f9fa;border-left:3px solid #4361ee;padding:.6rem;margin:.75rem 0;border-radius:0 4px 4px 0;font-size:.85rem;line-height:1.6;color:#333">
|
||||
低延迟 (23ms)，比全局均值高3.1个标准差；大数据包 (1520 bytes)，高2.3个标准差；系统端口 (443)，高1.8个标准差（簇内高度集中）。
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h3>📊 特征分析 <span style="font-weight:400;font-size:.8rem;color:#888">12 个特征</span></h3>
|
||||
<table class="feat-table">
|
||||
<thead><tr><th>#</th><th>特征名</th><th>区分度</th><th>均值</th> <th>标准差</th><th>中位数</th><th>P25</th><th>P75</th><th>离散度</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>1</td><td><code>延迟 (4dur)</code></td><td class="feat-p">+3.213</td><td>23.0ms</td><td>5.1</td><td>21ms</td><td>18ms</td><td>26ms</td><td><span style="color:#e65100;font-size:.72rem;">离散中</span></td></tr>
|
||||
<tr><td>2</td><td><code>数据包 (8ppk)</code></td><td class="feat-p">+2.346</td><td>1520.0B</td><td>128</td><td>1480B</td><td>1400B</td><td>1650B</td><td><span style="color:#e65100;font-size:.72rem;">离散中</span></td></tr>
|
||||
<tr><td>3</td><td><code>TLS版本 (0ver)</code></td><td class="feat-n">-1.845</td><td>1.2</td><td>0.3</td><td>1.2</td><td>1.2</td><td>1.3</td><td><span style="color:#43aa8b;font-size:.72rem;">高度集中</span></td></tr>
|
||||
<tr><td>4</td><td><code>目标端口 (:prd)</code></td><td class="feat-p">+1.821</td><td>443</td><td>12</td><td>443</td><td>443</td><td>443</td><td><span style="color:#1565c0;font-size:.72rem;">极集中</span></td></tr>
|
||||
<tr><td>5</td><td><code>会话时长 (8ses)</code></td><td class="feat-n">-1.523</td><td>15s</td><td>4s</td><td>14s</td><td>12s</td><td>18s</td><td><span style="color:#e65100;font-size:.72rem;">离散中</span></td></tr>
|
||||
<tr><td>6</td><td><code>密钥大小 (4ksz)</code></td><td class="feat-p">+1.234</td><td>256</td><td>0</td><td>256</td><td>256</td><td>256</td><td><span style="color:#1565c0;font-size:.72rem;">零离散</span></td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h3>🔬 SVD 关键特征 <span style="font-weight:400;font-size:.8rem;color:#888">Top 5 按区分度</span></h3>
|
||||
<div class="grid-2">
|
||||
<div class="svd-card"><h4>延迟 (4dur)</h4><div style="font-size:.75rem;color:#666;margin-top:.2rem">Score: <span class="feat-p">+3.21</span> | 均值: 23.00 | 标准差: 5.10</div></div>
|
||||
<div class="svd-card"><h4>数据包 (8ppk)</h4><div style="font-size:.75rem;color:#666;margin-top:.2rem">Score: <span class="feat-p">+2.35</span> | 均值: 1520.00 | 标准差: 128.00</div></div>
|
||||
<div class="svd-card"><h4>TLS版本 (0ver)</h4><div style="font-size:.75rem;color:#666;margin-top:.2rem">Score: <span class="feat-n">-1.85</span> | 均值: 1.20 | 标准差: 0.30</div></div>
|
||||
<div class="svd-card"><h4>目标端口 (:prd)</h4><div style="font-size:.75rem;color:#666;margin-top:.2rem">Score: <span class="feat-p">+1.82</span> | 均值: 443.00 | 标准差: 12.00</div></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h3>📋 数据行 <span style="font-weight:400;font-size:.8rem;color:#888">100 条</span></h3>
|
||||
<div class="scroll-x">
|
||||
<table class="data-table">
|
||||
<thead><tr><th>#</th><th>:ips</th><th>:ipd</th><th>:prs</th><th>:prd</th><th>0ver</th><th>8ack</th><th>4dur</th><th>timestamp</th><th>snam</th><th>cnam</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>1</td><td><code>192.168.1.42</code></td><td><code>10.0.0.5</code></td><td><code>49152</code></td><td><code>443</code></td><td><code>TLSv1.3</code></td><td><code>1420</code></td><td><code>23</code></td><td><code>2024-01-15 08:23</code></td><td><code>api.example.com</code></td><td><code>*.example.com</code></td></tr>
|
||||
<tr><td>2</td><td><code>10.0.0.100</code></td><td><code>203.0.113.50</code></td><td><code>34567</code></td><td><code>443</code></td><td><code>TLSv1.3</code></td><td><code>2048</code></td><td><code>18</code></td><td><code>2024-01-15 08:23</code></td><td><code>cdn.example.com</code></td><td><code>*.example.com</code></td></tr>
|
||||
<tr><td>3</td><td><code>172.16.0.8</code></td><td><code>198.51.100.22</code></td><td><code>50001</code></td><td><code>8443</code></td><td><code>TLSv1.2</code></td><td><code>890</code></td><td><code>35</code></td><td><code>2024-01-15 08:24</code></td><td><code>api.internal.net</code></td><td><code>*.internal.net</code></td></tr>
|
||||
<tr><td>4</td><td><code>192.168.1.100</code></td><td><code>10.0.0.1</code></td><td><code>44321</code></td><td><code>443</code></td><td><code>TLSv1.3</code></td><td><code>5600</code></td><td><code>12</code></td><td><code>2024-01-15 08:24</code></td><td><code>www.example.com</code></td><td><code>*.example.com</code></td></tr>
|
||||
<tr><td>5</td><td><code>10.0.1.50</code></td><td><code>203.0.113.100</code></td><td><code>51000</code></td><td><code>8080</code></td><td><code>TLSv1.2</code></td><td><code>320</code></td><td><code>45</code></td><td><code>2024-01-15 08:25</code></td><td><code>api.alt.net</code></td><td><code>*.alt.net</code></td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<p style="text-align:center;color:#999;font-size:.78rem;margin-top:.5rem">显示前 100 行，共 847,293 行</p>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,376 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-hans">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1.0">
|
||||
<title>聚类概览 — 预览</title>
|
||||
<style>
|
||||
*{margin:0;padding:0;box-sizing:border-box}
|
||||
body{font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;background:#f5f7fa;color:#1a1a2e}
|
||||
.header{background:#1a1a2e;color:#fff;padding:0.8rem 1.5rem;display:flex;justify-content:space-between;align-items:center}
|
||||
.header h1{font-size:1.1rem}.header span{font-size:0.8rem;color:#aaa}
|
||||
.main-layout{display:flex;position:relative;min-height:calc(100vh - 48px)}
|
||||
.left{flex:1;min-width:0;padding:1rem;overflow-y:auto;transition:flex .3s}
|
||||
.left.globe-open{flex:0 0 55%}
|
||||
.right{flex:0 0 0;overflow:hidden;transition:flex .3s}
|
||||
.right.globe-open{flex:0 0 45%}
|
||||
.toggle-handle{
|
||||
position:absolute;z-index:10;top:50%;width:20px;height:44px;background:#1a1a2e;color:#fff;
|
||||
border:none;border-radius:4px 0 0 4px;cursor:pointer;display:flex;align-items:center;
|
||||
justify-content:center;font-size:12px;transition:left .3s;left:calc(100% - 20px)
|
||||
}
|
||||
.toggle-handle:hover{background:#4361ee}
|
||||
.card{background:#fff;border-radius:8px;padding:1rem;margin-bottom:1rem;box-shadow:0 1px 3px rgba(0,0,0,.1)}
|
||||
.card h3{font-size:.95rem;margin-bottom:.5rem}
|
||||
.scatter-wrap{padding:.75rem}
|
||||
.scatter-wrap canvas{width:100%;height:380px;display:block;border-radius:4px;background:#fafbfc}
|
||||
.pill-row{display:flex;flex-wrap:wrap;gap:4px;margin-bottom:.75rem}
|
||||
.pill{display:inline-flex;align-items:center;gap:3px;padding:2px 10px;border-radius:12px;font-size:.75rem;cursor:pointer;border:2px solid transparent;color:#fff;white-space:nowrap}
|
||||
.pill.active{border-color:#333;font-weight:700}
|
||||
.card-grid{display:grid;grid-template-columns:1fr 1fr;gap:.75rem}
|
||||
.cluster-card{background:#fff;border:1px solid #e0e0e0;border-radius:8px;padding:.8rem;cursor:pointer;transition:box-shadow .15s}
|
||||
.cluster-card:hover{box-shadow:0 2px 8px rgba(0,0,0,.1)}
|
||||
.cluster-card.selected{border-color:#4361ee;box-shadow:0 0 0 2px rgba(67,97,238,.2)}
|
||||
.cluster-card .lbl{font-weight:700;font-size:1rem}
|
||||
.cluster-card .meta{font-size:.78rem;color:#666;margin-top:.2rem}
|
||||
.cluster-card .desc{font-size:.8rem;color:#333;margin-top:.4rem;padding:.4rem;background:#f8f9fa;border-left:3px solid #4361ee;border-radius:0 4px 4px 0;line-height:1.5}
|
||||
.detail-panel{position:fixed;bottom:0;left:0;right:0;z-index:100;transform:translateY(100%);transition:transform .35s cubic-bezier(.4,0,.2,1)}
|
||||
.detail-panel.open{transform:translateY(0)}
|
||||
.detail-body{background:#fff;border-radius:16px 16px 0 0;box-shadow:0 -4px 24px rgba(0,0,0,.15);max-height:40vh;overflow-y:auto;padding:.5rem 1.5rem 1.5rem}
|
||||
.detail-handle{width:36px;height:4px;background:#ccc;border-radius:2px;margin:8px auto;cursor:pointer}
|
||||
.detail-close{float:right;cursor:pointer;font-size:1.3rem;color:#999;border:none;background:none}
|
||||
.detail-close:hover{color:#333}
|
||||
.detail-body h3{font-size:1rem;margin-bottom:.3rem}
|
||||
.detail-body p{font-size:.85rem;color:#555;margin:.2rem 0}
|
||||
.detail-body table{width:100%;border-collapse:collapse;font-size:.8rem;margin-top:.5rem}
|
||||
.detail-body th{background:#f5f5f5;padding:.25rem .4rem;text-align:left;font-weight:600;border-bottom:2px solid #ddd;white-space:nowrap}
|
||||
.detail-body td{padding:.2rem .4rem;border-bottom:1px solid #eee}
|
||||
.feat-p{color:#2e7d32}.feat-n{color:#c62828}
|
||||
#globeCont{width:100%;height:calc(100vh - 48px);background:#0a0a1a;position:relative}
|
||||
.btn-sm{display:inline-block;padding:.25rem .6rem;border-radius:4px;font-size:.78rem;text-decoration:none;background:#4361ee;color:#fff}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="header">
|
||||
<h1>聚类概览 — 运行 #9</h1>
|
||||
<span>1,284,592 条流 · 3 个簇 · 12,847 个噪声</span>
|
||||
</div>
|
||||
|
||||
<div class="main-layout" id="main">
|
||||
<div class="left" id="leftPanel">
|
||||
<!-- UMAP 散点图 -->
|
||||
<div class="card scatter-wrap">
|
||||
<div style="display:flex;justify-content:space-between;align-items:center">
|
||||
<h3>UMAP 嵌入 <span style="font-weight:400;font-size:.8rem;color:#888" id="ptCount">加载中...</span></h3>
|
||||
<span id="viewToggle" style="display:none">
|
||||
<button id="btn2D" class="btn-sm" style="font-weight:700">2D</button>
|
||||
<button id="btn3D" class="btn-sm btn-outline">3D</button>
|
||||
</span>
|
||||
</div>
|
||||
<canvas id="scatterChart"></canvas>
|
||||
<div class="pill-row" id="legendPills"></div>
|
||||
</div>
|
||||
|
||||
<!-- 簇标签 -->
|
||||
<div class="pill-row" id="clusterPills"></div>
|
||||
|
||||
<!-- 簇卡片 -->
|
||||
<div class="card-grid" id="cardGrid"></div>
|
||||
</div>
|
||||
|
||||
<div class="right" id="rightPanel">
|
||||
<div id="globeCont"></div>
|
||||
</div>
|
||||
<button class="toggle-handle" id="globeHandle" onclick="toggleGlobe()">◀</button>
|
||||
</div>
|
||||
|
||||
<!-- 详情面板 -->
|
||||
<div class="detail-panel" id="detailPanel">
|
||||
<div class="detail-body">
|
||||
<button class="detail-close" onclick="closeDetail()">✕</button>
|
||||
<div class="detail-handle" onclick="closeDetail()"></div>
|
||||
<div id="detailContent"><p style="color:#999;padding:.5rem 0">选择一个簇或数据点查看详情</p></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script src="static/tianxuan/three.min.js"></script>
|
||||
<script>
|
||||
// ═══ 模拟数据 ═══
|
||||
const COLORS = ['#4361ee','#f72585','#7209b7','#4cc9f0','#f8961e','#43aa8b','#f94144','#577590','#e9c46a','#9b5de5'];
|
||||
const CLUSTERS = [
|
||||
{label:0,size:847293,prop:0.66,silh:0.71,
|
||||
desc:'低延迟 (23ms)，比全局均值高3.1个标准差；大数据包 (1520 bytes)，高2.3个标准差；系统端口 (443)',
|
||||
detail:'簇 #0 由 847,293 条流组成（66%）。\n特征：延迟(4dur)均值23ms，区分度+3.2；数据包大小(8ppk)均值1520B，区分度+2.3；端口(:prd)高度集中于443(92%)；TLS版本以1.3为主(78%)。'},
|
||||
|
||||
{label:1,size:311482,prop:0.24,silh:0.65,
|
||||
desc:'高TLS1.3比例 (92%)，比均值高1.8σ；短会话 (12s)，低1.5σ；ECDHE曲线 X25519',
|
||||
detail:'簇 #1 由 311,482 条流组成（24%）。\n特征：TLS1.3比率(0ver)92%，区分度+1.8；会话时长(8ses)均值12s，区分度-1.5；密钥交换曲线集中在X25519(88%)；目标端口包括443和8443。'},
|
||||
|
||||
{label:2,size:112970,prop:0.09,silh:0.52,
|
||||
desc:'高连接频率 (85次/min)，比均值高4.2σ；非标准端口 (8080/8443)；目标多域名',
|
||||
detail:'簇 #2 由 112,970 条流组成（9%）。\n特征：连接频率(8seq/min)均值85，区分度+4.2；非标准端口比例高(76%)，以8080/8443为主；SNI名称(snam)分散在8个不同域名；源IP来自不同/24网段。'},
|
||||
|
||||
{label:-1,size:12847,prop:0.01,silh:null,
|
||||
desc:'噪声 — 异常TLS指纹、零散目标、单次连接',
|
||||
detail:''},
|
||||
];
|
||||
|
||||
const RNG = (s)=>{let seed=s%2147483647;return()=>{seed=seed*16807%2147483647;return(seed-1)/2147483646;}};
|
||||
const r=RNG(42);
|
||||
const centers={0:[2.2,1.8],1:[-2.0,0.3],2:[0.3,-2.2],'-1':[3.8,-1.2]};
|
||||
const spreads={0:[0.7,0.6],1:[0.8,0.7],2:[0.6,0.8],'-1':[1.5,1.0]};
|
||||
const SCATTER=[];
|
||||
let totalPts=0;
|
||||
for(const c of CLUSTERS){
|
||||
const n=c.label===-1?200:Math.min(c.size/2000,200);
|
||||
totalPts+=n;
|
||||
for(let i=0;i<n;i++){
|
||||
const cx=centers[c.label][0],cy=centers[c.label][1];
|
||||
const sx=spreads[c.label][0],sy=spreads[c.label][1];
|
||||
SCATTER.push({
|
||||
x:cx+(r()-0.5)*2*sx, y:cy+(r()-0.5)*2*sy,
|
||||
z:(r()-0.5)*2*0.8, label:c.label,
|
||||
entity:c.label===-1?'噪声':`主机_${1000+i}`
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let selCluster=null,globeOpen=false,globeInit=false;
|
||||
|
||||
// ═══ 2D Canvas 散点 ═══
|
||||
function drawScatter(highlight){
|
||||
const c=document.getElementById('scatterChart');
|
||||
if(!c)return;
|
||||
const W=c.parentElement.clientWidth||700,H=380;
|
||||
c.width=W;c.height=H;
|
||||
const ctx=c.getContext('2d');
|
||||
const pad={top:25,bottom:30,left:50,right:20};
|
||||
const pw=W-pad.left-pad.right,ph=H-pad.top-pad.bottom;
|
||||
const xs=SCATTER.map(d=>d.x),ys=SCATTER.map(d=>d.y);
|
||||
const xMin=Math.min(...xs),xMax=Math.max(...xs),yMin=Math.min(...ys),yMax=Math.max(...ys);
|
||||
const xR=xMax-xMin||1,yR=yMax-yMin||1;
|
||||
const toX=v=>pad.left+(v-xMin)/xR*pw,toY=v=>pad.top+ph-(v-yMin)/yR*ph;
|
||||
ctx.clearRect(0,0,W,H);
|
||||
ctx.strokeStyle='#eee';ctx.lineWidth=1;
|
||||
for(let i=0;i<=5;i++){ctx.beginPath();ctx.moveTo(pad.left,pad.top+i*ph/5);ctx.lineTo(W-pad.right,pad.top+i*ph/5);ctx.stroke();}
|
||||
SCATTER.forEach(d=>{
|
||||
const cx=toX(d.x),cy=toY(d.y);
|
||||
let color=d.label===-1?'rgba(150,150,150,0.4)':COLORS[Math.abs(d.label)%COLORS.length];
|
||||
let r2=d.label===-1?2:4;
|
||||
if(highlight!==null&&highlight!==undefined){
|
||||
if(d.label===highlight){color=d.label===-1?'rgba(150,150,150,0.9)':COLORS[Math.abs(d.label)%COLORS.length];r2=6;}
|
||||
else{color='rgba(200,200,200,0.15)';r2=2;}
|
||||
}
|
||||
ctx.beginPath();ctx.arc(cx,cy,r2,0,Math.PI*2);ctx.fillStyle=color;ctx.fill();
|
||||
});
|
||||
ctx.fillStyle='#999';ctx.font='11px sans-serif';ctx.textAlign='center';
|
||||
ctx.fillText('UMAP-1',W/2,H-3);
|
||||
ctx.save();ctx.translate(14,H/2);ctx.rotate(-Math.PI/2);ctx.fillText('UMAP-2',0,0);ctx.restore();
|
||||
document.getElementById('ptCount').textContent=SCATTER.length+' 个点';
|
||||
// legend pills (from scatter data)
|
||||
const leg=document.getElementById('legendPills');leg.innerHTML='';
|
||||
const seen={};
|
||||
SCATTER.forEach(d=>{if(!seen[d.label]){seen[d.label]=true;
|
||||
const p=document.createElement('span');p.className='pill';
|
||||
p.style.background=d.label===-1?'#999':COLORS[Math.abs(d.label)%COLORS.length];
|
||||
p.textContent=d.label===-1?'噪声':'#'+d.label;
|
||||
p.onclick=()=>selectCluster(d.label);leg.appendChild(p);
|
||||
}});
|
||||
}
|
||||
|
||||
// ═══ 簇选择 ═══
|
||||
function selectCluster(lbl){
|
||||
selCluster=CLUSTERS.find(c=>c.label===lbl);
|
||||
drawScatter(lbl);
|
||||
document.querySelectorAll('#clusterPills .pill').forEach(p=>p.classList.toggle('active',parseInt(p.dataset.lbl)===lbl));
|
||||
document.querySelectorAll('.cluster-card').forEach(p=>p.classList.toggle('selected',parseInt(p.dataset.label)===lbl));
|
||||
if(globeOpen)globeFilter(lbl);
|
||||
showDetail(selCluster);
|
||||
}
|
||||
|
||||
function clearSel(){selCluster=null;drawScatter(null);closeDetail();
|
||||
document.querySelectorAll('.pill,.cluster-card').forEach(p=>p.classList.remove('active','selected'));
|
||||
if(globeOpen)globeFilter(null);
|
||||
}
|
||||
|
||||
// ═══ 详情面板 ═══
|
||||
function showDetail(c){
|
||||
const dc=document.getElementById('detailContent');
|
||||
if(!c||c.label===-1){dc.innerHTML='<p style="color:#999;">选择一个簇查看详情</p>';document.getElementById('detailPanel').classList.remove('open');return;}
|
||||
dc.innerHTML='<h3>簇 #'+c.label+'</h3><p>大小: '+c.size.toLocaleString()+' | 占比: '+(c.prop*100).toFixed(1)+'%'+(c.silh!==null?' | 轮廓系数: '+c.silh.toFixed(4):'')+'</p>'+
|
||||
'<p style="background:#f8f9fa;border-left:3px solid #4361ee;padding:.5rem;margin:.5rem 0;border-radius:0 4px 4px 0;font-size:.85rem;line-height:1.6;white-space:pre-wrap;">'+c.detail+'</p>'+
|
||||
'<table><thead><tr><th>特征</th><th>区分度</th><th>均值</th><th>标准差</th></tr></thead><tbody>'+
|
||||
(c.label===0?'<tr><td>延迟 (4dur)</td><td class="feat-p">+3.2</td><td>23.0ms</td><td>5.1</td></tr><tr><td>数据包 (8ppk)</td><td class="feat-p">+2.3</td><td>1520.0B</td><td>128</td></tr><tr><td>TLS版本 (0ver)</td><td class="feat-n">-1.8</td><td>1.2</td><td>0.3</td></tr>':
|
||||
c.label===1?'<tr><td>TLS1.3比率</td><td class="feat-p">+1.8</td><td>92%</td><td>8%</td></tr><tr><td>会话时长</td><td class="feat-n">-1.5</td><td>12s</td><td>4s</td></tr><tr><td>端口</td><td class="feat-p">+1.2</td><td>443</td><td>12</td></tr>':
|
||||
'<tr><td>连接频率</td><td class="feat-p">+4.2</td><td>85/min</td><td>12</td></tr><tr><td>非标准端口</td><td class="feat-p">+2.8</td><td>76%</td><td>9%</td></tr><tr><td>SNI域名数</td><td class="feat-p">+2.1</td><td>8</td><td>2</td></tr>')+
|
||||
'</tbody></table>';
|
||||
document.getElementById('detailPanel').classList.add('open');
|
||||
}
|
||||
function closeDetail(){document.getElementById('detailPanel').classList.remove('open');}
|
||||
|
||||
// ═══ 3D UMAP scatter ═══
|
||||
let s3DRenderer,s3DScene,s3DCamera,s3DPts,s3DInit=false,is3DMode=false;
|
||||
function hasWG2(){try{var c=document.createElement('canvas');return!!c.getContext('webgl2');}catch(e){return false;}}
|
||||
|
||||
(function(){
|
||||
if(!hasWG2())return;
|
||||
document.getElementById('viewToggle').style.display='inline';
|
||||
const b2=document.getElementById('btn2D'),b3=document.getElementById('btn3D');
|
||||
function show2D(){is3DMode=false;document.querySelector('#scatterChart').style.display='';
|
||||
document.getElementById('viewToggle').style.display='none';drawScatter(selCluster?selCluster.label:null);
|
||||
setTimeout(()=>document.getElementById('viewToggle').style.display='inline',50);
|
||||
b2.style.fontWeight='700';b3.style.fontWeight='normal';}
|
||||
function show3D(){is3DMode=true;document.querySelector('#scatterChart').style.display='none';
|
||||
if(!s3DInit)init3D();b3.style.fontWeight='700';b2.style.fontWeight='normal';}
|
||||
b2.onclick=show2D;b3.onclick=show3D;
|
||||
})();
|
||||
|
||||
function init3D(){
|
||||
if(s3DInit)return;
|
||||
const cont=document.getElementById('scatterChart').parentElement;
|
||||
const W=cont.clientWidth||700,H=450;
|
||||
const container=document.createElement('div');container.id='scatter3D';container.style.cssText='width:100%;height:450px';
|
||||
cont.appendChild(container);
|
||||
|
||||
const xs=SCATTER.map(d=>d.x),ys=SCATTER.map(d=>d.y),zs=SCATTER.map(d=>d.z||0);
|
||||
const xM=Math.min(...xs),xX=Math.max(...xs),yM=Math.min(...ys),yX=Math.max(...ys),zM=Math.min(...zs),zX=Math.max(...zs);
|
||||
const cx=(xM+xX)/2,cy=(yM+yX)/2,cz=(zM+zX)/2,rg=Math.max(xX-xM,yX-yM,zX-zM)||1;
|
||||
const nrm=(v,c,r)=>((v-c)/r)*3;
|
||||
const scene=new THREE.Scene();scene.background=new THREE.Color(0xf8f9fa);
|
||||
const camera=new THREE.PerspectiveCamera(50,W/H,0.1,50);camera.position.set(4.5,3.5,6);camera.lookAt(0,0,0);
|
||||
const renderer=new THREE.WebGLRenderer({antialias:true});renderer.setSize(W,H);renderer.setPixelRatio(Math.min(window.devicePixelRatio,2));
|
||||
container.appendChild(renderer.domElement);
|
||||
|
||||
const pos=new Float32Array(SCATTER.length*3);
|
||||
const cols=new Float32Array(SCATTER.length*3);
|
||||
SCATTER.forEach((d,i)=>{
|
||||
pos[i*3]=nrm(d.x,cx,rg);pos[i*3+1]=nrm(d.y,cy,rg);pos[i*3+2]=d.z!=null?nrm(d.z,cz,rg):0;
|
||||
const c3=new THREE.Color(d.label===-1?'#999999':COLORS[Math.abs(d.label)%COLORS.length]);
|
||||
cols[i*3]=c3.r;cols[i*3+1]=c3.g;cols[i*3+2]=c3.b;
|
||||
});
|
||||
const geo=new THREE.BufferGeometry();
|
||||
geo.setAttribute('position',new THREE.BufferAttribute(pos,3));
|
||||
geo.setAttribute('color',new THREE.BufferAttribute(cols,3));
|
||||
const mat=new THREE.PointsMaterial({size:0.12,vertexColors:true,sizeAttenuation:true,transparent:true,opacity:0.85});
|
||||
const pts=new THREE.Points(geo,mat);scene.add(pts);
|
||||
|
||||
let isD=false,pm={x:0,y:0};
|
||||
renderer.domElement.onmousedown=e=>{isD=true;pm={x:e.clientX,y:e.clientY};};
|
||||
window.onmouseup=()=>isD=false;
|
||||
window.onmousemove=e=>{if(!isD||!is3DMode)return;const dx=e.clientX-pm.x,dy=e.clientY-pm.y;
|
||||
pts.rotation.y+=dx*0.005;pts.rotation.x+=dy*0.005;pts.rotation.x=Math.max(-Math.PI/2,Math.min(Math.PI/2,pts.rotation.x));
|
||||
pm={x:e.clientX,y:e.clientY};};
|
||||
renderer.domElement.onwheel=e=>{if(!is3DMode)return;e.preventDefault();camera.position.z+=e.deltaY*0.008;camera.position.z=Math.max(1.5,Math.min(15,camera.position.z));};
|
||||
!function anim(){requestAnimationFrame(anim);if(!is3DMode)return;renderer.render(scene,camera);}();
|
||||
s3DRenderer=renderer;s3DScene=scene;s3DCamera=camera;s3DPts=pts;s3DInit=true;
|
||||
}
|
||||
|
||||
// ═══ 地球侧栏 ═══
|
||||
function toggleGlobe(){
|
||||
globeOpen=!globeOpen;
|
||||
document.getElementById('leftPanel').classList.toggle('globe-open',globeOpen);
|
||||
document.getElementById('rightPanel').classList.toggle('globe-open',globeOpen);
|
||||
const h=document.getElementById('globeHandle');
|
||||
h.innerHTML=globeOpen?'▶':'◀';
|
||||
h.style.left=globeOpen?'calc(55% - 20px)':'calc(100% - 20px)';
|
||||
if(globeOpen&&!globeInit)initGlobe();
|
||||
}
|
||||
function globeFilter(label){
|
||||
if(!window._arcs)return;
|
||||
window._arcs.forEach(m=>{
|
||||
if(label===null){m.material.opacity=0.5;m.material.color.setHex(parseInt(m.userData.c,16));}
|
||||
else{m.material.opacity=0.04;m.material.color.setHex(0x444444);}
|
||||
});
|
||||
}
|
||||
|
||||
function initGlobe(){
|
||||
if(globeInit)return;
|
||||
const cont=document.getElementById('globeCont');
|
||||
if(!cont)return;
|
||||
const W=cont.clientWidth||500,H=cont.clientHeight||500;
|
||||
const scene=new THREE.Scene();scene.background=new THREE.Color(0x0a0a1a);
|
||||
const cam=new THREE.PerspectiveCamera(45,W/H,0.1,100);cam.position.set(0,0,6);
|
||||
const ren=new THREE.WebGLRenderer({antialias:true});ren.setSize(W,H);ren.setPixelRatio(Math.min(window.devicePixelRatio,2));
|
||||
cont.appendChild(ren.domElement);
|
||||
|
||||
// Earth with texture
|
||||
const tex=new THREE.TextureLoader().load('static/tianxuan/earth_atmos_2048.jpg');
|
||||
const earth=new THREE.Mesh(new THREE.SphereGeometry(2,64,64),new THREE.MeshPhongMaterial({map:tex,specular:new THREE.Color(0x333333),shininess:5}));
|
||||
scene.add(earth);
|
||||
// Glow
|
||||
scene.add(new THREE.Mesh(new THREE.SphereGeometry(2.06,48,48),new THREE.MeshBasicMaterial({color:0x224488,transparent:true,opacity:0.12})));
|
||||
// Grid
|
||||
const gridMat=new THREE.LineBasicMaterial({color:0x6699cc,transparent:true,opacity:0.08});
|
||||
for(let lat=-80;lat<=80;lat+=20){const pts=[];for(let lon=0;lon<=360;lon+=5){const p=(90-lat)*Math.PI/180,t=(lon+180)*Math.PI/180;pts.push(new THREE.Vector3(-2.02*Math.sin(p)*Math.cos(t),2.02*Math.cos(p),2.02*Math.sin(p)*Math.sin(t)));}scene.add(new THREE.Line(new THREE.BufferGeometry().setFromPoints(pts),gridMat));}
|
||||
for(let lon=0;lon<360;lon+=20){const pts=[];for(let lat=-90;lat<=90;lat+=5){const p=(90-lat)*Math.PI/180,t=(lon+180)*Math.PI/180;pts.push(new THREE.Vector3(-2.02*Math.sin(p)*Math.cos(t),2.02*Math.cos(p),2.02*Math.sin(p)*Math.sin(t)));}scene.add(new THREE.Line(new THREE.BufferGeometry().setFromPoints(pts),gridMat));}
|
||||
|
||||
// Lights
|
||||
scene.add(new THREE.AmbientLight(0x222244));
|
||||
const dl=new THREE.DirectionalLight(0xffffff,1);dl.position.set(5,10,7);scene.add(dl);
|
||||
scene.add(new THREE.DirectionalLight(0x4488ff,0.3));
|
||||
// Stars
|
||||
const sg=new THREE.BufferGeometry();const sp=new Float32Array(600*3);
|
||||
for(let i=0;i<600*3;i++)sp[i]=(Math.random()-0.5)*100;
|
||||
sg.setAttribute('position',new THREE.BufferAttribute(sp,3));
|
||||
scene.add(new THREE.Points(sg,new THREE.PointsMaterial({color:0xffffff,size:0.15})));
|
||||
|
||||
// Flow arcs
|
||||
const tC={'TLSv1.3':0x4cc9f0,'TLSv1.2':0x43aa8b,'TLSv1.1':0xf8961e,'TLSv1.0':0xf94144};
|
||||
const ag=new THREE.Group();scene.add(ag);
|
||||
const arcs=[];
|
||||
const cities=[{lat:31.2,lon:121.5},{lat:39.9,lon:116.4},{lat:35.7,lon:139.7},{lat:37.6,lon:127.0},{lat:51.5,lon:-0.13},{lat:48.9,lon:2.35},{lat:40.7,lon:-74.0},{lat:37.8,lon:-122.4},{lat:1.35,lon:103.8},{lat:-33.9,lon:151.2}];
|
||||
const tls=['TLSv1.3','TLSv1.2','TLSv1.3','TLSv1.3','TLSv1.2','TLSv1.1','TLSv1.2','TLSv1.3','TLSv1.2','TLSv1.3'];
|
||||
function ll(lat,lon,r){const p=(90-lat)*Math.PI/180,t=(lon+180)*Math.PI/180;return new THREE.Vector3(-r*Math.sin(p)*Math.cos(t),r*Math.cos(p),r*Math.sin(p)*Math.sin(t));}
|
||||
for(let i=0;i<cities.length;i++){
|
||||
for(let j=0;j<cities.length;j++){
|
||||
if(i===j)continue;
|
||||
const src=cities[i],dst=cities[j];
|
||||
const f=ll(src.lat,src.lon,2),t=ll(dst.lat,dst.lon,2);
|
||||
const mid=new THREE.Vector3().addVectors(f,t).multiplyScalar(0.5).normalize().multiplyScalar(3.2);
|
||||
const curve=new THREE.QuadraticBezierCurve3(f,mid,t);
|
||||
const c=tls[(i+j)%tls.length];
|
||||
const mat=new THREE.MeshBasicMaterial({color:tC[c],transparent:true,opacity:0.4});
|
||||
const mesh=new THREE.Mesh(new THREE.TubeGeometry(curve,16,0.012,4,false),mat);
|
||||
mesh.userData={c:tC[c].toString(16)};ag.add(mesh);arcs.push(mesh);
|
||||
}
|
||||
}
|
||||
window._arcs=arcs;
|
||||
|
||||
let isD=false,pm={x:0,y:0};
|
||||
ren.domElement.onmousedown=e=>{isD=true;pm={x:e.clientX,y:e.clientY};};
|
||||
window.onmouseup=()=>isD=false;
|
||||
window.onmousemove=e=>{if(!isD)return;const dx=e.clientX-pm.x,dy=e.clientY-pm.y;earth.rotation.y+=dx*0.005;earth.rotation.x+=dy*0.005;ag.rotation.y=earth.rotation.y;ag.rotation.x=earth.rotation.x;pm={x:e.clientX,y:e.clientY};};
|
||||
|
||||
!function anim(){requestAnimationFrame(anim);if(!isD){earth.rotation.y+=0.003;ag.rotation.y=earth.rotation.y;}ren.render(scene,cam);}();
|
||||
globeInit=true;
|
||||
}
|
||||
|
||||
// ═══ 渲染 ═══
|
||||
(function init(){
|
||||
const pr=document.getElementById('clusterPills');
|
||||
const ap=document.createElement('span');ap.className='pill';ap.style.background='#e8e8e8';ap.style.color='#333';
|
||||
ap.textContent='全部';ap.onclick=clearSel;pr.appendChild(ap);
|
||||
CLUSTERS.forEach(c=>{
|
||||
if(c.label===-1)return;
|
||||
const p=document.createElement('span');p.className='pill';p.style.background=COLORS[c.label%COLORS.length];
|
||||
p.dataset.lbl=c.label;p.textContent='#'+c.label+' ('+(c.size/1000).toFixed(0)+'K)';p.onclick=()=>selectCluster(c.label);
|
||||
pr.appendChild(p);
|
||||
});
|
||||
const np=document.createElement('span');np.className='pill';np.style.background='#999';
|
||||
np.dataset.lbl='-1';np.textContent='噪声 (12.8K)';np.onclick=()=>selectCluster(-1);pr.appendChild(np);
|
||||
|
||||
const cg=document.getElementById('cardGrid');
|
||||
CLUSTERS.forEach(c=>{
|
||||
if(c.label===-1)return;
|
||||
const cd=document.createElement('div');cd.className='cluster-card';cd.dataset.label=c.label;
|
||||
cd.innerHTML='<div class="lbl">簇 #'+c.label+'</div><div class="meta">'+c.size.toLocaleString()+' 条流 · '+(c.prop*100).toFixed(0)+'%'+(c.silh!==null?' · 轮廓系数: '+c.silh.toFixed(4):'')+'</div><div class="desc">'+c.desc+'</div>';
|
||||
cd.onclick=()=>selectCluster(c.label);cg.appendChild(cd);
|
||||
});
|
||||
|
||||
drawScatter(null);
|
||||
document.getElementById('globeHandle').style.left='calc(100% - 20px)';
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,184 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-hans">
|
||||
<head><meta charset="UTF-8"><title>运行详情 — 预览</title>
|
||||
<style>
|
||||
*{margin:0;padding:0;box-sizing:border-box}
|
||||
body{font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;background:#f5f7fa;color:#1a1a2e;padding:1rem 2rem}
|
||||
.card{background:#fff;border-radius:8px;padding:1.5rem;margin-bottom:1rem;box-shadow:0 1px 3px rgba(0,0,0,.1)}
|
||||
h2{font-size:1.2rem;margin:0}h3{font-size:1rem;margin-bottom:.5rem}
|
||||
.badge{display:inline-block;padding:.15rem .5rem;border-radius:12px;font-size:.75rem;font-weight:600}
|
||||
.badge-success{background:#d4edda;color:#155724}
|
||||
.badge-danger{background:#f8d7da;color:#721c24}
|
||||
.badge-info{background:#d1ecf1;color:#0c5460}
|
||||
.stat{font-size:2rem;font-weight:700;color:#1a1a2e}
|
||||
.stat-label{font-size:.85rem;color:#666;margin-top:.25rem}
|
||||
.grid-3{display:grid;grid-template-columns:1fr 1fr 1fr;gap:1.5rem}
|
||||
.grid-svd{display:grid;grid-template-columns:1fr 1fr;gap:.75rem}
|
||||
.svd-card{background:#f8f9fa;border:1px solid #e0e0e0;border-radius:8px;padding:.8rem}
|
||||
.svd-card h4{margin:0 0 .3rem;font-size:.9rem}
|
||||
.var-bar{height:8px;border-radius:4px;background:#4361ee;margin:.2rem 0}
|
||||
.feat-tag{display:inline-block;background:#e8f0fe;padding:.1rem .4rem;border-radius:3px;font-size:.75rem;margin:.1rem;color:#333}
|
||||
.small{font-size:.8rem;color:#888}
|
||||
.btn{display:inline-block;padding:.4rem .8rem;border-radius:6px;text-decoration:none;font-size:.85rem;background:#4361ee;color:#fff}
|
||||
.tl-step{margin-bottom:.5rem}.tl-step .hdr{font-weight:700;font-size:.85rem;color:#4361ee;padding:.4rem 0 .2rem;border-top:1px solid #eee}
|
||||
.tl-think{background:#eef2ff;padding:.5rem;border-radius:6px;font-size:.78rem;line-height:1.6;color:#333;border:1px solid #d0d8f0;margin-bottom:.3rem}
|
||||
.tl-tool{border:1px solid #e0e0e0;border-radius:6px;margin-bottom:.3rem;overflow:hidden}
|
||||
.tl-tool .hd{background:#f8f9fa;padding:.35rem .6rem;cursor:pointer;display:flex;justify-content:space-between;align-items:center;font-size:.8rem;font-weight:600}
|
||||
.tl-tool .bd{display:none;padding:.4rem .6rem;background:#fafbfc;font-size:.72rem}
|
||||
.tl-tool .bd pre{background:#1a1a2e;color:#e0e0e0;padding:.4rem;border-radius:4px;font-size:.65rem;overflow:auto;margin:.2rem 0}
|
||||
.cluster-table{width:100%;border-collapse:collapse;font-size:.85rem;margin-top:.5rem}
|
||||
.cluster-table th{background:#f5f5f5;padding:.3rem .5rem;text-align:left;font-weight:600;border-bottom:2px solid #ddd}
|
||||
.cluster-table td{padding:.3rem .5rem;border-bottom:1px solid #eee}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="card">
|
||||
<div style="display:flex;justify-content:space-between;align-items:center;flex-wrap:wrap;gap:.5rem">
|
||||
<div>
|
||||
<h2>运行 #9</h2>
|
||||
<p style="color:#666;font-size:.9rem;margin-top:.25rem">2024-01-15 08:23 — 状态: <span class="badge badge-success">完成</span></p>
|
||||
</div>
|
||||
<a href="#" class="btn">聚类概览 →</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="grid-3">
|
||||
<div class="card" style="text-align:center"><div class="stat">1,284,592</div><div class="stat-label">总流数</div></div>
|
||||
<div class="card" style="text-align:center"><div class="stat">1,284,592</div><div class="stat-label">行数</div></div>
|
||||
<div class="card" style="text-align:center"><div class="stat">3</div><div class="stat-label">聚类数</div></div>
|
||||
</div>
|
||||
|
||||
<!-- 全域 SVD 特征分析 -->
|
||||
<div class="card">
|
||||
<div style="display:flex;justify-content:space-between;align-items:center">
|
||||
<h3>🔬 全域 SVD 特征分析 <span class="small">基于 1,284,592 行数据</span></h3>
|
||||
</div>
|
||||
<p style="font-size:.82rem;color:#666;margin-top:.3rem">奇异值分解 (TruncatedSVD) 降维后的各主成分解释方差比例与贡献特征。</p>
|
||||
<div class="grid-svd" style="margin-top:.75rem">
|
||||
<div class="svd-card">
|
||||
<h4>PC1</h4>
|
||||
<div style="display:flex;justify-content:space-between;font-size:.78rem;color:#555"><span>解释方差: 42.3%</span><span>累积: 42.3%</span></div>
|
||||
<div class="var-bar" style="width:42%"></div>
|
||||
<div style="margin-top:.4rem">
|
||||
<span class="feat-tag">8pak: 0.321</span>
|
||||
<span class="feat-tag">8ack: 0.284</span>
|
||||
<span class="feat-tag">4dur: 0.198</span>
|
||||
<span class="feat-tag">8ses: 0.156</span>
|
||||
<span class="feat-tag">2tmo: 0.112</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="svd-card">
|
||||
<h4>PC2</h4>
|
||||
<div style="display:flex;justify-content:space-between;font-size:.78rem;color:#555"><span>解释方差: 28.7%</span><span>累积: 71.0%</span></div>
|
||||
<div class="var-bar" style="width:29%"></div>
|
||||
<div style="margin-top:.4rem">
|
||||
<span class="feat-tag">:prd: 0.412</span>
|
||||
<span class="feat-tag">:prs: 0.334</span>
|
||||
<span class="feat-tag">1ipp: 0.187</span>
|
||||
<span class="feat-tag">0ver: 0.098</span>
|
||||
<span class="feat-tag">4dbn: 0.045</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="svd-card">
|
||||
<h4>PC3</h4>
|
||||
<div style="display:flex;justify-content:space-between;font-size:.78rem;color:#555"><span>解释方差: 15.2%</span><span>累积: 86.2%</span></div>
|
||||
<div class="var-bar" style="width:15%"></div>
|
||||
<div style="margin-top:.4rem">
|
||||
<span class="feat-tag">0ver: 0.502</span>
|
||||
<span class="feat-tag">4ksz: 0.321</span>
|
||||
<span class="feat-tag">0cph: 0.145</span>
|
||||
<span class="feat-tag">0crv: 0.088</span>
|
||||
<span class="feat-tag">cnam: 0.032</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="svd-card">
|
||||
<h4>PC4</h4>
|
||||
<div style="display:flex;justify-content:space-between;font-size:.78rem;color:#555"><span>解释方差: 8.9%</span><span>累积: 95.1%</span></div>
|
||||
<div class="var-bar" style="width:9%"></div>
|
||||
<div style="margin-top:.4rem">
|
||||
<span class="feat-tag">snam: 0.432</span>
|
||||
<span class="feat-tag">cnam: 0.356</span>
|
||||
<span class="feat-tag">ecdhe: 0.178</span>
|
||||
<span class="feat-tag">:ips: 0.089</span>
|
||||
<span class="feat-tag">:ipd: 0.034</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="svd-card">
|
||||
<h4>PC5</h4>
|
||||
<div style="display:flex;justify-content:space-between;font-size:.78rem;color:#555"><span>解释方差: 4.9%</span><span>累积: 100%</span></div>
|
||||
<div class="var-bar" style="width:5%"></div>
|
||||
<div style="margin-top:.4rem">
|
||||
<span class="feat-tag">timestamp: 0.523</span>
|
||||
<span class="feat-tag">8dbd: 0.312</span>
|
||||
<span class="feat-tag">8seq: 0.145</span>
|
||||
<span class="feat-tag">8did: 0.067</span>
|
||||
<span class="feat-tag">row: 0.021</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<p style="font-size:.78rem;color:#888;margin-top:.5rem">前 3 个主成分解释了 86.2% 的方差。SVD 去噪保留 95% 方差，去除 ~5% 作为底噪。</p>
|
||||
</div>
|
||||
|
||||
<!-- LLM 自动分析流程 -->
|
||||
<div class="card">
|
||||
<h3>🧠 LLM 自动分析流程</h3>
|
||||
<div>
|
||||
<div class="tl-step">
|
||||
<div class="hdr">步骤 0</div>
|
||||
<div class="tl-think">💭 思考：先对数据集进行概要分析，了解列类型和分布后再决定下一步。</div>
|
||||
<div class="tl-tool">
|
||||
<div class="hd"><span>🔧 profile_data</span><span>▶</span></div>
|
||||
<div class="bd"><div>输入:</div><pre>{"dataset_id": "upload_9"}</pre><div>输出:</div><pre>{"columns": 57, "numeric": 28, "string": 20, ...}</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tl-step">
|
||||
<div class="hdr">步骤 1</div>
|
||||
<div class="tl-think">💭 思考：数据包含 28 个数值列，选择聚类相关的特征列进行分析。</div>
|
||||
<div class="tl-tool">
|
||||
<div class="hd"><span>🔧 filter_data</span><span>▶</span></div>
|
||||
<div class="bd"><div>输入:</div><pre>{"dataset_id": "upload_9", "filters": [...]}</pre><div>输出:</div><pre>{"row_count": 1284592, "columns": 57}</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tl-step">
|
||||
<div class="hdr">步骤 2</div>
|
||||
<div class="tl-think">💭 思考：数据准备完成，对原始行进行聚类分析（KMeans，3 个簇）。</div>
|
||||
<div class="tl-tool">
|
||||
<div class="hd"><span>🔧 run_clustering</span><span>▶</span></div>
|
||||
<div class="bd"><div>输入:</div><pre>{"dataset_id": "upload_9", "algorithm": "kmeans", "cluster_columns": [...], "_timeout": 300}</pre><div>输出:</div><pre>{"n_clusters": 3, "n_noise": 0, "silhouette": 0.68}</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tl-step">
|
||||
<div class="hdr">步骤 3</div>
|
||||
<div class="tl-think">💭 思考：✅ 分析完成。聚类产生 3 个簇，轮廓系数 0.68。簇 #0（66%，低延迟大数据包）、簇 #1（24%，高TLS1.3短会话）、簇 #2（9%，高频非标准端口）。</div>
|
||||
<div style="background:#e8f5e9;padding:.5rem;border-radius:6px;font-size:.82rem;line-height:1.7;color:#333;border:1px solid #c8e6c9">
|
||||
<span style="font-weight:600;color:#2e7d32;display:block;margin-bottom:.25rem">✅ 回答</span>
|
||||
<strong>聚类分析完成</strong>，数据划分为 3 个行为簇：<br>
|
||||
• <strong>簇 #0</strong>（847,293 条，66%）：低延迟、大数据包、目标端口集中于443 —— 典型的 <strong>正常HTTPS流量</strong><br>
|
||||
• <strong>簇 #1</strong>（311,482 条，24%）：高TLS1.3比例、短会话、ECDHE X25519 —— <strong>现代浏览器访问</strong><br>
|
||||
• <strong>簇 #2</strong>（112,970 条，9%）：高连接频率、非标准端口、多域名 —— <strong>API/微服务调用</strong>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 聚类列表 -->
|
||||
<div class="card">
|
||||
<h3>📊 聚类结果</h3>
|
||||
<table class="cluster-table">
|
||||
<thead><tr><th>簇</th><th>大小</th><th>比例</th><th>轮廓系数</th><th></th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td><span class="badge badge-info">#0</span></td><td>847,293</td><td>0.66</td><td>0.7100</td><td><a href="#" class="btn" style="padding:.2rem .5rem;font-size:.75rem;">详情</a></td></tr>
|
||||
<tr><td><span class="badge badge-info">#1</span></td><td>311,482</td><td>0.24</td><td>0.6500</td><td><a href="#" class="btn" style="padding:.2rem .5rem;font-size:.75rem;">详情</a></td></tr>
|
||||
<tr><td><span class="badge badge-info">#2</span></td><td>112,970</td><td>0.09</td><td>0.5200</td><td><a href="#" class="btn" style="padding:.2rem .5rem;font-size:.75rem;">详情</a></td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// Toggle tool details on click
|
||||
document.querySelectorAll('.tl-tool .hd').forEach(function(el){
|
||||
el.onclick=function(){var b=this.nextElementSibling;if(b)b.style.display=b.style.display==='block'?'none':'block';};
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,356 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-hans">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1.0">
|
||||
<title>原型 — 聚类UI交互验证</title>
|
||||
<style>
|
||||
* { margin:0; padding:0; box-sizing:border-box; }
|
||||
body { font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif; background:#f5f7fa; color:#1a1a2e; }
|
||||
.header { background:#1a1a2e; color:#fff; padding:0.8rem 1.5rem; display:flex; justify-content:space-between; align-items:center; }
|
||||
.header h1 { font-size:1.1rem; }
|
||||
.header span { font-size:0.8rem; color:#aaa; }
|
||||
|
||||
.main-layout { display:flex; position:relative; transition:all 0.3s; min-height:calc(100vh - 48px); }
|
||||
.left-panel { flex:1; min-width:0; transition:flex 0.3s; padding:1rem; overflow-y:auto; }
|
||||
.left-panel.globe-open { flex:0 0 55%; }
|
||||
.right-panel { flex:0 0 0; overflow:hidden; transition:flex 0.3s; }
|
||||
.right-panel.globe-open { flex:0 0 45%; }
|
||||
|
||||
/* toggle handle */
|
||||
.toggle-handle { position:absolute; z-index:10; top:50%; width:20px; height:44px; background:#1a1a2e; color:#fff; border:none; border-radius:4px 0 0 4px; cursor:pointer; display:flex; align-items:center; justify-content:center; font-size:12px; transition:left 0.3s ease; }
|
||||
.toggle-handle:hover { background:#4361ee; }
|
||||
|
||||
/* scatter */
|
||||
.scatter-wrap { position:relative; background:#fff; border-radius:8px; padding:0.75rem; margin-bottom:1rem; box-shadow:0 1px 3px rgba(0,0,0,0.1); }
|
||||
.scatter-wrap h3 { font-size:0.95rem; margin-bottom:0.5rem; }
|
||||
.scatter-wrap canvas { width:100%; height:360px; display:block; cursor:crosshair; border-radius:4px; background:#fafbfc; }
|
||||
|
||||
/* pills */
|
||||
.pill-row { display:flex; flex-wrap:wrap; gap:4px; margin-bottom:0.75rem; }
|
||||
.pill { display:inline-flex; align-items:center; gap:3px; padding:2px 10px; border-radius:12px; font-size:0.75rem; cursor:pointer; border:2px solid transparent; transition:all 0.15s; color:#fff; white-space:nowrap; }
|
||||
.pill.all { background:#e8e8e8; color:#333; }
|
||||
.pill.active { border-color:#333; font-weight:700; }
|
||||
|
||||
/* cluster cards */
|
||||
.card-grid { display:grid; grid-template-columns:1fr 1fr; gap:0.75rem; margin-bottom:1rem; }
|
||||
.cluster-card { background:#fff; border:1px solid #e0e0e0; border-radius:8px; padding:0.8rem; cursor:pointer; transition:box-shadow 0.15s; }
|
||||
.cluster-card:hover { box-shadow:0 2px 8px rgba(0,0,0,0.1); }
|
||||
.cluster-card.selected { border-color:#4361ee; box-shadow:0 0 0 2px rgba(67,97,238,0.2); }
|
||||
.cluster-card .label { font-weight:700; font-size:1rem; }
|
||||
.cluster-card .meta { font-size:0.78rem; color:#666; margin-top:0.2rem; }
|
||||
.cluster-card .desc { font-size:0.8rem; color:#333; margin-top:0.4rem; padding:0.4rem; background:#f8f9fa; border-left:3px solid #4361ee; border-radius:0 4px 4px 0; line-height:1.5; }
|
||||
|
||||
/* detail panel */
|
||||
.detail-panel { position:fixed; bottom:0; left:0; right:0; z-index:100; transform:translateY(100%); transition:transform 0.35s cubic-bezier(.4,0,.2,1); }
|
||||
.detail-panel.open { transform:translateY(0); }
|
||||
.detail-body { background:#fff; border-radius:16px 16px 0 0; box-shadow:0 -4px 24px rgba(0,0,0,0.15); max-height:40vh; overflow-y:auto; padding:0.5rem 1.5rem 1.5rem; }
|
||||
.detail-handle { width:36px; height:4px; background:#ccc; border-radius:2px; margin:8px auto; cursor:pointer; }
|
||||
.detail-close { float:right; cursor:pointer; font-size:1.3rem; color:#999; line-height:1; border:none; background:none; }
|
||||
.detail-close:hover { color:#333; }
|
||||
.detail-body h3 { font-size:1rem; margin-bottom:0.3rem; }
|
||||
.detail-body p { font-size:0.85rem; color:#555; margin:0.2rem 0; }
|
||||
.detail-body table { width:100%; border-collapse:collapse; font-size:0.8rem; margin-top:0.5rem; }
|
||||
.detail-body th { background:#f5f5f5; padding:0.25rem 0.4rem; text-align:left; font-weight:600; border-bottom:2px solid #ddd; }
|
||||
.detail-body td { padding:0.2rem 0.4rem; border-bottom:1px solid #eee; }
|
||||
.detail-body td code { font-size:0.72rem; }
|
||||
|
||||
/* 3D地球容器 */
|
||||
#globeContainer { width:100%; height:calc(100vh - 48px); background:#0a0a1a; position:relative; }
|
||||
#globeContainer canvas { display:block; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="header">
|
||||
<h1>聚类概览 — 运行 #9</h1>
|
||||
<span>200 条流 · 3 个簇 · 12 个噪声点</span>
|
||||
</div>
|
||||
|
||||
<div class="main-layout" id="mainLayout">
|
||||
<!-- Left: cluster content -->
|
||||
<div class="left-panel" id="leftPanel">
|
||||
<!-- Scatter -->
|
||||
<div class="scatter-wrap">
|
||||
<h3>UMAP 嵌入 <span style="font-weight:400;font-size:0.8rem;color:#888;" id="ptCount">加载中...</span></h3>
|
||||
<canvas id="scatterChart"></canvas>
|
||||
</div>
|
||||
|
||||
<!-- Cluster pills -->
|
||||
<div class="pill-row" id="pillRow">
|
||||
<span class="pill all active" data-lbl="all" onclick="clearSel()">全部</span>
|
||||
</div>
|
||||
|
||||
<!-- Cluster cards -->
|
||||
<div class="card-grid" id="cardGrid"></div>
|
||||
</div>
|
||||
|
||||
<!-- Right: 3D Globe -->
|
||||
<div class="right-panel" id="rightPanel">
|
||||
<div id="globeContainer"></div>
|
||||
</div>
|
||||
|
||||
<!-- Toggle handle -->
|
||||
<button class="toggle-handle" id="globeHandle" onclick="toggleGlobe()">◀</button>
|
||||
</div>
|
||||
|
||||
<!-- Bottom detail panel -->
|
||||
<div class="detail-panel" id="detailPanel">
|
||||
<div class="detail-body" id="detailBody">
|
||||
<button class="detail-close" onclick="closeDetail()">✕</button>
|
||||
<div class="detail-handle" onclick="closeDetail()"></div>
|
||||
<div id="detailContent"><p style="color:#999;">选择一个簇或数据点查看详情</p></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script src="static/tianxuan/three.min.js"></script>
|
||||
<script>
|
||||
// ═══════════════════════════════════════════════
|
||||
// MOCK DATA
|
||||
// ═══════════════════════════════════════════════
|
||||
const MOCK_CLUSTERS = [
|
||||
{ label:0, size:78, prop:0.39, silh:0.72, desc:'低延迟 (23ms)，比全局均值高3.1σ；大数据包 (1520 bytes)，高2.3σ；系统端口 (443)' },
|
||||
{ label:1, size:65, prop:0.33, silh:0.68, desc:'高TLS1.3比例 (92%)，比均值高1.8σ；短会话 (12s)，低1.5σ；ECDHE曲线 X25519' },
|
||||
{ label:2, size:45, prop:0.23, silh:0.54, desc:'高连接频率 (85次/min)，比均值高4.2σ；非标准端口 (8080/8443)，目标多域名' },
|
||||
{ label:-1, size:12, prop:0.06, silh:null, desc:'噪声点 — 异常TLS指纹、零散目标、低流量' },
|
||||
];
|
||||
|
||||
const COLORS = ['#4361ee','#f72585','#7209b7','#4cc9f0','#f8961e','#43aa8b','#f94144','#577590','#e9c46a','#9b5de5'];
|
||||
function colorFor(lbl) { return lbl===-1?'rgba(150,150,150,0.7)':COLORS[Math.abs(lbl)%COLORS.length]; }
|
||||
|
||||
// Generate mock UMAP scatter data (200 points)
|
||||
const SCATTER = [];
|
||||
const RNG = (seed) => { let s=seed%2147483647; return ()=> { s=s*16807%2147483647; return (s-1)/2147483646; }; };
|
||||
const rng = RNG(42);
|
||||
const centers = {0:[2,1.5],1:[-1.8,0.5],2:[0.5,-2], '-1':[3.5,-1.5]};
|
||||
const spreads = {0:[0.6,0.5],1:[0.7,0.6],2:[0.5,0.7], '-1':[1.2,0.8]};
|
||||
for (const c of MOCK_CLUSTERS) {
|
||||
for (let i=0;i<c.size;i++) {
|
||||
const cx=centers[c.label][0], cy=centers[c.label][1];
|
||||
const sx=spreads[c.label][0], sy=spreads[c.label][1];
|
||||
SCATTER.push({
|
||||
x:cx+(rng()-0.5)*2*sx, y:cy+(rng()-0.5)*2*sy,
|
||||
label:c.label, entity:`模拟主机 ${1000+i}`
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════
|
||||
// SCATTER 2D CANVAS
|
||||
// ═══════════════════════════════════════════════
|
||||
let selCluster = null, selPoint = null;
|
||||
let globeOpen = false;
|
||||
|
||||
function drawScatter(highlight) {
|
||||
const c=document.getElementById('scatterChart');
|
||||
if(!c)return;
|
||||
const W=c.parentElement.clientWidth||700, H=360;
|
||||
c.width=W*2; c.height=H*2; c.style.width=W+'px'; c.style.height=H+'px';
|
||||
const ctx=c.getContext('2d'); ctx.scale(2,2);
|
||||
const pad={top:25,bottom:30,left:50,right:20};
|
||||
const pw=W-pad.left-pad.right, ph=H-pad.top-pad.bottom;
|
||||
const xs=SCATTER.map(d=>d.x), ys=SCATTER.map(d=>d.y);
|
||||
const xMin=Math.min(...xs), xMax=Math.max(...xs), yMin=Math.min(...ys), yMax=Math.max(...ys);
|
||||
const xR=xMax-xMin||1, yR=yMax-yMin||1;
|
||||
const toX=v=>pad.left+(v-xMin)/xR*pw, toY=v=>pad.top+ph-(v-yMin)/yR*ph;
|
||||
|
||||
ctx.clearRect(0,0,W,H);
|
||||
// grid
|
||||
ctx.strokeStyle='#eee'; ctx.lineWidth=1;
|
||||
for(let i=0;i<=5;i++){ctx.beginPath();ctx.moveTo(pad.left,pad.top+i*ph/5);ctx.lineTo(W-pad.right,pad.top+i*ph/5);ctx.stroke();}
|
||||
// points
|
||||
SCATTER.forEach(d=>{
|
||||
const cx=toX(d.x),cy=toY(d.y);
|
||||
let radius=3,color=colorFor(d.label);
|
||||
if(highlight!==null&&highlight!==undefined){
|
||||
if(d.label===highlight){color=colorFor(d.label);radius=5;}
|
||||
else{color='rgba(200,200,200,0.15)';radius=2;}
|
||||
}
|
||||
ctx.beginPath();ctx.arc(cx,cy,radius,0,Math.PI*2);ctx.fillStyle=color;ctx.fill();
|
||||
});
|
||||
// labels
|
||||
ctx.fillStyle='#999'; ctx.font='11px sans-serif'; ctx.textAlign='center';
|
||||
ctx.fillText('UMAP-1',W/2,H-3);
|
||||
ctx.save();ctx.translate(14,H/2);ctx.rotate(-Math.PI/2);ctx.fillText('UMAP-2',0,0);ctx.restore();
|
||||
|
||||
c.onclick=function(e){
|
||||
const r=c.getBoundingClientRect();
|
||||
const mx=e.clientX-r.left, my=e.clientY-r.top;
|
||||
const sx=mx/W*pw, sy=my/H*ph;
|
||||
let best=null,bestD=10;
|
||||
for(const d of SCATTER){
|
||||
const dx=toX(d.x)-mx, dy=toY(d.y)-my;
|
||||
const dist=Math.sqrt(dx*dx+dy*dy);
|
||||
if(dist<bestD){bestD=dist;best=d;}
|
||||
}
|
||||
if(best){selCluster=MOCK_CLUSTERS.find(c=>c.label===best.label);selPoint=best;showDetail(selCluster,selPoint);}
|
||||
};
|
||||
document.getElementById('ptCount').textContent=SCATTER.length+' 个点';
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════
|
||||
// CLUSTER SELECTION
|
||||
// ═══════════════════════════════════════════════
|
||||
function selectCluster(lbl){
|
||||
selCluster=MOCK_CLUSTERS.find(c=>c.label===lbl);
|
||||
selPoint=null;
|
||||
drawScatter(lbl);
|
||||
document.querySelectorAll('.pill').forEach(p=>p.classList.toggle('active',parseInt(p.dataset.lbl)===lbl));
|
||||
document.querySelectorAll('.cluster-card').forEach(p=>p.classList.toggle('selected',parseInt(p.dataset.label)===lbl));
|
||||
if(globeOpen)globeFilter(lbl);
|
||||
showDetail(selCluster,null);
|
||||
}
|
||||
function clearSel(){
|
||||
selCluster=null;selPoint=null;
|
||||
drawScatter(null);
|
||||
document.querySelectorAll('.pill,.cluster-card').forEach(p=>p.classList.remove('active','selected'));
|
||||
closeDetail();
|
||||
if(globeOpen)globeFilter(null);
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════
|
||||
// DETAIL PANEL
|
||||
// ═══════════════════════════════════════════════
|
||||
function showDetail(cluster,point){
|
||||
const dc=document.getElementById('detailContent');
|
||||
let html='';
|
||||
if(point){
|
||||
html+='<h3>数据点详情</h3><table><thead><tr><th>字段</th><th>值</th></tr></thead><tbody>'+
|
||||
'<tr><td>标识</td><td><code>'+point.entity+'</code></td></tr>'+
|
||||
'<tr><td>所属簇</td><td>#'+(point.label===-1?'噪声':point.label)+'</td></tr>'+
|
||||
'<tr><td>UMAP-x</td><td>'+point.x.toFixed(4)+'</td></tr>'+
|
||||
'<tr><td>UMAP-y</td><td>'+point.y.toFixed(4)+'</td></tr>'+
|
||||
'</tbody></table>';
|
||||
}else if(cluster){
|
||||
html+='<h3>簇 #'+(cluster.label===-1?'噪声':cluster.label)+'</h3>'+
|
||||
'<p>大小: '+cluster.size+' | 占比: '+(cluster.prop*100).toFixed(0)+'%'+
|
||||
(cluster.silh!==null?' | 轮廓系数: '+cluster.silh.toFixed(4):'')+'</p>'+
|
||||
'<p style="background:#f8f9fa;border-left:3px solid #4361ee;padding:0.5rem;margin:0.5rem 0;border-radius:0 4px 4px 0;font-size:0.85rem;line-height:1.6;">'+cluster.desc+'</p>'+
|
||||
'<table><thead><tr><th>特征</th><th>区分度</th><th>均值</th><th>标准差</th></tr></thead><tbody>'+
|
||||
'<tr><td>延迟 (4dur)</td><td style="color:#2e7d32;">+3.2</td><td>23ms</td><td>5.1</td></tr>'+
|
||||
'<tr><td>数据包大小 (8ppk)</td><td style="color:#2e7d32;">+2.3</td><td>1520B</td><td>128</td></tr>'+
|
||||
'<tr><td>TLS 版本 (0ver)</td><td style="color:#c62828;">-1.8</td><td>1.2</td><td>0.3</td></tr>'+
|
||||
'</tbody></table>';
|
||||
}
|
||||
dc.innerHTML=html;
|
||||
document.getElementById('detailPanel').classList.add('open');
|
||||
}
|
||||
function closeDetail(){document.getElementById('detailPanel').classList.remove('open');}
|
||||
|
||||
// ═══════════════════════════════════════════════
|
||||
// GLOBE SIDEBAR
|
||||
// ═══════════════════════════════════════════════
|
||||
function toggleGlobe(){
|
||||
globeOpen=!globeOpen;
|
||||
document.getElementById('leftPanel').classList.toggle('globe-open',globeOpen);
|
||||
document.getElementById('rightPanel').classList.toggle('globe-open',globeOpen);
|
||||
const h=document.getElementById('globeHandle');
|
||||
h.innerHTML=globeOpen?'▶':'◀';
|
||||
h.style.left=globeOpen?'calc(55% - 20px)':'calc(100% - 20px)';
|
||||
if(globeOpen&&!window._globeInit)initGlobe3D();
|
||||
}
|
||||
|
||||
function globeFilter(label){
|
||||
if(!window._arcMeshes)return;
|
||||
window._arcMeshes.forEach(m=>{
|
||||
if(label===null){m.material.opacity=0.5;m.material.color.setHex(parseInt(m.userData.color,16));}
|
||||
else{m.material.opacity=0.06;m.material.color.setHex(0x666666);}
|
||||
});
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════
|
||||
// 3D GLOBE (Three.js, simplified)
|
||||
// ═══════════════════════════════════════════════
|
||||
function initGlobe3D(){
|
||||
if(window._globeInit)return;
|
||||
const cont=document.getElementById('globeContainer');
|
||||
if(!cont)return;
|
||||
const W=cont.clientWidth||450, H=cont.clientHeight||500;
|
||||
|
||||
const scene=new THREE.Scene();scene.background=new THREE.Color(0x0a0a1a);
|
||||
const cam=new THREE.PerspectiveCamera(45,W/H,0.1,100);cam.position.set(0,0,6);
|
||||
const ren=new THREE.WebGLRenderer({antialias:true});ren.setSize(W,H);ren.setPixelRatio(Math.min(window.devicePixelRatio,2));
|
||||
cont.appendChild(ren.domElement);
|
||||
|
||||
// earth
|
||||
const earth=new THREE.Mesh(new THREE.SphereGeometry(2,48,48),new THREE.MeshPhongMaterial({color:0x1a3a5c,emissive:0x0a1a2a}));
|
||||
scene.add(earth);
|
||||
// glow
|
||||
scene.add(new THREE.Mesh(new THREE.SphereGeometry(2.06,48,48),new THREE.MeshBasicMaterial({color:0x224488,transparent:true,opacity:0.12})));
|
||||
// lights
|
||||
scene.add(new THREE.AmbientLight(0x222244));
|
||||
const dl=new THREE.DirectionalLight(0xffffff,1);dl.position.set(5,10,7);scene.add(dl);
|
||||
scene.add(new THREE.DirectionalLight(0x4488ff,0.3));
|
||||
// stars
|
||||
const s=new THREE.BufferGeometry();const p=new Float32Array(600*3);
|
||||
for(let i=0;i<600*3;i++)p[i]=(Math.random()-0.5)*100;
|
||||
s.setAttribute('position',new THREE.BufferAttribute(p,3));
|
||||
scene.add(new THREE.Points(s,new THREE.PointsMaterial({color:0xffffff,size:0.15})));
|
||||
|
||||
// mock flows → arcs
|
||||
const tlsC={'TLSv1.3':0x4cc9f0,'TLSv1.2':0x43aa8b,'TLSv1.1':0xf8961e,'TLSv1.0':0xf94144};
|
||||
const arcGrp=new THREE.Group();scene.add(arcGrp);
|
||||
const arcs=[];
|
||||
function llz(lat,lon,r){const p=(90-lat)*Math.PI/180,t=(lon+180)*Math.PI/180;return new THREE.Vector3(-r*Math.sin(p)*Math.cos(t),r*Math.cos(p),r*Math.sin(p)*Math.sin(t));}
|
||||
|
||||
// generate ~30 mock flows
|
||||
const cities=[{lat:31.2,lon:121.5},{lat:39.9,lon:116.4},{lat:35.7,lon:139.7},{lat:37.6,lon:127.0},{lat:51.5,lon:-0.13},{lat:48.9,lon:2.35},{lat:40.7,lon:-74.0},{lat:37.8,lon:-122.4},{lat:1.35,lon:103.8},{lat:-33.9,lon:151.2}];
|
||||
const tlsOpts=['TLSv1.3','TLSv1.2','TLSv1.3','TLSv1.2','TLSv1.1'];
|
||||
for(let i=0;i<30;i++){
|
||||
const src=cities[i%cities.length],dst=cities[(i+3)%cities.length];
|
||||
const f=llz(src.lat,src.lon,2),t=llz(dst.lat,dst.lon,2);
|
||||
const mid=new THREE.Vector3().addVectors(f,t).multiplyScalar(0.5).normalize().multiplyScalar(3.2);
|
||||
const curve=new THREE.QuadraticBezierCurve3(f,mid,t);
|
||||
const c=tlsOpts[i%tlsOpts.length];
|
||||
const mat=new THREE.MeshBasicMaterial({color:tlsC[c],transparent:true,opacity:0.5});
|
||||
const mesh=new THREE.Mesh(new THREE.TubeGeometry(curve,16,0.015,4,false),mat);
|
||||
mesh.userData={color:tlsC[c].toString(16)};
|
||||
arcGrp.add(mesh);arcs.push(mesh);
|
||||
}
|
||||
window._arcMeshes=arcs;
|
||||
|
||||
// drag
|
||||
let isD=false,pm={x:0,y:0};
|
||||
ren.domElement.onmousedown=e=>{isD=true;pm={x:e.clientX,y:e.clientY};};
|
||||
window.onmouseup=()=>isD=false;
|
||||
window.onmousemove=e=>{if(!isD)return;const dx=e.clientX-pm.x,dy=e.clientY-pm.y;earth.rotation.y+=dx*0.005;earth.rotation.x+=dy*0.005;arcGrp.rotation.y=earth.rotation.y;arcGrp.rotation.x=earth.rotation.x;pm={x:e.clientX,y:e.clientY};};
|
||||
|
||||
!function anim(){requestAnimationFrame(anim);if(!isD){earth.rotation.y+=0.003;arcGrp.rotation.y=earth.rotation.y;}ren.render(scene,cam);}();
|
||||
window._globeInit=true;
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════
|
||||
// RENDER PILLS & CARDS
|
||||
// ═══════════════════════════════════════════════
|
||||
(function init(){
|
||||
const pr=document.getElementById('pillRow');
|
||||
MOCK_CLUSTERS.forEach(c=>{
|
||||
if(c.label===-1)return;
|
||||
const p=document.createElement('span');p.className='pill';p.style.background=colorFor(c.label);
|
||||
p.dataset.lbl=c.label;p.textContent='#'+c.label+' ('+c.size+')';p.onclick=()=>selectCluster(c.label);
|
||||
pr.appendChild(p);
|
||||
});
|
||||
// noise pill
|
||||
const np=document.createElement('span');np.className='pill';np.style.background='#999';
|
||||
np.dataset.lbl='-1';np.textContent='噪声 ('+MOCK_CLUSTERS.find(c=>c.label===-1).size+')';np.onclick=()=>selectCluster(-1);
|
||||
pr.appendChild(np);
|
||||
|
||||
// cards
|
||||
const cg=document.getElementById('cardGrid');
|
||||
MOCK_CLUSTERS.forEach(c=>{
|
||||
if(c.label===-1)return;
|
||||
const card=document.createElement('div');card.className='cluster-card';card.dataset.label=c.label;
|
||||
card.innerHTML='<div class="label">簇 #'+c.label+'</div>'+
|
||||
'<div class="meta">'+c.size+' 个实体 · '+(c.prop*100).toFixed(0)+'%'+(c.silh!==null?' · 轮廓系数: '+c.silh.toFixed(4):'')+'</div>'+
|
||||
'<div class="desc">'+c.desc+'</div>';
|
||||
card.onclick=()=>selectCluster(c.label);
|
||||
cg.appendChild(card);
|
||||
});
|
||||
|
||||
drawScatter(null);
|
||||
// set initial handle pos
|
||||
document.getElementById('globeHandle').style.left='calc(100% - 20px)';
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,27 +1,12 @@
|
||||
@echo off
|
||||
set PYTHONUTF8=1
|
||||
cd /d "%~dp0"
|
||||
|
||||
:: Start server in a separate window (detached, survives parent exit)
|
||||
:: /MIN runs minimized so it doesn't pop up a console window
|
||||
start "" /MIN runtime\python\python.exe scripts\start_server.py
|
||||
|
||||
:: Wait for PID file (up to 10 seconds)
|
||||
set WAIT=0
|
||||
:waitloop
|
||||
if exist .server_pid goto gotpid
|
||||
ping -n 2 127.0.0.1 >nul
|
||||
set /a WAIT+=1
|
||||
if %WAIT% lss 5 goto waitloop
|
||||
|
||||
:gotpid
|
||||
powershell -NoProfile -Command "Start-Process -FilePath 'runtime\python\python.exe' -WindowStyle Hidden -ArgumentList 'scripts\start_server.py'"
|
||||
ping -n 4 127.0.0.1 >nul 2>&1
|
||||
if exist .server_pid (
|
||||
set /p PID=<.server_pid
|
||||
)
|
||||
if defined PID (
|
||||
echo TianXuan started. PID: %PID%
|
||||
echo Visit: http://127.0.0.1:8000/
|
||||
echo.
|
||||
echo To stop: taskkill /F /PID %PID%
|
||||
) else (
|
||||
echo TianXuan started.
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
"""Fix the unresolved merge conflict in views/clustering.py."""
|
||||
import re
|
||||
with open('analysis/views/clustering.py', 'r', encoding='utf-8-sig') as f:
|
||||
lines = f.readlines()
|
||||
|
||||
# Find and remove lines containing conflict markers
|
||||
new_lines = [l for l in lines if '<<<<<<<' not in l and '>>>>>>>' not in l and '=======' not in l]
|
||||
with open('analysis/views/clustering.py', 'w', encoding='utf-8') as f:
|
||||
f.writelines(new_lines)
|
||||
print(f'Removed {len(lines) - len(new_lines)} conflict marker lines')
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
"""Generate test CSVs with alternating high/low column coverage.
|
||||
Output files: 0.csv .. N-1.csv in specified directory.
|
||||
Columns that are all-blank in a CSV are dropped from that file.
|
||||
Usage:
|
||||
runtime\python\python.exe scripts\gen_coverage_test.py --files 2500 --rows 10000 --output-dir ../test_data
|
||||
"""
|
||||
import argparse, csv, os, random, sys
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
# ── Column spec with coverage profiles ──
|
||||
# HIGH = present in 95-100% of files, MEDIUM = 40-60%, LOW = 5-15%
|
||||
COLUMNS = [
|
||||
# HIGH coverage — always present
|
||||
(':ips', 1.0), (':ipd', 1.0), (':prs', 1.0), (':prd', 1.0),
|
||||
('scnt', 1.0), ('dcnt', 1.0), ('tabl', 1.0), ('name', 1.0),
|
||||
('source-node', 1.0), ('server-ip', 1.0), ('client-ip', 1.0),
|
||||
('timestamp', 1.0), ('time', 1.0),
|
||||
('1ipp', 1.0), ('4dbn', 1.0),
|
||||
# MEDIUM-HIGH coverage
|
||||
(':ips.latd', 0.85), (':ips.lond', 0.85),
|
||||
(':ipd.latd', 0.85), (':ipd.lond', 0.85),
|
||||
(':ips.ispn', 0.80), (':ipd.ispn', 0.80),
|
||||
(':ips.orgn', 0.70), (':ipd.orgn', 0.70),
|
||||
(':ips.city', 0.85), (':ipd.city', 0.85),
|
||||
('4dur', 0.95), ('8seq', 0.95), ('2tmo', 0.90),
|
||||
('8ack', 0.95), ('8pak', 0.90), ('8byt', 0.85), ('8ppk', 0.80),
|
||||
('8ses', 0.80), ('8did', 0.85), ('4srs', 0.80),
|
||||
# MEDIUM coverage — present in about half the files
|
||||
('0rnd', 0.55), ('0rnt', 0.55), ('0ver', 0.60),
|
||||
('snam', 0.50), ('cnam', 0.35),
|
||||
('0cph', 0.45), ('0crv', 0.35),
|
||||
('cnrs', 0.50), ('isrs', 0.50),
|
||||
# LOW coverage — rarely present
|
||||
(':ips.anon', 0.12), (':ipd.anon', 0.12),
|
||||
(':ips.doma', 0.15), (':ipd.doma', 0.15),
|
||||
('4ksz', 0.20), ('cipher-suite', 0.15),
|
||||
('ecdhe-named-curve', 0.12),
|
||||
('crcc', 0.06), ('orga', 0.06), ('orgu', 0.05),
|
||||
('eiph', 0.02), ('@iph', 0.02),
|
||||
]
|
||||
|
||||
def generate_row(row_idx: int, base_time: datetime, file_rng: random.Random,
|
||||
present_cols: set) -> dict:
|
||||
"""Generate a single row of TLS flow data."""
|
||||
row = {}
|
||||
src_octets = file_rng.randint(1, 223)
|
||||
src_ip = f'{src_octets}.{file_rng.randint(0,255)}.{file_rng.randint(0,255)}.{file_rng.randint(1,254)}'
|
||||
dst_octets = file_rng.randint(1, 223)
|
||||
dst_ip = f'{dst_octets}.{file_rng.randint(0,255)}.{file_rng.randint(0,255)}.{file_rng.randint(1,254)}'
|
||||
ts_val = base_time.timestamp() + file_rng.randint(0, 86400 * 7) + row_idx * 0.1
|
||||
|
||||
for col, _ in COLUMNS:
|
||||
if col not in present_cols:
|
||||
continue
|
||||
v = ''
|
||||
if col in (':ips', 'server-ip'):
|
||||
v = src_ip
|
||||
elif col in (':ipd', 'client-ip'):
|
||||
v = dst_ip
|
||||
elif col == ':prs':
|
||||
v = str(file_rng.randint(1024, 65535))
|
||||
elif col == ':prd':
|
||||
v = str(file_rng.choice([80, 443, 8080, 8443, 53, 22, 25, 993, 3306, 6379]))
|
||||
elif col in ('scnt', 'dcnt'):
|
||||
v = file_rng.choice(['CN', 'US', 'JP', 'KR', 'GB', 'DE', 'SG', 'HK', 'AU', 'RU'])
|
||||
elif col == 'tabl':
|
||||
v = file_rng.choice(['TlsC', 'TlsS'])
|
||||
elif col == 'name':
|
||||
v = file_rng.choice(['流量_01', '流量_02', '流量_03', '流量_04', '流量_05'])
|
||||
elif col == 'source-node':
|
||||
v = file_rng.choice(['node-a', 'node-b', 'node-c', 'node-d'])
|
||||
elif col in ('4dur', '8ses', '2tmo'):
|
||||
v = f'{file_rng.uniform(1, 300):.2f}' if file_rng.random() < 0.95 else ''
|
||||
elif col in ('8ack', '8pak', '8byt', '8ppk', '8did', '4srs'):
|
||||
v = str(file_rng.randint(50, 100000))
|
||||
elif col == '8seq':
|
||||
v = f'{file_rng.uniform(1, 1000):.2f}' if file_rng.random() < 0.95 else ''
|
||||
elif col in ('timestamp',):
|
||||
v = f'{ts_val:.2f}'
|
||||
elif col == 'time':
|
||||
dt = datetime.fromtimestamp(ts_val)
|
||||
v = dt.strftime('%Y-%m-%d-%H-%M-%S')
|
||||
elif col == '1ipp':
|
||||
v = str(file_rng.choice([6, 17, 1, 58]))
|
||||
elif col == '4dbn':
|
||||
v = str(file_rng.randint(1, 200))
|
||||
elif col in (':ips.latd', ':ipd.latd'):
|
||||
v = f'{file_rng.uniform(-90, 90):.4f}' if file_rng.random() < 0.98 else ''
|
||||
elif col in (':ips.lond', ':ipd.lond'):
|
||||
v = f'{file_rng.uniform(-180, 180):.4f}' if file_rng.random() < 0.98 else ''
|
||||
elif col in (':ips.ispn', ':ipd.ispn'):
|
||||
v = file_rng.choices(['电信', '联通', '移动', 'AWS', 'Azure', 'GCP', 'Cloudflare', ''], weights=[0.25,0.2,0.15,0.1,0.1,0.05,0.05,0.1], k=1)[0]
|
||||
elif col in (':ips.orgn', ':ipd.orgn'):
|
||||
v = file_rng.choices(['阿里巴巴', '腾讯', '华为', 'Amazon', 'Microsoft', 'Google', ''], weights=[0.2,0.15,0.1,0.15,0.1,0.05,0.25], k=1)[0]
|
||||
elif col in (':ips.city', ':ipd.city'):
|
||||
v = file_rng.choice(['北京', '上海', '深圳', '广州', '杭州', '东京', '新加坡', '伦敦', '纽约', '硅谷', '首尔', '悉尼', '法兰克福', ''])
|
||||
elif col in (':ips.anon', ':ipd.anon'):
|
||||
v = file_rng.choices(['', '+'], weights=[0.87, 0.13], k=1)[0]
|
||||
elif col in (':ips.doma', ':ipd.doma'):
|
||||
v = file_rng.choice(['example.com', 'api.example.com', 'cdn.example.net', 'db.internal', ''])
|
||||
elif col in ('cnrs', 'isrs'):
|
||||
v = file_rng.choices(['', '+', ''], weights=[0.5, 0.2, 0.3], k=1)[0]
|
||||
elif col in ('0ver',):
|
||||
v = file_rng.choices(['0303', '0304', '0302', ''], weights=[0.6, 0.3, 0.05, 0.05], k=1)[0]
|
||||
elif col in ('snam',):
|
||||
v = file_rng.choice(['www.example.com', 'api.server.com', 'cdn.cdn.net', 'mail.host.com', 'db.internal.net', ''])
|
||||
elif col in ('cnam',):
|
||||
v = file_rng.choice(['*.example.com', '*.server.com', 'www', ''])
|
||||
elif col in ('0rnd',):
|
||||
v = ''.join(file_rng.choices('0123456789abcdef', k=28)) if file_rng.random() < 0.99 else ''
|
||||
elif col in ('0rnt',):
|
||||
v = f'{file_rng.uniform(1, 2**32):.0f}' if file_rng.random() < 0.99 else ''
|
||||
elif col in ('4ksz',):
|
||||
v = str(file_rng.choice([128, 256, 1024, 2048, 4096]))
|
||||
elif col in ('0cph', '0crv'):
|
||||
v = file_rng.choice(['0e a6 3f 2b', 'ab cd ef 01', '12 34 56 78', '00 00 00 00', ''])
|
||||
elif col in ('cipher-suite',):
|
||||
v = file_rng.choice(['TLS_AES_128_GCM_SHA256', 'TLS_AES_256_GCM_SHA384', 'TLS_CHACHA20_POLY1305_SHA256', ''])
|
||||
elif col in ('ecdhe-named-curve',):
|
||||
v = file_rng.choice(['X25519', 'secp256r1', 'secp384r1', ''])
|
||||
elif col in ('crcc',):
|
||||
v = file_rng.choices(['AB', 'CD', 'EF', ''], weights=[0.4, 0.3, 0.1, 0.2], k=1)[0]
|
||||
elif col in ('orga', 'orgu'):
|
||||
v = file_rng.choice(['ORG_A', 'ORG_B', 'ORG_C', 'ORG_D', ''])
|
||||
elif col in ('eiph', '@iph'):
|
||||
v = file_rng.choices(['', '+', '', ''], weights=[0.02, 0.02, 0.48, 0.48], k=1)[0]
|
||||
row[col] = v
|
||||
return row
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Generate test CSVs with alternating column coverage')
|
||||
parser.add_argument('--files', type=int, default=100, help='Number of CSV files')
|
||||
parser.add_argument('--rows', type=int, default=10000, help='Rows per file')
|
||||
parser.add_argument('--output-dir', type=str, default='../test_data', help='Output directory')
|
||||
parser.add_argument('--seed', type=int, default=42, help='Random seed')
|
||||
args = parser.parse_args()
|
||||
|
||||
out_dir = Path(args.output_dir)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
base_time = datetime(2026, 1, 1, 0, 0, 0)
|
||||
total_rows = 0
|
||||
col_names = [c[0] for c in COLUMNS]
|
||||
|
||||
for file_idx in range(args.files):
|
||||
file_rng = random.Random(args.seed + file_idx * 7)
|
||||
# Decide per-file which LOW/MEDIUM columns are present
|
||||
present = set()
|
||||
for col, cov in COLUMNS:
|
||||
if file_rng.random() < cov:
|
||||
present.add(col)
|
||||
|
||||
# Generate rows, track which columns actually have data
|
||||
non_blank = {c: False for c in present}
|
||||
rows = []
|
||||
for row_i in range(args.rows):
|
||||
row = generate_row(total_rows + row_i, base_time, file_rng, present)
|
||||
rows.append(row)
|
||||
for col in present:
|
||||
if row.get(col, '') and not non_blank[col]:
|
||||
non_blank[col] = True
|
||||
|
||||
# Determine final column list: only columns that have at least some data
|
||||
final_cols = [c for c in col_names if non_blank.get(c, False)]
|
||||
if not final_cols:
|
||||
final_cols = [c for c in present] # fallback
|
||||
|
||||
# Write file
|
||||
out_path = out_dir / f'{file_idx}.csv'
|
||||
with open(out_path, 'w', newline='', encoding='utf-8') as f:
|
||||
writer = csv.DictWriter(f, fieldnames=final_cols, extrasaction='ignore')
|
||||
writer.writeheader()
|
||||
for row in rows:
|
||||
writer.writerow(row)
|
||||
|
||||
total_rows += args.rows
|
||||
if (file_idx + 1) % 100 == 0 or file_idx == 0 or file_idx == args.files - 1:
|
||||
print(f'[{file_idx+1}/{args.files}] {total_rows} rows, {len(final_cols)} cols, file={file_idx}.csv', flush=True)
|
||||
|
||||
print(f'\nDone: {args.files} files x {args.rows} rows = {total_rows} total rows')
|
||||
print(f'Output: {out_dir.resolve()}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -7,7 +7,6 @@ can run in the background without blocking any terminal.
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import webbrowser
|
||||
from pathlib import Path
|
||||
|
||||
_project_root = Path(__file__).resolve().parent.parent
|
||||
@@ -32,10 +31,8 @@ def main() -> None:
|
||||
# Write PID so run.bat can print it
|
||||
pid_file.write_text(str(os.getpid()), encoding='utf-8')
|
||||
|
||||
# Open browser
|
||||
webbrowser.open(f'http://127.0.0.1:{port}/')
|
||||
|
||||
# Run server with ALL output redirected to files
|
||||
# Run server (blocks until server stops).
|
||||
# Browser NOT opened here — webbrowser.open() hangs in hidden-window mode.
|
||||
with open(stdout_log, 'a', encoding='utf-8') as out_fh, \
|
||||
open(stderr_log, 'a', encoding='utf-8') as err_fh:
|
||||
rc = subprocess.call(
|
||||
|
||||
+27
-143
@@ -1,143 +1,27 @@
|
||||
Restored 1 datasets from disk
|
||||
[2026-07-24 12:30:18,755] INFO analysis.data_loader: [LOAD] files=1 total_rows_est=200
|
||||
[2026-07-24 12:30:18,742] INFO django: Restored 1 datasets from disk
|
||||
[2026-07-24 12:30:18,811] INFO analysis.data_loader: [LOAD_SCHEMA] col=0cph dtype=String
|
||||
[2026-07-24 12:30:18,811] INFO analysis.data_loader: [LOAD_SCHEMA] col=0crv dtype=String
|
||||
[2026-07-24 12:30:18,811] INFO analysis.data_loader: [LOAD_SCHEMA] col=0rnd dtype=String
|
||||
[2026-07-24 12:30:18,811] INFO analysis.data_loader: [LOAD_SCHEMA] col=0rnt dtype=String
|
||||
[2026-07-24 12:30:18,811] INFO analysis.data_loader: [LOAD_SCHEMA] col=0ver dtype=String
|
||||
[2026-07-24 12:30:18,811] INFO analysis.data_loader: [LOAD_SCHEMA] col=1ipp dtype=Int64
|
||||
[2026-07-24 12:30:18,811] INFO analysis.data_loader: [LOAD_SCHEMA] col=2tmo dtype=Float64
|
||||
[2026-07-24 12:30:18,811] INFO analysis.data_loader: [LOAD_SCHEMA] col=4dbn dtype=Int64
|
||||
[2026-07-24 12:30:18,817] INFO analysis.data_loader: [LOAD_SCHEMA] col=4dur dtype=Float64
|
||||
[2026-07-24 12:30:18,817] INFO analysis.data_loader: [LOAD_SCHEMA] col=4ksz dtype=Int64
|
||||
[2026-07-24 12:30:18,817] INFO analysis.data_loader: [LOAD_SCHEMA] col=4srs dtype=Int64
|
||||
[2026-07-24 12:30:18,817] INFO analysis.data_loader: [LOAD_SCHEMA] col=8ack dtype=Int64
|
||||
[2026-07-24 12:30:18,817] INFO analysis.data_loader: [LOAD_SCHEMA] col=8byt dtype=Int64
|
||||
[2026-07-24 12:30:18,817] INFO analysis.data_loader: [LOAD_SCHEMA] col=8dbd dtype=Int64
|
||||
[2026-07-24 12:30:18,818] INFO analysis.data_loader: [LOAD_SCHEMA] col=8did dtype=Int64
|
||||
[2026-07-24 12:30:18,818] INFO analysis.data_loader: [LOAD_SCHEMA] col=8pak dtype=Int64
|
||||
[2026-07-24 12:30:18,818] INFO analysis.data_loader: [LOAD_SCHEMA] col=8ppk dtype=Int64
|
||||
[2026-07-24 12:30:18,818] INFO analysis.data_loader: [LOAD_SCHEMA] col=8seq dtype=Float64
|
||||
[2026-07-24 12:30:18,818] INFO analysis.data_loader: [LOAD_SCHEMA] col=8ses dtype=Float64
|
||||
[2026-07-24 12:30:18,818] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ipd dtype=String
|
||||
[2026-07-24 12:30:18,818] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ipd.anon dtype=String
|
||||
[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ipd.city dtype=String
|
||||
[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ipd.doma dtype=String
|
||||
[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ipd.ispn dtype=String
|
||||
[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ipd.latd dtype=String
|
||||
[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ipd.lond dtype=String
|
||||
[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ipd.orgn dtype=String
|
||||
[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ips dtype=String
|
||||
[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ips.anon dtype=String
|
||||
[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ips.city dtype=String
|
||||
[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ips.doma dtype=String
|
||||
[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ips.ispn dtype=String
|
||||
[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ips.latd dtype=String
|
||||
[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ips.lond dtype=String
|
||||
[2026-07-24 12:30:18,822] INFO analysis.data_loader: [LOAD_SCHEMA] col=:ips.orgn dtype=String
|
||||
[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=:prd dtype=Int64
|
||||
[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=:prs dtype=Int64
|
||||
[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=cipher-suite dtype=String
|
||||
[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=client-ip dtype=String
|
||||
[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=cnam dtype=String
|
||||
[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=crcc dtype=String
|
||||
[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=dcnt dtype=String
|
||||
[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=ecdhe-named-curve dtype=String
|
||||
[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=name dtype=String
|
||||
[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=orga dtype=String
|
||||
[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=orgu dtype=Int64
|
||||
[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=scnt dtype=String
|
||||
[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=server-ip dtype=String
|
||||
[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=snam dtype=String
|
||||
[2026-07-24 12:30:18,827] INFO analysis.data_loader: [LOAD_SCHEMA] col=source-node dtype=String
|
||||
[2026-07-24 12:30:18,832] INFO analysis.data_loader: [LOAD_SCHEMA] col=tabl dtype=String
|
||||
[2026-07-24 12:30:18,832] INFO analysis.data_loader: [LOAD_SCHEMA] col=time dtype=String
|
||||
[2026-07-24 12:30:18,832] INFO analysis.data_loader: [LOAD_SCHEMA] col=timestamp dtype=Float64
|
||||
[2026-07-24 12:30:18,840] INFO analysis.data_loader: [CLEAN] Generic numeric coercion applied: [':ipd.lond', ':ipd.doma', ':ipd.ispn', 'cnam', ':ips.ispn', ':ips.doma', 'orga', 'snam', 'time', ':ips.orgn', ':ipd.orgn', 'crcc', ':ips.latd', ':ipd.latd', ':ips.city', ':ipd.city', ':ips.lond']
|
||||
[2026-07-24 12:30:18,840] INFO analysis.data_loader: [CLEAN] HEX columns added hex_pairs_count: ['0cph', '0crv', '0ver', '0rnt', '0rnd']
|
||||
[24/Jul/2026 12:30:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:31:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:31:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:32:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:32:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:33:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:33:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:34:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:34:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:35:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:35:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:36:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:36:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:37:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:37:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:38:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:38:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:39:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:39:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:40:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:40:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:41:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:41:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:42:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:42:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:43:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:43:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:44:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:44:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:45:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:45:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:46:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:46:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:47:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:47:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:48:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:48:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:49:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:49:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:50:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:50:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:51:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:51:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:52:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:52:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:53:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:53:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:54:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:54:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:55:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:55:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:56:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:56:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:57:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:57:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:58:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:58:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:59:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 12:59:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 13:00:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 13:00:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 13:01:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 13:01:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 13:02:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 13:02:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 13:03:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 13:03:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 13:04:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 13:04:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 13:05:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 13:05:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 13:06:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 13:06:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 13:07:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 13:07:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 13:08:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 13:08:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 13:09:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 13:09:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 13:10:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 13:10:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 13:11:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 13:11:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 13:12:07] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 13:12:37] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 23:20:02] "GET / HTTP/1.1" 200 21770
|
||||
[24/Jul/2026 23:20:33] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 23:21:03] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 23:21:33] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 23:22:03] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 23:22:33] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 23:23:03] "HEAD / HTTP/1.1" 200 0
|
||||
[24/Jul/2026 23:24:21] "HEAD / HTTP/1.1" 200 0
|
||||
[25/Jul/2026 09:15:14] "GET / HTTP/1.1" 200 21910
|
||||
[25/Jul/2026 09:15:38] "HEAD / HTTP/1.1" 200 0
|
||||
[25/Jul/2026 09:15:44] "HEAD / HTTP/1.1" 200 0
|
||||
[25/Jul/2026 09:16:14] "HEAD / HTTP/1.1" 200 0
|
||||
[25/Jul/2026 09:19:20] "HEAD / HTTP/1.1" 200 0
|
||||
[25/Jul/2026 09:19:25] "GET / HTTP/1.1" 200 21910
|
||||
[25/Jul/2026 09:19:55] "HEAD / HTTP/1.1" 200 0
|
||||
[25/Jul/2026 09:20:20] "HEAD / HTTP/1.1" 200 0
|
||||
[25/Jul/2026 09:20:25] "HEAD / HTTP/1.1" 200 0
|
||||
[25/Jul/2026 09:20:55] "HEAD / HTTP/1.1" 200 0
|
||||
[25/Jul/2026 09:21:20] "HEAD / HTTP/1.1" 200 0
|
||||
[25/Jul/2026 09:21:26] "HEAD / HTTP/1.1" 200 0
|
||||
[25/Jul/2026 09:21:55] "HEAD / HTTP/1.1" 200 0
|
||||
[25/Jul/2026 09:22:20] "HEAD / HTTP/1.1" 200 0
|
||||
[25/Jul/2026 09:22:25] "HEAD / HTTP/1.1" 200 0
|
||||
[25/Jul/2026 09:22:55] "HEAD / HTTP/1.1" 200 0
|
||||
[25/Jul/2026 09:23:20] "HEAD / HTTP/1.1" 200 0
|
||||
[25/Jul/2026 09:23:25] "HEAD / HTTP/1.1" 200 0
|
||||
[25/Jul/2026 09:23:56] "HEAD / HTTP/1.1" 200 0
|
||||
|
||||
+48
-2
@@ -1,8 +1,54 @@
|
||||
Performing system checks...
|
||||
|
||||
System check identified no issues (0 silenced).
|
||||
July 24, 2026 - 12:30:18
|
||||
July 24, 2026 - 23:20:02
|
||||
Django version 4.2.30, using settings 'tianxuan.settings'
|
||||
Starting development server at http://127.0.0.1:8765/
|
||||
Starting development server at http://0.0.0.0:8000/
|
||||
Quit the server with CTRL-BREAK.
|
||||
|
||||
Performing system checks...
|
||||
|
||||
System check identified no issues (0 silenced).
|
||||
Performing system checks...
|
||||
|
||||
System check identified no issues (0 silenced).
|
||||
Performing system checks...
|
||||
|
||||
System check identified no issues (0 silenced).
|
||||
July 25, 2026 - 09:14:52
|
||||
Django version 4.2.30, using settings 'tianxuan.settings'
|
||||
Starting development server at http://0.0.0.0:8000/
|
||||
Quit the server with CTRL-BREAK.
|
||||
|
||||
Performing system checks...
|
||||
|
||||
System check identified no issues (0 silenced).
|
||||
July 25, 2026 - 09:19:16
|
||||
Django version 4.2.30, using settings 'tianxuan.settings'
|
||||
Starting development server at http://0.0.0.0:8000/
|
||||
Quit the server with CTRL-BREAK.
|
||||
|
||||
Performing system checks...
|
||||
|
||||
System check identified no issues (0 silenced).
|
||||
July 25, 2026 - 09:57:53
|
||||
Django version 4.2.30, using settings 'tianxuan.settings'
|
||||
Starting development server at http://0.0.0.0:8000/
|
||||
Quit the server with CTRL-BREAK.
|
||||
|
||||
Performing system checks...
|
||||
|
||||
System check identified no issues (0 silenced).
|
||||
July 25, 2026 - 09:58:39
|
||||
Django version 4.2.30, using settings 'tianxuan.settings'
|
||||
Starting development server at http://0.0.0.0:8000/
|
||||
Quit the server with CTRL-BREAK.
|
||||
|
||||
Performing system checks...
|
||||
|
||||
System check identified no issues (0 silenced).
|
||||
July 25, 2026 - 09:59:44
|
||||
Django version 4.2.30, using settings 'tianxuan.settings'
|
||||
Starting development server at http://0.0.0.0:8000/
|
||||
Quit the server with CTRL-BREAK.
|
||||
|
||||
|
||||
@@ -1,10 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64">
|
||||
<defs>
|
||||
<linearGradient id="g" x1="0" y1="0" x2="1" y2="1">
|
||||
<stop offset="0%" stop-color="#4361ee"/>
|
||||
<stop offset="100%" stop-color="#7b2ff7"/>
|
||||
</linearGradient>
|
||||
</defs>
|
||||
<circle cx="32" cy="32" r="28" fill="url(#g)"/>
|
||||
<text x="32" y="42" text-anchor="middle" font-size="32" font-family="sans-serif" fill="#fff">璇</text>
|
||||
</svg>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32"><rect width="32" height="32" rx="4" fill="#1a1a2e"/><circle cx="16" cy="12" r="2" fill="#f72585"/><circle cx="23" cy="16" r="2" fill="#4cc9f0"/><circle cx="16" cy="21" r="2" fill="#43aa8b"/><circle cx="9" cy="16" r="2" fill="#f8961e"/><line x1="16" y1="14" x2="22" y2="16" stroke="#4cc9f0" stroke-width="0.6" opacity="0.6"/><line x1="22" y1="17" x2="17" y2="20" stroke="#4cc9f0" stroke-width="0.6" opacity="0.6"/><line x1="15" y1="20" x2="10" y2="17" stroke="#4cc9f0" stroke-width="0.6" opacity="0.6"/><line x1="10" y1="15" x2="15" y2="13" stroke="#4cc9f0" stroke-width="0.6" opacity="0.6"/><circle cx="16" cy="16" r="1" fill="#fff"/></svg>
|
||||
|
Before Width: | Height: | Size: 417 B After Width: | Height: | Size: 703 B |
@@ -113,11 +113,11 @@ USE_TZ = True
|
||||
|
||||
DATA_UPLOAD_MAX_NUMBER_FIELDS = 10000000
|
||||
DATA_UPLOAD_MAX_NUMBER_FILES = 10000
|
||||
DATA_UPLOAD_MAX_MEMORY_SIZE = 256 * 1024 * 1024
|
||||
FILE_UPLOAD_MAX_MEMORY_SIZE = 100 * 1024 * 1024
|
||||
DATA_UPLOAD_MAX_MEMORY_SIZE = 0 # 0 = no limit (stream to disk)
|
||||
FILE_UPLOAD_MAX_MEMORY_SIZE = 0 # stream all file uploads to disk
|
||||
|
||||
# Upload temp dir — use D: (3TB) to avoid filling C: (500GB OS drive)
|
||||
FILE_UPLOAD_TEMP_DIR = r'D:\TianXuan\temp'
|
||||
# Upload temp dir — use E: project drive to avoid filling C: (500GB OS drive)
|
||||
FILE_UPLOAD_TEMP_DIR = str(BASE_DIR / "tmp_uploads")
|
||||
os.makedirs(FILE_UPLOAD_TEMP_DIR, exist_ok=True)
|
||||
|
||||
STATIC_URL = "static/"
|
||||
|
||||
Reference in New Issue
Block a user