Compare commits
43 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 PM-pinou
|
8477291f53 | ||
|
PM-pinou
|
464642288e | ||
|
PM-pinou
|
a75c5a7a0e | ||
|
PM-pinou
|
10c9c46a84 | ||
|
PM-pinou
|
92e1305c07 | ||
|
PM-pinou
|
656f0ae2ab | ||
|
PM-pinou
|
806cfc0a64 | ||
|
PM-pinou
|
e54499fa78 | ||
|
PM-pinou
|
9f06e40a41 | ||
|
PM-pinou
|
6cbbc24f14 | ||
|
PM-pinou
|
c54917791e | ||
|
PM-pinou
|
3fd3896aba | ||
|
PM-pinou
|
66f68a2062 | ||
|
PM-pinou
|
b768d02987 | ||
|
PM-pinou
|
908bfcaa32 | ||
|
PM-pinou
|
0ca1e87b46 | ||
|
PM-pinou
|
30e48ba282 | ||
|
PM-pinou
|
df0f7d621f | ||
|
PM-pinou
|
92c1b0f768 | ||
|
PM-pinou
|
d24507e541 | ||
|
PM-pinou
|
f683a084f3 | ||
|
PM-pinou
|
1463e56a42 | ||
|
PM-pinou
|
244967bad5 | ||
|
PM-pinou
|
d2e34ca43d | ||
|
PM-pinou
|
b135b18779 | ||
|
PM-pinou
|
2c1aad6bd1 | ||
|
PM-pinou
|
5280ce5819 | ||
|
PM-pinou
|
e2c64fd068 | ||
|
PM-pinou
|
fa13ebcf72 | ||
|
PM-pinou
|
d7f9db098c | ||
|
PM-pinou
|
982cdcf83e | ||
|
PM-pinou
|
8e320b0059 | ||
|
PM-pinou
|
1105f12d89 | ||
|
PM-pinou
|
e340b49c71 | ||
|
PM-pinou
|
9508cc8d58 | ||
|
PM-pinou
|
88b26f16fe | ||
|
PM-pinou
|
ef60e3e489 | ||
|
PM-pinou
|
885da89b6f | ||
|
PM-pinou
|
d4c82768a8 | ||
|
PM-pinou
|
123db6b08f | ||
|
PM-pinou
|
02176cf6f4 | ||
|
PM-pinou
|
8bac406116 | ||
|
PM-pinou
|
12cd607837 |
+57
-50
@@ -1,51 +1,58 @@
|
||||
# Python-generated files
|
||||
__pycache__/
|
||||
*.py[oc]
|
||||
build/
|
||||
dist/
|
||||
wheels/
|
||||
*.egg-info
|
||||
.pytest_cache/
|
||||
# Python-generated files
|
||||
__pycache__/
|
||||
*.py[oc]
|
||||
build/
|
||||
dist/
|
||||
wheels/
|
||||
*.egg-info
|
||||
.pytest_cache/
|
||||
|
||||
# Python runtime (portable)
|
||||
runtime/
|
||||
|
||||
# Virtual environments
|
||||
.venv
|
||||
|
||||
# Database
|
||||
*.sqlite3
|
||||
*.sqlite3-wal
|
||||
*.sqlite3-shm
|
||||
|
||||
# Logs
|
||||
logs/
|
||||
*.log
|
||||
|
||||
# System
|
||||
Thumbs.db
|
||||
.DS_Store
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
|
||||
# OpenCode state
|
||||
.omo/
|
||||
|
||||
# Update cache (auto-generated by build_update.py)
|
||||
.update_cache/
|
||||
RUNTIME_CHANGED.txt
|
||||
|
||||
# Test data (auto-generated CSV files)
|
||||
data/*.csv
|
||||
data/multi_upload/
|
||||
|
||||
# User config (contains API keys)
|
||||
config/config.yaml
|
||||
|
||||
data/test_csvs/
|
||||
|
||||
data/geoip_chunks/
|
||||
*.mmdb.gz
|
||||
*.mmdb
|
||||
|
||||
# Playwright MCP sessions
|
||||
.playwright-mcp/
|
||||
|
||||
# Python runtime (portable)
|
||||
runtime/
|
||||
|
||||
# Virtual environments
|
||||
.venv
|
||||
|
||||
# Database
|
||||
*.sqlite3
|
||||
*.sqlite3-wal
|
||||
*.sqlite3-shm
|
||||
|
||||
# GeoIP MMDB (generated from geoip_chunks/)
|
||||
*.mmdb
|
||||
*.mmdb.gz
|
||||
data/geoip_chunks/
|
||||
|
||||
# Logs
|
||||
logs/
|
||||
*.log
|
||||
|
||||
# System
|
||||
Thumbs.db
|
||||
.DS_Store
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
|
||||
# OpenCode state
|
||||
.omo/
|
||||
|
||||
# Update cache (auto-generated by build_update.py)
|
||||
.update_cache/
|
||||
RUNTIME_CHANGED.txt
|
||||
|
||||
# Test data (auto-generated CSV files)
|
||||
data/**/*.csv
|
||||
data/multi_upload/
|
||||
data/gen_multi2/
|
||||
node_modules/
|
||||
package-lock.json
|
||||
package.json
|
||||
tmp_uploads/
|
||||
*.bat
|
||||
*.py
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
30672
|
||||
@@ -0,0 +1 @@
|
||||
16884
|
||||
@@ -0,0 +1,370 @@
|
||||
# 天璇 (TianXuan) — Agent Knowledge Base
|
||||
|
||||
## Project Identity
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| 项目名称 | 天璇 (TianXuan) |
|
||||
| 原始名称 | tls-analyzer |
|
||||
| 核心功能 | TLS 流数据分析、实体画像、聚类 3D 地球可视化、UMAP/SVD 降维散点图 |
|
||||
| Python 版本 | 3.12 (embedded portable at `runtime/python/python.exe`) |
|
||||
| Polars 版本 | 1.42.1 (精确锁定) |
|
||||
| 数据框架 | Polars (LazyFrame + streaming) |
|
||||
| 机器学习 | scikit-learn (HDBSCAN, KMeans, MiniBatchKMeans, IsolationForest), UMAP, TruncatedSVD, StandardScaler |
|
||||
| Web 框架 | Django 4.2 + SQLite WAL |
|
||||
| 前端 3D | Three.js (603KB, 离线, 地球贴图 1.4MB) |
|
||||
| 前端图表 | 纯 Canvas 2D 散点图 / Canvas 地理分布 |
|
||||
| LLM 协议 | MCP (stdio transport) + OpenAI 兼容 API |
|
||||
| 启动方式 | `run.bat` (双击即用, 非阻塞, 打印 PID 后退出) / `runtime\python\python.exe manage.py runserver` (开发) |
|
||||
| 目标设备 | Windows 10 1909+, 8GB RAM, 无独立显卡 (核显), 支持 2500 CSV × 10000 行 × 50~58 列 |
|
||||
|
||||
## Project Structure
|
||||
|
||||
```
|
||||
天璇/
|
||||
├── tianxuan/ Django 项目配置
|
||||
│ ├── settings.py ALLOWED_HOSTS 自动检测, LOGGING 文件+stderr
|
||||
│ ├── wsgi.py SQLite 启动自修复
|
||||
│ ├── urls.py 路由汇总
|
||||
│ └── llm_orchestrator.py LLM 编排 (策略式提示词, 自动错误恢复)
|
||||
├── analysis/ 核心分析模块 (Django app)
|
||||
│ ├── views/ 视图包 (12 模块, 原 views.py 拆分)
|
||||
│ │ ├── __init__.py 重导出所有视图函数
|
||||
│ │ ├── helpers.py 辅助函数
|
||||
│ │ ├── dashboard.py 首页 + 运行记录
|
||||
│ │ ├── pipeline.py 管道执行器
|
||||
│ │ ├── clustering.py 聚类概览/详情/地球数据
|
||||
│ │ ├── entity.py 实体画像页
|
||||
│ │ ├── upload.py CSV 上传
|
||||
│ │ ├── manual.py 手动分析工作流构建器
|
||||
│ │ ├── auto.py LLM 自动分析
|
||||
│ │ ├── globe.py 3D 地球数据接口
|
||||
│ │ ├── config.py 配置编辑 + LLM 测试
|
||||
│ │ ├── log_viewer.py 日志查看器
|
||||
│ │ └── tools.py 工具实验室
|
||||
│ ├── tools/ 工具包 (17 模块, 30 个 MCP 工具实现)
|
||||
│ │ ├── __init__.py 重导出所有 handler
|
||||
│ │ ├── _registry.py 工具元数据 (Tool descriptor + get_tools_meta)
|
||||
│ │ ├── _dispatch.py 工具调用分发 (handle_call)
|
||||
│ │ ├── _helpers.py 共享辅助 (过滤表达式构建/响应截断/数据集决议/数值列检测)
|
||||
│ │ ├── load_data.py 工具 1: load_data
|
||||
│ │ ├── profile.py 工具 2: profile_data
|
||||
│ │ ├── filter.py 工具 3: filter_data
|
||||
│ │ ├── preprocess.py 工具 4: preprocess_data
|
||||
│ │ ├── clustering.py 工具 5+8: run_clustering, filter_and_cluster
|
||||
│ │ ├── evaluate.py 工具 6: evaluate_clustering
|
||||
│ │ ├── features.py 工具 7: extract_features (+ UMAP 嵌入)
|
||||
│ │ ├── entities.py 工具 9+10: build_entity_profiles, compute_scores
|
||||
│ │ ├── anomalies.py 工具 11+12: detect_anomalies, visualize_anomalies
|
||||
│ │ ├── export.py 工具 13: export_results
|
||||
│ │ ├── analysis.py 工具 14-19: analyze_patterns/temporal/fft/tls_health/geo_distribution/entity_detail
|
||||
│ │ ├── diagnostics.py 工具 20-26: validate_data/explore_distributions/find_outliers/diagnose_clustering/compare_datasets/export_debug_sample/repair_schema
|
||||
│ │ ├── data_mgmt.py 工具 27-29: list_datasets/drop_dataset/clone_dataset
|
||||
│ │ └── distance_matrix.py 工具 30: compute_distance_matrix
|
||||
│ ├── tool_registry.py 旧编排层 (薄包装, 重导出 tools/)
|
||||
│ ├── mcp_server.py MCP stdio server
|
||||
│ ├── data_loader.py CSV 加载底层 (BOM/编码/schema 容错/递归 glob/ZIP 解压)
|
||||
│ ├── data_profiler.py 列统计 + 相关性矩阵
|
||||
│ ├── data_validator.py 列校验 (缺失/异常值/IP 有效性)
|
||||
│ ├── type_classifier.py 值优先类型检测 (MAC/端口/IPv4/URL/HEX/ENUM/LAT_LON/BOOL_ENUM/TIMESTAMP)
|
||||
│ ├── geoip.py GeoIP 经纬度查询
|
||||
│ ├── ip_clustering.py IP 子网聚类与转换
|
||||
│ ├── session_store.py 线程安全单例内存存储
|
||||
│ ├── distance.py 距离计算 (IP 子网/地理/Levenshtein/Hamming/FFT 相位)
|
||||
│ ├── models.py ORM: AnalysisRun, ClusterResult, EntityProfile, ClusterFeature
|
||||
│ ├── urls.py 路由注册
|
||||
│ ├── nl_describe.py 自然语言数据描述
|
||||
│ ├── profile_util.py 画像工具函数
|
||||
│ ├── tls_ref.py TLS 版本/加密套件参考数据
|
||||
│ ├── db_utils.py SQLite WAL 工具 + 锁重试
|
||||
│ ├── constants.py 全局常量
|
||||
│ ├── appdata.py 应用数据路径 (%APPDATA%/TianXuan)
|
||||
│ └── management/commands/
|
||||
│ ├── start_mcp.py MCP 服务器启动命令
|
||||
│ ├── run_pipeline.py 一键 CLI 管道命令
|
||||
│ └── import_tlsdb.py TLS 数据库导入
|
||||
├── config/ 配置管理
|
||||
│ ├── config.yaml 自动生成默认配置 (entity, server, data, clustering, llm)
|
||||
│ ├── loader.py Pydantic 配置加载 + 文件修改检测 + 缓存
|
||||
│ └── __init__.py
|
||||
├── templates/ 页面模板 (13 个 HTML)
|
||||
│ ├── base.html 导航: 首页/上传/手动/LLM/记录/地球/配置
|
||||
│ ├── analysis/ 分析页面 (6 个)
|
||||
│ │ ├── dashboard.html 首页 — 最近运行
|
||||
│ │ ├── run_list.html 运行记录列表
|
||||
│ │ ├── run_detail.html 运行详情摘要
|
||||
│ │ ├── cluster_overview.html 聚类概览 — Canvas 散点图 (PC1/PC2) + 地理分布 + Silhouette
|
||||
│ │ ├── cluster_detail.html 簇详情 — 特征实体列表
|
||||
│ │ └── entity_profile.html 实体画像 — 特征偏离 + Canvas 单点
|
||||
│ └── tianxuan/ 功能页面 (6 个)
|
||||
│ ├── upload.html CSV 上传 — 拖拽+多文件删除+进度
|
||||
│ ├── manual.html 手动分析 — 工作流构建器 (添加/排序/保存/执行)
|
||||
│ ├── auto.html LLM 自动分析 — 实时日志, thinking panel, 工具调用折叠
|
||||
│ ├── globe.html 3D 地球 — Three.js 流量弧线 + 经纬线 + 国境线
|
||||
│ ├── config.html 配置编辑 — LLM 连通性测试
|
||||
│ └── log_viewer.html 日志查看
|
||||
├── static/tianxuan/ 离线静态资源
|
||||
│ ├── three.min.js Three.js 3D 引擎 (603KB)
|
||||
│ ├── earth_atmos_2048.jpg 地球贴图 (1.4MB)
|
||||
│ └── world_borders.js 177 国完整国境线 (269KB)
|
||||
├── runtime/python/ 便携 Python 3.12 运行时 (~593MB)
|
||||
├── scripts/ 工具脚本
|
||||
│ ├── gen_test_data.py 简单测试数据生成
|
||||
│ ├── gen_complex_test.py 复杂数据生成 (5000 行 含 55% 缺失)
|
||||
│ ├── column_survey.py CSV 列结构调查
|
||||
│ ├── diagnose_compare.py 跨机日志对比诊断
|
||||
│ └── debug_db.py DB 持久化调试
|
||||
├── tests/ 测试
|
||||
│ ├── test_e2e_full.py 全端到端测试
|
||||
│ ├── test_e2e_batch.py 批量端到端测试
|
||||
│ ├── start_srv.py 测试服务器启动辅助
|
||||
│ └── _test_hdbscan.py HDBSCAN 零样本降级测试
|
||||
├── data/ 数据目录
|
||||
│ └── geoip_data.txt GeoIP 离线数据库
|
||||
├── logs/ 日志输出
|
||||
│ └── tianxuan.log 主日志文件
|
||||
├── docs/
|
||||
│ ├── 工作流.md 完整分析流水线文档
|
||||
│ └── 故障诊断手册.md 跨机问题诊断指南
|
||||
├── run.bat 用户启动入口 (含 PYTHONUTF8=1)
|
||||
├── shell.bat Django shell
|
||||
├── update.bat 增量更新
|
||||
├── rollback.bat 更新回滚
|
||||
├── build.bat 构建打包
|
||||
├── TlsDB.csv 参考表头名清单
|
||||
├── README.md
|
||||
├── AGENTS.md
|
||||
└── manage.py
|
||||
```
|
||||
|
||||
## 30 MCP Tools
|
||||
|
||||
系统现有 **30 个 MCP 工具**, 分四层, 实现于 `analysis/tools/` 包的 14 个 handler 模块:
|
||||
|
||||
### ⚡ 核心工具 (14 个) — 执行分析, 可修改数据
|
||||
|
||||
| # | 工具 | 实现模块 | 功能 | 必需参数 |
|
||||
|---|------|---------|------|----------|
|
||||
| 1 | `load_data` | tools/load_data.py | 加载 CSV 文件 (glob/递归/ZIP 解压/schema 容错) | `csv_glob` |
|
||||
| 2 | `profile_data` | tools/profile.py | 数据集概要统计 + 相关性矩阵 | `dataset_id` |
|
||||
| 3 | `filter_data` | tools/filter.py | 按条件过滤 (12 操作符 + AND/OR 逻辑) | `dataset_id`, `filters` |
|
||||
| 4 | `preprocess_data` | tools/preprocess.py | 预处理 (StandardScaler/OneHot/填充/列删除) | `dataset_id`, `columns` |
|
||||
| 5 | `run_clustering` | tools/clustering.py | 执行聚类 (HDBSCAN/KMeans + 自动特征过滤 + 质量评估) | `dataset_id`, `cluster_columns` |
|
||||
| 6 | `evaluate_clustering` | tools/evaluate.py | 评估聚类质量 (Silhouette/DB/CH/噪声比/簇大小) | `cluster_result_id`, `dataset_id` |
|
||||
| 7 | `extract_features` | tools/features.py | 提取各聚类区分特征 (Z-Score/ANOVA) + UMAP-2D 嵌入 | `dataset_id`, `cluster_result_id` |
|
||||
| 8 | `filter_and_cluster` | tools/clustering.py | 一步完成: 过滤 + 自动选数值列 + 聚类 | `dataset_id`, `filters` |
|
||||
| 9 | `build_entity_profiles` | tools/entities.py | 自动检测实体列 + 聚合特征 (多列复合键支持) | `dataset_id` |
|
||||
| 10 | `compute_scores` | tools/entities.py | 自适应代理/异常/威胁评分 (SVD 权重 + IQR 阈值) | `dataset_id` |
|
||||
| 11 | `detect_anomalies` | tools/anomalies.py | Isolation Forest 异常检测 (分块处理大数据集) | `dataset_id` |
|
||||
| 12 | `visualize_anomalies` | tools/anomalies.py | UMAP-2D 嵌入 + 聚类可视化 (按风险等级/聚类着色) | `dataset_id` |
|
||||
| 13 | `export_results` | tools/export.py | 导出结果到磁盘 (CSV/Parquet/JSON) | `result_id`, `output_path` |
|
||||
| - | `export_entities_json` | tools/export.py | 导出实体画像为 JSON (附加工具) | `dataset_id`, `output_path` |
|
||||
|
||||
### 🔍 分析工具 (6 个) — 只读深潜, 安全随时调用
|
||||
|
||||
| # | 工具 | 实现模块 | 功能 |
|
||||
|---|------|---------|------|
|
||||
| 14 | `analyze_patterns` | tools/analysis.py | 流量模式: top 源/目标 IP、端口/TLS 版本/协议分布 |
|
||||
| 15 | `analyze_temporal` | tools/analysis.py | 时间维度流量分析: 小时/日流量计数、繁忙/空闲时段 |
|
||||
| 16 | `analyze_fft` | tools/analysis.py | FFT 频谱分析: 周期模式、频率特征、周期性评分 |
|
||||
| 17 | `analyze_tls_health` | tools/analysis.py | TLS 安全态势: 旧版比例、弱加密、证书问题、SNI 异常 |
|
||||
| 18 | `analyze_geo_distribution` | tools/analysis.py | 地理分布: top 来源国家、不可能旅行、ISP 多样性 |
|
||||
| 19 | `analyze_entity_detail` | tools/analysis.py | 单实体深度调查: 流/版本/目标/时间模式/异常评分 |
|
||||
|
||||
### 🩺 诊断工具 (7 个) — 排查问题
|
||||
|
||||
| # | 工具 | 实现模块 | 功能 |
|
||||
|---|------|---------|------|
|
||||
| 20 | `validate_data` | tools/diagnostics.py | 数据质量: schema 一致性、缺失率、列类型冲突 |
|
||||
| 21 | `explore_distributions` | tools/diagnostics.py | 列分布统计: 唯一值、空值率、min/max/mean/std、直方图 |
|
||||
| 22 | `find_outliers` | tools/diagnostics.py | IQR 方法检测数值列统计异常值 |
|
||||
| 23 | `diagnose_clustering` | tools/diagnostics.py | 聚类效果差时诊断: 方差/相关分析 + 参数建议 |
|
||||
| 24 | `compare_datasets` | tools/diagnostics.py | 对比两数据集: schema 差异、行数变化、值分布偏移 |
|
||||
| 25 | `export_debug_sample` | tools/diagnostics.py | 导出原始数据样本 JSON 供外部调试 |
|
||||
| 26 | `repair_schema` | tools/diagnostics.py | 修复 schema 不匹配: 列名对齐、大小写合并、填充缺失 |
|
||||
|
||||
### 📦 数据管理 (3 个)
|
||||
|
||||
| # | 工具 | 实现模块 | 功能 |
|
||||
|---|------|---------|------|
|
||||
| 27 | `list_datasets` | tools/data_mgmt.py | 列出所有活跃数据集和结果 |
|
||||
| 28 | `drop_dataset` | tools/data_mgmt.py | 删除数据集释放内存 (支持 dataset/cluster/feature 三种类型) |
|
||||
| 29 | `clone_dataset` | tools/data_mgmt.py | 浅拷贝数据集 (LazyFrame 查询计划, 不复制数据) |
|
||||
|
||||
### 🧠 LLM 驱动 (1 个)
|
||||
|
||||
| # | 工具 | 实现模块 | 功能 |
|
||||
|---|------|---------|------|
|
||||
| 30 | `compute_distance_matrix` | tools/distance_matrix.py | LLM 编写 Python 距离函数, 逐行执行并返回评分摘要 |
|
||||
|
||||
**工具注册流程**: `tools/_registry.py` (Tool 元数据) → `tools/_dispatch.py` (分发到对应 handler) → `analysis/mcp_server.py` (暴露 MCP stdio 接口)
|
||||
|
||||
### 手动分析工作流
|
||||
|
||||
手动分析页面 (manual.html) 为**工作流构建器**:
|
||||
- 添加/移除/重排步骤
|
||||
- 每步选择工具 + 填写参数
|
||||
- **保存/加载/删除**方案 (存储于 `.omo/plans/<name>.json`)
|
||||
- 单步执行 / **全部执行** (自动串行)
|
||||
- 执行结果显示在每一步下面
|
||||
|
||||
### 标准分析流程
|
||||
|
||||
```
|
||||
load_data → filter_data → profile_data → build_entity_profiles → compute_scores → run_clustering → extract_features
|
||||
```
|
||||
|
||||
LLM 自动模式由 `tianxuan/llm_orchestrator.py` 驱动, 策略式提示词: 先 profile 了解数据 → 根据数据决策 → 失败自动诊断 → 重复调用检测 + 自动错误恢复。
|
||||
|
||||
## Config.Entity
|
||||
|
||||
`config/loader.py` 中的 Pydantic 模型:
|
||||
|
||||
```python
|
||||
class Entity(BaseModel):
|
||||
subnet_masks: list[int] = [] # 子网掩码列表, 如 [24, 28]
|
||||
ip_columns: list[str] = ['src_ip', 'dst_ip'] # 需聚合的 IP 列
|
||||
```
|
||||
|
||||
config.yaml 默认配置:
|
||||
```yaml
|
||||
entity:
|
||||
subnet_masks: [24, 28]
|
||||
ip_columns: ["src_ip", "dst_ip"]
|
||||
```
|
||||
|
||||
IP 子网聚合逻辑位于 `analysis/ip_clustering.py`, `tools/entities.py::_handle_build_entity_profiles` 在 group_by 前对 IP 列自动应用子网掩码。
|
||||
|
||||
---
|
||||
|
||||
## Architecture
|
||||
|
||||
- **views 包拆分**: 原单体 `views.py` (2000+ 行) 拆分为 12 个模块: dashboard, pipeline, clustering, entity, upload, manual, auto, globe, config, log_viewer, tools, helpers
|
||||
- **tools 包拆分**: 原 `tool_registry.py` 拆分为 17 个模块 (14 个 handler + 3 个基础设施: _registry/_dispatch/_helpers)
|
||||
- **tuple 替代 frozenset**: 避免跨机 PYTHONHASHSEED 差异
|
||||
- **PYTHONUTF8=1**: 跨机编码一致性 (run.bat 内置)
|
||||
- **类型安全聚合**: 聚合前检查 dtype, 非数值列 cast
|
||||
- **sync_to_async ORM**: 异步上下文中 Django ORM 安全调用
|
||||
- **纯 Canvas 图表**: 零外部 JS 依赖 (散点图/地理分布/Silhouette 柱状图)
|
||||
- **上传目录隔离**: `%APPDATA%/TianXuan/data/uploads/`, 与项目路径无关
|
||||
- **通用数值清理 _coerce_to_float**: 处理 `+` / `-` / 空白 / N/A 等伪值
|
||||
- **值优先类型检测**: config → 值 → 名称 → STRING (type_classifier.py)
|
||||
- **地球深度测试**: depthTest: true, depthWrite: false, r=5.5
|
||||
- **完整国境线**: Natural Earth 110m, 177 国 286 多边形 269KB
|
||||
- **UMAP 降维**: extract_features 中为实体数据计算 UMAP-2D 嵌入, 存储到 EntityProfile 表 (embedding_x/embedding_y), 训练 10K 样本 + 批量变换 1K 批次
|
||||
- **TruncatedSVD**: 聚类时特征 > 50 维自动降维; compute_scores 中学习自适应评分权重
|
||||
- **LLM 驱动距离计算**: compute_distance_matrix 工具允许 LLM 编写 Python 函数在受限命名空间 (math, numpy) 中逐行执行
|
||||
- **距离计算多策略**: tools/distance_matrix.py 中的 compute_distance_matrix 和 analysis/distance.py 中的 IP 子网/地理/Levenshtein/Hamming/FFT 相位距离
|
||||
|
||||
---
|
||||
|
||||
## Testing Protocol
|
||||
|
||||
### 运行测试
|
||||
|
||||
```bash
|
||||
# 端到端测试 (自动启动后端、模拟前端行为、监控 stderr)
|
||||
runtime\python\python.exe tests\test_e2e_full.py
|
||||
runtime\python\python.exe tests\test_e2e_batch.py
|
||||
|
||||
# HDBSCAN 零样本降级测试
|
||||
runtime\python\python.exe tests\_test_hdbscan.py
|
||||
```
|
||||
|
||||
### 跨机一致性测试
|
||||
|
||||
```bash
|
||||
# 两台机器各自执行
|
||||
runtime\python\python.exe manage.py run_pipeline data\complex_test.csv
|
||||
|
||||
# 对比日志
|
||||
runtime\python\python.exe scripts\diagnose_compare.py log_a.txt log_b.txt
|
||||
```
|
||||
|
||||
### DB 持久化测试
|
||||
|
||||
```bash
|
||||
runtime\python\python.exe scripts\debug_db.py
|
||||
# 期望: n_features=30 db_saved=True
|
||||
# 验证: AnalysisRun, ClusterFeature 表有数据
|
||||
```
|
||||
|
||||
### LLM 编排测试
|
||||
|
||||
```bash
|
||||
runtime\python\python.exe scripts\test_llm_full.py
|
||||
```
|
||||
|
||||
### 生成测试数据
|
||||
|
||||
```bash
|
||||
# 简单数据
|
||||
runtime\python\python.exe scripts\gen_test_data.py --rows 1000
|
||||
|
||||
# Globe 模式 (IP 在真实 GeoIP 范围)
|
||||
runtime\python\python.exe scripts\gen_test_data.py --globe
|
||||
|
||||
# 复杂数据 (5000 行 4 列含 55% 缺失)
|
||||
runtime\python\python.exe scripts\gen_complex_test.py --rows 5000
|
||||
```
|
||||
|
||||
### 列结构调查
|
||||
|
||||
```bash
|
||||
runtime\python\python.exe scripts\column_survey.py --csv "data/*.csv"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Polars Version Compatibility
|
||||
|
||||
| API | 旧版 (<1.0) | 新版 (>=1.0) |
|
||||
|-----|-----------|-----------|
|
||||
| mode | `pl.mode()` | `pl.col(col).mode()` |
|
||||
| encoding | `encoding='utf-8'` | `encoding='utf8'` |
|
||||
| value_counts col | `'index'` | 原列名 |
|
||||
| streaming | `collect(streaming=True)` | `collect(engine='streaming')` |
|
||||
| truth value | `if series:` | `if series.item():` |
|
||||
|
||||
## 32B LLM Optimization
|
||||
|
||||
- **Step guidance**: 每步告诉模型下一步做什么, 减少 32B 模型推理发散
|
||||
- **Truncation**: 工具结果截断 1200 chars (避免上下文窗口溢出)
|
||||
- **Timeout**: 90s (32B 推理更慢)
|
||||
- **Max steps**: 15
|
||||
- **Tool description**: 一行简洁描述 (中文)
|
||||
- **策略式提示词**: 先 profile → 根据数据决策 → 失败自动诊断 → 重复调用检测 + 自动错误恢复
|
||||
- **工具结果自动截断**: `_truncate_response()` 确保单次返回不超过 64 KB, 长列表/矩阵自动压缩
|
||||
|
||||
---
|
||||
|
||||
## Operating Procedures
|
||||
|
||||
1. **非阻塞启动后端** — `run.bat` 使用 `start /B` 启动 Django, 打印 PID 后退出, 不阻塞终端。日志写入 `logs/tianxuan.log`。
|
||||
|
||||
2. **全量测试** — 所有改动完成后, 运行完整 E2E 测试: 生成测试 CSV, 上传前端, 验证全流水线。
|
||||
|
||||
3. **更新文档和 Git** — 测试通过后: 更新 AGENTS.md, 提交所有更改, 清理临时文件和残留进程。不留脏状态 (孤儿进程、临时文件、未提交更改)。
|
||||
|
||||
4. **同步依赖** — 使用 `git pull` 同步最新代码, 然后 `runtime\python\python.exe -m pip install -r requirements.txt` 安装新依赖。每次新工作会话前执行。
|
||||
|
||||
5. **编码前缀** — 所有终端命令加 `chcp 65001` 避免 Opencode 编码错误。PowerShell/CMD 编码问题是命令失败的首要原因。
|
||||
|
||||
6. **Python UTF-8** — 运行 Python 时始终加 `set PYTHONUTF8=1`。确保跨机 Unicode 处理一致, 防止编码漂移。
|
||||
|
||||
## 近期变更 (beta-clean 分支)
|
||||
|
||||
- views.py (2000+ 行单体) → `analysis/views/` 包 (13 模块)
|
||||
- tool_registry.py (3000+ 行) → `analysis/tools/` 包 (18 模块)
|
||||
- 默认聚类算法改为 AgglomerativeClustering
|
||||
- 移除实体概念，直接对原始行聚类 (entity_value = row_N)
|
||||
- cluster_detail 显示原始数据行而非实体画像
|
||||
- run_detail 新增全域 SVD 特征分析 + LLM 工作流时间线
|
||||
- 3D 地球侧栏复用 `/globe/?embed=1`（iframe + postMessage 联动）
|
||||
- UI 全面中文化
|
||||
- 非阻塞启动 (run.bat 打印 PID 后退出)
|
||||
|
||||
## 待办
|
||||
|
||||
- [ ] **前端 UI 验证**: 打开 `prototype_cluster_ui.html` 查看交互设计是否符合要求
|
||||
- [ ] **更新前端原型**后，将验证通过的交互逻辑合并到正式模板
|
||||
@@ -1,100 +0,0 @@
|
||||
CLAUDE.md — 天璇 (TianXuan) 系统
|
||||
|
||||
项目概览
|
||||
天璇 是一个 TLS 加密流量数据分析与实体画像系统，基于 Django + Polars + scikit-learn + Three.js 构建，支持从 CSV 导入、实体检测、聚合、聚类、特征提取到 3D 可视化的全自动离线分析流水线。
|
||||
|
||||
- 版本: v2.0.0beta
|
||||
- 运行方式: 离线桌面应用，内置 Python 3.12 运行时，双击 run.bat 即可启动，无需安装任何依赖。
|
||||
- 数据存储: SQLite (WAL 模式) + 内存缓存
|
||||
- 分支策略: 后续所有改动直接提交到 main 分支，并推送到远程仓库 https://gitea.cattysteve.top/HJQ/tianxuan。
|
||||
|
||||
核心模块
|
||||
| 模块 | 功能 |
|
||||
|------|------|
|
||||
| analysis/data_loader.py | CSV 加载、合并、列名规范化、GeoIP 解析 |
|
||||
| analysis/entity_detector.py | 基于列名关键词和唯一值比率的实体列自动检测 |
|
||||
| analysis/entity_aggregator.py | 按实体分组计算流量统计、TLS特征、时间模式等 |
|
||||
| analysis/tool_registry.py | 12 个 MCP 工具注册（145KB） |
|
||||
| simple_analysis/ | 新增的“简单分析”模块，独立 Django App，提供上传→筛选→聚类→地图可视化一站式工作流 |
|
||||
| templates/ | 传统功能页面（仪表盘、上传、手动分析、LLM分析、3D地球等） |
|
||||
| data/ | 离线 GeoIP 数据库 (GeoLite2-City.mmdb、GeoLite2-ASN.mmdb)、自定义 IP 数据 geoip_data.txt |
|
||||
|
||||
项目结构要点
|
||||
天璇/
|
||||
├── analysis/ # 主分析 App
|
||||
├── simple_analysis/ # 简单分析 App (新增)
|
||||
│ ├── views.py # API 视图 (upload, filter, cluster, edges)
|
||||
│ ├── urls.py # 路由 /simple/
|
||||
│ ├── templates/simple_analysis/
|
||||
│ │ └── simple_analysis.html # 单页应用
|
||||
│ └── static/
|
||||
├── config/
|
||||
│ ├── config.yaml # 全局配置（含 API Key、参数）
|
||||
│ └── loader.py
|
||||
├── data/
|
||||
│ ├── GeoLite2-City.mmdb # 130MB，城市级 GeoIP
|
||||
│ ├── GeoLite2-ASN.mmdb # 9.5MB，ASN 数据库
|
||||
│ └── geoip_data.txt # 补充 IP 地理位置
|
||||
├── runtime/python/ # 嵌入式 Python 3.12 运行时
|
||||
├── db.sqlite3 # 主数据库
|
||||
├── main.py / manage.py
|
||||
└── run.bat
|
||||
|
||||
当前开发焦点：简单分析模块 (v3)
|
||||
目标：构建一个完全离线、高性能的网络流量 IP 通信社区发现与可视化工具。
|
||||
工作流（4 步）：
|
||||
1. 上传与预筛选：拖拽上传 CSV，支持高级筛选器（AND/OR/NOT 树状表达式），上传时立即过滤以减少内存占用。
|
||||
2. 高级手动筛选：可视化布尔表达式编辑器，对已加载数据进一步精筛。
|
||||
3. 网络聚类：基于 IP 通信图进行社区发现：
|
||||
- 提取所有唯一 :ips / :ipd 作为节点。
|
||||
- 先按 /24 子网分组，子网内部使用 HDBSCAN (haversine 距离) 二次聚类。
|
||||
- 最终每个节点（IP）获得一个 cluster_id。
|
||||
4. 地图可视化与交互：
|
||||
- 离线地图：使用 Leaflet + 离线瓦片缓存（leaflet-offline），无网络时回退到本地低级别瓦片。
|
||||
- 节点按簇着色，支持 Canvas 渲染器和聚类插件优化性能。
|
||||
- 点击节点展示：IP、运营商、组织、城市、关联对端IP、host/text 等。
|
||||
- 点击边（IP 间连线）展示：通信次数、总流量、首次/末次北京时间、时间规律。
|
||||
- 导出点表/边表 CSV。
|
||||
|
||||
重要技术约定
|
||||
- 列名规范化：所有 CSV 列名通过 normalize_columns() 处理成小写+下划线格式，内部使用标准列名（如 src_ip, dst_ip, cnam 等）。
|
||||
- GeoIP：完全抛弃 CSV 中自带经纬度（:ips.latd 等），所有坐标均使用离线 GeoLite2-City.mmdb 解析。
|
||||
- 时间处理：原始时间戳/时间列统一转为北京时间 (UTC+8) 展示。
|
||||
- 内存管理：上传后的数据保存在内存缓存（_cache dict），以 session_id（12位 hex）索引，无持久化。
|
||||
- 性能要求：i7-10700 + 8GB RAM，十万级 IP 节点需在 1 分钟内完成聚类，内存占用 <2GB。
|
||||
|
||||
前端注意事项
|
||||
- 所有静态资源（Leaflet, Chart.js, Three.js 等）必须本地托管，确保离线可用。
|
||||
- 地图瓦片离线方案必须实现：在线时缓存 OSM 瓦片到 IndexedDB，离线时使用缓存或内置低级别瓦片。
|
||||
- 按钮必须防重复点击，操作栏固定在顶部，调试信息面板实时显示。
|
||||
- 需修复 Cannot set properties of null 类 DOM 操作错误。
|
||||
|
||||
API 端点 (简单分析)
|
||||
| 方法 | 路径 | 作用 |
|
||||
|------|------|------|
|
||||
| GET | /simple/ | 主页面 |
|
||||
| POST | /simple/upload/ | 上传文件 + 预设筛选 + GeoIP 解析 |
|
||||
| POST | /simple/filter/advanced/ | 树状高级筛选 |
|
||||
| GET | /simple/column-values/ | 列 Top 50 唯一值（供下拉菜单） |
|
||||
| POST | /simple/cluster/ | 执行网络聚类 |
|
||||
| GET | /simple/cluster/<label>/ | 簇详情 |
|
||||
| POST | /simple/edges/ | 获取边关系 |
|
||||
|
||||
已知问题与待办
|
||||
- [ ] 离线地图瓦片缓存机制完善（目前仅低级别内置）。
|
||||
- [ ] 边渲染时坐标查找需使用节点真实坐标，而非簇中心近似。
|
||||
- [ ] 大数据量时地图标记性能优化（可换用 WebGL 渲染器）。
|
||||
- [ ] 预设筛选器与 Step 2 筛选器 UI 统一为同一个高级筛选组件。
|
||||
|
||||
开发命令
|
||||
- 启动开发服务器：python manage.py runserver
|
||||
- 运行测试：python manage.py test simple_analysis
|
||||
- 生成要求：pip freeze > requirements.txt (注意剔除内部路径)
|
||||
|
||||
提交规范
|
||||
- 直接提交到 main 分支，git push origin main。
|
||||
- 提交信息格式：feat: xxx / fix: xxx / refactor: xxx，简要描述改动。
|
||||
|
||||
---
|
||||
最后更新: 2026-07-24
|
||||
维护者: 天璇开发团队
|
||||
@@ -1,541 +1,190 @@
|
||||
# 天璇 (TianXuan) — TLS 流数据分析与实体画像系统
|
||||
# 天璇 (TianXuan) — TLS 流数据分析平台
|
||||
|
||||
基于 Polars + scikit-learn + Three.js 的 TLS 流数据分析工具链。支持 CSV 批量导入、自动实体检测、多维聚合、聚类分析、特征提取、**3D 地球流量可视化**。提供 Django Web 操作界面和 MCP 协议接口，支持 32B 大语言模型编排分析流程。**全离线运行**，所有 JS 库/地图贴图/GeoIP 数据库均已内嵌。
|
||||
基于 Polars + scikit-learn + Three.js 的 TLS 流量分析工具。CSV 上传后支持手动分步分析（工作流构建器）、LLM 全自动分析（MCP 工具链）、聚类 3D 地球可视化。全离线运行，内嵌 Python 3.12 运行时。
|
||||
|
||||
## TlsDB.csv — 参考表头名清单
|
||||
## 功能概述
|
||||
|
||||
`TlsDB.csv` 是 TLS 流量数据的**参考表头名清单**，包含常见的列名、含义和数据类型对照表。
|
||||
**此文件为参考对照表，非实际数据文件。**
|
||||
实际 CSV 文件的列名可能与表头名不同，分析工具会自动匹配。
|
||||
|
||||
---
|
||||
|
||||
## 目录
|
||||
|
||||
1. [系统概述](#1-系统概述)
|
||||
2. [快速开始](#2-快速开始)
|
||||
3. [用户指南](#3-用户指南)
|
||||
4. [页面功能说明](#4-页面功能说明)
|
||||
5. [CLI 管道](#5-cli-管道)
|
||||
6. [MCP 工具文档](#6-mcp-工具文档)
|
||||
7. [测试](#7-测试)
|
||||
8. [项目结构](#8-项目结构)
|
||||
9. [常见问题](#9-常见问题)
|
||||
|
||||
---
|
||||
|
||||
## 1. 系统概述
|
||||
|
||||
### 1.1 解决的问题
|
||||
|
||||
企业网络中 TLS 流量通常以 CSV 形式导出（每行一条流记录），数据量大、列名不统一、缺乏标签。本系统提供一条从原始 CSV 到实体画像的自动化分析流水线：
|
||||
|
||||
```
|
||||
CSV 文件 → 上传/加载 → 自动检测实体列 → 按实体聚合特征
|
||||
→ 实体级聚类 → 提取每簇区分特征 → Web 可视化 / MCP 接口
|
||||
```
|
||||
|
||||
### 1.2 核心能力
|
||||
|
||||
| 模块 | 功能说明 |
|
||||
|------|---------|
|
||||
| **数据加载** | 多文件合并、BOM 自动检测、跨文件 schema 容错、递归 `**/*.csv`、中文路径支持 |
|
||||
| **实体检测** | 基于列名关键词 + 唯一值比率 + 数据类型的三维打分，自动推荐实体列（如 `src_ip`、`sni`、`user`） |
|
||||
| **实体聚合** | 按实体分组计算流统计、目标多样性、TLS 特征、协议特征、时间模式、地理位置（经纬度） |
|
||||
| **IP 子网聚合** | 支持 /24 和 /28 子网掩码，同一子网下所有 IP 合并为一个实体节点 |
|
||||
| **多列复合键** | 支持 `(src_ip, dst_ip)` 等多列组合作为实体标识 |
|
||||
| **聚类分析** | HDBSCAN（自动确定簇数）/ KMeans，零样本自动降级，Silhouette / Davies-Bouldin 质量评估 |
|
||||
| **特征提取** | Z-Score / ANOVA 方法计算每个聚类的区分性特征 |
|
||||
| **3D 地球** | 基于 Three.js 的 TLS 流量弧线可视化，多数据源叠加，经纬线网格，国境线轮廓 |
|
||||
| **地理可视化** | PCA 散点图 + Canvas 原生渲染（无外部 JS 依赖） |
|
||||
| **LLM 集成** | OpenAI 兼容 API、LLM 自动编排分析流程（profile → 实体 → 聚类 → 特征） |
|
||||
| **Web 界面** | CSV 上传拖拽、手动分步分析向导、LLM 自动分析、配置管理、运行记录查看 |
|
||||
| **MCP 协议** | 12 个 JSON-RPC 工具，供外部 LLM 编排调用 |
|
||||
| **便携运行时** | 项目内置 Python 3.12 运行时（593MB），零系统依赖，`run.bat` 双击即用 |
|
||||
|
||||
### 1.3 技术栈
|
||||
|
||||
| 组件 | 技术选型 |
|
||||
|------|---------|
|
||||
| **后端框架** | Django 4.2 (SQLite WAL) |
|
||||
| **数据处理** | Polars 1.42.1 (LazyFrame 延迟计算 + streaming) |
|
||||
| **机器学习** | scikit-learn (StandardScaler, HDBSCAN, KMeans, PCA) |
|
||||
| **MCP 协议** | Python MCP SDK (stdio 传输) |
|
||||
| **前端 3D** | Three.js (603KB, 离线) |
|
||||
| **前端图表** | 纯 Canvas 2D API 原生绘制 |
|
||||
| **配置** | Pydantic + YAML |
|
||||
| **LLM 接口** | OpenAI chat/completions 兼容 API |
|
||||
| **运行时** | 嵌入式 Python 3.12 (Win7 兼容版 adang1345) |
|
||||
|
||||
---
|
||||
|
||||
## 2. 快速开始
|
||||
|
||||
### 2.1 用户模式（无需预装 Python）
|
||||
|
||||
```bat
|
||||
1. 解压项目文件夹到任意目录
|
||||
2. 双击 run.bat
|
||||
3. 等待命令行显示 "Starting TianXuan..."
|
||||
4. 浏览器自动打开 http://127.0.0.1:8000/
|
||||
5. 点击「上传数据」→ 选择 CSV 文件 → 上传 → 等待预处理完成
|
||||
6. 进入「手动分析」→ 选择数据集 → 配置参数 → 运行分析
|
||||
7. 查看结果：聚类概览、簇详情、实体画像、地球分布图
|
||||
```
|
||||
|
||||
### 2.2 开发者模式
|
||||
|
||||
```bash
|
||||
# 初始化数据库
|
||||
runtime\python\python.exe manage.py migrate
|
||||
|
||||
# 生成测试数据
|
||||
runtime\python\python.exe scripts\gen_test_data.py --rows 1000
|
||||
|
||||
# 启动开发服务器
|
||||
runtime\python\python.exe manage.py runserver
|
||||
|
||||
# 浏览器打开 http://127.0.0.1:8000/
|
||||
```
|
||||
|
||||
### 2.3 脚本一览
|
||||
|
||||
| 脚本 | 功能 |
|
||||
| 模块 | 说明 |
|
||||
|------|------|
|
||||
| `run.bat` | 启动 Django Web 界面（:8000），自动打开浏览器 |
|
||||
| `shell.bat` | 打开 Django Shell（Python 交互式环境） |
|
||||
|
||||
### 2.4 一键分析管道
|
||||
|
||||
```bash
|
||||
# 加载 CSV → 自动检测实体 → 聚合 → 聚类 → 特征提取 → 写入数据库
|
||||
runtime\python\python.exe manage.py run_pipeline "data/input.csv"
|
||||
|
||||
# 指定聚类算法
|
||||
runtime\python\python.exe manage.py run_pipeline "data/*.csv" --algo kmeans
|
||||
|
||||
# 手动指定实体列（跳过自动检测）
|
||||
runtime\python\python.exe manage.py run_pipeline "data/*.csv" --entity-col src_ip
|
||||
|
||||
# 导出结果
|
||||
runtime\python\python.exe manage.py run_pipeline "data/*.csv" --output ./results/
|
||||
```
|
||||
|
||||
输出示例：
|
||||
```
|
||||
[1/5] 加载 CSV: data/test_flows.csv → 500 rows, 1 files, 0.2 MB
|
||||
[2/5] 检测实体列 → 自动检测: src_ip (共 6 候选)
|
||||
[3/5] 实体聚合 → 50 个实体, 18 维特征
|
||||
[4/5] 聚类 (hdbscan) → 2 个簇, 噪声比 0.24, Silhouette: 0.2557
|
||||
[5/5] 特征提取 → 30 个特征已保存到数据库
|
||||
*** 分析完成! Run ID: #1
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. 用户指南
|
||||
|
||||
### 3.1 上传数据
|
||||
|
||||
1. 点击导航栏「上传数据」
|
||||
2. 拖拽 CSV 文件到虚线区域，或点击选择文件
|
||||
3. 支持多文件上传，自动拼接相同 schema 的文件
|
||||
4. 支持 `schema_strict=false` 模式（默认）：不同列名的文件会自动合并，缺失列填 null
|
||||
5. 支持 `.zip` 压缩包自动解压
|
||||
6. 上传后自动在后台进行预处理（加载 → 检测实体列 → 聚合），完成后状态变为 `ready`
|
||||
7. 进度条实时显示处理状态
|
||||
|
||||
**数据格式要求**：
|
||||
- 每行一条 TLS 流记录
|
||||
- 建议包含列：`src_ip`、`dst_ip`、`proto`、`bytes_sent`、`bytes_rev`、`duration`、`packets` 等
|
||||
- 可选地理列：`latitude` / `lat` / `y` 和 `longitude` / `lon` / `lng` / `x`（自动识别）
|
||||
- 可选时间列：`timestamp` / `ts` / `time`（自动识别）
|
||||
|
||||
### 3.2 手动分析
|
||||
|
||||
3 步向导完成分析：
|
||||
|
||||
**Step 1 - 选择数据集**：从已预处理完成的数据集中选择一个（显示行数、已检测的实体列）
|
||||
|
||||
**Step 2 - 配置参数**：
|
||||
- 聚类算法：HDBSCAN（自动确定簇数，推荐）/ KMeans（需预设 k 值）
|
||||
- 最小簇大小（min_cluster_size）：默认 5，数据不足时自动降级
|
||||
|
||||
**Step 3 - 确认运行**：回顾配置，点击「开始分析」。后台自动完成聚类 + 特征提取，完成后跳转到结果页。
|
||||
|
||||
### 3.3 LLM 自动分析
|
||||
|
||||
需要先在「配置」页面设置 LLM 的 base_url 和 API Key：
|
||||
|
||||
1. 进入「LLM 分析」页面
|
||||
2. 选择一个已预处理的数据集
|
||||
3. 点击「开始自动分析」
|
||||
4. 后端 LLM 编排器会自动调用 profile → build_entity_profiles → run_clustering → extract_features
|
||||
5. 进度条实时显示当前步骤
|
||||
|
||||
### 3.4 查看结果
|
||||
|
||||
#### 运行详情页
|
||||
- 总流数、实体数、簇数 摘要卡片
|
||||
- 簇列表：每个簇的大小、占比、Silhouette 分数
|
||||
|
||||
#### 聚类概览页
|
||||
- **PCA 散点图**：实体在二维主成分空间中的分布，颜色区分聚类
|
||||
- **地理分布图**：实体在地理坐标上的分布（需 CSV 包含经纬度列），支持缺失值
|
||||
- 每个簇的详情卡片：Top 5 区分特征
|
||||
|
||||
#### 簇详情页
|
||||
- 特征表：每列的均值、标准差、中位数、区分度分数（Top 50）
|
||||
- 实体列表：该簇包含的实体（Top 50），可点击查看详情
|
||||
|
||||
#### 实体画像页
|
||||
- 实体标识、所属簇
|
||||
- 聚合特征值和相对簇均值的偏离（Z-score）
|
||||
- 偏差显著的特征以颜色高亮（|Z| > 2）
|
||||
|
||||
#### 3D 地球页
|
||||
- Three.js 3D 地球，TLS 流量弧线连接源目 IP
|
||||
- 弧线颜色标识 TLS 版本（TLSv1.3 蓝 / v1.2 粉 / 其他青）
|
||||
- 鼠标拖拽旋转、滚轮缩放
|
||||
- 经纬线网格 + 国境线轮廓
|
||||
- 多数据源叠加复选框
|
||||
- 脉冲光点动画（背面剔除）
|
||||
|
||||
---
|
||||
|
||||
## 4. 页面功能说明
|
||||
|
||||
| 页面 | 路由 | 功能 |
|
||||
|------|------|------|
|
||||
| **首页** | `/` | 最近运行记录概览 |
|
||||
| **上传数据** | `/upload/` | 拖拽上传 CSV，自动预处理 |
|
||||
| **手动分析** | `/analyze/manual/` | 3 步向导：选数据→设参数→运行 |
|
||||
| **LLM 分析** | `/analyze/auto/` | LLM 自动编排分析流程 |
|
||||
| **运行记录** | `/runs/` | 所有分析运行列表 |
|
||||
| **运行详情** | `/runs/<id>/` | 单次运行摘要 + 簇列表 |
|
||||
| **聚类概览** | `/clusters/<run_id>/` | PCA 散点图 + 每簇区分特征 |
|
||||
| **簇详情** | `/clusters/<run_id>/<label>/` | 特征表 + 实体列表 |
|
||||
| **实体画像** | `/entities/<id>/` | 单实体特征 + 簇偏离 |
|
||||
| **3D 地球** | `/globe/` | Three.js 流量弧线可视化 |
|
||||
| **配置** | `/config/` | 系统配置编辑 + LLM 测试 |
|
||||
| **日志** | `/logs/` | 运行日志查看 |
|
||||
|
||||
---
|
||||
|
||||
## 5. CLI 管道
|
||||
|
||||
```bash
|
||||
# 完整管道（HDBSCAN 默认）
|
||||
runtime\python\python.exe manage.py run_pipeline data/complex_test.csv
|
||||
|
||||
# 指定 KMeans
|
||||
runtime\python\python.exe manage.py run_pipeline data/*.csv --algo kmeans
|
||||
|
||||
# 手动指定实体列
|
||||
runtime\python\python.exe manage.py run_pipeline data/*.csv --entity-col src_ip
|
||||
|
||||
# 设置子网聚合掩码
|
||||
runtime\python\python.exe manage.py run_pipeline data/*.csv --subnet-masks 24 28
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. MCP 工具文档
|
||||
|
||||
### 6.1 工具总览
|
||||
|
||||
系统通过 MCP (Model Context Protocol) 暴露 12 个工具，支持 LLM 以 JSON-RPC 方式编排调用。
|
||||
|
||||
| # | 工具名 | 功能 | 必需参数 |
|
||||
|---|--------|------|----------|
|
||||
| 1 | `load_data` | 加载 CSV 文件（支持 glob、递归、schema 容错） | `csv_glob` |
|
||||
| 2 | `profile_data` | 数据集概要统计 + 相关性矩阵 | `dataset_id` |
|
||||
| 3 | `filter_data` | 按条件过滤（12 操作符 + AND/OR） | `dataset_id`, `filters` |
|
||||
| 4 | `preprocess_data` | 预处理（标准化/编码/填充） | `dataset_id`, `columns` |
|
||||
| 5 | `run_clustering` | 执行聚类（HDBSCAN/KMeans，带质量评估） | `dataset_id`, `cluster_columns` |
|
||||
| 6 | `evaluate_clustering` | 评估聚类质量（Silhouette/DB/CH） | `cluster_result_id`, `dataset_id` |
|
||||
| 7 | `extract_features` | 提取各聚类区分特征（Z-Score/ANOVA） | `dataset_id`, `cluster_result_id` |
|
||||
| 8 | `export_results` | 导出结果到磁盘（CSV/Parquet/JSON） | `result_id`, `output_path` |
|
||||
| 9 | `list_datasets` | 列出所有活跃数据集 | 无 |
|
||||
| 10 | `drop_dataset` | 删除数据集释放内存 | `dataset_id` |
|
||||
| 11 | `clone_dataset` | 浅拷贝数据集（不复制数据） | `dataset_id` |
|
||||
| 12 | `build_entity_profiles` | 自动检测实体列 + 聚合画像 | `dataset_id` |
|
||||
|
||||
### 6.2 响应约定
|
||||
|
||||
- 成功时返回 JSON 对象，含 `truncated: false`
|
||||
- 超过阈值时自动截断，设置 `truncated: true` + `omitted_columns`
|
||||
- 错误时返回 `{ "error": "描述", "truncated": false }`
|
||||
- 所有响应通过 `TextContent` 包装返回
|
||||
|
||||
### 6.3 JSON-RPC 调用示例
|
||||
|
||||
```
|
||||
→ {"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"load_data","arguments":{"csv_glob":"data/*.csv"}}}
|
||||
← {"jsonrpc":"2.0","id":1,"result":{"content":[{"type":"text","text":"{\"dataset_id\":\"ds_...\",\"row_count\":500}"}]}}
|
||||
|
||||
→ {"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"build_entity_profiles","arguments":{"dataset_id":"ds_...","auto_detect":true}}}
|
||||
← {"jsonrpc":"2.0","id":2,"result":{"content":[{"type":"text","text":"{\"entity_dataset_id\":\"entity_...\",\"entity_count\":50,\"entity_column_used\":\"src_ip\"}"}]}}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 7. 测试
|
||||
|
||||
```bash
|
||||
# 运行全部测试
|
||||
runtime\python\python.exe -m pytest tests -q
|
||||
|
||||
# 运行特定测试文件
|
||||
runtime\python\python.exe -m pytest tests/test_entity.py -v
|
||||
|
||||
# 运行端到端测试
|
||||
runtime\python\python.exe -m pytest tests/test_e2e.py -v
|
||||
```
|
||||
|
||||
### 生成测试数据
|
||||
```bash
|
||||
# 简单数据
|
||||
runtime\python\python.exe scripts\gen_test_data.py --rows 1000
|
||||
|
||||
# Globe 模式（IP 在真实 GeoIP 范围内，弧线可见）
|
||||
runtime\python\python.exe scripts\gen_test_data.py --globe
|
||||
|
||||
# 复杂数据（5000 行 24 列含 55% 缺失）
|
||||
runtime\python\python.exe scripts\gen_complex_test.py --rows 5000
|
||||
```
|
||||
|
||||
### 列结构调查
|
||||
```bash
|
||||
# 统计各 CSV 文件的列名、类型、分布
|
||||
runtime\python\python.exe scripts\column_survey.py --csv "data/*.csv"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 8. 项目结构
|
||||
|
||||
```
|
||||
tianxuan/
|
||||
├── tianxuan/ # Django 项目配置
|
||||
│ ├── settings.py # Django 配置（数据库、中间件）
|
||||
│ ├── urls.py # 根路由
|
||||
│ └── llm_orchestrator.py # LLM 编排后端（tool_calls 循环）
|
||||
│
|
||||
├── analysis/ # 核心分析模块（Django app）
|
||||
│ ├── data_loader.py # CSV 加载、BOM 检测、schema 容错、递归 glob
|
||||
│ ├── data_profiler.py # 数据集概要统计与相关性矩阵
|
||||
│ ├── entity_detector.py # 实体列自动检测（关键词 + unique_ratio 打分）
|
||||
│ ├── entity_aggregator.py # 实体聚合（流→实体画像），含 IP 子网聚合
|
||||
│ ├── type_classifier.py # 值优先类型检测（MAC/端口/IPv4/URL/HEX/ENUM/LAT_LON）
|
||||
│ ├── data_validator.py # 列校验（缺失率/异常值/IP有效性）
|
||||
│ ├── geoip.py # GeoIP 经纬度查询
|
||||
│ ├── session_store.py # 线程安全内存会话存储（Singleton + RLock）
|
||||
│ ├── tool_registry.py # 12 个 MCP 工具定义 + 异步处理函数
|
||||
│ ├── mcp_server.py # MCP stdio 服务器
|
||||
│ ├── views.py # 所有 Django 视图
|
||||
│ ├── urls.py # 16 条 URL 路由
|
||||
│ ├── models.py # ORM 模型
|
||||
│ └── management/commands/
|
||||
│ ├── start_mcp.py # manage.py start_mcp
|
||||
│ └── run_pipeline.py # manage.py run_pipeline（一键 CLI 管道）
|
||||
│
|
||||
├── config/
|
||||
│ ├── config.yaml # 配置文件（自动生成，带注释）
|
||||
│ └── loader.py # Pydantic 配置加载器
|
||||
│
|
||||
├── templates/tianxuan/ # 11 个页面模板
|
||||
├── static/tianxuan/ # Three.js + 地球贴图 + 国境线
|
||||
├── runtime/python/ # 便携 Python 3.12 运行时（593MB）
|
||||
├── tests/ # 53 个测试 (pytest)
|
||||
├── scripts/ # 工具脚本
|
||||
├── docs/ # 文档
|
||||
├── run.bat # 用户启动脚本（双击即用）
|
||||
└── manage.py # Django 管理入口
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 9. 常见问题
|
||||
|
||||
### Q1: 双击 run.bat 后浏览器没打开？
|
||||
A: 手动打开浏览器访问 `http://127.0.0.1:8000/`。如果也无法访问，检查命令行是否有报错。
|
||||
|
||||
### Q2: 压缩时提示 `db.sqlite3-wal 被占用`？
|
||||
A: 有残留的 Python 进程持有 SQLite 连接。运行以下命令后重试：
|
||||
```powershell
|
||||
Get-Process python -ErrorAction SilentlyContinue | Stop-Process -Force
|
||||
Remove-Item db.sqlite3-wal, db.sqlite3-shm -Force -ErrorAction SilentlyContinue
|
||||
```
|
||||
|
||||
### Q3: 地图不显示？
|
||||
A: CSV 需包含 lat/lon 列（列名含 `lat`/`latitude`/`lon`/`longitude`/`lng` 自动识别），缺失值不显示。页面会提示跳过的数量。
|
||||
|
||||
### Q4: 上传 CSV 后显示 "处理失败"？
|
||||
A: 在运行详情页查看错误消息。常见原因：
|
||||
- CSV 编码不是 UTF-8
|
||||
- 所有列都是非数值类型（无法聚类，返回 0 簇）
|
||||
- 数据量太小（< 3 条实体记录，HDBSCAN 自动跳过）
|
||||
|
||||
### Q5: 如何配置 LLM？
|
||||
A: 导航到「配置」页面，填写 base_url 和 API Key，点击「测试连通性」验证。支持 OpenAI 兼容的任何 API。
|
||||
|
||||
### Q6: 支持哪些 CSV 编码？
|
||||
A: 自动检测 UTF-8 BOM、UTF-16LE BOM、UTF-16BE BOM。无 BOM 时默认 UTF-8。中文路径完全支持。
|
||||
|
||||
### Q7: 不同 CSV 文件的列名不一致怎么办？
|
||||
A: 默认 `schema_strict=false`，系统自动取所有文件的列名并集，缺失列填 null。如需严格模式，在配置中将 `schema_strict` 改为 `true`。
|
||||
|
||||
### Q8: 项目太大，如何缩小？
|
||||
```powershell
|
||||
Remove-Item -Recurse *.pyc, __pycache__ -Force
|
||||
Remove-Item db.sqlite3-wal, db.sqlite3-shm -Force -ErrorAction SilentlyContinue
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 10. 增量更新（部署到离线机）
|
||||
|
||||
天璇支持**增量更新**，只需将变更代码打包成小 zip 包，复制到离线机双击 `update.bat` 即可完成更新，无需每次传输整个项目（1.5GB）。
|
||||
|
||||
### 10.1 首次部署
|
||||
|
||||
在离线机上第一次部署时，需要完成数据库迁移：
|
||||
| **CSV 上传** | 多文件拖拽、BOM 自动检测、schema 容错、ZIP 自动解压 |
|
||||
| **手动分析** | 工作流构建器：添加/排序/保存工具链，串行执行 |
|
||||
| **LLM 自动分析** | OpenAI 兼容 API，自动编排 filter→profile→entity→cluster→feature |
|
||||
| **聚类可视化** | HDBSCAN/KMeans，Silhouette/DB 评估，Canvas 散点图 + 地理分布 |
|
||||
| **3D 地球** | Three.js 流量弧线，TLS 版本着色，经纬线+国境线 |
|
||||
| **降维** | UMAP + TruncatedSVD 用于高维数据降维与可视化 |
|
||||
| **实体画像** | 聚合特征 + Z-Score 簇偏离量，自定义实体列/IP 子网掩码 |
|
||||
| **MCP 协议** | 30 个 JSON-RPC 工具供外部 LLM 编排调用 |
|
||||
| **增量更新** | update.bat + rollback.bat，用户数据/代码分离 |
|
||||
|
||||
## 技术栈
|
||||
|
||||
| 组件 | 选型 |
|
||||
|------|------|
|
||||
| 后端 | Django 4.2 + SQLite WAL |
|
||||
| 数据处理 | Polars 1.42 (LazyFrame + streaming) |
|
||||
| 机器学习 | scikit-learn (HDBSCAN, KMeans, MiniBatchKMeans, IsolationForest), UMAP, TruncatedSVD, StandardScaler |
|
||||
| 前端 3D | Three.js (603KB, 离线, 地球贴图 1.4MB) |
|
||||
| 前端图表 | 纯 Canvas 2D (散点图/地理分布/Silhouette) |
|
||||
| 配置管理 | Pydantic + YAML |
|
||||
| LLM 协议 | MCP SDK (stdio) + OpenAI 兼容 API |
|
||||
| 运行时 | 内嵌 Python 3.12 (~593MB) |
|
||||
|
||||
## 安装
|
||||
|
||||
项目自包含，无需预装 Python。解压后即可使用。
|
||||
|
||||
```bat
|
||||
:: 确保数据库在 %APPDATA%/TianXuan/ 下
|
||||
runtime\python\python.exe scripts\migrate_db_to_appdata.py
|
||||
|
||||
:: 初始化数据库表
|
||||
:: 首次部署 — 初始化数据库
|
||||
runtime\python\python.exe manage.py migrate
|
||||
|
||||
:: 导入 TLS 引用数据
|
||||
runtime\python\python.exe manage.py import_tlsdb
|
||||
```
|
||||
|
||||
> 如果之前已在使用天璇，`migrate_db_to_appdata.py` 会自动将已有 `db.sqlite3` 复制到新位置。
|
||||
## 使用方法
|
||||
|
||||
### 10.2 开发机：构建更新包
|
||||
```bat
|
||||
:: 双击启动 Web 界面 (http://127.0.0.1:8000/)
|
||||
run.bat :: 非阻塞启动，打印 PID 后退出，日志写入 logs/tianxuan.log
|
||||
|
||||
在**开发机**上完成代码修改并提交后：
|
||||
:: 命令行一键管道
|
||||
runtime\python\python.exe manage.py run_pipeline "data/*.csv"
|
||||
|
||||
```bash
|
||||
# 确保工作树干净（无未提交变更）
|
||||
git status
|
||||
|
||||
# 构建增量更新包
|
||||
python scripts/build_update.py
|
||||
:: 开发模式启动
|
||||
set PYTHONUTF8=1
|
||||
runtime\python\python.exe manage.py runserver
|
||||
```
|
||||
|
||||
脚本会自动：
|
||||
1. 检测当前 git tag 与最近一次 tag 之间的文件变更
|
||||
2. 计算每个文件的 SHA256 校验和
|
||||
3. 打包为 `tianxuan_update_<旧版本>-<新版本>.zip`
|
||||
4. 更新 `VERSION` 文件
|
||||
5. 如果检测到依赖变更，会输出提示信息
|
||||
**Web 工作流:** 上传 CSV → 手动分析（工作流构建器）或 LLM 自动分析 → 检视聚类概览/簇详情/实体画像/3D 地球
|
||||
|
||||
**输出示例**：
|
||||
```
|
||||
==================================================
|
||||
天璇 增量更新包构建工具
|
||||
==================================================
|
||||
**CLI 管道:** 加载 CSV → 类型检测 → 实体聚合 → 聚类 → 特征提取 → 写入数据库
|
||||
|
||||
当前 VERSION: v1.0.0
|
||||
最新 git tag: v1.0
|
||||
|
||||
变更文件: 5 个修改/新增, 1 个删除
|
||||
|
||||
==================================================
|
||||
✅ 增量更新包构建完成！
|
||||
包: tianxuan_update_v1.0-v1.1.zip
|
||||
大小: 0.05 MB
|
||||
从: v1.0
|
||||
到: v1.1
|
||||
文件数: 5
|
||||
删除: 1
|
||||
==================================================
|
||||
```
|
||||
|
||||
### 10.3 离线机：应用更新
|
||||
|
||||
在**离线机**上：
|
||||
|
||||
1. 将 `tianxuan_update_*.zip` 复制到天璇项目目录（与 `run.bat` 同目录）
|
||||
2. **双击 `update.bat`**（或拖拽 zip 到 `update.bat` 上）
|
||||
3. 等待更新完成
|
||||
4. 关闭窗口，重新双击 `run.bat` 启动新版本
|
||||
|
||||
更新脚本会自动：
|
||||
1. ✅ 验证更新包完整性（SHA256 校验）
|
||||
2. ✅ 备份当前配置（`config/config.yaml`）
|
||||
3. ✅ 删除废弃文件
|
||||
4. ✅ 覆盖更新文件
|
||||
5. ✅ 智能合并配置（保留你的设置，仅增加新配置项）
|
||||
6. ✅ 清理 Python 缓存
|
||||
7. ✅ 运行数据库迁移
|
||||
8. ✅ 更新版本号
|
||||
|
||||
### 10.4 回滚
|
||||
|
||||
如果更新后出现问题：
|
||||
## 项目结构
|
||||
|
||||
```
|
||||
双击 rollback.bat
|
||||
→ 输入 y 确认回滚
|
||||
→ 等待恢复完成
|
||||
→ 重新双击 run.bat 启动旧版本
|
||||
天璇/
|
||||
├── analysis/ 核心分析模块 (Django app)
|
||||
│ ├── views/ 视图包 (12 模块)
|
||||
│ │ ├── __init__.py 重导出所有视图函数
|
||||
│ │ ├── helpers.py 辅助函数 (_extract_lat, _extract_lon)
|
||||
│ │ ├── dashboard.py 首页 + 运行记录
|
||||
│ │ ├── pipeline.py 管道执行器 (_run_pipeline_worker)
|
||||
│ │ ├── clustering.py 聚类概览/详情/3D 地球数据
|
||||
│ │ ├── entity.py 实体画像页
|
||||
│ │ ├── upload.py CSV 上传 (拖拽/多文件/进度)
|
||||
│ │ ├── manual.py 手动分析工作流构建器
|
||||
│ │ ├── auto.py LLM 自动分析 (SSE 流式)
|
||||
│ │ ├── globe.py 3D 地球数据接口
|
||||
│ │ ├── config.py 配置编辑 + LLM 连通性测试
|
||||
│ │ ├── log_viewer.py 日志查看器
|
||||
│ │ └── tools.py 工具实验室
|
||||
│ ├── tools/ 工具包 (17 模块, 30 个 MCP 工具实现)
|
||||
│ │ ├── __init__.py 重导出所有 handler
|
||||
│ │ ├── _registry.py 工具元数据定义 (Tool schema)
|
||||
│ │ ├── _dispatch.py 工具调用分发器
|
||||
│ │ ├── _helpers.py 共享辅助 (过滤/截断/数据集决议)
|
||||
│ │ ├── load_data.py CSV 加载 (glob/递归/ZIP/schema 容错)
|
||||
│ │ ├── profile.py 数据概要统计 + 相关性矩阵
|
||||
│ │ ├── filter.py 数据过滤 (12 操作符 + AND/OR)
|
||||
│ │ ├── preprocess.py 预处理 (StandardScaler/OneHot/填充)
|
||||
│ │ ├── clustering.py 聚类执行 (HDBSCAN/KMeans + 自动降维)
|
||||
│ │ ├── evaluate.py 聚类质量评估 (Silhouette/DB/CH)
|
||||
│ │ ├── features.py 特征提取 (Z-Score/ANOVA/UMAP)
|
||||
│ │ ├── entities.py 实体画像 + 自适应评分 (SVD 权重)
|
||||
│ │ ├── anomalies.py 异常检测 (IsolationForest) + UMAP 可视化
|
||||
│ │ ├── export.py 结果导出 (CSV/Parquet/JSON)
|
||||
│ │ ├── analysis.py 只读分析 (6 工具: 模式/时序/FFT/TLS/地理/实体)
|
||||
│ │ ├── diagnostics.py 诊断 (7 工具: 校验/分布/异常值/聚类诊断/对比/导出/修复)
|
||||
│ │ ├── data_mgmt.py 数据管理 (列表/删除/克隆)
|
||||
│ │ └── distance_matrix.py LLM 驱动距离计算
|
||||
│ ├── tool_registry.py 旧编排层 (薄包装，重导出 tools/)
|
||||
│ ├── mcp_server.py MCP stdio 服务器
|
||||
│ ├── data_loader.py CSV 加载底层实现
|
||||
│ ├── data_profiler.py 列统计 + 相关性矩阵
|
||||
│ ├── data_validator.py 列校验 (缺失/异常值/IP 有效性)
|
||||
│ ├── type_classifier.py 值优先类型检测 (MAC/端口/IPv4/URL/HEX/经纬度)
|
||||
│ ├── geoip.py GeoIP 经纬度查询
|
||||
│ ├── ip_clustering.py IP 子网聚类与转换
|
||||
│ ├── session_store.py 线程安全内存会话存储
|
||||
│ ├── distance.py 距离计算 (IP 子网/地理/Levenshtein/Hamming/FFT)
|
||||
│ ├── models.py ORM: AnalysisRun, ClusterResult, EntityProfile, ClusterFeature
|
||||
│ ├── urls.py 路由注册
|
||||
│ ├── nl_describe.py 自然语言数据描述
|
||||
│ ├── profile_util.py 画像工具函数
|
||||
│ ├── tls_ref.py TLS 参考数据
|
||||
│ ├── db_utils.py SQLite 工具
|
||||
│ ├── constants.py 全局常量
|
||||
│ ├── appdata.py 应用数据路径管理
|
||||
│ └── management/commands/ CLI 命令
|
||||
│ ├── start_mcp.py MCP 服务器启动
|
||||
│ ├── run_pipeline.py 一键 CLI 管道
|
||||
│ └── import_tlsdb.py TLS 数据库导入
|
||||
├── tianxuan/ Django 项目配置
|
||||
│ ├── settings.py ALLOWED_HOSTS 自动检测, LOGGING 文件+stderr
|
||||
│ ├── wsgi.py SQLite 启动自修复
|
||||
│ ├── urls.py 路由汇总
|
||||
│ └── llm_orchestrator.py LLM 编排 (策略式提示词, 自动错误恢复)
|
||||
├── config/ 配置管理
|
||||
│ ├── config.yaml 自动生成默认配置 (entity, server, data, clustering, llm)
|
||||
│ ├── loader.py Pydantic 配置加载 + 文件修改检测
|
||||
│ └── __init__.py
|
||||
├── templates/ 页面模板 (13 个 HTML)
|
||||
│ ├── base.html 导航: 首页/上传/手动/LLM/记录/地球/配置
|
||||
│ ├── analysis/ 分析页面 (6 个)
|
||||
│ │ ├── dashboard.html 首页
|
||||
│ │ ├── run_list.html 运行记录列表
|
||||
│ │ ├── run_detail.html 运行详情
|
||||
│ │ ├── cluster_overview.html 聚类概览
|
||||
│ │ ├── cluster_detail.html 簇详情
|
||||
│ │ └── entity_profile.html 实体画像
|
||||
│ └── tianxuan/ 功能页面 (6 个)
|
||||
│ ├── upload.html CSV 上传
|
||||
│ ├── manual.html 手动分析
|
||||
│ ├── auto.html LLM 自动分析
|
||||
│ ├── globe.html 3D 地球
|
||||
│ ├── config.html 配置编辑
|
||||
│ └── log_viewer.html 日志查看
|
||||
├── static/tianxuan/ 离线静态资源
|
||||
│ ├── three.min.js Three.js 3D 引擎 (603KB)
|
||||
│ ├── earth_atmos_2048.jpg 地球贴图 (1.4MB)
|
||||
│ └── world_borders.js 177 国完整国境线 (269KB)
|
||||
├── runtime/python/ 便携 Python 3.12 运行时 (~593MB)
|
||||
├── scripts/ 工具脚本
|
||||
│ ├── gen_test_data.py 测试数据生成
|
||||
│ ├── gen_complex_test.py 复杂数据生成
|
||||
│ ├── column_survey.py CSV 列结构调查
|
||||
│ ├── diagnose_compare.py 跨机日志对比诊断
|
||||
│ └── debug_db.py DB 持久化调试
|
||||
├── tests/ 端到端测试
|
||||
│ ├── test_e2e_full.py
|
||||
│ ├── test_e2e_batch.py
|
||||
│ └── _test_hdbscan.py
|
||||
├── docs/ 文档
|
||||
│ ├── 工作流.md 完整分析流水线
|
||||
│ └── 故障诊断手册.md 问题排查指南
|
||||
├── run.bat 用户启动入口 (含 PYTHONUTF8=1)
|
||||
├── shell.bat Django shell
|
||||
├── update.bat 增量更新
|
||||
├── rollback.bat 更新回滚
|
||||
├── build.bat 构建打包
|
||||
└── manage.py Django 管理入口
|
||||
```
|
||||
|
||||
回滚会从 `%APPDATA%/TianXuan/backups/` 恢复最近一次备份。
|
||||
## 架构要点
|
||||
|
||||
### 10.5 依赖变更时的特殊处理
|
||||
- **views 包**: 原单体 `views.py` (2000+ 行) 拆分为 12 个模块，按功能域分离 (dashboard, pipeline, clustering, entity, upload, manual, auto, globe, config, log_viewer, tools, helpers)
|
||||
- **tools 包**: 原 `tool_registry.py` 拆分为 17 个模块 (14 个 handler 模块 + 3 个基础设施模块)，每个工具领域独立文件
|
||||
- **30 个 MCP 工具** 分四层: 核心 (14 个)、只读分析 (6 个)、诊断 (7 个)、数据管理 (3 个)
|
||||
- **纯 Canvas 2D 图表**: 零外部 JS 依赖，散点图/地理分布/Silhouette 均纯 Canvas 绘制
|
||||
- **tuple 替代 frozenset**: 避免跨机 PYTHONHASHSEED 差异
|
||||
- **值优先类型检测**: config → 值采样 → 名称推断 → STRING 回退 (type_classifier.py)
|
||||
- **TruncatedSVD 降维**: 特征 >50 维时自动降维至 50；compute_scores 中学习自适应评分权重
|
||||
- **UMAP 2D 嵌入**: 训练 10K 样本 + 批量变换 1K 批次，存入 EntityProfile 表
|
||||
- **上传目录隔离**: `%APPDATA%/TianXuan/data/uploads/`，与项目路径无关
|
||||
|
||||
当 `pyproject.toml` 或 `requirements.txt` 变更时（即添加/更新了 Python 依赖），构建脚本会输出警告：
|
||||
## 目标设备
|
||||
|
||||
```
|
||||
⚠️ 依赖已变更！请手动将 runtime/ 目录同步到离线机
|
||||
```
|
||||
|
||||
此时需要额外步骤：
|
||||
|
||||
1. 在开发机上将 `runtime/` 目录**完整压缩**（约 645MB）
|
||||
2. 复制到离线机，覆盖 `runtime/` 目录
|
||||
3. 或者使用差异更新：仅覆盖变更的包目录（`runtime/python/Lib/site-packages/`）
|
||||
|
||||
> 依赖变更是低频操作（通常只在添加新功能时需要），所以这不会影响日常的代码增量更新。
|
||||
|
||||
### 10.6 文件说明
|
||||
|
||||
| 文件 | 用途 | 需要 git 跟踪？ |
|
||||
|------|------|----------------|
|
||||
| `VERSION` | 记录当前版本号 | ✅ 是 |
|
||||
| `scripts/build_update.py` | 开发机：构建增量包 | ✅ 是 |
|
||||
| `scripts/apply_update.py` | 离线机：应用更新 | ✅ 是 |
|
||||
| `scripts/merge_config.py` | 合并用户配置与源码配置 | ✅ 是 |
|
||||
| `scripts/migrate_db_to_appdata.py` | 首次迁移数据库到 APPDATA | ✅ 是 |
|
||||
| `scripts/version_utils.py` | 版本工具函数 | ✅ 是 |
|
||||
| `update.bat` | 离线机双击入口 | ✅ 是 |
|
||||
| `rollback.bat` | 回滚入口 | ✅ 是 |
|
||||
| `analysis/appdata.py` | APPDATA 路径共享模块 | ✅ 是 |
|
||||
| `.update_cache/runtime_snapshot.json` | 运行时依赖快照（自动生成）| ❌ 否 |
|
||||
| `RUNTIME_CHANGED.txt` | 依赖变更提示（自动生成）| ❌ 否 |
|
||||
|
||||
### 10.7 目录结构变化
|
||||
|
||||
更新系统将用户数据与项目代码分离：
|
||||
|
||||
```
|
||||
项目目录（可安全覆盖） 用户数据目录（永不覆盖）
|
||||
天璇/ %APPDATA%/TianXuan/
|
||||
├── analysis/ ← 代码 ├── db.sqlite3 ← 数据库
|
||||
├── config/config.yaml ← 配置 ├── data/uploads/ ← 上传的 CSV
|
||||
├── runtime/ ← 运行时 ├── .session_store.json
|
||||
├── templates/ ← 模板 └── backups/ ← 更新备份
|
||||
├── static/ ← 静态资源
|
||||
├── scripts/ ← 工具脚本
|
||||
├── update.bat ← 更新入口
|
||||
└── VERSION ← 版本号
|
||||
```
|
||||
|
||||
|
||||
```
|
||||
| 条件 | 规格 |
|
||||
|------|------|
|
||||
| 操作系统 | Windows 10 1909+ |
|
||||
| 内存 | 8 GB |
|
||||
| GPU | 无独立显卡（核显可运行 Three.js） |
|
||||
| 磁盘 | SSD 推荐（SQLite WAL 优化） |
|
||||
| 数据规模 | 支持 2500 个 CSV × 10000 行 × 50~58 列 |
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import subprocess, time, urllib.request, os, sys
|
||||
|
||||
PROJ_DIR = r"C:\Users\25044\Desktop\Proj\天璇"
|
||||
PYTHON_EXE = os.path.join(PROJ_DIR, "runtime", "python", "python.exe")
|
||||
os.chdir(PROJ_DIR)
|
||||
|
||||
server = subprocess.Popen(
|
||||
[PYTHON_EXE, "manage.py", "runserver", "127.0.0.1:8765", "--noreload"],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL
|
||||
)
|
||||
|
||||
print(f"SERVER_PID={server.pid}")
|
||||
|
||||
# Poll with backoff: fast early, then slower
|
||||
deadline = time.time() + 60
|
||||
attempt = 0
|
||||
while time.time() < deadline:
|
||||
attempt += 1
|
||||
try:
|
||||
resp = urllib.request.urlopen("http://127.0.0.1:8765/", timeout=3)
|
||||
if resp.status == 200:
|
||||
print("SERVER_READY")
|
||||
sys.exit(0)
|
||||
except Exception as e:
|
||||
if attempt % 10 == 0:
|
||||
print(f"WAITING... attempt={attempt} err={type(e).__name__}")
|
||||
time.sleep(1)
|
||||
|
||||
print("SERVER_START_TIMEOUT")
|
||||
server.kill()
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,162 @@
|
||||
"""Unified column processor — dispatches per-type distance computation.
|
||||
|
||||
Replaces the if-elif chain in :func:`distance.run_preprocessing_pipeline`
|
||||
with :data:`~analysis.types.TYPE_REGISTRY` dispatch.
|
||||
|
||||
Usage::
|
||||
|
||||
from analysis.types import classify_schema
|
||||
from analysis.column_processor import ColumnProcessor
|
||||
|
||||
column_types = classify_schema(lf)
|
||||
proc = ColumnProcessor(lf, column_types)
|
||||
lf = proc.compute_distances() # Applies each type's .distance()
|
||||
lf = proc.normalize() # StandardScaler on numeric dist cols
|
||||
lf, noise = proc.svd_denoise() # SVD with noise profile
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
from sklearn.decomposition import TruncatedSVD
|
||||
|
||||
|
||||
# ── Prefixes produced by each type's .distance() ──
|
||||
_DIST_PREFIXES: tuple[str, ...] = (
|
||||
'_ip_dist_', '_str_dist_', '_bool_dist_', '_bytes_dist_',
|
||||
'_ts_dist_', '_enum_dist_', '_raw_',
|
||||
)
|
||||
|
||||
# ── Types that produce numeric distance/raw output (all 8 do) ──
|
||||
_ALL_TYPES: set[str] = {
|
||||
"枚举", "字节", "浮点", "整数", "布尔", "字符串", "IPv4", "时间戳",
|
||||
}
|
||||
|
||||
|
||||
class ColumnProcessor:
|
||||
"""Processes all columns in a dataset by their registered types.
|
||||
|
||||
Groups columns by type, dispatches each group to the corresponding
|
||||
:class:`~analysis.types.ColumnType` instance in
|
||||
:data:`~analysis.types.TYPE_REGISTRY`, then chains normalisation
|
||||
and SVD denoising.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
lf:
|
||||
Input LazyFrame with raw data columns.
|
||||
column_types:
|
||||
``{col_name: ColumnType}`` mapping from
|
||||
:func:`~analysis.types.classify_schema`.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
lf: pl.LazyFrame,
|
||||
column_types: dict,
|
||||
) -> None:
|
||||
self._lf = lf
|
||||
self._column_types = column_types
|
||||
self._feature_cols: list[str] = []
|
||||
self._noise_profile: dict = {}
|
||||
|
||||
# ── public API ──────────────────────────────────────────────────────
|
||||
|
||||
@property
|
||||
def feature_cols(self) -> list[str]:
|
||||
"""Numeric feature columns available after :meth:`compute_distances`."""
|
||||
return list(self._feature_cols)
|
||||
|
||||
@property
|
||||
def noise_profile(self) -> dict:
|
||||
"""SVD noise profile populated by :meth:`svd_denoise`."""
|
||||
return dict(self._noise_profile)
|
||||
|
||||
@property
|
||||
def lf(self) -> pl.LazyFrame:
|
||||
"""Current LazyFrame state."""
|
||||
return self._lf
|
||||
|
||||
def compute_distances(self) -> pl.LazyFrame:
|
||||
"""Group columns by type and dispatch to TYPE_REGISTRY.
|
||||
|
||||
Each :class:`~analysis.types.ColumnType` subclass handles all
|
||||
columns of that type in one call. IP columns with fewer than
|
||||
2 entries are skipped (they need pairs).
|
||||
|
||||
Returns the LazyFrame with distance/raw columns appended.
|
||||
"""
|
||||
from analysis.types import TYPE_REGISTRY
|
||||
|
||||
# Group columns by type name
|
||||
groups: dict[str, list[str]] = {}
|
||||
for col_name, ct in self._column_types.items():
|
||||
groups.setdefault(ct.name, []).append(col_name)
|
||||
|
||||
result = self._lf
|
||||
|
||||
# Dispatch each group to its type's distance method
|
||||
for type_name, cols in groups.items():
|
||||
type_obj = TYPE_REGISTRY.get(type_name)
|
||||
if type_obj is None:
|
||||
continue
|
||||
# IP distance needs at least 2 columns (pairs)
|
||||
if type_name == "IPv4" and len(cols) < 2:
|
||||
continue
|
||||
result = type_obj.distance(result, cols)
|
||||
|
||||
# ── Collect feature columns produced by distance computation ──
|
||||
schema = result.collect_schema()
|
||||
self._feature_cols = [
|
||||
c for c in schema.names()
|
||||
if any(c.startswith(p) for p in _DIST_PREFIXES)
|
||||
]
|
||||
|
||||
self._lf = result
|
||||
return result
|
||||
|
||||
def normalize(self) -> pl.LazyFrame:
|
||||
"""Apply StandardScaler normalisation to feature columns.
|
||||
|
||||
Uses :func:`analysis.distance.normalize_features` internally.
|
||||
Requires :meth:`compute_distances` to have been called first.
|
||||
"""
|
||||
if not self._feature_cols:
|
||||
return self._lf
|
||||
|
||||
from analysis.distance import normalize_features
|
||||
self._lf = normalize_features(self._lf, self._feature_cols)
|
||||
return self._lf
|
||||
|
||||
def svd_denoise(self, variance_threshold: float = 0.95) -> tuple[pl.LazyFrame, dict]:
|
||||
"""SVD noise removal on normalised feature columns.
|
||||
|
||||
Uses :func:`analysis.distance.svd_denoise` internally.
|
||||
Requires :meth:`normalize` to have been called first.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
variance_threshold:
|
||||
Fraction of total variance to preserve (0.0–1.0).
|
||||
|
||||
Returns
|
||||
-------
|
||||
tuple[pl.LazyFrame, dict]
|
||||
``(denoised_lf, noise_profile)``.
|
||||
"""
|
||||
if not self._feature_cols:
|
||||
self._noise_profile = {
|
||||
'kept_components': 0, 'total_components': 0,
|
||||
'kept_variance': 0.0, 'noise_components': [],
|
||||
}
|
||||
return self._lf, self._noise_profile
|
||||
|
||||
from analysis.distance import svd_denoise as _svd_denoise
|
||||
norm_cols = [f'_norm_{c}' for c in self._feature_cols]
|
||||
self._lf, self._noise_profile = _svd_denoise(
|
||||
self._lf, norm_cols, variance_threshold,
|
||||
)
|
||||
return self._lf, self._noise_profile
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Centralized constants for the TianXuan analysis module.
|
||||
|
||||
All magic numbers from analysis/tools/ and analysis/views/ are defined here
|
||||
so they can be updated in one place. Import with::
|
||||
|
||||
from analysis.constants import RANDOM_SEED, MAX_SAMPLE_ROWS, ...
|
||||
"""
|
||||
|
||||
import polars as pl
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Clustering / UMAP — tuned for 25M rows × 50 cols on 8GB RAM
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
MAX_SAMPLE_ROWS = 200000
|
||||
"""Maximum rows for clustering sample. 200K ≈ 0.8% of 25M rows, fits 8GB RAM."""
|
||||
|
||||
MAX_DISTRIBUTION_SAMPLE = 50000
|
||||
"""Default sample for distribution exploration — 50K provides stable statistics."""
|
||||
|
||||
UMAP_TRAIN_SAMPLE = 30000
|
||||
"""Max rows for UMAP training. 30K is a balance: UMAP O(n²) for neighbors."""
|
||||
# Note: UMAP neighbors search is O(n * n_neighbors * log n). 30K × 15 ≈ fine.
|
||||
# Going above 50K causes UMAP to allocate several GB for the nearest-neighbor index.
|
||||
|
||||
UMAP_BATCH_SIZE = 2000
|
||||
"""Batch size for UMAP transform when dataset exceeds UMAP_TRAIN_SAMPLE."""
|
||||
|
||||
LOW_MEMORY_THRESHOLD_GB = 2
|
||||
"""If available memory < this (in GiB), enable low-memory clustering path."""
|
||||
|
||||
LOW_MEMORY_THRESHOLD_ROWS = 500000
|
||||
"""Row count above which low-memory downsampling is triggered.
|
||||
500K gives 2% coverage of 25M rows while keeping numpy matrix < 200 MB."""
|
||||
|
||||
RANDOM_SEED = 42
|
||||
"""Default random seed for all sklearn/numpy operations."""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Globe — 8GB RAM / 25M rows constraints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
GLOBE_MAX_ROWS = 500
|
||||
"""Max rows to load per run for 3D globe visualization. 300→500 for better
|
||||
coverage on the globe, still < 1 MB of flow data."""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Filter
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
FILTER_MAX_ROWS = 5
|
||||
"""Maximum visible filter rows in the manual analysis UI (deprecated / reserved)."""
|
||||
|
||||
MAX_COLUMNS_PROFILE = 200
|
||||
"""Max columns to include in a dataset profile summary."""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# File upload limits
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
FILE_UPLOAD_MAX_SIZE = 500 * 1024 * 1024 # 500 MB
|
||||
"""Maximum size for a single uploaded file."""
|
||||
|
||||
TOTAL_UPLOAD_MAX_SIZE = 50 * 1024 * 1024 * 1024 # 50 GB
|
||||
"""Maximum total upload size across all files."""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool limits (numeric type tuples used across tools/)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
NUMERIC_DTYPES: tuple[type[pl.DataType], ...] = (
|
||||
pl.Int8, pl.Int16, pl.Int32, pl.Int64,
|
||||
pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
|
||||
pl.Float32, pl.Float64,
|
||||
)
|
||||
"""Polars numeric data types — used for column selection in clustering,
|
||||
entity aggregation, anomaly detection, and feature extraction."""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Diagnostics / Validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
MAX_VALIDATE_SAMPLE = 5000
|
||||
"""Max rows sampled during validate_data."""
|
||||
|
||||
MAX_DIAGNOSE_SAMPLE = 50000
|
||||
"""Max rows sampled for clustering diagnosis (SVD). 50K for stable SVD."""
|
||||
|
||||
MAX_CLUSTER_FEATURES = 50
|
||||
"""Feature count threshold above which TruncatedSVD is applied."""
|
||||
|
||||
CLUSTER_MAX_FEATURES_TOP = 10
|
||||
"""Max feature columns selected by auto-detection in clustering tools."""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Globe / Flow extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
GLOBE_FLOW_MAX_ROWS = 500
|
||||
"""Alias for GLOBE_MAX_ROWS — max rows loaded for flow extraction."""
|
||||
+15
-697
@@ -1,700 +1,18 @@
|
||||
"""CSV loading with per-file schema validation, BOM detection, and multi-file merge.
|
||||
"""Thin re-export wrapper — implementation moved to analysis/data_loader/ package.
|
||||
|
||||
Supports multiple encodings and delimiter-separated files. The main entry
|
||||
point is :func:`load_csv_directory`, which globs files, detects BOM per file,
|
||||
validates schema consistency, and returns a merged :class:`pl.LazyFrame`.
|
||||
When both ``analysis/data_loader.py`` and ``analysis/data_loader/`` exist,
|
||||
Python prefers the package directory, so this file is effectively dead code.
|
||||
Kept for git history continuity — all real code lives in the package.
|
||||
|
||||
If you need to restore the monolithic version, see git history.
|
||||
"""
|
||||
import glob
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import polars as pl
|
||||
import yaml
|
||||
|
||||
from analysis.type_classifier import classify_schema, DataType
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
SUPPORTED_ENCODINGS = ['utf-8', 'utf-8-sig', 'utf-16le', 'utf-16be', 'latin-1']
|
||||
|
||||
|
||||
# ── numeric coercion helper ──────────────────────────────────────────────
|
||||
|
||||
def _coerce_to_float(series: pl.Series) -> pl.Series:
|
||||
"""Convert string series to Float64.
|
||||
|
||||
Every entry that doesn't parse as a valid float becomes null.
|
||||
Handles blank strings, standalone '+'/'-', 'N/A', 'null', 'None', etc.
|
||||
"""
|
||||
series = series.cast(pl.Utf8).str.strip_chars()
|
||||
artifacts = ('', '+', '-', '.', '..', 'N/A', 'n/a', 'NA', 'null', 'None', 'NULL', '?', '---', '--', '/')
|
||||
is_artifact = series.is_in(pl.Series(artifacts).implode())
|
||||
result = []
|
||||
for v, bad in zip(series.to_list(), is_artifact.to_list()):
|
||||
if v is None or bad:
|
||||
result.append(None)
|
||||
else:
|
||||
# Check for "+"/"-" prefix artifacts (e.g., "+abc", "-xyz")
|
||||
# where the value isn't a genuine numeric string. Values like
|
||||
# "+5" or "-3.14" are still accepted as valid floats below.
|
||||
if len(v) > 1 and v[0] in '+-':
|
||||
try:
|
||||
float(v[1:]) # validate suffix minus prefix
|
||||
except ValueError:
|
||||
result.append(None)
|
||||
continue
|
||||
try:
|
||||
result.append(float(v))
|
||||
except (ValueError, TypeError):
|
||||
result.append(None)
|
||||
return pl.Series(result, dtype=pl.Float64)
|
||||
|
||||
|
||||
# ── BOM detection ───────────────────────────────────────────────────────
|
||||
|
||||
def detect_bom(path: str) -> str:
|
||||
"""Read the first 4 bytes of *path* and return the detected encoding.
|
||||
|
||||
Returns ``'utf-8'`` when no BOM is found.
|
||||
|
||||
Note: Python 3 ``open()`` fully supports Unicode/Chinese paths on all
|
||||
platforms (no encoding dance needed).
|
||||
"""
|
||||
with open(path, 'rb') as f:
|
||||
header = f.read(4)
|
||||
|
||||
if header[:3] == b'\xef\xbb\xbf':
|
||||
return 'utf-8-sig'
|
||||
if header[:2] == b'\xff\xfe':
|
||||
return 'utf-16le'
|
||||
if header[:2] == b'\xfe\xff':
|
||||
return 'utf-16be'
|
||||
return 'utf-8'
|
||||
|
||||
|
||||
# ── schema utilities ────────────────────────────────────────────────────
|
||||
|
||||
def collect_schema(
|
||||
path: str,
|
||||
encoding: str = 'utf-8',
|
||||
delimiter: str = ',',
|
||||
sample_rows: int = 10000,
|
||||
) -> dict:
|
||||
"""Return {column_name: dtype_string} for a single CSV file.
|
||||
|
||||
Uses :func:`pl.scan_csv` with ``infer_schema_length=0`` to skip type
|
||||
inference. All columns are forced to ``Utf8`` for cross-file compatibility,
|
||||
avoiding type-inference failures when one file's column is ``Int64`` and
|
||||
another's is ``String``.
|
||||
"""
|
||||
lf = pl.scan_csv(
|
||||
str(path), # str() ensures Unicode/Chinese paths resolve correctly
|
||||
encoding=encoding,
|
||||
separator=delimiter,
|
||||
infer_schema_length=0,
|
||||
)
|
||||
schema = lf.collect_schema()
|
||||
result = {}
|
||||
for name, dtype in zip(schema.names(), schema.dtypes()):
|
||||
result[name] = 'Utf8' # force Utf8 for cross-file compatibility
|
||||
return result
|
||||
|
||||
|
||||
def _merge_schema_entries(
|
||||
schemas: list[dict],
|
||||
) -> dict:
|
||||
"""Merge multiple schema dicts, raising on type conflicts.
|
||||
|
||||
Returns a single {name: dtype} dict. If a column appears in multiple
|
||||
files with different dtypes the *last* non-null dtype wins (with a
|
||||
warning embedded in the result).
|
||||
"""
|
||||
merged: dict[str, str] = {}
|
||||
conflicts: list[str] = []
|
||||
for s in schemas:
|
||||
for name, dtype in s.items():
|
||||
if name in merged and merged[name] != dtype:
|
||||
conflicts.append(name)
|
||||
merged[name] = dtype # last wins
|
||||
return merged, conflicts
|
||||
|
||||
|
||||
def validate_schemas(schemas: list[dict], strict: bool = True) -> None:
|
||||
"""Compare schemas across files and raise or warn on column mismatch.
|
||||
|
||||
Checks that every file has the same set of column names. Type
|
||||
differences across files are **not** considered an error (Polars can
|
||||
cast), but differences in *column sets* are.
|
||||
|
||||
Args:
|
||||
schemas: List of ``{column_name: dtype_string}`` dicts, one per file.
|
||||
strict: If ``True`` (default), raise :class:`ValueError` on any
|
||||
column mismatch. If ``False``, log a warning instead.
|
||||
"""
|
||||
if not schemas:
|
||||
return
|
||||
|
||||
col_sets = [set(s.keys()) for s in schemas]
|
||||
reference = col_sets[0]
|
||||
|
||||
for i, cs in enumerate(col_sets[1:], start=1):
|
||||
only_in_ref = reference - cs
|
||||
only_in_current = cs - reference
|
||||
if only_in_ref or only_in_current:
|
||||
msg_parts = []
|
||||
if only_in_ref:
|
||||
msg_parts.append(f"File #{i} missing columns: {sorted(only_in_ref)}")
|
||||
if only_in_current:
|
||||
msg_parts.append(f"File #{i} has extra columns: {sorted(only_in_current)}")
|
||||
full_msg = f"Schema mismatch: {'; '.join(msg_parts)}"
|
||||
if strict:
|
||||
raise ValueError(full_msg)
|
||||
else:
|
||||
logger.warning(full_msg)
|
||||
|
||||
|
||||
# ── dtype inference from schema ─────────────────────────────────────────
|
||||
|
||||
def detect_schema_dtypes(
|
||||
lf: pl.LazyFrame,
|
||||
sample_rows: int = 10000,
|
||||
) -> dict:
|
||||
"""Infer Polars dtype strings from a LazyFrame's schema.
|
||||
|
||||
The schema is read without collecting the full data (the scan scans
|
||||
a small sample).
|
||||
"""
|
||||
schema = lf.collect_schema()
|
||||
return {name: str(dtype) for name, dtype in zip(schema.names(), schema.dtypes())}
|
||||
|
||||
|
||||
# ── config loading ──────────────────────────────────────────────────────
|
||||
|
||||
def load_config(config_path: Optional[str]) -> dict:
|
||||
"""Load a YAML config file; returns an empty dict if *config_path* is None."""
|
||||
if config_path is None:
|
||||
return {}
|
||||
path = Path(config_path)
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Config file not found: {config_path}")
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
return yaml.safe_load(f) or {}
|
||||
|
||||
|
||||
# ── SQLite persistence ──────────────────────────────────────────────────
|
||||
|
||||
def _dtype_to_sqlite(dtype: pl.DataType) -> str:
|
||||
"""Map a Polars dtype to a SQLite column type affinity."""
|
||||
if dtype in (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
|
||||
pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64):
|
||||
return 'INTEGER'
|
||||
if dtype in (pl.Float32, pl.Float64):
|
||||
return 'REAL'
|
||||
if dtype == pl.Boolean:
|
||||
return 'INTEGER'
|
||||
return 'TEXT' # String, Date, Datetime, etc. all map to TEXT
|
||||
|
||||
|
||||
def save_to_db(run_id: int, lf: pl.LazyFrame, mode: str = 'replace') -> str:
|
||||
"""Write a LazyFrame to a SQLite table ``_data_{run_id}``.
|
||||
|
||||
Collects the LazyFrame (streaming if possible), creates or replaces the
|
||||
table, bulk-inserts rows in chunks, and returns the table name.
|
||||
|
||||
Args:
|
||||
run_id: Analysis run ID (used as ``_data_{run_id}``).
|
||||
lf: LazyFrame to persist.
|
||||
mode: ``'replace'`` (default) → drop+create+insert.
|
||||
``'append'`` → skip DDL, only INSERT (table must exist).
|
||||
|
||||
This is a *persistence copy* — the original CSV files remain untouched.
|
||||
"""
|
||||
from django.conf import settings
|
||||
|
||||
table_name = f"_data_{run_id}"
|
||||
db_path = settings.DATABASES['default']['NAME']
|
||||
|
||||
df = lf.collect(streaming=True)
|
||||
conn = sqlite3.connect(db_path)
|
||||
|
||||
try:
|
||||
if mode == 'replace':
|
||||
# Drop existing table for retry/idempotency safety
|
||||
conn.execute(f'DROP TABLE IF EXISTS "{table_name}"')
|
||||
|
||||
# Build CREATE TABLE from Polars schema
|
||||
col_defs = [f'"{name}" {_dtype_to_sqlite(dtype)}'
|
||||
for name, dtype in zip(df.columns, df.dtypes)]
|
||||
conn.execute(f'CREATE TABLE "{table_name}" ({", ".join(col_defs)})')
|
||||
|
||||
# Bulk insert in chunks
|
||||
col_names = [f'"{c}"' for c in df.columns]
|
||||
placeholders = ','.join(['?' for _ in df.columns])
|
||||
insert_sql = f'INSERT INTO "{table_name}" ({", ".join(col_names)}) VALUES ({placeholders})'
|
||||
|
||||
total_rows = len(df)
|
||||
chunk_size = 10000
|
||||
for i in range(0, total_rows, chunk_size):
|
||||
chunk = df.slice(i, chunk_size)
|
||||
conn.executemany(insert_sql, chunk.rows())
|
||||
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
logger.info('[SAVE_TO_DB] table=%s rows=%d mode=%s', table_name, total_rows, mode)
|
||||
return table_name
|
||||
|
||||
|
||||
_STR_TO_DTYPE = {
|
||||
'Float64': pl.Float64, 'Float32': pl.Float32,
|
||||
'Int64': pl.Int64, 'Int32': pl.Int32, 'Int16': pl.Int16, 'Int8': pl.Int8,
|
||||
'UInt64': pl.UInt64, 'UInt32': pl.UInt32, 'UInt16': pl.UInt16, 'UInt8': pl.UInt8,
|
||||
'Utf8': pl.Utf8, 'String': pl.Utf8, 'Categorical': pl.Categorical,
|
||||
'Boolean': pl.Boolean, 'Bool': pl.Boolean,
|
||||
'Date': pl.Date, 'Datetime': pl.Datetime, 'Time': pl.Time,
|
||||
}
|
||||
|
||||
|
||||
def _str_to_polars_dtype(s: str) -> pl.DataType:
|
||||
"""Convert a Polars dtype string (e.g. ``'Float64'``, ``'Int32'``) to a ``pl.DataType``.
|
||||
|
||||
Strips trailing metadata like ``Datetime(time_unit='us', time_zone=None)``.
|
||||
"""
|
||||
base = s.split('(')[0].strip()
|
||||
return _STR_TO_DTYPE.get(base, pl.Utf8)
|
||||
|
||||
|
||||
def _rows_to_csv_df(rows: list[sqlite3.Row]) -> pl.DataFrame:
|
||||
"""Convert a batch of sqlite3.Row objects to a DataFrame via CSV fallback.
|
||||
|
||||
Loads all columns as Utf8 to handle mixed-type SQLite columns.
|
||||
"""
|
||||
import csv as _csv_mod
|
||||
|
||||
buf = io.StringIO()
|
||||
writer = _csv_mod.writer(buf)
|
||||
writer.writerow(rows[0].keys())
|
||||
for r in rows:
|
||||
writer.writerow(str(v) if v is not None else '' for v in r)
|
||||
buf.seek(0)
|
||||
return pl.read_csv(buf, infer_schema_length=0)
|
||||
|
||||
|
||||
def load_from_db(
|
||||
table_name: str,
|
||||
schema_overrides: dict[str, pl.DataType | str] | None = None,
|
||||
) -> Optional[pl.LazyFrame]:
|
||||
"""Load a SQLite table into a LazyFrame.
|
||||
|
||||
Args:
|
||||
table_name: SQLite table name.
|
||||
schema_overrides: Optional dict of ``{column_name: dtype}`` to cast
|
||||
columns after loading. Values can be ``pl.DataType`` instances
|
||||
(e.g. ``pl.Float64``) or dtype strings (e.g. ``'Float64'``).
|
||||
Use when SQLite ``TEXT`` columns need to be restored to
|
||||
``Float64`` / ``Int64`` etc.
|
||||
|
||||
Returns ``None`` if the table does not exist.
|
||||
"""
|
||||
from django.conf import settings
|
||||
|
||||
db_path = settings.DATABASES['default']['NAME']
|
||||
conn = sqlite3.connect(db_path)
|
||||
|
||||
try:
|
||||
# Check table exists
|
||||
cursor = conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name=?",
|
||||
(table_name,),
|
||||
)
|
||||
if cursor.fetchone() is None:
|
||||
return None
|
||||
|
||||
conn.row_factory = sqlite3.Row
|
||||
cursor = conn.execute(f'SELECT * FROM "{table_name}"')
|
||||
|
||||
# Peek first row to detect if table is empty
|
||||
first_row = cursor.fetchone()
|
||||
if first_row is None:
|
||||
return pl.DataFrame().lazy()
|
||||
|
||||
batch_rows = [first_row]
|
||||
dfs = []
|
||||
_use_csv_fallback = False
|
||||
|
||||
while True:
|
||||
chunk = cursor.fetchmany(10000)
|
||||
if not chunk:
|
||||
# Flush remaining rows
|
||||
if batch_rows:
|
||||
if _use_csv_fallback:
|
||||
dfs.append(_rows_to_csv_df(batch_rows))
|
||||
else:
|
||||
try:
|
||||
dfs.append(pl.from_dicts([dict(r) for r in batch_rows]))
|
||||
except Exception:
|
||||
_use_csv_fallback = True
|
||||
dfs.append(_rows_to_csv_df(batch_rows))
|
||||
break
|
||||
|
||||
batch_rows.extend(chunk)
|
||||
if len(batch_rows) >= 10000:
|
||||
if _use_csv_fallback:
|
||||
dfs.append(_rows_to_csv_df(batch_rows))
|
||||
else:
|
||||
try:
|
||||
dfs.append(pl.from_dicts([dict(r) for r in batch_rows]))
|
||||
except Exception:
|
||||
_use_csv_fallback = True
|
||||
# Re-process current batch with CSV fallback
|
||||
dfs.append(_rows_to_csv_df(batch_rows))
|
||||
batch_rows = []
|
||||
|
||||
# Concat all batches
|
||||
if len(dfs) == 1:
|
||||
df = dfs[0]
|
||||
else:
|
||||
df = pl.concat(dfs, how='vertical')
|
||||
|
||||
# Apply type overrides to restore original column types
|
||||
if schema_overrides:
|
||||
casts = {}
|
||||
for col, dtype in schema_overrides.items():
|
||||
if col not in df.columns:
|
||||
continue
|
||||
dtype_obj = dtype if isinstance(dtype, pl.DataType) else _str_to_polars_dtype(dtype)
|
||||
if df[col].dtype != dtype_obj:
|
||||
try:
|
||||
casts[col] = pl.col(col).cast(dtype_obj)
|
||||
except Exception:
|
||||
# If a cast fails (e.g. "A" → Int64), keep the column as-is
|
||||
pass
|
||||
if casts:
|
||||
df = df.with_columns(list(casts.values()))
|
||||
|
||||
return df.lazy()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def drop_sqlite_table(table_name: str) -> None:
|
||||
"""Drop a SQLite data table silently if it exists."""
|
||||
from django.conf import settings
|
||||
|
||||
db_path = settings.DATABASES['default']['NAME']
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
conn.execute(f'DROP TABLE IF EXISTS "{table_name}"')
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ── helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
def _file_count(paths: list[str]) -> int:
|
||||
return len(paths)
|
||||
|
||||
|
||||
def _resolve_encoding(encoding: str, file_encoding: str) -> str:
|
||||
"""BOM-detected encoding overrides user-supplied encoding.
|
||||
|
||||
Polars >= 1.0 requires ``'utf8'`` (not ``'utf-8'``).
|
||||
"""
|
||||
resolved = file_encoding if file_encoding != 'utf-8' else encoding
|
||||
return resolved.replace('-', '') if resolved in ('utf-8', 'utf-8-sig') else resolved
|
||||
|
||||
|
||||
# ── main entry ──────────────────────────────────────────────────────────
|
||||
|
||||
def load_csv_directory(
|
||||
glob_pattern: str,
|
||||
encoding: str = 'utf-8',
|
||||
delimiter: str = ',',
|
||||
config_path: Optional[str] = None,
|
||||
schema_override: Optional[dict] = None,
|
||||
sample_rows: int = 10000,
|
||||
schema_strict: bool = False,
|
||||
recursive: bool = False,
|
||||
head: Optional[int] = None,
|
||||
) -> tuple[pl.LazyFrame, dict, int, int, float]:
|
||||
"""Load, validate, and merge CSV files matching *glob_pattern*.
|
||||
|
||||
Args:
|
||||
glob_pattern: Glob pattern for CSV files (e.g. ``data/*.csv``).
|
||||
encoding: Fallback encoding when no BOM is detected.
|
||||
delimiter: Column delimiter character.
|
||||
config_path: Optional path to a YAML config (merged into metadata).
|
||||
schema_override: If given, force these {column: dtype} overrides.
|
||||
sample_rows: Rows to scan for dtype inference per file.
|
||||
schema_strict: When True, raise ValueError on column mismatch across
|
||||
files. When False (default), log warnings and merge with
|
||||
``diagonal_relaxed`` (missing columns get null).
|
||||
recursive: When True, use ``glob.glob(..., recursive=True)`` to
|
||||
support ``**/*.csv`` patterns.
|
||||
head: Optional row limit for preview scenarios. When set, applies
|
||||
``lf.head(head)`` before returning.
|
||||
|
||||
Returns:
|
||||
A tuple of ``(lazyframe, schema_dict, total_row_count, file_count,
|
||||
memory_estimate_mb)``.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If no files match the glob pattern.
|
||||
ValueError: If schemas are inconsistent across files and
|
||||
*schema_strict* is True.
|
||||
"""
|
||||
# Normalise path separators for cross-platform Unicode support
|
||||
glob_pattern = os.path.normpath(glob_pattern)
|
||||
|
||||
# Resolve file list
|
||||
paths = sorted(glob.glob(glob_pattern, recursive=recursive))
|
||||
if not paths and not recursive:
|
||||
# Fallback: try recursive pattern
|
||||
paths = sorted(glob.glob(glob_pattern, recursive=True))
|
||||
if not paths:
|
||||
raise FileNotFoundError(f"No files match glob pattern: {glob_pattern}")
|
||||
|
||||
# Load config (user-provided or fall back to project default)
|
||||
config = load_config(config_path)
|
||||
if not config:
|
||||
default_path = Path(__file__).parent.parent / 'config' / 'config.yaml'
|
||||
if default_path.exists():
|
||||
config = load_config(str(default_path))
|
||||
|
||||
# Collect per-file schema
|
||||
schemas: list[dict] = []
|
||||
lazyframes: list[pl.LazyFrame] = []
|
||||
for path in paths:
|
||||
file_encoding = detect_bom(path)
|
||||
actual_encoding = _resolve_encoding(encoding, file_encoding)
|
||||
|
||||
lf = pl.scan_csv(
|
||||
str(path), # str() ensures Unicode/Chinese paths resolve correctly
|
||||
encoding=actual_encoding,
|
||||
separator=delimiter,
|
||||
infer_schema_length=max(sample_rows, 1000),
|
||||
)
|
||||
schema = detect_schema_dtypes(lf, sample_rows=sample_rows)
|
||||
|
||||
# Apply schema override for this file
|
||||
if schema_override:
|
||||
for col, dtype in schema_override.items():
|
||||
if col in schema:
|
||||
schema[col] = dtype
|
||||
|
||||
schemas.append(schema)
|
||||
lazyframes.append(lf)
|
||||
|
||||
if schema_strict:
|
||||
# Raise on column mismatch
|
||||
validate_schemas(schemas)
|
||||
else:
|
||||
# Lenient mode: log warnings for column mismatches
|
||||
col_sets = [set(s.keys()) for s in schemas]
|
||||
reference = col_sets[0]
|
||||
for i, cs in enumerate(col_sets[1:], start=1):
|
||||
only_in_ref = reference - cs
|
||||
only_in_current = cs - reference
|
||||
if only_in_ref:
|
||||
logger.warning(
|
||||
"File #%d missing columns: %s (will be null)",
|
||||
i + 1, sorted(only_in_ref),
|
||||
)
|
||||
if only_in_current:
|
||||
logger.warning(
|
||||
"File #%d has extra columns: %s (will be null in other files)",
|
||||
i + 1, sorted(only_in_current),
|
||||
)
|
||||
|
||||
# Merge schemas (unify column ordering — first file wins order)
|
||||
merged_schema, _ = _merge_schema_entries(schemas)
|
||||
|
||||
# Concatenate all LazyFrames (union vertically)
|
||||
merged_lf: pl.LazyFrame = pl.concat(lazyframes, how='diagonal_relaxed')
|
||||
|
||||
# Log schema and size estimate
|
||||
total_row_count_est = len(paths) * (lazyframes[0].collect(streaming=True).height if lazyframes else 0)
|
||||
logger.info(f'[LOAD] files={len(paths)} total_rows_est={total_row_count_est}')
|
||||
for col_name, dtype in sorted(merged_schema.items()):
|
||||
logger.info(f'[LOAD_SCHEMA] col={col_name} dtype={dtype}')
|
||||
|
||||
# Apply schema override on merged frame (cast columns)
|
||||
if schema_override:
|
||||
casts = {}
|
||||
for col, dtype_str in schema_override.items():
|
||||
if col in merged_schema:
|
||||
try:
|
||||
polars_dtype = getattr(pl, dtype_str.split('(')[0], None)
|
||||
if polars_dtype:
|
||||
casts[col] = pl.col(col).cast(polars_dtype)
|
||||
except Exception:
|
||||
pass # skip uncastable overrides silently
|
||||
if casts:
|
||||
merged_lf = merged_lf.with_columns(list(casts.values()))
|
||||
|
||||
# ── Cleaning: drop outdated columns ──────────────────────────────────
|
||||
geoip_cfg = config.get('data', {}).get('geoip', {})
|
||||
drop_cols = geoip_cfg.get(
|
||||
'drop_columns',
|
||||
[], # Empty by default — lat/lon columns are now cleaned, not dropped
|
||||
)
|
||||
existing_drop = [c for c in drop_cols if c in merged_schema]
|
||||
if existing_drop:
|
||||
merged_lf = merged_lf.drop(existing_drop)
|
||||
merged_schema = {k: v for k, v in merged_schema.items() if k not in existing_drop}
|
||||
logger.info('[CLEAN] dropped columns: %s', existing_drop)
|
||||
|
||||
# ── Cleaning: BOOL_ENUM normalisation ────────────────────────────────
|
||||
try:
|
||||
type_map = classify_schema(
|
||||
merged_lf.head(1000),
|
||||
config_overrides=config.get('columns', {}),
|
||||
)
|
||||
except Exception:
|
||||
type_map = {}
|
||||
|
||||
bool_enum_cols = [
|
||||
c for c, t in type_map.items()
|
||||
if t == DataType.BOOL_ENUM and c in merged_schema
|
||||
]
|
||||
if bool_enum_cols:
|
||||
# Only clean string-typed BOOL_ENUM columns; native Boolean is already clean
|
||||
schema_dtypes = dict(zip(
|
||||
merged_lf.collect_schema().names(),
|
||||
merged_lf.collect_schema().dtypes(),
|
||||
))
|
||||
bool_exprs = []
|
||||
cleaned_names: list[str] = []
|
||||
for c in bool_enum_cols:
|
||||
if schema_dtypes.get(c) not in (pl.Utf8, pl.String):
|
||||
continue # native Boolean / numeric — skip
|
||||
c_str = pl.col(c).cast(pl.Utf8).str.strip_chars()
|
||||
cleaned = (
|
||||
pl.when(c_str == "")
|
||||
.then(pl.lit("False"))
|
||||
.when(c_str.str.to_lowercase() == "+")
|
||||
.then(pl.lit("True"))
|
||||
.when(c_str.str.to_lowercase() == "-")
|
||||
.then(pl.lit("False"))
|
||||
.otherwise(c_str)
|
||||
)
|
||||
bool_exprs.append(cleaned.alias(c))
|
||||
cleaned_names.append(c)
|
||||
if bool_exprs:
|
||||
merged_lf = merged_lf.with_columns(bool_exprs)
|
||||
for c in cleaned_names:
|
||||
merged_schema[c] = 'Utf8'
|
||||
logger.info('[CLEAN] BOOL_ENUM columns normalised: %s', cleaned_names)
|
||||
|
||||
# ── Cleaning: LAT_LON normalisation ────────────────────────────────
|
||||
latlon_cols = [
|
||||
c for c, t in type_map.items()
|
||||
if t == DataType.LAT_LON and c in merged_schema
|
||||
]
|
||||
if latlon_cols:
|
||||
schema_dtypes_ll = dict(zip(
|
||||
merged_lf.collect_schema().names(),
|
||||
merged_lf.collect_schema().dtypes(),
|
||||
))
|
||||
latlon_exprs = []
|
||||
for c in latlon_cols:
|
||||
dtype = schema_dtypes_ll.get(c)
|
||||
if dtype in (pl.Utf8, pl.String):
|
||||
latlon_exprs.append(
|
||||
pl.col(c).map_batches(_coerce_to_float, return_dtype=pl.Float64).alias(c)
|
||||
)
|
||||
merged_schema[c] = 'Float64'
|
||||
if latlon_exprs:
|
||||
merged_lf = merged_lf.with_columns(latlon_exprs)
|
||||
logger.info('[CLEAN] LAT_LON columns normalised to Float64: %s', latlon_cols)
|
||||
|
||||
# ── Cleaning: Generic numeric coercion ────────────────────────────
|
||||
# For any remaining string column (not IPv4/URL/HEX/ENUM/etc.),
|
||||
# try to coerce to Float64. Whatever can't parse becomes null.
|
||||
# This prevents "mean on str column" crashes for ANY column.
|
||||
_SKIP_NUM_COERCE = (DataType.IPv4, DataType.URL, DataType.HEX,
|
||||
DataType.ENUM, DataType.LAT_LON, DataType.BOOL_ENUM)
|
||||
# Also protect columns with these name patterns (MAC addresses, etc.)
|
||||
_SKIP_NAME_PATTERNS = ('mac', 'mac_', '_mac', 'addr', 'address')
|
||||
schema_dtypes_gn = dict(zip(
|
||||
merged_lf.collect_schema().names(),
|
||||
merged_lf.collect_schema().dtypes(),
|
||||
))
|
||||
generic_numeric_cols = []
|
||||
for c, t in type_map.items():
|
||||
if t in _SKIP_NUM_COERCE:
|
||||
continue
|
||||
if c not in merged_schema:
|
||||
continue
|
||||
dtype = schema_dtypes_gn.get(c)
|
||||
if dtype not in (pl.Utf8, pl.String):
|
||||
continue # already numeric
|
||||
# Skip columns with protected name patterns
|
||||
cl = c.lower().replace('-', '_').replace(' ', '_')
|
||||
if any(p in cl for p in _SKIP_NAME_PATTERNS):
|
||||
continue
|
||||
generic_numeric_cols.append(c)
|
||||
|
||||
if generic_numeric_cols:
|
||||
num_exprs = []
|
||||
for c in generic_numeric_cols:
|
||||
num_exprs.append(
|
||||
pl.col(c).map_batches(_coerce_to_float, return_dtype=pl.Float64).alias(c)
|
||||
)
|
||||
if num_exprs:
|
||||
merged_lf = merged_lf.with_columns(num_exprs)
|
||||
for c in generic_numeric_cols:
|
||||
merged_schema[c] = 'Float64'
|
||||
logger.info('[CLEAN] Generic numeric coercion applied: %s', generic_numeric_cols)
|
||||
|
||||
# ── Cleaning: HEX preprocessing (hex_pairs_count) ────────────────────
|
||||
hex_cols = [
|
||||
c for c, t in type_map.items()
|
||||
if t == DataType.HEX and c in merged_schema
|
||||
]
|
||||
if hex_cols:
|
||||
hex_exprs = []
|
||||
for c in hex_cols:
|
||||
hex_exprs.append(
|
||||
pl.col(c).str.split(" ").list.len().alias(f"{c}_hex_pairs_count")
|
||||
)
|
||||
merged_lf = merged_lf.with_columns(hex_exprs)
|
||||
for c in hex_cols:
|
||||
merged_schema[f"{c}_hex_pairs_count"] = 'UInt32'
|
||||
logger.info('[CLEAN] HEX columns added hex_pairs_count: %s', hex_cols)
|
||||
|
||||
# Row count estimate (fast path: use first file row count × files)
|
||||
# For accurate count we'd need to collect, but that defeats lazy.
|
||||
# We estimate from the first file's scanned schema info.
|
||||
first_rows = lazyframes[0].collect(streaming=True).height if lazyframes else 0
|
||||
total_row_count = first_rows * len(paths)
|
||||
|
||||
# Memory estimate
|
||||
bytes_per_row = 0
|
||||
for col_name, dtype_str in merged_schema.items():
|
||||
numeric_part = dtype_str.split('(')[0].lower()
|
||||
size_map = {
|
||||
'int8': 1, 'int16': 2, 'int32': 4, 'int64': 8,
|
||||
'uint8': 1, 'uint16': 2, 'uint32': 4, 'uint64': 8,
|
||||
'float32': 4, 'float64': 8,
|
||||
'boolean': 1, 'bool': 1,
|
||||
'utf8': 50, 'string': 50, 'str': 50,
|
||||
'categorical': 8, 'cat': 8,
|
||||
'date': 4, 'datetime': 8, 'time': 8, 'duration': 8,
|
||||
}
|
||||
bytes_per_row += size_map.get(numeric_part, 8)
|
||||
memory_mb = (bytes_per_row * total_row_count) / (1024 * 1024)
|
||||
|
||||
# Apply head limit for preview (keeps LazyFrame lazy — rows are limited on collect)
|
||||
if head is not None:
|
||||
merged_lf = merged_lf.head(head)
|
||||
total_row_count = min(total_row_count, head)
|
||||
|
||||
return merged_lf, merged_schema, total_row_count, len(paths), memory_mb
|
||||
from analysis.data_loader._csv import load_csv_directory
|
||||
from analysis.data_loader._sqlite import (
|
||||
load_from_db,
|
||||
load_from_db_lazy,
|
||||
save_to_db,
|
||||
drop_sqlite_table,
|
||||
)
|
||||
from analysis.data_loader._clean import _coerce_to_float
|
||||
from analysis.data_loader._sqlite import _dtype_to_sqlite
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
"""CSV data loading, schema handling, cleaning, and SQLite persistence.
|
||||
|
||||
Package layout:
|
||||
_csv.py — CSV scanning, BOM/encoding detection, :func:`load_csv_directory`
|
||||
_schema.py — Schema merging and cross-file validation
|
||||
_clean.py — Data cleaning helpers (:func:`_coerce_to_float`)
|
||||
_sqlite.py — SQLite persistence (save/load/drop tables)
|
||||
"""
|
||||
|
||||
from ._csv import (
|
||||
SUPPORTED_ENCODINGS,
|
||||
detect_bom,
|
||||
collect_schema,
|
||||
detect_schema_dtypes,
|
||||
load_config,
|
||||
_file_count,
|
||||
_resolve_encoding,
|
||||
load_csv_directory,
|
||||
)
|
||||
from ._schema import _merge_schema_entries, validate_schemas
|
||||
from ._clean import _coerce_to_float
|
||||
from ._sqlite import (
|
||||
_dtype_to_sqlite,
|
||||
_str_to_polars_dtype,
|
||||
_rows_to_csv_df,
|
||||
save_to_db,
|
||||
load_from_db,
|
||||
load_from_db_lazy,
|
||||
drop_sqlite_table,
|
||||
)
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Data cleaning helpers: numeric coercion for string columns."""
|
||||
import polars as pl
|
||||
|
||||
|
||||
def _coerce_to_float(series: pl.Series) -> pl.Series:
|
||||
"""Convert string series to Float64.
|
||||
|
||||
Every entry that doesn't parse as a valid float becomes null.
|
||||
Handles blank strings, standalone '+'/'-', 'N/A', 'null', 'None', etc.
|
||||
"""
|
||||
series = series.cast(pl.Utf8).str.strip_chars()
|
||||
artifacts = ('', '+', '-', '.', '..', 'N/A', 'n/a', 'NA', 'null', 'None', 'NULL', '?', '---', '--', '/')
|
||||
is_artifact = series.is_in(pl.Series(artifacts).implode())
|
||||
result = []
|
||||
for v, bad in zip(series.to_list(), is_artifact.to_list()):
|
||||
if v is None or bad:
|
||||
result.append(None)
|
||||
else:
|
||||
# Check for "+"/"-" prefix artifacts (e.g., "+abc", "-xyz")
|
||||
# where the value isn't a genuine numeric string. Values like
|
||||
# "+5" or "-3.14" are still accepted as valid floats below.
|
||||
if len(v) > 1 and v[0] in '+-':
|
||||
try:
|
||||
float(v[1:]) # validate suffix minus prefix
|
||||
except ValueError:
|
||||
result.append(None)
|
||||
continue
|
||||
try:
|
||||
result.append(float(v))
|
||||
except (ValueError, TypeError):
|
||||
result.append(None)
|
||||
return pl.Series(result, dtype=pl.Float64)
|
||||
@@ -0,0 +1,375 @@
|
||||
"""CSV loading: glob, BOM detection, encoding, schema collection, multi-file merge.
|
||||
|
||||
Main entry: :func:`load_csv_directory`.
|
||||
"""
|
||||
import glob
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import polars as pl
|
||||
import yaml
|
||||
|
||||
from analysis.types import classify_schema, TYPE_REGISTRY
|
||||
from ._clean import _coerce_to_float
|
||||
from ._schema import _merge_schema_entries, validate_schemas
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SUPPORTED_ENCODINGS = ['utf-8', 'utf-8-sig', 'utf-16le', 'utf-16be', 'latin-1']
|
||||
|
||||
|
||||
# ── BOM detection ───────────────────────────────────────────────────────
|
||||
|
||||
def detect_bom(path: str) -> str:
|
||||
"""Read the first 4 bytes of *path* and return the detected encoding.
|
||||
|
||||
Returns ``'utf-8'`` when no BOM is found.
|
||||
|
||||
Note: Python 3 ``open()`` fully supports Unicode/Chinese paths on all
|
||||
platforms (no encoding dance needed).
|
||||
"""
|
||||
with open(path, 'rb') as f:
|
||||
header = f.read(4)
|
||||
|
||||
if header[:3] == b'\xef\xbb\xbf':
|
||||
return 'utf-8-sig'
|
||||
if header[:2] == b'\xff\xfe':
|
||||
return 'utf-16le'
|
||||
if header[:2] == b'\xfe\xff':
|
||||
return 'utf-16be'
|
||||
return 'utf-8'
|
||||
|
||||
|
||||
# ── schema utilities ────────────────────────────────────────────────────
|
||||
|
||||
def collect_schema(
|
||||
path: str,
|
||||
encoding: str = 'utf-8',
|
||||
delimiter: str = ',',
|
||||
sample_rows: int = 10000,
|
||||
) -> dict:
|
||||
"""Return {column_name: dtype_string} for a single CSV file.
|
||||
|
||||
Uses :func:`pl.scan_csv` with ``infer_schema_length=0`` to skip type
|
||||
inference. All columns are forced to ``Utf8`` for cross-file compatibility,
|
||||
avoiding type-inference failures when one file's column is ``Int64`` and
|
||||
another's is ``String``.
|
||||
"""
|
||||
lf = pl.scan_csv(
|
||||
str(path), # str() ensures Unicode/Chinese paths resolve correctly
|
||||
encoding=encoding,
|
||||
separator=delimiter,
|
||||
infer_schema_length=0,
|
||||
)
|
||||
schema = lf.collect_schema()
|
||||
result = {}
|
||||
for name, dtype in zip(schema.names(), schema.dtypes()):
|
||||
result[name] = 'Utf8' # force Utf8 for cross-file compatibility
|
||||
return result
|
||||
|
||||
|
||||
# ── dtype inference from schema ─────────────────────────────────────────
|
||||
|
||||
def detect_schema_dtypes(
|
||||
lf: pl.LazyFrame,
|
||||
sample_rows: int = 10000,
|
||||
) -> dict:
|
||||
"""Infer Polars dtype strings from a LazyFrame's schema.
|
||||
|
||||
The schema is read without collecting the full data (the scan scans
|
||||
a small sample).
|
||||
"""
|
||||
schema = lf.collect_schema()
|
||||
return {name: str(dtype) for name, dtype in zip(schema.names(), schema.dtypes())}
|
||||
|
||||
|
||||
# ── config loading ──────────────────────────────────────────────────────
|
||||
|
||||
def load_config(config_path: Optional[str]) -> dict:
|
||||
"""Load a YAML config file; returns an empty dict if *config_path* is None."""
|
||||
if config_path is None:
|
||||
return {}
|
||||
path = Path(config_path)
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Config file not found: {config_path}")
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
return yaml.safe_load(f) or {}
|
||||
|
||||
|
||||
# ── helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
def _file_count(paths: list[str]) -> int:
|
||||
return len(paths)
|
||||
|
||||
|
||||
def _resolve_encoding(encoding: str, file_encoding: str) -> str:
|
||||
"""BOM-detected encoding overrides user-supplied encoding.
|
||||
|
||||
Polars >= 1.0 requires ``'utf8'`` (not ``'utf-8'``).
|
||||
"""
|
||||
resolved = file_encoding if file_encoding != 'utf-8' else encoding
|
||||
return resolved.replace('-', '') if resolved in ('utf-8', 'utf-8-sig') else resolved
|
||||
|
||||
|
||||
# ── main entry ──────────────────────────────────────────────────────────
|
||||
|
||||
def load_csv_directory(
|
||||
glob_pattern: str,
|
||||
encoding: str = 'utf-8',
|
||||
delimiter: str = ',',
|
||||
config_path: Optional[str] = None,
|
||||
schema_override: Optional[dict] = None,
|
||||
sample_rows: int = 10000,
|
||||
schema_strict: bool = False,
|
||||
recursive: bool = False,
|
||||
head: Optional[int] = None,
|
||||
) -> tuple[pl.LazyFrame, dict, int, int, float]:
|
||||
"""Load, validate, and merge CSV files matching *glob_pattern*.
|
||||
|
||||
Args:
|
||||
glob_pattern: Glob pattern for CSV files (e.g. ``data/*.csv``).
|
||||
encoding: Fallback encoding when no BOM is detected.
|
||||
delimiter: Column delimiter character.
|
||||
config_path: Optional path to a YAML config (merged into metadata).
|
||||
schema_override: If given, force these {column: dtype} overrides.
|
||||
sample_rows: Rows to scan for dtype inference per file.
|
||||
schema_strict: When True, raise ValueError on column mismatch across
|
||||
files. When False (default), log warnings and merge with
|
||||
``diagonal_relaxed`` (missing columns get null).
|
||||
recursive: When True, use ``glob.glob(..., recursive=True)`` to
|
||||
support ``**/*.csv`` patterns.
|
||||
head: Optional row limit for preview scenarios. When set, applies
|
||||
``lf.head(head)`` before returning.
|
||||
|
||||
Returns:
|
||||
A tuple of ``(lazyframe, schema_dict, total_row_count, file_count,
|
||||
memory_estimate_mb)``.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If no files match the glob pattern.
|
||||
ValueError: If schemas are inconsistent across files and
|
||||
*schema_strict* is True.
|
||||
"""
|
||||
# Normalise path separators for cross-platform Unicode support
|
||||
glob_pattern = os.path.normpath(glob_pattern)
|
||||
|
||||
# Resolve file list
|
||||
paths = sorted(glob.glob(glob_pattern, recursive=recursive))
|
||||
if not paths and not recursive:
|
||||
# Fallback: try recursive pattern
|
||||
paths = sorted(glob.glob(glob_pattern, recursive=True))
|
||||
if not paths:
|
||||
raise FileNotFoundError(f"No files match glob pattern: {glob_pattern}")
|
||||
|
||||
# Load config (user-provided or fall back to project default)
|
||||
config = load_config(config_path)
|
||||
if not config:
|
||||
default_path = Path(__file__).parent.parent.parent / 'config' / 'config.yaml'
|
||||
if default_path.exists():
|
||||
config = load_config(str(default_path))
|
||||
|
||||
# Collect per-file schema
|
||||
schemas: list[dict] = []
|
||||
lazyframes: list[pl.LazyFrame] = []
|
||||
for path in paths:
|
||||
file_encoding = detect_bom(path)
|
||||
actual_encoding = _resolve_encoding(encoding, file_encoding)
|
||||
|
||||
lf = pl.scan_csv(
|
||||
str(path), # str() ensures Unicode/Chinese paths resolve correctly
|
||||
encoding=actual_encoding,
|
||||
separator=delimiter,
|
||||
infer_schema_length=max(sample_rows, 1000),
|
||||
)
|
||||
schema = detect_schema_dtypes(lf, sample_rows=sample_rows)
|
||||
|
||||
# Apply schema override for this file
|
||||
if schema_override:
|
||||
for col, dtype in schema_override.items():
|
||||
if col in schema:
|
||||
schema[col] = dtype
|
||||
|
||||
schemas.append(schema)
|
||||
lazyframes.append(lf)
|
||||
|
||||
if schema_strict:
|
||||
# Raise on column mismatch
|
||||
validate_schemas(schemas)
|
||||
else:
|
||||
# Lenient mode: log warnings for column mismatches
|
||||
col_sets = [set(s.keys()) for s in schemas]
|
||||
reference = col_sets[0]
|
||||
for i, cs in enumerate(col_sets[1:], start=1):
|
||||
only_in_ref = reference - cs
|
||||
only_in_current = cs - reference
|
||||
if only_in_ref:
|
||||
logger.warning(
|
||||
"File #%d missing columns: %s (will be null)",
|
||||
i + 1, sorted(only_in_ref),
|
||||
)
|
||||
if only_in_current:
|
||||
logger.warning(
|
||||
"File #%d has extra columns: %s (will be null in other files)",
|
||||
i + 1, sorted(only_in_current),
|
||||
)
|
||||
|
||||
# Merge schemas (unify column ordering — first file wins order)
|
||||
merged_schema, _ = _merge_schema_entries(schemas)
|
||||
|
||||
# Concatenate all LazyFrames (union vertically)
|
||||
merged_lf: pl.LazyFrame = pl.concat(lazyframes, how='diagonal_relaxed')
|
||||
|
||||
# Log schema and size estimate
|
||||
total_row_count_est = len(paths) * (lazyframes[0].collect(streaming=True).height if lazyframes else 0)
|
||||
logger.info(f'[LOAD] files={len(paths)} total_rows_est={total_row_count_est}')
|
||||
for col_name, dtype in sorted(merged_schema.items()):
|
||||
logger.info(f'[LOAD_SCHEMA] col={col_name} dtype={dtype}')
|
||||
|
||||
# Apply schema override on merged frame (cast columns)
|
||||
if schema_override:
|
||||
casts = {}
|
||||
for col, dtype_str in schema_override.items():
|
||||
if col in merged_schema:
|
||||
try:
|
||||
polars_dtype = getattr(pl, dtype_str.split('(')[0], None)
|
||||
if polars_dtype:
|
||||
casts[col] = pl.col(col).cast(polars_dtype)
|
||||
except Exception:
|
||||
pass # skip uncastable overrides silently
|
||||
if casts:
|
||||
merged_lf = merged_lf.with_columns(list(casts.values()))
|
||||
|
||||
# ── Cleaning: drop outdated columns ──────────────────────────────────
|
||||
geoip_cfg = config.get('data', {}).get('geoip', {})
|
||||
drop_cols = geoip_cfg.get(
|
||||
'drop_columns',
|
||||
[], # Empty by default — lat/lon columns are now cleaned, not dropped
|
||||
)
|
||||
existing_drop = [c for c in drop_cols if c in merged_schema]
|
||||
if existing_drop:
|
||||
merged_lf = merged_lf.drop(existing_drop)
|
||||
merged_schema = {k: v for k, v in merged_schema.items() if k not in existing_drop}
|
||||
logger.info('[CLEAN] dropped columns: %s', existing_drop)
|
||||
|
||||
# ── Cleaning: BOOL normalisation ─────────────────────────────────────
|
||||
try:
|
||||
type_map = classify_schema(
|
||||
merged_lf.head(1000),
|
||||
config_overrides=config.get('columns', {}),
|
||||
)
|
||||
except Exception:
|
||||
type_map = {}
|
||||
|
||||
bool_cols = [
|
||||
c for c, t in type_map.items()
|
||||
if t.name == "布尔" and c in merged_schema
|
||||
]
|
||||
if bool_cols:
|
||||
schema_dtypes = dict(zip(
|
||||
merged_lf.collect_schema().names(),
|
||||
merged_lf.collect_schema().dtypes(),
|
||||
))
|
||||
bool_exprs = []
|
||||
cleaned_names: list[str] = []
|
||||
for c in bool_cols:
|
||||
if schema_dtypes.get(c) not in (pl.Utf8, pl.String):
|
||||
continue
|
||||
c_str = pl.col(c).cast(pl.Utf8).str.strip_chars()
|
||||
cleaned = (
|
||||
pl.when(c_str == "")
|
||||
.then(pl.lit("False"))
|
||||
.when(c_str.str.to_lowercase() == "+")
|
||||
.then(pl.lit("True"))
|
||||
.when(c_str.str.to_lowercase() == "-")
|
||||
.then(pl.lit("False"))
|
||||
.otherwise(c_str)
|
||||
)
|
||||
bool_exprs.append(cleaned.alias(c))
|
||||
cleaned_names.append(c)
|
||||
if bool_exprs:
|
||||
merged_lf = merged_lf.with_columns(bool_exprs)
|
||||
for c in cleaned_names:
|
||||
merged_schema[c] = 'Utf8'
|
||||
logger.info('[CLEAN] BOOL columns normalised: %s', cleaned_names)
|
||||
|
||||
# ── Generic numeric coercion ────────────────────────────────────────
|
||||
# For remaining string columns (not IPv4/BYTES/ENUM/BOOL etc.),
|
||||
# try to coerce to Float64.
|
||||
_SKIP_NUM_COERCE = {"IPv4", "字节", "枚举", "布尔"}
|
||||
_SKIP_NAME_PATTERNS = ('mac', 'mac_', '_mac', 'addr', 'address')
|
||||
schema_dtypes_gn = dict(zip(
|
||||
merged_lf.collect_schema().names(),
|
||||
merged_lf.collect_schema().dtypes(),
|
||||
))
|
||||
generic_numeric_cols = []
|
||||
for c, t in type_map.items():
|
||||
if t.name in _SKIP_NUM_COERCE:
|
||||
continue
|
||||
if c not in merged_schema:
|
||||
continue
|
||||
dtype = schema_dtypes_gn.get(c)
|
||||
if dtype not in (pl.Utf8, pl.String):
|
||||
continue
|
||||
cl = c.lower().replace('-', '_').replace(' ', '_')
|
||||
if any(p in cl for p in _SKIP_NAME_PATTERNS):
|
||||
continue
|
||||
generic_numeric_cols.append(c)
|
||||
|
||||
if generic_numeric_cols:
|
||||
num_exprs = []
|
||||
for c in generic_numeric_cols:
|
||||
num_exprs.append(
|
||||
pl.col(c).map_batches(_coerce_to_float, return_dtype=pl.Float64).alias(c)
|
||||
)
|
||||
if num_exprs:
|
||||
merged_lf = merged_lf.with_columns(num_exprs)
|
||||
for c in generic_numeric_cols:
|
||||
merged_schema[c] = 'Float64'
|
||||
logger.info('[CLEAN] Generic numeric coercion applied: %s', generic_numeric_cols)
|
||||
|
||||
# ── Cleaning: BYTES (hex) preprocessing ─────────────────────────────
|
||||
hex_cols = [
|
||||
c for c, t in type_map.items()
|
||||
if t.name == "字节" and c in merged_schema
|
||||
]
|
||||
if hex_cols:
|
||||
hex_exprs = []
|
||||
for c in hex_cols:
|
||||
hex_exprs.append(
|
||||
pl.col(c).str.split(" ").list.len().alias(f"{c}_hex_pairs_count")
|
||||
)
|
||||
merged_lf = merged_lf.with_columns(hex_exprs)
|
||||
for c in hex_cols:
|
||||
merged_schema[f"{c}_hex_pairs_count"] = 'UInt32'
|
||||
logger.info('[CLEAN] BYTES columns added hex_pairs_count: %s', hex_cols)
|
||||
|
||||
# Row count estimate (fast path: use first file row count × files)
|
||||
# For accurate count we'd need to collect, but that defeats lazy.
|
||||
# We estimate from the first file's scanned schema info.
|
||||
first_rows = lazyframes[0].collect(streaming=True).height if lazyframes else 0
|
||||
total_row_count = first_rows * len(paths)
|
||||
|
||||
# Memory estimate
|
||||
bytes_per_row = 0
|
||||
for col_name, dtype_str in merged_schema.items():
|
||||
numeric_part = dtype_str.split('(')[0].lower()
|
||||
size_map = {
|
||||
'int8': 1, 'int16': 2, 'int32': 4, 'int64': 8,
|
||||
'uint8': 1, 'uint16': 2, 'uint32': 4, 'uint64': 8,
|
||||
'float32': 4, 'float64': 8,
|
||||
'boolean': 1, 'bool': 1,
|
||||
'utf8': 50, 'string': 50, 'str': 50,
|
||||
'categorical': 8, 'cat': 8,
|
||||
'date': 4, 'datetime': 8, 'time': 8, 'duration': 8,
|
||||
}
|
||||
bytes_per_row += size_map.get(numeric_part, 8)
|
||||
memory_mb = (bytes_per_row * total_row_count) / (1024 * 1024)
|
||||
|
||||
# Apply head limit for preview (keeps LazyFrame lazy — rows are limited on collect)
|
||||
if head is not None:
|
||||
merged_lf = merged_lf.head(head)
|
||||
total_row_count = min(total_row_count, head)
|
||||
|
||||
return merged_lf, merged_schema, total_row_count, len(paths), memory_mb
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Schema detection, validation, and merging across CSV files."""
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _merge_schema_entries(
|
||||
schemas: list[dict],
|
||||
) -> tuple[dict[str, str], list[str]]:
|
||||
"""Merge multiple schema dicts, raising on type conflicts.
|
||||
|
||||
Returns a single {name: dtype} dict. If a column appears in multiple
|
||||
files with different dtypes the *last* non-null dtype wins (with a
|
||||
warning embedded in the result).
|
||||
"""
|
||||
merged: dict[str, str] = {}
|
||||
conflicts: list[str] = []
|
||||
for s in schemas:
|
||||
for name, dtype in s.items():
|
||||
if name in merged and merged[name] != dtype:
|
||||
conflicts.append(name)
|
||||
merged[name] = dtype # last wins
|
||||
return merged, conflicts
|
||||
|
||||
|
||||
def validate_schemas(schemas: list[dict], strict: bool = True) -> None:
|
||||
"""Compare schemas across files and raise or warn on column mismatch.
|
||||
|
||||
Checks that every file has the same set of column names. Type
|
||||
differences across files are **not** considered an error (Polars can
|
||||
cast), but differences in *column sets* are.
|
||||
|
||||
Args:
|
||||
schemas: List of ``{column_name: dtype_string}`` dicts, one per file.
|
||||
strict: If ``True`` (default), raise :class:`ValueError` on any
|
||||
column mismatch. If ``False``, log a warning instead.
|
||||
"""
|
||||
if not schemas:
|
||||
return
|
||||
|
||||
col_sets = [set(s.keys()) for s in schemas]
|
||||
reference = col_sets[0]
|
||||
|
||||
for i, cs in enumerate(col_sets[1:], start=1):
|
||||
only_in_ref = reference - cs
|
||||
only_in_current = cs - reference
|
||||
if only_in_ref or only_in_current:
|
||||
msg_parts = []
|
||||
if only_in_ref:
|
||||
msg_parts.append(f"File #{i} missing columns: {sorted(only_in_ref)}")
|
||||
if only_in_current:
|
||||
msg_parts.append(f"File #{i} has extra columns: {sorted(only_in_current)}")
|
||||
full_msg = f"Schema mismatch: {'; '.join(msg_parts)}"
|
||||
if strict:
|
||||
raise ValueError(full_msg)
|
||||
else:
|
||||
logger.warning(full_msg)
|
||||
@@ -0,0 +1,292 @@
|
||||
"""SQLite persistence layer for LazyFrame data."""
|
||||
import csv as _csv_mod
|
||||
import io
|
||||
import logging
|
||||
import sqlite3
|
||||
from typing import Optional
|
||||
|
||||
import polars as pl
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── dtype mapping ─────────────────────────────────────────────────────────
|
||||
|
||||
_STR_TO_DTYPE = {
|
||||
'Float64': pl.Float64, 'Float32': pl.Float32,
|
||||
'Int64': pl.Int64, 'Int32': pl.Int32, 'Int16': pl.Int16, 'Int8': pl.Int8,
|
||||
'UInt64': pl.UInt64, 'UInt32': pl.UInt32, 'UInt16': pl.UInt16, 'UInt8': pl.UInt8,
|
||||
'Utf8': pl.Utf8, 'String': pl.Utf8, 'Categorical': pl.Categorical,
|
||||
'Boolean': pl.Boolean, 'Bool': pl.Boolean,
|
||||
'Date': pl.Date, 'Datetime': pl.Datetime, 'Time': pl.Time,
|
||||
}
|
||||
|
||||
|
||||
def _dtype_to_sqlite(dtype: pl.DataType) -> str:
|
||||
"""Map a Polars dtype to a SQLite column type affinity."""
|
||||
if dtype in (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
|
||||
pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64):
|
||||
return 'INTEGER'
|
||||
if dtype in (pl.Float32, pl.Float64):
|
||||
return 'REAL'
|
||||
if dtype == pl.Boolean:
|
||||
return 'INTEGER'
|
||||
return 'TEXT' # String, Date, Datetime, etc. all map to TEXT
|
||||
|
||||
|
||||
def _str_to_polars_dtype(s: str) -> pl.DataType:
|
||||
"""Convert a Polars dtype string (e.g. ``'Float64'``, ``'Int32'``) to a ``pl.DataType``.
|
||||
|
||||
Strips trailing metadata like ``Datetime(time_unit='us', time_zone=None)``.
|
||||
"""
|
||||
base = s.split('(')[0].strip()
|
||||
return _STR_TO_DTYPE.get(base, pl.Utf8)
|
||||
|
||||
|
||||
def _rows_to_csv_df(rows: list[sqlite3.Row]) -> pl.DataFrame:
|
||||
"""Convert a batch of sqlite3.Row objects to a DataFrame via CSV fallback.
|
||||
|
||||
Loads all columns as Utf8 to handle mixed-type SQLite columns.
|
||||
"""
|
||||
buf = io.StringIO()
|
||||
writer = _csv_mod.writer(buf)
|
||||
writer.writerow(rows[0].keys())
|
||||
for r in rows:
|
||||
writer.writerow(str(v) if v is not None else '' for v in r)
|
||||
buf.seek(0)
|
||||
return pl.read_csv(buf, infer_schema_length=0)
|
||||
|
||||
|
||||
# ── public API ───────────────────────────────────────────────────────────
|
||||
|
||||
def save_to_db(run_id: int, lf: pl.LazyFrame, mode: str = 'replace') -> str:
|
||||
"""Write a LazyFrame to a SQLite table ``_data_{run_id}``.
|
||||
|
||||
Collects the LazyFrame (streaming if possible), creates or replaces the
|
||||
table, bulk-inserts rows in chunks, and returns the table name.
|
||||
|
||||
Args:
|
||||
run_id: Analysis run ID (used as ``_data_{run_id}``).
|
||||
lf: LazyFrame to persist.
|
||||
mode: ``'replace'`` (default) → drop+create+insert.
|
||||
``'append'`` → skip DDL, only INSERT (table must exist).
|
||||
|
||||
This is a *persistence copy* — the original CSV files remain untouched.
|
||||
"""
|
||||
from django.conf import settings
|
||||
|
||||
table_name = f"_data_{run_id}"
|
||||
db_path = settings.DATABASES['default']['NAME']
|
||||
|
||||
df = lf.collect(streaming=True)
|
||||
conn = sqlite3.connect(db_path)
|
||||
|
||||
try:
|
||||
if mode == 'replace':
|
||||
# Drop existing table for retry/idempotency safety
|
||||
conn.execute(f'DROP TABLE IF EXISTS "{table_name}"')
|
||||
|
||||
# Build CREATE TABLE from Polars schema
|
||||
col_defs = [f'"{name}" {_dtype_to_sqlite(dtype)}'
|
||||
for name, dtype in zip(df.columns, df.dtypes)]
|
||||
conn.execute(f'CREATE TABLE "{table_name}" ({", ".join(col_defs)})')
|
||||
|
||||
# Bulk insert in chunks
|
||||
col_names = [f'"{c}"' for c in df.columns]
|
||||
placeholders = ','.join(['?' for _ in df.columns])
|
||||
insert_sql = f'INSERT INTO "{table_name}" ({", ".join(col_names)}) VALUES ({placeholders})'
|
||||
|
||||
total_rows = len(df)
|
||||
chunk_size = 10000
|
||||
for i in range(0, total_rows, chunk_size):
|
||||
chunk = df.slice(i, chunk_size)
|
||||
conn.executemany(insert_sql, chunk.rows())
|
||||
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
logger.info('[SAVE_TO_DB] table=%s rows=%d mode=%s', table_name, total_rows, mode)
|
||||
return table_name
|
||||
|
||||
|
||||
def load_from_db(
|
||||
table_name: str,
|
||||
schema_overrides: dict[str, pl.DataType | str] | None = None,
|
||||
) -> Optional[pl.LazyFrame]:
|
||||
"""Load a SQLite table into a LazyFrame.
|
||||
|
||||
Args:
|
||||
table_name: SQLite table name.
|
||||
schema_overrides: Optional dict of ``{column_name: dtype}`` to cast
|
||||
columns after loading. Values can be ``pl.DataType`` instances
|
||||
(e.g. ``pl.Float64``) or dtype strings (e.g. ``'Float64'``).
|
||||
Use when SQLite ``TEXT`` columns need to be restored to
|
||||
``Float64`` / ``Int64`` etc.
|
||||
|
||||
Returns ``None`` if the table does not exist.
|
||||
"""
|
||||
from django.conf import settings
|
||||
|
||||
db_path = settings.DATABASES['default']['NAME']
|
||||
conn = sqlite3.connect(db_path)
|
||||
|
||||
try:
|
||||
# Check table exists
|
||||
cursor = conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name=?",
|
||||
(table_name,),
|
||||
)
|
||||
if cursor.fetchone() is None:
|
||||
return None
|
||||
|
||||
conn.row_factory = sqlite3.Row
|
||||
cursor = conn.execute(f'SELECT * FROM "{table_name}"')
|
||||
|
||||
# Peek first row to detect if table is empty
|
||||
first_row = cursor.fetchone()
|
||||
if first_row is None:
|
||||
return pl.DataFrame().lazy()
|
||||
|
||||
batch_rows = [first_row]
|
||||
dfs = []
|
||||
_use_csv_fallback = False
|
||||
|
||||
while True:
|
||||
chunk = cursor.fetchmany(10000)
|
||||
if not chunk:
|
||||
# Flush remaining rows
|
||||
if batch_rows:
|
||||
if _use_csv_fallback:
|
||||
dfs.append(_rows_to_csv_df(batch_rows))
|
||||
else:
|
||||
try:
|
||||
dfs.append(pl.from_dicts([dict(r) for r in batch_rows]))
|
||||
except Exception:
|
||||
_use_csv_fallback = True
|
||||
dfs.append(_rows_to_csv_df(batch_rows))
|
||||
break
|
||||
|
||||
batch_rows.extend(chunk)
|
||||
if len(batch_rows) >= 10000:
|
||||
if _use_csv_fallback:
|
||||
dfs.append(_rows_to_csv_df(batch_rows))
|
||||
else:
|
||||
try:
|
||||
dfs.append(pl.from_dicts([dict(r) for r in batch_rows]))
|
||||
except Exception:
|
||||
_use_csv_fallback = True
|
||||
# Re-process current batch with CSV fallback
|
||||
dfs.append(_rows_to_csv_df(batch_rows))
|
||||
batch_rows = []
|
||||
|
||||
# Concat all batches
|
||||
if len(dfs) == 1:
|
||||
df = dfs[0]
|
||||
else:
|
||||
df = pl.concat(dfs, how='vertical')
|
||||
|
||||
# Apply type overrides to restore original column types
|
||||
if schema_overrides:
|
||||
casts = {}
|
||||
for col, dtype in schema_overrides.items():
|
||||
if col not in df.columns:
|
||||
continue
|
||||
dtype_obj = dtype if isinstance(dtype, pl.DataType) else _str_to_polars_dtype(dtype)
|
||||
if df[col].dtype != dtype_obj:
|
||||
try:
|
||||
casts[col] = pl.col(col).cast(dtype_obj)
|
||||
except Exception:
|
||||
# If a cast fails (e.g. "A" → Int64), keep the column as-is
|
||||
pass
|
||||
if casts:
|
||||
df = df.with_columns(list(casts.values()))
|
||||
|
||||
return df.lazy()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def load_from_db_lazy(
|
||||
table_name: str,
|
||||
schema_overrides: dict[str, pl.DataType | str] | None = None,
|
||||
) -> Optional[pl.LazyFrame]:
|
||||
"""Load a SQLite table into a LazyFrame using Polars native DB reader.
|
||||
|
||||
Uses ``pl.read_database`` for efficient bulk reading — an order of
|
||||
magnitude faster than the manual batch-processing path in
|
||||
:func:`load_from_db`. Returns a :class:`pl.LazyFrame` so downstream
|
||||
operations remain lazy (query-plan, not materialised).
|
||||
|
||||
Args:
|
||||
table_name: SQLite table name.
|
||||
schema_overrides: Optional ``{column_name: dtype}`` dict to cast
|
||||
columns after loading. Values can be ``pl.DataType`` instances
|
||||
(e.g. ``pl.Float64``) or dtype strings (e.g. ``'Float64'``).
|
||||
|
||||
Returns:
|
||||
``None`` if the table does not exist or is empty.
|
||||
"""
|
||||
from django.conf import settings
|
||||
|
||||
db_path = settings.DATABASES['default']['NAME']
|
||||
conn = sqlite3.connect(db_path)
|
||||
|
||||
try:
|
||||
# Check table existence (fast metadata query)
|
||||
cursor = conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name=?",
|
||||
(table_name,),
|
||||
)
|
||||
if cursor.fetchone() is None:
|
||||
return None
|
||||
|
||||
# Bulk-read with Polars native engine — leverages the DBAPI2
|
||||
# connection directly for efficient C-level data transfer.
|
||||
query = f'SELECT * FROM "{table_name}"'
|
||||
df = pl.read_database(connection=conn, query=query)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
if df.height == 0:
|
||||
return None
|
||||
|
||||
# Apply type overrides to restore original column types
|
||||
# (SQLite stores everything as TEXT/INTEGER/REAL/BLOB;
|
||||
# we need to cast back to Float64 / Int64 / etc.)
|
||||
if schema_overrides:
|
||||
casts = {}
|
||||
for col, dtype in schema_overrides.items():
|
||||
if col not in df.columns:
|
||||
continue
|
||||
dtype_obj = (
|
||||
dtype if isinstance(dtype, pl.DataType)
|
||||
else _str_to_polars_dtype(dtype)
|
||||
)
|
||||
if df[col].dtype != dtype_obj:
|
||||
try:
|
||||
casts[col] = pl.col(col).cast(dtype_obj)
|
||||
except Exception:
|
||||
pass # uncastable column — keep as-is
|
||||
if casts:
|
||||
df = df.with_columns(list(casts.values()))
|
||||
|
||||
rows = df.height
|
||||
cols = df.width
|
||||
logger.info(
|
||||
'[LOAD_FROM_DB_LAZY] table=%s rows=%d cols=%d',
|
||||
table_name, rows, cols,
|
||||
)
|
||||
return df.lazy()
|
||||
|
||||
|
||||
def drop_sqlite_table(table_name: str) -> None:
|
||||
"""Drop a SQLite data table silently if it exists."""
|
||||
from django.conf import settings
|
||||
|
||||
db_path = settings.DATABASES['default']['NAME']
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
conn.execute(f'DROP TABLE IF EXISTS "{table_name}"')
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Distance computation package for TianXuan analysis pipeline.
|
||||
|
||||
Provides distance functions for various data types (IP, string, boolean,
|
||||
bytes, timestamp), plus normalisation, SVD denoising, UMAP reduction,
|
||||
and per-cluster SVD feature extraction.
|
||||
|
||||
Public API:
|
||||
ip_distance — subnet-mask + Haversine geographic distance for IP pairs
|
||||
string_distance — Levenshtein / enum-like distance for string columns
|
||||
bool_distance — 0/1 match-to-mode distance for boolean columns
|
||||
bytes_distance — Hamming (popcount) distance for hex byte columns
|
||||
timestamp_fft_distance — FFT phase distance for timestamp columns
|
||||
normalize_features — StandardScaler normalisation (μ=0, σ=1)
|
||||
svd_denoise — TruncatedSVD noise removal (returns noise_profile)
|
||||
umap_reduce — UMAP dimensionality reduction (3D→2D fallback)
|
||||
cluster_svd_extract — Per-cluster SVD distinguishing feature extraction
|
||||
"""
|
||||
|
||||
from analysis.distance._geo import ip_distance
|
||||
from analysis.distance._text import bool_distance, string_distance
|
||||
from analysis.distance._numeric import bytes_distance, timestamp_fft_distance
|
||||
from analysis.distance._normalize import normalize_features
|
||||
from analysis.distance._svd import svd_denoise
|
||||
from analysis.distance._umap import umap_reduce
|
||||
from analysis.distance._cluster_svd import cluster_svd_extract
|
||||
|
||||
__all__ = [
|
||||
"ip_distance",
|
||||
"string_distance",
|
||||
"bool_distance",
|
||||
"bytes_distance",
|
||||
"timestamp_fft_distance",
|
||||
"normalize_features",
|
||||
"svd_denoise",
|
||||
"umap_reduce",
|
||||
"cluster_svd_extract",
|
||||
]
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Per-cluster SVD feature extraction."""
|
||||
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
from sklearn.decomposition import TruncatedSVD
|
||||
|
||||
|
||||
def cluster_svd_extract(
|
||||
lf: pl.LazyFrame,
|
||||
labels: list[int],
|
||||
feature_cols: list[str],
|
||||
) -> dict[int, dict]:
|
||||
"""Per-cluster SVD-based distinguishing feature extraction.
|
||||
|
||||
For each cluster label:
|
||||
1. Extracts rows belonging to that cluster.
|
||||
2. Fits ``TruncatedSVD(n_components=min(N, len(feature_cols)))``.
|
||||
3. The absolute loadings on the first SVD component serve as
|
||||
*feature_strength* — higher values mean more distinguishing.
|
||||
|
||||
Returns a mapping ``{cluster_label: {...}}`` where each value is::
|
||||
|
||||
{
|
||||
'features': [...], # top-10 feature names by strength
|
||||
'explained_variance_ratio': [...], # per-component ratios
|
||||
'feature_strength': [...], # per-feature loading magnitude
|
||||
}
|
||||
|
||||
Parameters
|
||||
----------
|
||||
lf:
|
||||
LazyFrame with rows aligned to *labels*.
|
||||
labels:
|
||||
Cluster labels (same length as *lf* row count, ``-1`` = noise).
|
||||
feature_cols:
|
||||
Numeric column names to analyse.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict[int, dict]
|
||||
Per-cluster SVD extraction results.
|
||||
"""
|
||||
available = [c for c in feature_cols if c in lf.collect_schema().names()]
|
||||
if not available:
|
||||
return {}
|
||||
|
||||
df = lf.select(available).collect(streaming=True)
|
||||
mat = df.to_numpy().astype(np.float64)
|
||||
mat = np.nan_to_num(mat, nan=0.0, posinf=0.0, neginf=0.0)
|
||||
|
||||
if len(labels) != len(mat):
|
||||
labels = labels[:len(mat)]
|
||||
|
||||
unique_labels = sorted(set(labels))
|
||||
result: dict[int, dict] = {}
|
||||
|
||||
for label in unique_labels:
|
||||
mask = np.array(labels) == label
|
||||
n = int(mask.sum())
|
||||
if n < 2:
|
||||
continue
|
||||
|
||||
cluster_mat = mat[mask]
|
||||
n_comp = min(n, len(available))
|
||||
svd = TruncatedSVD(n_components=n_comp, random_state=42)
|
||||
svd.fit(cluster_mat)
|
||||
|
||||
# First component loadings → feature strength
|
||||
if n_comp > 0 and svd.components_.shape[0] > 0:
|
||||
feature_strength = np.abs(svd.components_[0]).astype(np.float64)
|
||||
else:
|
||||
feature_strength = np.zeros(len(available), dtype=np.float64)
|
||||
|
||||
total_strength = float(feature_strength.sum())
|
||||
if total_strength > 1e-10:
|
||||
feature_strength = feature_strength / total_strength
|
||||
|
||||
top_indices = np.argsort(-feature_strength)
|
||||
top_k = min(10, len(available))
|
||||
|
||||
result[int(label)] = {
|
||||
'features': [available[i] for i in top_indices[:top_k]],
|
||||
'explained_variance_ratio': svd.explained_variance_ratio_.tolist(),
|
||||
'feature_strength': feature_strength.tolist(),
|
||||
}
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,170 @@
|
||||
"""IP distance functions for Polars LazyFrames.
|
||||
|
||||
Provides subnet-mask distance and Haversine geographic distance
|
||||
between pairs of IP columns. All distances are normalised to [0, 1];
|
||||
invalid / non-geolocatable IPs produce ``NaN``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Optional
|
||||
|
||||
import polars as pl
|
||||
|
||||
from analysis.geoip import ip_to_int, lookup
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Earth constant
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_HALF_EARTH_CIRCUMFERENCE_KM = 20_037.508 # equatorial / 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Core distance helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _subnet_mask_distance(ip1: Optional[str], ip2: Optional[str], mask: int) -> float:
|
||||
"""Normalised subnet-mask distance between two IP strings.
|
||||
|
||||
Computes ``abs(int(ip1)/2**mask - int(ip2)/2**mask) / 2**(32-mask)``.
|
||||
Returns ``NaN`` when either IP is ``None`` or unparseable.
|
||||
"""
|
||||
if ip1 is None or ip2 is None:
|
||||
return float("nan")
|
||||
if not isinstance(ip1, str):
|
||||
ip1 = str(ip1)
|
||||
if not isinstance(ip2, str):
|
||||
ip2 = str(ip2)
|
||||
|
||||
int1 = ip_to_int(ip1)
|
||||
int2 = ip_to_int(ip2)
|
||||
if int1 is None or int2 is None:
|
||||
return float("nan")
|
||||
|
||||
divisor = 2.0 ** mask
|
||||
max_val = 2.0 ** (32 - mask)
|
||||
return abs(int1 / divisor - int2 / divisor) / max_val
|
||||
|
||||
|
||||
def _haversine_km(lat1: float, lon1: float, lat2: float, lon2: float) -> float:
|
||||
"""Great-circle distance (km) via the Haversine formula."""
|
||||
dlat = math.radians(lat2 - lat1)
|
||||
dlon = math.radians(lon2 - lon1)
|
||||
a = (
|
||||
math.sin(dlat / 2) ** 2
|
||||
+ math.cos(math.radians(lat1))
|
||||
* math.cos(math.radians(lat2))
|
||||
* math.sin(dlon / 2) ** 2
|
||||
)
|
||||
return 6371.0 * 2.0 * math.asin(math.sqrt(a))
|
||||
|
||||
|
||||
def _geo_distance(ip1: Optional[str], ip2: Optional[str]) -> float:
|
||||
"""Normalised [0, 1] geographic distance between two IP strings.
|
||||
|
||||
Uses ``analysis.geoip.lookup()`` to obtain lat/lon, then Haversine.
|
||||
Returns ``NaN`` when either IP cannot be geolocated.
|
||||
"""
|
||||
if ip1 is None or ip2 is None:
|
||||
return float("nan")
|
||||
if not isinstance(ip1, str):
|
||||
ip1 = str(ip1)
|
||||
if not isinstance(ip2, str):
|
||||
ip2 = str(ip2)
|
||||
|
||||
geo1 = lookup(ip1)
|
||||
geo2 = lookup(ip2)
|
||||
if geo1 is None or geo2 is None:
|
||||
return float("nan")
|
||||
|
||||
km = _haversine_km(geo1["lat"], geo1["lon"], geo2["lat"], geo2["lon"])
|
||||
return km / _HALF_EARTH_CIRCUMFERENCE_KM
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def ip_distance(
|
||||
lf: pl.LazyFrame,
|
||||
ip_columns: list[str],
|
||||
subnet_masks: Optional[list[int]] = None,
|
||||
) -> pl.LazyFrame:
|
||||
"""Append IP distance columns to a LazyFrame.
|
||||
|
||||
For each **consecutive pair** of *ip_columns*
|
||||
(``ip_columns[0]`` vs ``ip_columns[1]``,
|
||||
``ip_columns[2]`` vs ``ip_columns[3]``, …) computes:
|
||||
|
||||
* one subnet-mask distance per entry in *subnet_masks*
|
||||
* one Haversine geographic distance
|
||||
|
||||
All distances are normalised to [0, 1]. Invalid IPs or IPs not
|
||||
found in the GeoIP database produce ``NaN``.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
lf:
|
||||
LazyFrame containing the IP columns.
|
||||
ip_columns:
|
||||
List of IP column names (used in consecutive pairs).
|
||||
subnet_masks:
|
||||
Subnet masks for network distance (default ``[24, 28]``).
|
||||
|
||||
Returns
|
||||
-------
|
||||
pl.LazyFrame
|
||||
Input frame with ``_ip_dist_0``, ``_ip_dist_1``, …
|
||||
columns appended.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> lf = pl.LazyFrame({
|
||||
... "src_ip": ["8.8.8.8", "1.1.1.1"],
|
||||
... "dst_ip": ["8.8.4.4", "1.0.0.1"],
|
||||
... })
|
||||
>>> result = ip_distance(lf, ["src_ip", "dst_ip"], subnet_masks=[24])
|
||||
>>> result.collect().columns
|
||||
['src_ip', 'dst_ip', '_ip_dist_0', '_ip_dist_1']
|
||||
"""
|
||||
if subnet_masks is None:
|
||||
subnet_masks = [24, 28]
|
||||
|
||||
result = lf
|
||||
dist_idx = 0
|
||||
|
||||
# Walk consecutive pairs
|
||||
for i in range(0, len(ip_columns) - 1, 2):
|
||||
col_a = ip_columns[i]
|
||||
col_b = ip_columns[i + 1]
|
||||
|
||||
# -- subnet-mask distances --
|
||||
for mask in subnet_masks:
|
||||
name = f"_ip_dist_{dist_idx}"
|
||||
result = result.with_columns(
|
||||
pl.struct([col_a, col_b])
|
||||
.map_elements(
|
||||
lambda s, _mask=mask, _ca=col_a, _cb=col_b: (
|
||||
_subnet_mask_distance(s[_ca], s[_cb], _mask)
|
||||
),
|
||||
return_dtype=pl.Float64,
|
||||
)
|
||||
.alias(name)
|
||||
)
|
||||
dist_idx += 1
|
||||
|
||||
# -- geographic distance --
|
||||
name = f"_ip_dist_{dist_idx}"
|
||||
result = result.with_columns(
|
||||
pl.struct([col_a, col_b])
|
||||
.map_elements(
|
||||
lambda s, _ca=col_a, _cb=col_b: _geo_distance(s[_ca], s[_cb]),
|
||||
return_dtype=pl.Float64,
|
||||
)
|
||||
.alias(name)
|
||||
)
|
||||
dist_idx += 1
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,52 @@
|
||||
"""StandardScaler normalisation (step 2 of preprocessing pipeline)."""
|
||||
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
|
||||
|
||||
def normalize_features(lf: pl.LazyFrame, feature_cols: list[str]) -> pl.LazyFrame:
|
||||
"""Apply StandardScaler normalisation to numeric feature columns.
|
||||
|
||||
Collects *feature_cols* into a numpy array, replaces ``NaN`` with
|
||||
zero, centres to μ=0 σ=1 via ``sklearn.preprocessing.StandardScaler``,
|
||||
then returns the result as *lf* extended with ``_norm_*`` prefixed
|
||||
columns.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
lf:
|
||||
LazyFrame containing *feature_cols*.
|
||||
feature_cols:
|
||||
Numeric column names to normalise. Columns not present in
|
||||
*lf* are silently skipped; non-numeric columns raise an error.
|
||||
|
||||
Returns
|
||||
-------
|
||||
pl.LazyFrame
|
||||
Input frame with additional ``_norm_{col}`` columns.
|
||||
|
||||
Memory
|
||||
------
|
||||
Peak memory ≈ collected numpy matrix + scaler workspace
|
||||
(well under the 800 MB budget for typical inputs).
|
||||
"""
|
||||
available = [c for c in feature_cols if c in lf.collect_schema().names()]
|
||||
if not available:
|
||||
return lf
|
||||
|
||||
df = lf.select(available).collect(streaming=True)
|
||||
mat = df.to_numpy().astype(np.float64)
|
||||
mat = np.nan_to_num(mat, nan=0.0)
|
||||
|
||||
scaler = StandardScaler()
|
||||
scaled = scaler.fit_transform(mat)
|
||||
|
||||
# Build new columns with _norm_ prefix
|
||||
df_result = lf.collect(streaming=True)
|
||||
for i, col in enumerate(available):
|
||||
df_result = df_result.with_columns(
|
||||
pl.Series(f'_norm_{col}', scaled[:, i])
|
||||
)
|
||||
|
||||
return df_result.lazy()
|
||||
@@ -0,0 +1,278 @@
|
||||
"""Bytes/hex distance (Hamming/popcount) and timestamp FFT phase distance."""
|
||||
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bytes Hamming distance
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_HEX_POPCOUNT: dict[str, int] = {
|
||||
'0': 0, '1': 1, '2': 1, '3': 2,
|
||||
'4': 1, '5': 2, '6': 2, '7': 3,
|
||||
'8': 1, '9': 2,
|
||||
'a': 2, 'b': 3, 'c': 2, 'd': 3, 'e': 3, 'f': 4,
|
||||
'A': 2, 'B': 3, 'C': 2, 'D': 3, 'E': 3, 'F': 4,
|
||||
}
|
||||
|
||||
|
||||
def _hex_popcount_normalized(val: str | None) -> float:
|
||||
"""Return the proportion of 1-bits in a hex byte string (0.0–1.0).
|
||||
|
||||
This is the Hamming distance from the all-zero bit string,
|
||||
normalised by total bits. Returns ``NaN`` for null or
|
||||
non-hex input.
|
||||
"""
|
||||
if val is None:
|
||||
return float("nan")
|
||||
s = str(val).replace(' ', '').replace('\t', '').replace('\n', '')
|
||||
if not s:
|
||||
return 0.0
|
||||
|
||||
total_ones = 0
|
||||
for ch in s:
|
||||
ones = _HEX_POPCOUNT.get(ch)
|
||||
if ones is None:
|
||||
return float("nan") # non-hex character
|
||||
total_ones += ones
|
||||
|
||||
total_bits = len(s) * 4
|
||||
return total_ones / total_bits
|
||||
|
||||
|
||||
def bytes_distance(
|
||||
lf: pl.LazyFrame,
|
||||
byte_columns: list[str],
|
||||
) -> pl.LazyFrame:
|
||||
"""Append normalised Hamming-distance columns for hex byte columns.
|
||||
|
||||
Each byte column is converted from hex to binary, and the
|
||||
proportion of 1-bits (Hamming distance from the all-zero
|
||||
bit vector) is computed as a ``_bytes_dist_N`` Float64 column
|
||||
normalised to [0, 1]. Non-hex / null values produce ``NaN``.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
lf:
|
||||
LazyFrame containing the byte columns.
|
||||
byte_columns:
|
||||
Column names whose values are hex byte strings
|
||||
(with optional whitespace separators, e.g. ``"A3 F2"``).
|
||||
|
||||
Returns
|
||||
-------
|
||||
pl.LazyFrame
|
||||
Input frame with ``_bytes_dist_0``, ``_bytes_dist_1``, …
|
||||
appended. Columns not present in *lf* are silently skipped.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> df = pl.DataFrame({"payload": ["A3F2", "FFFF", None]})
|
||||
>>> result = bytes_distance(df.lazy(), ["payload"])
|
||||
>>> result.collect()
|
||||
shape: (3, 2)
|
||||
┌─────────┬────────────────┐
|
||||
│ payload ┆ _bytes_dist_0 │
|
||||
│ --- ┆ --- │
|
||||
│ str ┆ f64 │
|
||||
╞═════════╪════════════════╡
|
||||
│ A3F2 ┆ 0.625 │
|
||||
│ FFFF ┆ 1.0 │
|
||||
│ null ┆ NaN │
|
||||
└─────────┴────────────────┘
|
||||
"""
|
||||
existing = set(lf.collect_schema().names())
|
||||
result = lf
|
||||
|
||||
for i, col_name in enumerate(byte_columns):
|
||||
if col_name not in existing:
|
||||
continue
|
||||
result = result.with_columns(
|
||||
pl.col(col_name)
|
||||
.cast(pl.Utf8)
|
||||
.map_elements(_hex_popcount_normalized, return_dtype=pl.Float64)
|
||||
.alias(f'_bytes_dist_{i}')
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Timestamp FFT phase distance
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# ── threshold: minimum fraction of repeated intervals to use FFT path ──
|
||||
_FFT_PERIODICITY_THRESHOLD = 0.05
|
||||
# ── minimum samples before FFT is attempted ──
|
||||
_FFT_MIN_SAMPLES = 4
|
||||
|
||||
|
||||
def _parse_timestamp_to_epoch(series: pl.Series) -> pl.Series:
|
||||
"""Convert a timestamp series to Unix epoch seconds (float64).
|
||||
|
||||
Handles int/float (assumed already epoch), pl.Datetime/pl.Date
|
||||
(ns→seconds), and str/pl.Utf8 (strptime→ns→seconds, with direct
|
||||
float-cast fallback on parse failure).
|
||||
"""
|
||||
dtype = series.dtype
|
||||
if dtype in (pl.Utf8, pl.String):
|
||||
try:
|
||||
parsed = series.str.strptime(pl.Datetime, format=None, strict=False)
|
||||
except Exception:
|
||||
return series.cast(pl.Float64)
|
||||
return parsed.cast(pl.Int64).cast(pl.Float64) / 1_000_000_000.0
|
||||
elif dtype in (pl.Datetime, pl.Date):
|
||||
return series.cast(pl.Int64).cast(pl.Float64) / 1_000_000_000.0
|
||||
else:
|
||||
return series.cast(pl.Float64)
|
||||
|
||||
|
||||
def _compute_fft_distances(values: np.ndarray) -> np.ndarray:
|
||||
"""Compute per-row phase distances for a 1-D array.
|
||||
|
||||
Sorts *values*, checks for periodicity, then either:
|
||||
* FFT path: ``np.fft.fft()`` → ``np.angle()``, mapped back to
|
||||
original (unsorted) row order via rank lookup.
|
||||
* Statistical fallback: interval z-scores mapped to row order.
|
||||
|
||||
Returns an ``(n,)`` float64 array, normalised to zero-mean
|
||||
unit-variance.
|
||||
"""
|
||||
n = len(values)
|
||||
if n < _FFT_MIN_SAMPLES:
|
||||
return np.full(n, np.nan, dtype=np.float64)
|
||||
|
||||
# Sort once; record rank→position mapping for unsorting later
|
||||
sorted_order = np.argsort(values)
|
||||
ranks = np.empty(n, dtype=np.int64)
|
||||
ranks[sorted_order] = np.arange(n)
|
||||
sorted_vals = values[sorted_order]
|
||||
|
||||
# ── periodicity check ──────────────────────────────────────────
|
||||
intervals = np.diff(sorted_vals)
|
||||
unique_int, counts = np.unique(
|
||||
np.round(intervals, 6), return_counts=True
|
||||
)
|
||||
repeat_ratio = float(np.sum(counts[counts > 1])) / max(len(intervals), 1)
|
||||
|
||||
if repeat_ratio >= _FFT_PERIODICITY_THRESHOLD:
|
||||
# ── FFT phase path ─────────────────────────────────────────
|
||||
centered = sorted_vals - np.mean(sorted_vals)
|
||||
fft_result = np.fft.fft(centered)
|
||||
phases = np.angle(fft_result)
|
||||
dist_values = phases[ranks]
|
||||
else:
|
||||
# ── statistical fallback ───────────────────────────────────
|
||||
mean_int = float(np.mean(intervals))
|
||||
std_int = float(np.std(intervals))
|
||||
if std_int > 1e-10:
|
||||
z_scores = (intervals - mean_int) / std_int
|
||||
else:
|
||||
z_scores = np.zeros_like(intervals)
|
||||
|
||||
# Row at sorted position r gets the z-score of interval r-1
|
||||
# (preceding interval); row at position 0 reuses last interval.
|
||||
dist_values = np.empty(n, dtype=np.float64)
|
||||
for r in range(n):
|
||||
if r > 0:
|
||||
dist_values[ranks[r]] = z_scores[r - 1]
|
||||
else:
|
||||
dist_values[ranks[0]] = (
|
||||
z_scores[-1] if n > 1 else 0.0
|
||||
)
|
||||
|
||||
# Normalise to zero-mean unit-variance
|
||||
d_std = float(np.std(dist_values))
|
||||
d_mean = float(np.mean(dist_values))
|
||||
if d_std > 1e-10:
|
||||
dist_values = (dist_values - d_mean) / d_std
|
||||
else:
|
||||
dist_values = dist_values - d_mean
|
||||
|
||||
return dist_values
|
||||
|
||||
|
||||
def timestamp_fft_distance(
|
||||
lf: pl.LazyFrame, ts_columns: list[str]
|
||||
) -> pl.LazyFrame:
|
||||
"""Append FFT-based phase-distance columns for timestamp columns.
|
||||
|
||||
For each column in *ts_columns* that exists in the schema:
|
||||
|
||||
1. Parse values to Unix epoch float seconds (handles int, float,
|
||||
datetime strings, and ``pl.Datetime``).
|
||||
2. Collect the column to numpy, sort, then run ``np.fft.fft``
|
||||
followed by ``np.angle`` to extract per-row phase values.
|
||||
3. If fewer than 5 % of timestamp intervals repeat (weak
|
||||
periodicity), fall back to statistical features derived
|
||||
from interval mean, standard deviation, and z-scores.
|
||||
|
||||
Columns are appended as ``_ts_dist_0``, ``_ts_dist_1``, … in the
|
||||
order *ts_columns* are supplied. Rows whose timestamp is null
|
||||
receive ``NaN``.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
lf : pl.LazyFrame
|
||||
The input LazyFrame with at least one timestamp column.
|
||||
ts_columns : list[str]
|
||||
Column names to process.
|
||||
|
||||
Returns
|
||||
-------
|
||||
pl.LazyFrame
|
||||
A new LazyFrame with ``_ts_dist_N`` columns appended.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> import polars as pl
|
||||
>>> from analysis.distance import timestamp_fft_distance
|
||||
>>> df = pl.DataFrame({
|
||||
... 'ts': [0, 60, 120, 180, 240, 300],
|
||||
... })
|
||||
>>> result = timestamp_fft_distance(df.lazy(), ['ts'])
|
||||
>>> result.collect().columns
|
||||
['ts', '_ts_dist_0']
|
||||
"""
|
||||
schema = lf.collect_schema()
|
||||
available = [c for c in ts_columns if c in schema.names()]
|
||||
if not available:
|
||||
return lf
|
||||
|
||||
# Collect all needed columns once
|
||||
df = lf.select(available).collect(streaming=True)
|
||||
n_rows = len(df)
|
||||
|
||||
dist_arrays: list[np.ndarray] = []
|
||||
|
||||
for col in available:
|
||||
series = df[col]
|
||||
ts_numeric = _parse_timestamp_to_epoch(series)
|
||||
|
||||
# Identify valid (non-null) rows
|
||||
valid = ~ts_numeric.is_null()
|
||||
valid_indices = np.where(valid.to_numpy())[0]
|
||||
values = ts_numeric.filter(valid).to_numpy().astype(np.float64)
|
||||
|
||||
if len(values) < _FFT_MIN_SAMPLES:
|
||||
continue
|
||||
|
||||
phase_values = _compute_fft_distances(values)
|
||||
|
||||
# Build full-length array; invalid rows stay NaN
|
||||
full_dist = np.full(n_rows, np.nan, dtype=np.float64)
|
||||
full_dist[valid_indices] = phase_values
|
||||
dist_arrays.append(full_dist)
|
||||
|
||||
if not dist_arrays:
|
||||
return lf
|
||||
|
||||
# Re-materialise the original frame and bolt on distance columns
|
||||
result_df = lf.collect(streaming=True)
|
||||
for i, arr in enumerate(dist_arrays):
|
||||
result_df = result_df.with_columns(
|
||||
pl.Series(f'_ts_dist_{i}', arr)
|
||||
)
|
||||
|
||||
return result_df.lazy()
|
||||
@@ -0,0 +1,116 @@
|
||||
"""SVD noise removal (step 3 of preprocessing pipeline)."""
|
||||
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
|
||||
|
||||
def svd_denoise(
|
||||
lf: pl.LazyFrame,
|
||||
feature_cols: list[str],
|
||||
variance_threshold: float = 0.95,
|
||||
) -> tuple[pl.LazyFrame, dict]:
|
||||
"""Remove noise from numeric features via TruncatedSVD.
|
||||
|
||||
1. Fits TruncatedSVD on *feature_cols*.
|
||||
2. Keeps *k* components where cumulative explained variance
|
||||
``trace(S[:k]) / trace(S) ≥ variance_threshold`` (default 0.95).
|
||||
3. Inverse-transforms to reconstruct the denoised matrix.
|
||||
4. **Replaces** the original *feature_cols* with denoised values.
|
||||
|
||||
The noise components (removed by SVD) are returned as a dict for
|
||||
display, NOT as new feature columns.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
lf:
|
||||
LazyFrame containing *feature_cols*.
|
||||
feature_cols:
|
||||
Numeric column names to denoise. These columns are replaced
|
||||
in-place by their denoised reconstructions.
|
||||
variance_threshold:
|
||||
Fraction of total variance to preserve (0.0–1.0).
|
||||
Default 0.95 keeps 95 % of signal and treats 5 % as noise.
|
||||
|
||||
Returns
|
||||
-------
|
||||
tuple[pl.LazyFrame, dict]
|
||||
``(denoised_lf, noise_profile)`` where *noise_profile* contains:
|
||||
- ``kept_components`` : int — number of components kept
|
||||
- ``total_components`` : int — max possible components
|
||||
- ``kept_variance`` : float — fraction of variance kept
|
||||
- ``noise_components`` : list[dict] — per-component noise info
|
||||
(component index, explained variance, top-5 contributing features)
|
||||
"""
|
||||
noise_profile = {
|
||||
'kept_components': 0,
|
||||
'total_components': 0,
|
||||
'kept_variance': 0.0,
|
||||
'noise_components': [],
|
||||
}
|
||||
|
||||
available = [c for c in feature_cols if c in lf.collect_schema().names()]
|
||||
if not available:
|
||||
return lf, noise_profile
|
||||
|
||||
df = lf.select(available).collect(streaming=True)
|
||||
mat = df.to_numpy().astype(np.float64)
|
||||
mat = np.nan_to_num(mat, nan=0.0)
|
||||
|
||||
n, d = mat.shape
|
||||
max_components = min(n, d - 1)
|
||||
if max_components < 1:
|
||||
return lf, noise_profile
|
||||
|
||||
from sklearn.decomposition import TruncatedSVD
|
||||
svd = TruncatedSVD(n_components=max_components, random_state=42)
|
||||
_ = svd.fit_transform(mat)
|
||||
|
||||
total_variance = float(np.sum(svd.explained_variance_))
|
||||
if total_variance <= 1e-10:
|
||||
return lf, noise_profile
|
||||
|
||||
cumsum = np.cumsum(svd.explained_variance_)
|
||||
for k in range(1, max_components + 1):
|
||||
if cumsum[k - 1] / total_variance >= variance_threshold:
|
||||
break
|
||||
else:
|
||||
k = max_components
|
||||
|
||||
# Re-fit with chosen k
|
||||
svd_k = TruncatedSVD(n_components=k, random_state=42)
|
||||
reduced = svd_k.fit_transform(mat)
|
||||
reconstructed = svd_k.inverse_transform(reduced)
|
||||
|
||||
# --- Build noise profile (removed components) ---
|
||||
# The "noise" is the variance in components k..max_components
|
||||
kept_var_ratio = cumsum[k - 1] / total_variance if k > 0 else 0.0
|
||||
noise_profile = {
|
||||
'kept_components': k,
|
||||
'total_components': max_components,
|
||||
'kept_variance': round(float(kept_var_ratio), 4),
|
||||
}
|
||||
|
||||
# Per-component noise info for removed components (k and beyond)
|
||||
noise_info = []
|
||||
for comp_idx in range(k, min(k + 10, max_components)):
|
||||
loading = np.abs(svd.components_[comp_idx])
|
||||
top5_idx = np.argsort(loading)[-5:][::-1]
|
||||
top5 = [
|
||||
{'feature': available[i], 'strength': round(float(loading[i]), 4)}
|
||||
for i in top5_idx
|
||||
]
|
||||
noise_info.append({
|
||||
'component': int(comp_idx + 1),
|
||||
'explained_var_ratio': round(float(svd.explained_variance_ratio_[comp_idx]) * 100, 3),
|
||||
'top_features': top5,
|
||||
})
|
||||
noise_profile['noise_components'] = noise_info
|
||||
|
||||
# --- Replace original columns with denoised values ---
|
||||
df_result = lf.collect(streaming=True)
|
||||
for i, col in enumerate(available):
|
||||
df_result = df_result.with_columns(
|
||||
pl.Series(col, reconstructed[:, i])
|
||||
)
|
||||
|
||||
return df_result.lazy(), noise_profile
|
||||
@@ -0,0 +1,164 @@
|
||||
"""String distance, enum detection, and boolean distance functions."""
|
||||
|
||||
import polars as pl
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Enum-like detection helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _is_enum_like(s: pl.Series) -> bool:
|
||||
"""Check if ≥50% of rows have values appearing ≥2 times."""
|
||||
total = len(s)
|
||||
if total == 0:
|
||||
return False
|
||||
non_null = s.drop_nulls()
|
||||
if len(non_null) == 0:
|
||||
return False
|
||||
vc = non_null.value_counts()
|
||||
if 'count' not in vc.columns:
|
||||
return False
|
||||
duplicated = vc.filter(pl.col('count') >= 2)
|
||||
if len(duplicated) == 0:
|
||||
return False
|
||||
duplicated_count: int = duplicated['count'].sum() # type: ignore[union-attr]
|
||||
return duplicated_count >= 0.5 * total
|
||||
|
||||
|
||||
def _levenshtein_normalized(s1: str | None, s2: str) -> float:
|
||||
"""Normalised Levenshtein distance between *s1* and *s2*.
|
||||
|
||||
Empty or ``None`` values return 1.0 (maximum distance).
|
||||
"""
|
||||
if s1 is None or s1 == '':
|
||||
return 1.0
|
||||
if s2 is None or s2 == '':
|
||||
return 1.0
|
||||
s1_str = str(s1)
|
||||
s2_str = str(s2)
|
||||
max_len = max(len(s1_str), len(s2_str))
|
||||
if max_len == 0:
|
||||
return 0.0
|
||||
import Levenshtein
|
||||
return Levenshtein.distance(s1_str, s2_str) / max_len
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# String distance
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def string_distance(lf: pl.LazyFrame, str_columns: list[str]) -> pl.LazyFrame:
|
||||
"""Compute string distance columns.
|
||||
|
||||
For each column in *str_columns*:
|
||||
|
||||
* If **≥50%** of rows have values appearing ≥2 times, the column is
|
||||
treated as an enum: 0 when the value equals the mode, 1 otherwise.
|
||||
* Otherwise, the normalised Levenshtein distance to the mode is
|
||||
computed for each row.
|
||||
|
||||
Empty or ``None`` values always produce the maximum distance (1.0).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
lf:
|
||||
Input LazyFrame.
|
||||
str_columns:
|
||||
List of string column names to compute distances for.
|
||||
|
||||
Returns
|
||||
-------
|
||||
pl.LazyFrame
|
||||
Input LazyFrame extended with ``_str_dist_0``, ``_str_dist_1``,
|
||||
... columns (one per entry in *str_columns*).
|
||||
"""
|
||||
df = lf.collect(streaming=True)
|
||||
dist_exprs: list[pl.Expr] = []
|
||||
|
||||
for idx, col in enumerate(str_columns):
|
||||
if col not in df.columns:
|
||||
continue
|
||||
|
||||
series = df[col].cast(pl.Utf8)
|
||||
enum_like = _is_enum_like(series)
|
||||
|
||||
non_null = series.drop_nulls()
|
||||
if len(non_null) > 0:
|
||||
mode_val = non_null.value_counts().sort('count', descending=True).row(0)[0]
|
||||
mode_str = str(mode_val)
|
||||
else:
|
||||
mode_str = ''
|
||||
|
||||
col_name = f'_str_dist_{idx}'
|
||||
|
||||
if enum_like:
|
||||
dist_exprs.append(
|
||||
pl.when(pl.col(col).is_null())
|
||||
.then(pl.lit(1.0))
|
||||
.when(pl.col(col).cast(pl.Utf8) == pl.lit(mode_str))
|
||||
.then(pl.lit(0.0))
|
||||
.otherwise(pl.lit(1.0))
|
||||
.alias(col_name)
|
||||
)
|
||||
else:
|
||||
# Capture in closure for map_elements
|
||||
_mode = mode_str
|
||||
dist_exprs.append(
|
||||
pl.col(col).map_elements(
|
||||
lambda v: _levenshtein_normalized(v, _mode),
|
||||
return_dtype=pl.Float64,
|
||||
).alias(col_name)
|
||||
)
|
||||
|
||||
return lf.with_columns(dist_exprs)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Boolean distance
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def bool_distance(lf: pl.LazyFrame, bool_columns: list[str]) -> pl.LazyFrame:
|
||||
"""Compute boolean distance columns.
|
||||
|
||||
For each column in *bool_columns*, each row gets:
|
||||
0 if its value matches the column mode, 1 otherwise.
|
||||
``None`` values always produce 1.0 (maximum distance).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
lf:
|
||||
Input LazyFrame.
|
||||
bool_columns:
|
||||
List of boolean column names to compute distances for.
|
||||
|
||||
Returns
|
||||
-------
|
||||
pl.LazyFrame
|
||||
Input LazyFrame extended with ``_bool_dist_0``, ``_bool_dist_1``,
|
||||
... columns (one per entry in *bool_columns*).
|
||||
"""
|
||||
df = lf.collect(streaming=True)
|
||||
dist_exprs: list[pl.Expr] = []
|
||||
|
||||
for idx, col in enumerate(bool_columns):
|
||||
if col not in df.columns:
|
||||
continue
|
||||
|
||||
series = df[col]
|
||||
non_null = series.drop_nulls()
|
||||
col_name = f'_bool_dist_{idx}'
|
||||
|
||||
if len(non_null) == 0:
|
||||
dist_exprs.append(pl.lit(0.0).alias(col_name))
|
||||
else:
|
||||
mode_val = non_null.value_counts().sort('count', descending=True).row(0)[0]
|
||||
dist_exprs.append(
|
||||
pl.when(pl.col(col).is_null())
|
||||
.then(pl.lit(1.0))
|
||||
.when(pl.col(col) == pl.lit(mode_val))
|
||||
.then(pl.lit(0.0))
|
||||
.otherwise(pl.lit(1.0))
|
||||
.alias(col_name)
|
||||
)
|
||||
|
||||
return lf.with_columns(dist_exprs)
|
||||
@@ -0,0 +1,122 @@
|
||||
"""UMAP dimensionality reduction (3D → 2D fallback)."""
|
||||
|
||||
import logging
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
import umap
|
||||
|
||||
|
||||
def umap_reduce(
|
||||
lf: pl.LazyFrame,
|
||||
feature_cols: list[str],
|
||||
n_components: int = 3,
|
||||
) -> tuple[pl.LazyFrame, dict]:
|
||||
"""UMAP dimensionality reduction with automatic 3D→2D fallback.
|
||||
|
||||
Trains on a ≤10K-row sample (when *lf* exceeds that limit), then
|
||||
batch-transforms the remaining rows in 1K-row batches.
|
||||
|
||||
If ``n_components=3`` fails (e.g. too few rows or numerical issues),
|
||||
falls back to ``n_components=2`` automatically.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
lf:
|
||||
LazyFrame containing the numeric feature columns.
|
||||
feature_cols:
|
||||
Numeric column names to use for UMAP projection. Columns not
|
||||
present in *lf* are silently skipped.
|
||||
n_components:
|
||||
Target dimensionality (2 or 3). Default 3 with fallback to 2.
|
||||
|
||||
Returns
|
||||
-------
|
||||
tuple[pl.LazyFrame, dict]
|
||||
- LazyFrame: input data extended with ``_umap_x``, ``_umap_y``
|
||||
(and ``_umap_z`` when ``n_components ≥ 3``).
|
||||
- dict: metadata with keys ``n_components`` and ``reducer``.
|
||||
"""
|
||||
logger = logging.getLogger(__name__)
|
||||
MAX_TRAIN = 10000
|
||||
BATCH_SIZE = 1000
|
||||
|
||||
available = [c for c in feature_cols if c in lf.collect_schema().names()]
|
||||
if not available:
|
||||
return lf, {'n_components': 0, 'reducer': 'umap', 'error': 'no feature columns available'}
|
||||
|
||||
df = lf.collect(streaming=True)
|
||||
n_total = len(df)
|
||||
|
||||
if n_total < 2:
|
||||
return lf, {'n_components': 0, 'reducer': 'umap', 'error': 'too few rows'}
|
||||
|
||||
mat = df.select(available).to_numpy().astype(np.float64)
|
||||
mat = np.nan_to_num(mat, nan=0.0, posinf=0.0, neginf=0.0)
|
||||
|
||||
scaler = StandardScaler()
|
||||
if n_total > MAX_TRAIN:
|
||||
rng = np.random.RandomState(42)
|
||||
idx_sample = rng.choice(n_total, size=MAX_TRAIN, replace=False)
|
||||
mat_sample = mat[idx_sample]
|
||||
scaler.fit(mat_sample)
|
||||
mat_sample_scaled = np.nan_to_num(scaler.transform(mat_sample), nan=0.0)
|
||||
else:
|
||||
scaler.fit(mat)
|
||||
mat_sample_scaled = np.nan_to_num(scaler.transform(mat), nan=0.0)
|
||||
|
||||
# ── Train UMAP; fallback 3→2 on failure ──
|
||||
reducer = None
|
||||
coords = None
|
||||
effective_n = n_components
|
||||
|
||||
# Try requested n_components first, then fall back to 2 (no duplicates)
|
||||
attempts = sorted(set((n_components, 2)), reverse=True)
|
||||
for attempt_n in attempts:
|
||||
if attempt_n != n_components:
|
||||
logger.warning(f'UMAP {n_components}D failed, falling back to 2D')
|
||||
effective_n = attempt_n
|
||||
try:
|
||||
reducer = umap.UMAP(
|
||||
n_components=effective_n, random_state=42,
|
||||
n_neighbors=15, min_dist=0.1, metric='euclidean',
|
||||
)
|
||||
if n_total > MAX_TRAIN:
|
||||
reducer.fit(mat_sample_scaled)
|
||||
else:
|
||||
coords = reducer.fit_transform(mat_sample_scaled)
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt_n == 2:
|
||||
raise
|
||||
logger.warning(f'UMAP {attempt_n}D failed: {e}')
|
||||
|
||||
# ── Batch transform if dataset was larger than training sample ──
|
||||
if n_total > MAX_TRAIN and reducer is not None:
|
||||
coords_list: list[np.ndarray] = []
|
||||
for start in range(0, n_total, BATCH_SIZE):
|
||||
end = min(start + BATCH_SIZE, n_total)
|
||||
mat_batch = scaler.transform(mat[start:end])
|
||||
mat_batch = np.nan_to_num(mat_batch, nan=0.0, posinf=0.0, neginf=0.0)
|
||||
coords_list.append(reducer.transform(mat_batch))
|
||||
coords = np.vstack(coords_list)
|
||||
|
||||
if coords is None:
|
||||
return lf, {'n_components': 0, 'reducer': 'umap', 'error': 'UMAP produced no output'}
|
||||
|
||||
# ── Build result LazyFrame with coordinate columns ──
|
||||
coord_series = [
|
||||
pl.Series('_umap_x', coords[:, 0]),
|
||||
pl.Series('_umap_y', coords[:, 1]),
|
||||
]
|
||||
if effective_n >= 3 and coords.shape[1] >= 3:
|
||||
coord_series.append(pl.Series('_umap_z', coords[:, 2]))
|
||||
|
||||
df_result = df.with_columns(coord_series)
|
||||
|
||||
metadata: dict = {
|
||||
'n_components': effective_n,
|
||||
'reducer': 'umap',
|
||||
}
|
||||
|
||||
return df_result.lazy(), metadata
|
||||
+133
-239
@@ -1,18 +1,18 @@
|
||||
"""
|
||||
离线 GeoIP 查询 — 支持 MMDB (GeoLite2-City / GeoLite2-ASN) 和文本格式回退。
|
||||
离线 GeoIP 查询。纯本地运行，无网络请求。
|
||||
|
||||
优先级：
|
||||
1. GeoLite2-City.mmdb (MaxMind MMDB 格式，城市级精度)
|
||||
2. GeoLite2-ASN.mmdb (可选，提供 ISP / 组织信息)
|
||||
3. geoip_data.txt (文本格式回退，803 段覆盖 30 城市)
|
||||
三级存储：
|
||||
1. 快速缓存 (data/geoip_cache.json) — 已查过的 IP，磁盘持久化
|
||||
2. 离线数据库 (data/geoip_data.txt) — 手动整理的优先 IP 段
|
||||
3. MMDB 数据库 — 若 python_geoip_geolite2 包已安装，加载其
|
||||
内置的 GeoLite2-City.mmdb 作为最全面回退。
|
||||
|
||||
MMDB 查询使用 ``maxminddb`` 库，文本格式使用二分查找 O(log N)。
|
||||
查询链：缓存 > geoip_data.txt > MMDB > 写入缓存。
|
||||
"""
|
||||
import bisect
|
||||
import gzip
|
||||
import json
|
||||
import logging
|
||||
import shutil
|
||||
import tempfile
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
@@ -22,26 +22,22 @@ logger = logging.getLogger(__name__)
|
||||
# Data structures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# MMDB readers (lazy-loaded, None if unavailable)
|
||||
_mmdb_city: Optional[object] = None # maxminddb.Reader for City DB
|
||||
_mmdb_asn: Optional[object] = None # maxminddb.Reader for ASN DB
|
||||
|
||||
# Text-format data (fallback)
|
||||
# [(start_int, end_int, lat, lon, city, country), ...]
|
||||
_GEO_DATA: list[tuple[int, int, float, float, str, str]] = []
|
||||
_STARTS: list[int] = []
|
||||
_RANGES: list[tuple[int, int, float, float, str, str]] = []
|
||||
|
||||
_MMDB_READER = None
|
||||
|
||||
_CACHE: dict[str, Optional[dict]] = {}
|
||||
_CACHE_DIRTY = False
|
||||
_CACHE_PATH = Path(__file__).resolve().parent.parent / 'data' / 'geoip_cache.json'
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# IP conversion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def ip_to_int(ip_str: str) -> Optional[int]:
|
||||
"""Convert an IPv4 dotted-string to its 32-bit integer representation.
|
||||
|
||||
Returns ``None`` if *ip_str* is not a valid IPv4 address.
|
||||
"""
|
||||
parts = ip_str.strip().split('.')
|
||||
if len(parts) != 4:
|
||||
return None
|
||||
@@ -55,89 +51,10 @@ def ip_to_int(ip_str: str) -> Optional[int]:
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MMDB loading
|
||||
# geoip_data.txt loader
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _load_mmdb() -> None:
|
||||
"""Try to load MaxMind MMDB databases from ``data/`` directory.
|
||||
|
||||
Auto-decompresses .mmdb.gz if the uncompressed file is missing.
|
||||
Looks for:
|
||||
- ``data/GeoLite2-City.mmdb`` (primary — city, country, lat/lon)
|
||||
- ``data/GeoLite2-ASN.mmdb`` (optional — ISP, organization, ASN)
|
||||
"""
|
||||
global _mmdb_city, _mmdb_asn
|
||||
|
||||
data_dir = Path(__file__).parent.parent / 'data'
|
||||
|
||||
city_path = data_dir / 'GeoLite2-City.mmdb'
|
||||
asn_path = data_dir / 'GeoLite2-ASN.mmdb'
|
||||
|
||||
# Auto-decompress gzipped databases (reassemble chunks first if needed)
|
||||
chunks_dir = data_dir / 'geoip_chunks'
|
||||
for mmdb_path in (city_path, asn_path):
|
||||
gz_path = mmdb_path.with_suffix(mmdb_path.suffix + '.gz')
|
||||
if not mmdb_path.exists() and not gz_path.exists() and chunks_dir.exists():
|
||||
# Try reassembling from chunks
|
||||
prefix = gz_path.name + '.part'
|
||||
chunks = sorted(p for p in chunks_dir.iterdir() if p.name.startswith(prefix))
|
||||
if chunks:
|
||||
logger.info('[GEOIP] reassembling %d chunks → %s', len(chunks), gz_path.name)
|
||||
try:
|
||||
with open(gz_path, 'wb') as out:
|
||||
for chunk in chunks:
|
||||
with open(chunk, 'rb') as f:
|
||||
shutil.copyfileobj(f, out, length=16 * 1024 * 1024)
|
||||
logger.info('[GEOIP] reassembled %s', gz_path.name)
|
||||
except Exception as e:
|
||||
logger.warning('[GEOIP] failed to reassemble %s: %s', gz_path.name, e)
|
||||
|
||||
if not mmdb_path.exists() and gz_path.exists():
|
||||
logger.info('[GEOIP] decompressing %s …', gz_path.name)
|
||||
try:
|
||||
with gzip.open(gz_path, 'rb') as f_in:
|
||||
with open(mmdb_path, 'wb') as f_out:
|
||||
shutil.copyfileobj(f_in, f_out, length=16 * 1024 * 1024)
|
||||
logger.info('[GEOIP] decompressed → %s', mmdb_path.name)
|
||||
except Exception as e:
|
||||
logger.warning('[GEOIP] failed to decompress %s: %s', gz_path.name, e)
|
||||
|
||||
if not city_path.exists():
|
||||
logger.info('[GEOIP] GeoLite2-City.mmdb not found at %s — will use text fallback', city_path)
|
||||
return
|
||||
|
||||
try:
|
||||
import maxminddb
|
||||
|
||||
# maxminddb C extension cannot handle non-ASCII Windows paths.
|
||||
# Workaround: copy MMDB files to a temp dir with ASCII-only path.
|
||||
_tmp_dir = Path(tempfile.gettempdir()) / 'tianxuan_geoip'
|
||||
_tmp_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
_tmp_city = _tmp_dir / 'GeoLite2-City.mmdb'
|
||||
if not _tmp_city.exists() or _tmp_city.stat().st_size != city_path.stat().st_size:
|
||||
logger.info('[GEOIP] copying MMDB to ASCII-safe temp dir …')
|
||||
shutil.copy2(city_path, _tmp_city)
|
||||
_mmdb_city = maxminddb.open_database(str(_tmp_city))
|
||||
logger.info('[GEOIP] loaded GeoLite2-City.mmdb (MMDB format)')
|
||||
|
||||
if asn_path.exists():
|
||||
_tmp_asn = _tmp_dir / 'GeoLite2-ASN.mmdb'
|
||||
if not _tmp_asn.exists() or _tmp_asn.stat().st_size != asn_path.stat().st_size:
|
||||
shutil.copy2(asn_path, _tmp_asn)
|
||||
_mmdb_asn = maxminddb.open_database(str(_tmp_asn))
|
||||
logger.info('[GEOIP] loaded GeoLite2-ASN.mmdb (ISP/ORG)')
|
||||
except Exception as e:
|
||||
logger.warning('[GEOIP] failed to open MMDB: %s', e)
|
||||
_mmdb_city = None
|
||||
_mmdb_asn = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Text-format loading (fallback)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _build_index() -> tuple[list[int], list[tuple[int, int, float, float, str, str]]]:
|
||||
def _build_index():
|
||||
if not _GEO_DATA:
|
||||
return [], []
|
||||
sorted_data = sorted(_GEO_DATA, key=lambda x: x[0])
|
||||
@@ -146,16 +63,12 @@ def _build_index() -> tuple[list[int], list[tuple[int, int, float, float, str, s
|
||||
|
||||
|
||||
def _load_data_file() -> None:
|
||||
"""Load IP range data from ``data/geoip_data.txt`` (text fallback)."""
|
||||
data_path = Path(__file__).parent.parent / 'data' / 'geoip_data.txt'
|
||||
|
||||
data_path = Path(__file__).resolve().parent.parent / 'data' / 'geoip_data.txt'
|
||||
if not data_path.exists():
|
||||
logger.warning('[GEOIP] text data file not found: %s', data_path)
|
||||
logger.warning('[GEOIP] data file not found: %s', data_path)
|
||||
return
|
||||
|
||||
loaded: list[tuple[int, int, float, float, str, str]] = []
|
||||
loaded = []
|
||||
skipped = 0
|
||||
|
||||
with open(data_path, 'r', encoding='utf-8') as f:
|
||||
for raw in f:
|
||||
stripped = raw.strip()
|
||||
@@ -165,121 +78,117 @@ def _load_data_file() -> None:
|
||||
if len(parts) != 6:
|
||||
skipped += 1
|
||||
continue
|
||||
start_ip_str, end_ip_str, lat_str, lon_str, city, country = parts
|
||||
start_int = ip_to_int(start_ip_str)
|
||||
end_int = ip_to_int(end_ip_str)
|
||||
if start_int is None or end_int is None:
|
||||
sip, eip, lat_s, lon_s, city, country = parts
|
||||
si = ip_to_int(sip)
|
||||
ei = ip_to_int(eip)
|
||||
if si is None or ei is None or si > ei:
|
||||
skipped += 1
|
||||
continue
|
||||
try:
|
||||
lat = float(lat_str)
|
||||
lon = float(lon_str)
|
||||
lat = float(lat_s)
|
||||
lon = float(lon_s)
|
||||
except ValueError:
|
||||
skipped += 1
|
||||
continue
|
||||
loaded.append((start_int, end_int, lat, lon, city, country))
|
||||
|
||||
global _GEO_DATA, _STARTS, _RANGES
|
||||
loaded.append((si, ei, lat, lon, city, country))
|
||||
_GEO_DATA.clear()
|
||||
_GEO_DATA.extend(loaded)
|
||||
global _STARTS, _RANGES
|
||||
_STARTS, _RANGES = _build_index()
|
||||
|
||||
logger.info('[GEOIP] text fallback: loaded %d ranges from %s (%d skipped)',
|
||||
len(loaded), data_path, skipped)
|
||||
logger.info('[GEOIP] loaded %d ranges (%d skipped)', len(loaded), skipped)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MMDB lookup helpers
|
||||
# MMDB reader (GeoLite2-City.mmdb bundled with python_geoip_geolite2)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _lookup_mmdb(ip_str: str) -> Optional[dict]:
|
||||
"""Query MMDB database for an IP. Returns rich result with ISP/ORG when
|
||||
ASN database is also available."""
|
||||
if _mmdb_city is None:
|
||||
return None
|
||||
|
||||
def _get_mmdb_reader():
|
||||
global _MMDB_READER
|
||||
if _MMDB_READER is not None:
|
||||
return _MMDB_READER
|
||||
# Try multiple locations for the MMDB file
|
||||
candidates = []
|
||||
try:
|
||||
city_result = _mmdb_city.get(ip_str)
|
||||
if city_result is None:
|
||||
return None
|
||||
import _geoip_geolite2
|
||||
candidates.append(Path(_geoip_geolite2.__file__).parent / 'GeoLite2-City.mmdb')
|
||||
except Exception:
|
||||
pass
|
||||
# Fallback: temp directory without Chinese chars in path
|
||||
candidates.append(Path(os.environ.get('TEMP', 'C:/temp')) / 'opencode' / 'GeoLite2-City.mmdb')
|
||||
candidates.append(Path.home() / 'AppData' / 'Local' / 'Temp' / 'opencode' / 'GeoLite2-City.mmdb')
|
||||
|
||||
# Extract location
|
||||
location = city_result.get('location', {})
|
||||
for mmdb_path in candidates:
|
||||
if mmdb_path.exists():
|
||||
try:
|
||||
import maxminddb
|
||||
_MMDB_READER = maxminddb.open_database(mmdb_path)
|
||||
logger.info('[GEOIP] loaded MMDB: %s (%.1f MB)',
|
||||
mmdb_path.name, mmdb_path.stat().st_size / 1e6)
|
||||
break
|
||||
except Exception as exc:
|
||||
logger.debug('[GEOIP] failed to open %s: %s', mmdb_path, exc)
|
||||
continue
|
||||
|
||||
if _MMDB_READER is None:
|
||||
logger.debug('[GEOIP] MMDB not available — falling back to txt database only')
|
||||
return _MMDB_READER
|
||||
|
||||
|
||||
def _mmdb_lookup(ip_str: str) -> Optional[dict]:
|
||||
reader = _get_mmdb_reader()
|
||||
if reader is None:
|
||||
return None
|
||||
try:
|
||||
result = reader.get(ip_str)
|
||||
if result is None:
|
||||
return None
|
||||
location = result.get('location', {})
|
||||
city_data = result.get('city', {})
|
||||
country_data = result.get('country', {})
|
||||
lat = location.get('latitude')
|
||||
lon = location.get('longitude')
|
||||
|
||||
if lat is None or lon is None:
|
||||
return None
|
||||
|
||||
# Extract names (English preferred)
|
||||
city = None
|
||||
city_names = city_result.get('city', {}).get('names', {})
|
||||
if city_names:
|
||||
city = city_names.get('en') or next(iter(city_names.values()), None)
|
||||
|
||||
country = None
|
||||
country_info = city_result.get('country', {})
|
||||
country = country_info.get('iso_code') or ''
|
||||
country_names = country_info.get('names', {})
|
||||
country_name = country_names.get('en') or next(iter(country_names.values()), None) if country_names else None
|
||||
|
||||
# Subdivisions
|
||||
subdivisions = city_result.get('subdivisions', [])
|
||||
region = None
|
||||
if subdivisions:
|
||||
sub_names = subdivisions[0].get('names', {})
|
||||
region = sub_names.get('en') or next(iter(sub_names.values()), None) if sub_names else None
|
||||
|
||||
result = {
|
||||
'lat': round(float(lat), 4),
|
||||
'lon': round(float(lon), 4),
|
||||
'city': city or region or '',
|
||||
'country': country,
|
||||
'country_name': country_name or '',
|
||||
'region': region or '',
|
||||
}
|
||||
|
||||
# Try ASN database for ISP/ORG
|
||||
if _mmdb_asn is not None:
|
||||
try:
|
||||
asn_result = _mmdb_asn.get(ip_str)
|
||||
if asn_result:
|
||||
result['isp'] = asn_result.get('autonomous_system_organization', '')
|
||||
result['asn'] = asn_result.get('autonomous_system_number', '')
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.debug('[GEOIP] MMDB lookup error for %s: %s', ip_str, e)
|
||||
city = ''
|
||||
if city_data and 'names' in city_data:
|
||||
city = city_data['names'].get('zh-CN') or city_data['names'].get('en', '')
|
||||
country = ''
|
||||
if country_data and 'names' in country_data:
|
||||
country = country_data['names'].get('zh-CN') or country_data['names'].get('en', '')
|
||||
return {'lat': float(lat), 'lon': float(lon), 'city': city, 'country': country}
|
||||
except Exception as exc:
|
||||
logger.debug('[GEOIP] MMDB lookup failed for %s: %s', ip_str, exc)
|
||||
return None
|
||||
|
||||
|
||||
def _lookup_text(ip_str: str) -> Optional[dict]:
|
||||
"""Query text-format database (fallback)."""
|
||||
target = ip_to_int(ip_str)
|
||||
if target is None:
|
||||
return None
|
||||
if not _STARTS:
|
||||
return None
|
||||
# ---------------------------------------------------------------------------
|
||||
# Disk cache
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
idx = bisect.bisect_right(_STARTS, target) - 1
|
||||
if idx < 0:
|
||||
return None
|
||||
def _load_cache() -> None:
|
||||
if not _CACHE_PATH.exists():
|
||||
return
|
||||
try:
|
||||
data = json.loads(_CACHE_PATH.read_text(encoding='utf-8'))
|
||||
_CACHE.update(data)
|
||||
logger.info('[GEOIP] loaded %d cached entries', len(data))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
start, end, lat, lon, city, country = _RANGES[idx]
|
||||
if start <= target <= end:
|
||||
return {
|
||||
'lat': lat,
|
||||
'lon': lon,
|
||||
'city': city,
|
||||
'country': country,
|
||||
'country_name': '',
|
||||
'region': '',
|
||||
}
|
||||
|
||||
return None
|
||||
def _save_cache() -> None:
|
||||
global _CACHE_DIRTY
|
||||
if not _CACHE_DIRTY:
|
||||
return
|
||||
try:
|
||||
_CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
_CACHE_PATH.write_text(
|
||||
json.dumps(_CACHE, ensure_ascii=False, indent=2),
|
||||
encoding='utf-8',
|
||||
)
|
||||
_CACHE_DIRTY = False
|
||||
except Exception as exc:
|
||||
logger.warning('[GEOIP] cache write failed: %s', exc)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -287,59 +196,44 @@ def _lookup_text(ip_str: str) -> Optional[dict]:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def lookup(ip_str: str) -> Optional[dict]:
|
||||
"""查询 IP 对应的地理位置。
|
||||
"""查询 IP 地理位置。纯本地，无网络。
|
||||
|
||||
Tries MMDB first (if available), then falls back to text format.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
ip_str:
|
||||
IPv4 address as a dotted string (e.g. ``'8.8.8.8'``).
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict or None
|
||||
Keys: ``lat``, ``lon``, ``city``, ``country`` (ISO code),
|
||||
``country_name``, ``region``.
|
||||
When ASN database is available, also includes: ``isp``, ``asn``.
|
||||
查询链：磁盘缓存 > geoip_data.txt (O(log N)) > MMDB (GeoLite2) > 写入缓存。
|
||||
"""
|
||||
if not ip_str or ip_str in ('None', '', '+', '-'):
|
||||
# 1. Cache hit
|
||||
cached = _CACHE.get(ip_str)
|
||||
if cached is not None:
|
||||
return cached
|
||||
if ip_str in _CACHE:
|
||||
return None
|
||||
|
||||
# MMDB first
|
||||
if _mmdb_city is not None:
|
||||
result = _lookup_mmdb(ip_str)
|
||||
if result is not None:
|
||||
return result
|
||||
# MMDB miss — fall through to text
|
||||
result = None
|
||||
|
||||
# Text fallback
|
||||
return _lookup_text(ip_str)
|
||||
# 2. geoip_data.txt
|
||||
target = ip_to_int(ip_str)
|
||||
if target is not None and _STARTS:
|
||||
idx = bisect.bisect_right(_STARTS, target) - 1
|
||||
if idx >= 0:
|
||||
start, end, lat, lon, city, country = _RANGES[idx]
|
||||
if start <= target <= end:
|
||||
result = {'lat': lat, 'lon': lon, 'city': city, 'country': country}
|
||||
|
||||
# 3. MMDB fallback (GeoLite2, ~3M entries world-wide)
|
||||
if result is None:
|
||||
result = _mmdb_lookup(ip_str)
|
||||
|
||||
def is_available() -> bool:
|
||||
"""Return True if at least one GeoIP source is loaded."""
|
||||
return _mmdb_city is not None or bool(_STARTS)
|
||||
# 4. Cache for next time
|
||||
global _CACHE_DIRTY
|
||||
_CACHE[ip_str] = result
|
||||
_CACHE_DIRTY = True
|
||||
_save_cache()
|
||||
|
||||
|
||||
def source_name() -> str:
|
||||
"""Return human-readable name of the active GeoIP source."""
|
||||
if _mmdb_city is not None:
|
||||
name = 'GeoLite2-City.mmdb'
|
||||
if _mmdb_asn is not None:
|
||||
name += ' + GeoLite2-ASN.mmdb'
|
||||
return name
|
||||
if _STARTS:
|
||||
return 'geoip_data.txt (text fallback)'
|
||||
return '(none)'
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Module-level initialisation
|
||||
# Module-level init
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_load_mmdb()
|
||||
if _mmdb_city is None:
|
||||
_load_data_file()
|
||||
|
||||
logger.info('[GEOIP] active source: %s', source_name())
|
||||
_load_data_file()
|
||||
_load_cache()
|
||||
|
||||
@@ -67,8 +67,8 @@ class Command(BaseCommand):
|
||||
_self.stdout.write(f' → 聚类特征: {feature_cols}')
|
||||
run.total_flows = row_count
|
||||
run.save(update_fields=['total_flows'])
|
||||
from analysis.views import _run_clustering_pipeline
|
||||
_run_clustering_pipeline(
|
||||
from analysis.services.clustering import run_clustering_pipeline
|
||||
run_clustering_pipeline(
|
||||
run=run, store=store, entity_ds_id=ds_id,
|
||||
feature_columns=feature_cols,
|
||||
algorithm=algo, min_cluster_size=5,
|
||||
|
||||
+60
-1
@@ -1,11 +1,19 @@
|
||||
# Generated by Django 4.2.30 on 2026-07-13 14:36
|
||||
# Generated by Django 4.2.30 on 2026-07-23 11:15
|
||||
|
||||
from django.db import migrations, models
|
||||
from django.db.models import F
|
||||
import django.db.migrations.operations.special
|
||||
import django.db.models.deletion
|
||||
|
||||
|
||||
def populate_display_id(apps, schema_editor):
|
||||
AnalysisRun = apps.get_model('tianxuan_analysis', 'AnalysisRun')
|
||||
AnalysisRun.objects.filter(display_id__isnull=True).update(display_id=F('id'))
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
replaces = [('tianxuan_analysis', '0001_initial'), ('tianxuan_analysis', '0002_analysisrun_progress_msg_analysisrun_progress_pct_and_more'), ('tianxuan_analysis', '0003_analysisrun_sqlite_table'), ('tianxuan_analysis', '0004_tls_ref_data'), ('tianxuan_analysis', '0005_remove_entity_column'), ('tianxuan_analysis', '0006_analysisrun_display_id_analysisrun_llm_thinking_and_more'), ('tianxuan_analysis', '0007_populate_display_id'), ('tianxuan_analysis', '0008_entityprofile_embedding_z_and_more')]
|
||||
|
||||
initial = True
|
||||
|
||||
dependencies = [
|
||||
@@ -26,6 +34,10 @@ class Migration(migrations.Migration):
|
||||
('entity_count', models.IntegerField(blank=True, null=True)),
|
||||
('cluster_count', models.IntegerField(blank=True, null=True)),
|
||||
('entity_column', models.CharField(blank=True, default='', help_text='Auto-detected entity column name', max_length=256)),
|
||||
('progress_msg', models.CharField(blank=True, default='', max_length=256)),
|
||||
('progress_pct', models.IntegerField(default=0)),
|
||||
('run_log', models.TextField(blank=True, default='')),
|
||||
('sqlite_table', models.CharField(blank=True, default='', help_text='SQLite table name for persisted raw data', max_length=256)),
|
||||
],
|
||||
options={
|
||||
'ordering': ['-created_at'],
|
||||
@@ -81,4 +93,51 @@ class Migration(migrations.Migration):
|
||||
'unique_together': {('cluster', 'feature_name')},
|
||||
},
|
||||
),
|
||||
migrations.RunSQL(
|
||||
sql="\nCREATE TABLE IF NOT EXISTS tls_ref_data (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n field_code TEXT NOT NULL UNIQUE,\n meaning TEXT NOT NULL DEFAULT '',\n data_type TEXT NOT NULL DEFAULT ''\n);\n",
|
||||
reverse_sql='\nDROP TABLE IF EXISTS tls_ref_data;\n',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='analysisrun',
|
||||
name='entity_column',
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='analysisrun',
|
||||
name='display_id',
|
||||
field=models.IntegerField(blank=True, help_text='User-facing numeric ID (recycles gaps)', null=True, unique=True),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='analysisrun',
|
||||
name='llm_thinking',
|
||||
field=models.TextField(blank=True, default='', help_text='Accumulated LLM reasoning / thinking text'),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='analysisrun',
|
||||
name='run_type',
|
||||
field=models.CharField(choices=[('upload', '手动上传'), ('manual', '手动分析'), ('auto', 'LLM自动')], default='upload', max_length=16),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='analysisrun',
|
||||
name='tool_calls_json',
|
||||
field=models.JSONField(blank=True, default=list, help_text='Structured tool call list: [{step, name, input, output}, ...]'),
|
||||
),
|
||||
migrations.RunPython(
|
||||
code=populate_display_id,
|
||||
reverse_code=django.db.migrations.operations.special.RunPython.noop,
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='entityprofile',
|
||||
name='embedding_z',
|
||||
field=models.FloatField(blank=True, help_text='UMAP-3D Z coordinate', null=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='entityprofile',
|
||||
name='embedding_x',
|
||||
field=models.FloatField(blank=True, help_text='UMAP-2D X coordinate', null=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='entityprofile',
|
||||
name='embedding_y',
|
||||
field=models.FloatField(blank=True, help_text='UMAP-2D Y coordinate', null=True),
|
||||
),
|
||||
]
|
||||
-28
@@ -1,28 +0,0 @@
|
||||
# Generated by Django 4.2.30 on 2026-07-16 05:11
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('tianxuan_analysis', '0001_initial'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='analysisrun',
|
||||
name='progress_msg',
|
||||
field=models.CharField(blank=True, default='', max_length=256),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='analysisrun',
|
||||
name='progress_pct',
|
||||
field=models.IntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='analysisrun',
|
||||
name='run_log',
|
||||
field=models.TextField(blank=True, default=''),
|
||||
),
|
||||
]
|
||||
@@ -1,18 +0,0 @@
|
||||
# Generated by Django 4.2.30 on 2026-07-20 05:11
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('tianxuan_analysis', '0002_analysisrun_progress_msg_analysisrun_progress_pct_and_more'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='analysisrun',
|
||||
name='sqlite_table',
|
||||
field=models.CharField(blank=True, default='', help_text='SQLite table name for persisted raw data', max_length=256),
|
||||
),
|
||||
]
|
||||
@@ -1,38 +0,0 @@
|
||||
"""Migration 0004 — create raw SQLite table ``tls_ref_data``.
|
||||
|
||||
Replaces the ``TlsDB.csv`` reference file with a proper SQLite table.
|
||||
The table stores TLS field-code metadata: human-readable meaning and
|
||||
data type for each field code used in TLS flow data.
|
||||
"""
|
||||
|
||||
from django.db import migrations
|
||||
|
||||
# Raw SQL for creating the tls_ref_data table
|
||||
CREATE_TLS_REF_DATA = """
|
||||
CREATE TABLE IF NOT EXISTS tls_ref_data (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
field_code TEXT NOT NULL UNIQUE,
|
||||
meaning TEXT NOT NULL DEFAULT '',
|
||||
data_type TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
"""
|
||||
|
||||
DROP_TLS_REF_DATA = """
|
||||
DROP TABLE IF EXISTS tls_ref_data;
|
||||
"""
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
"""Create ``tls_ref_data`` table via raw SQL — no Django model needed."""
|
||||
|
||||
dependencies = [
|
||||
('tianxuan_analysis', '0003_analysisrun_sqlite_table'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RunSQL(
|
||||
sql=CREATE_TLS_REF_DATA,
|
||||
reverse_sql=DROP_TLS_REF_DATA,
|
||||
state_operations=None,
|
||||
),
|
||||
]
|
||||
@@ -1,17 +0,0 @@
|
||||
# Generated by Django 4.2.30 on 2026-07-20 05:20
|
||||
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('tianxuan_analysis', '0004_tls_ref_data'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RemoveField(
|
||||
model_name='analysisrun',
|
||||
name='entity_column',
|
||||
),
|
||||
]
|
||||
@@ -1,33 +0,0 @@
|
||||
# Generated by Django 4.2.30 on 2026-07-20 05:25
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('tianxuan_analysis', '0005_remove_entity_column'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='analysisrun',
|
||||
name='display_id',
|
||||
field=models.IntegerField(blank=True, help_text='User-facing numeric ID (recycles gaps)', null=True, unique=True),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='analysisrun',
|
||||
name='llm_thinking',
|
||||
field=models.TextField(blank=True, default='', help_text='Accumulated LLM reasoning / thinking text'),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='analysisrun',
|
||||
name='run_type',
|
||||
field=models.CharField(choices=[('upload', '手动上传'), ('manual', '手动分析'), ('auto', 'LLM自动')], default='upload', max_length=16),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='analysisrun',
|
||||
name='tool_calls_json',
|
||||
field=models.JSONField(blank=True, default=list, help_text='Structured tool call list: [{step, name, input, output}, ...]'),
|
||||
),
|
||||
]
|
||||
@@ -1,20 +0,0 @@
|
||||
# Generated by Django 4.2.30 on 2026-07-20 05:26
|
||||
|
||||
from django.db import migrations
|
||||
from django.db.models import F
|
||||
|
||||
|
||||
def populate_display_id(apps, schema_editor):
|
||||
AnalysisRun = apps.get_model('tianxuan_analysis', 'AnalysisRun')
|
||||
AnalysisRun.objects.filter(display_id__isnull=True).update(display_id=F('id'))
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('tianxuan_analysis', '0006_analysisrun_display_id_analysisrun_llm_thinking_and_more'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RunPython(populate_display_id, migrations.RunPython.noop),
|
||||
]
|
||||
@@ -96,6 +96,7 @@ class EntityProfile(models.Model):
|
||||
help_text="Aggregated entity features as dict")
|
||||
embedding_x = models.FloatField(null=True, blank=True, help_text="UMAP-2D X coordinate")
|
||||
embedding_y = models.FloatField(null=True, blank=True, help_text="UMAP-2D Y coordinate")
|
||||
embedding_z = models.FloatField(null=True, blank=True, help_text="UMAP-3D Z coordinate")
|
||||
|
||||
class Meta:
|
||||
unique_together = ['run', 'entity_value']
|
||||
|
||||
@@ -0,0 +1,412 @@
|
||||
"""
|
||||
Template-based natural language feature descriptions.
|
||||
|
||||
Uses value-range → text mappings to generate Chinese natural language
|
||||
descriptions of cluster features. Consumed by cluster_overview.html and
|
||||
entity_profile.html for human-readable cluster summaries.
|
||||
|
||||
Data sources
|
||||
------------
|
||||
- ``analysis.tls_ref.get_tls_ref_dict()`` : ``field_code → {meaning, data_type}``
|
||||
- ``ClusterFeature`` model rows : feature_name, mean, std, distinguishing_score
|
||||
|
||||
Usage::
|
||||
|
||||
from analysis.nl_describe import describe_feature, describe_cluster
|
||||
|
||||
text = describe_feature("8ack", 720.5, std_dev=2.1)
|
||||
# "中等数据包 (720 bytes)，比全局均值高2.1个标准差"
|
||||
|
||||
summary = describe_cluster(features, top_n=5)
|
||||
# "此簇特征: 低延迟(23ms), 大数据包(1520 bytes), ..."
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional, Sequence
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Category mapping: field_code → semantic category
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
CATEGORY_EXACT: dict[str, str] = {
|
||||
# ── latency ──
|
||||
"4dur": "latency",
|
||||
"2tmo": "latency",
|
||||
"8dbd": "latency",
|
||||
# ── payload bytes ──
|
||||
"8ack": "packet_bytes",
|
||||
"8byt": "packet_bytes",
|
||||
# ── packet count ──
|
||||
"8pak": "packet_count",
|
||||
"8ppk": "packet_count",
|
||||
# ── port ──
|
||||
":prs": "port",
|
||||
":prd": "port",
|
||||
# ── TLS version ──
|
||||
"0ver": "tls_version",
|
||||
# ── key size ──
|
||||
"4ksz": "key_size",
|
||||
# ── cipher code ──
|
||||
"0cph": "cipher_code",
|
||||
# ── curve code ──
|
||||
"0crv": "curve_code",
|
||||
# ── session / offset ──
|
||||
"8seq": "session_offset",
|
||||
"8ses": "session_offset",
|
||||
"8did": "session_offset",
|
||||
# ── IP protocol ──
|
||||
"1ipp": "ip_protocol",
|
||||
# ── TLS random ──
|
||||
"0rnt": "tls_random_time",
|
||||
"0rnd": "tls_random",
|
||||
# ── database refs ──
|
||||
"4dbn": "database_ref",
|
||||
"4srs": "device_ref",
|
||||
# ── recovery flags ──
|
||||
"cnrs": "recovery",
|
||||
"isrs": "recovery",
|
||||
}
|
||||
|
||||
# Substring-based matching (checked after exact match fails)
|
||||
CATEGORY_SUBSTR: list[tuple[str, str]] = [
|
||||
("latd", "latitude"),
|
||||
("lond", "longitude"),
|
||||
("orgn", "organization"),
|
||||
("ispn", "isp"),
|
||||
("anon", "anonymity"),
|
||||
("doma", "domain"),
|
||||
("city", "city"),
|
||||
("_lat", "latitude"),
|
||||
("_lon", "longitude"),
|
||||
("latitude", "latitude"),
|
||||
("longitude", "longitude"),
|
||||
("duration", "latency"),
|
||||
("delay", "latency"),
|
||||
("bytes", "packet_bytes"),
|
||||
("packets", "packet_count"),
|
||||
("count", "count"),
|
||||
("ratio", "ratio"),
|
||||
("tls_ver", "tls_version"),
|
||||
("version", "tls_version"),
|
||||
]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TLS version code → human description
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
TLS_VERSION_MAP: dict[int, str] = {
|
||||
0x0000: "SSL 2.0",
|
||||
0x0300: "SSL 3.0",
|
||||
0x0301: "TLS 1.0",
|
||||
0x0302: "TLS 1.1",
|
||||
0x0303: "TLS 1.2",
|
||||
0x0304: "TLS 1.3",
|
||||
0x7F00: "TLS 1.3 (draft)",
|
||||
0x7F12: "TLS 1.3 (draft 18)",
|
||||
0xFEFF: "DTLS 1.0",
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Templates: category → [(predicate, template_string), ...]
|
||||
#
|
||||
# Template variables available:
|
||||
# {v} — value (formatted to 1 decimal unless overridden)
|
||||
# {v:.0f} — value with custom format
|
||||
# {sv} — signed value (e.g. "+2.1" for positive zscore)
|
||||
# {av} — absolute value (e.g. "2.1")
|
||||
# {z:.1f} — z-score signed (preserved in deviation suffix)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
TEMPLATES: dict[str, list[tuple]] = {
|
||||
"latency": [
|
||||
(lambda v: v > 1000, "极高延迟 ({v:.0f}ms)"),
|
||||
(lambda v: v > 500, "高延迟 ({v:.0f}ms)"),
|
||||
(lambda v: v > 200, "中等延迟 ({v:.0f}ms)"),
|
||||
(lambda v: v > 50, "较低延迟 ({v:.0f}ms)"),
|
||||
(lambda v: True, "低延迟 ({v:.0f}ms)"),
|
||||
],
|
||||
"packet_bytes": [
|
||||
(lambda v: v > 1500, "大数据包 ({v:.0f} bytes, 超过MTU)"),
|
||||
(lambda v: v > 1000, "较大数据包 ({v:.0f} bytes)"),
|
||||
(lambda v: v > 400, "中等数据包 ({v:.0f} bytes)"),
|
||||
(lambda v: True, "小数据包 ({v:.0f} bytes)"),
|
||||
],
|
||||
"packet_count": [
|
||||
(lambda v: v > 200, "大量数据包 ({v:.0f} 个)"),
|
||||
(lambda v: v > 50, "中等数据包数 ({v:.0f} 个)"),
|
||||
(lambda v: True, "少量数据包 ({v:.0f} 个)"),
|
||||
],
|
||||
"port": [
|
||||
(lambda v: v <= 1024, "系统端口 ({v:.0f})"),
|
||||
(lambda v: v <= 49151, "注册端口 ({v:.0f})"),
|
||||
(lambda v: True, "动态/私有端口 ({v:.0f})"),
|
||||
],
|
||||
"tls_version": [
|
||||
(lambda v: 0x0304 <= v <= 0x0304, "TLS 1.3 (最新)"),
|
||||
(lambda v: 0x0303 <= v < 0x0304, "TLS 1.2 (主流)"),
|
||||
(lambda v: 0x0302 <= v < 0x0303, "TLS 1.1 (已弃用)"),
|
||||
(lambda v: 0x0301 <= v < 0x0302, "TLS 1.0 (过时)"),
|
||||
(lambda v: 0x0300 <= v < 0x0301, "SSL 3.0 (不安全)"),
|
||||
(lambda v: v < 0x0300, "SSL 2.0 (极不安全)"),
|
||||
(lambda v: v > 0x0304, "DTLS/实验性TLS (0x{v:.0X})"),
|
||||
(lambda v: True, "未知TLS版本 (0x{v:.0X})"),
|
||||
],
|
||||
"key_size": [
|
||||
(lambda v: v >= 384, "高强度密钥 ({v:.0f} bits)"),
|
||||
(lambda v: v >= 256, "强密钥 ({v:.0f} bits)"),
|
||||
(lambda v: v >= 128, "标准密钥 ({v:.0f} bits)"),
|
||||
(lambda v: v >= 64, "弱密钥 ({v:.0f} bits)"),
|
||||
(lambda v: True, "极弱密钥 ({v:.0f} bits)"),
|
||||
],
|
||||
"ip_protocol": [
|
||||
(lambda v: abs(v - 6) < 0.5, "TCP 协议 ({v:.0f})"),
|
||||
(lambda v: abs(v - 17) < 0.5, "UDP 协议 ({v:.0f})"),
|
||||
(lambda v: abs(v - 1) < 0.5, "ICMP 协议 ({v:.0f})"),
|
||||
(lambda v: abs(v - 2) < 0.5, "IGMP 协议 ({v:.0f})"),
|
||||
(lambda v: True, "IP 协议 #{v:.0f}"),
|
||||
],
|
||||
"session_offset": [
|
||||
(lambda v: v > 10000, "大型会话 (偏移 {v:.0f})"),
|
||||
(lambda v: v > 1000, "中型会话 (偏移 {v:.0f})"),
|
||||
(lambda v: True, "小型会话 (偏移 {v:.0f})"),
|
||||
],
|
||||
"latitude": [
|
||||
(lambda v: v > 60, "高纬度地区 ({v:.1f}°)"),
|
||||
(lambda v: v > 23.5, "中纬度地区 — 北温带 ({v:.1f}°)"),
|
||||
(lambda v: v >= -23.5, "低纬度地区 — 热带 ({v:.1f}°)"),
|
||||
(lambda v: v >= -60, "中纬度地区 — 南温带 ({v:.1f}°)"),
|
||||
(lambda v: True, "高纬度地区 — 南极 ({v:.1f}°)"),
|
||||
],
|
||||
"longitude": [
|
||||
(lambda v: v > 120 or v < -120, "远东地区 ({v:.1f}°)"),
|
||||
(lambda v: v > 60 or v < -60, "中亚/东欧地区 ({v:.1f}°)"),
|
||||
(lambda v: v > 0 or v < 0, "西欧/非洲地区 ({v:.1f}°)"),
|
||||
(lambda v: True, "本初子午线附近 ({v:.1f}°)"),
|
||||
],
|
||||
"cipher_code": [
|
||||
(lambda v: True, "加密套件代码 0x{v:.0X}"),
|
||||
],
|
||||
"curve_code": [
|
||||
(lambda v: True, "椭圆曲线代码 0x{v:.0X}"),
|
||||
],
|
||||
"tls_random_time": [
|
||||
(lambda v: True, "TLS 随机时间戳 ({v:.0f})"),
|
||||
],
|
||||
"tls_random": [
|
||||
(lambda v: True, "TLS 随机值"),
|
||||
],
|
||||
"database_ref": [
|
||||
(lambda v: True, "数据库引用 #{v:.0f}"),
|
||||
],
|
||||
"device_ref": [
|
||||
(lambda v: True, "设备标识 #{v:.0f}"),
|
||||
],
|
||||
"recovery": [
|
||||
(lambda v: v >= 0.5, "可恢复流"),
|
||||
(lambda v: True, "不可恢复流"),
|
||||
],
|
||||
"organization": [
|
||||
(lambda v: True, "组织指标 ({v:.1f})"),
|
||||
],
|
||||
"isp": [
|
||||
(lambda v: True, "ISP 指标 ({v:.1f})"),
|
||||
],
|
||||
"anonymity": [
|
||||
(lambda v: True, "匿名状态指标 ({v:.1f})"),
|
||||
],
|
||||
"domain": [
|
||||
(lambda v: True, "域名指标 ({v:.1f})"),
|
||||
],
|
||||
"city": [
|
||||
(lambda v: True, "城市指标 ({v:.1f})"),
|
||||
],
|
||||
"count": [
|
||||
(lambda v: v > 100, "计数很高 ({v:.0f})"),
|
||||
(lambda v: v > 50, "计数较高 ({v:.0f})"),
|
||||
(lambda v: True, "计数正常 ({v:.0f})"),
|
||||
],
|
||||
"ratio": [
|
||||
(lambda v: v > 0.8, "占比极高 ({v:.1%})"),
|
||||
(lambda v: v > 0.5, "占比偏高 ({v:.1%})"),
|
||||
(lambda v: v >= 0.0, "占比较低 ({v:.1%})"),
|
||||
(lambda v: True, "占比 ({v:.1%})"),
|
||||
],
|
||||
}
|
||||
|
||||
# Default template for unrecognised categories / fallback
|
||||
DEFAULT_TEMPLATES: list[tuple] = [
|
||||
(lambda v: True, "{meaning} = {v:.2f}"),
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _resolve_category(feature_name: str) -> str | None:
|
||||
"""Map a ``feature_name`` (column name) to a template category string.
|
||||
|
||||
Tries exact match against ``CATEGORY_EXACT`` first, then substring
|
||||
match against ``CATEGORY_SUBSTR``. Returns ``None`` if no match.
|
||||
"""
|
||||
# Exact match
|
||||
cat = CATEGORY_EXACT.get(feature_name)
|
||||
if cat is not None:
|
||||
return cat
|
||||
|
||||
# Substring match
|
||||
lowered = feature_name.lower()
|
||||
for sub, cat_label in CATEGORY_SUBSTR:
|
||||
if sub in lowered:
|
||||
return cat_label
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _lookup_meaning(feature_name: str) -> str:
|
||||
"""Return the Chinese meaning of *feature_name* from the TLS reference DB.
|
||||
|
||||
Falls back to *feature_name* itself when no entry is found.
|
||||
"""
|
||||
try:
|
||||
from analysis.tls_ref import get_tls_ref_dict
|
||||
|
||||
ref = get_tls_ref_dict()
|
||||
entry = ref.get(feature_name)
|
||||
if entry is not None:
|
||||
return entry.get("meaning", feature_name)
|
||||
except Exception:
|
||||
pass
|
||||
return feature_name
|
||||
|
||||
|
||||
def describe_feature(
|
||||
feature_name: str,
|
||||
value: float,
|
||||
std_dev: float | None = None,
|
||||
cluster_std: float | None = None,
|
||||
) -> str:
|
||||
"""Generate a Chinese natural-language description for a single feature.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
feature_name : str
|
||||
Column name (e.g. ``"8ack"``, ``":ips.latd"``).
|
||||
value : float
|
||||
The feature's mean value in the current context (cluster mean).
|
||||
std_dev : float | None
|
||||
Optional Z-score magnitude (``abs(distinguishing_score)``). When
|
||||
provided and >= 1.0, appends a deviation suffix.
|
||||
cluster_std : float | None
|
||||
Optional within-cluster standard deviation. Describes how spread
|
||||
the values are within this cluster (high = scattered, low = tight).
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
Natural-language description string.
|
||||
"""
|
||||
meaning = _lookup_meaning(feature_name)
|
||||
category = _resolve_category(feature_name)
|
||||
|
||||
# Select templates
|
||||
tmpl_list = TEMPLATES.get(category) if category else None
|
||||
if tmpl_list is None:
|
||||
tmpl_list = DEFAULT_TEMPLATES
|
||||
|
||||
# Evaluate predicates
|
||||
text = ""
|
||||
for predicate, template in tmpl_list:
|
||||
try:
|
||||
if predicate(value):
|
||||
text = template.format(v=value, sv=_signed(value), av=abs(value),
|
||||
meaning=meaning)
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if not text:
|
||||
text = f"{meaning} = {value:.2f}"
|
||||
|
||||
# Append deviation context
|
||||
if std_dev is not None and std_dev >= 1.0:
|
||||
text += f"，比全局均值高{std_dev:.1f}个标准差"
|
||||
|
||||
# Append within-cluster variance context
|
||||
if cluster_std is not None and cluster_std > 0:
|
||||
if cluster_std > 2.0:
|
||||
text += "（簇内离散度高）"
|
||||
elif cluster_std < 0.3:
|
||||
text += "（簇内高度集中）"
|
||||
|
||||
return text
|
||||
|
||||
|
||||
def describe_cluster(
|
||||
features: Sequence[dict],
|
||||
top_n: int = 5,
|
||||
) -> str:
|
||||
"""Generate a 2-3 sentence natural-language summary for a cluster.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
features : list[dict]
|
||||
Each dict must include:
|
||||
|
||||
- ``feature_name`` : str
|
||||
- ``mean`` : float (cluster mean)
|
||||
- ``distinguishing_score`` : float (Z-score; may be signed)
|
||||
|
||||
Optional keys used when present:
|
||||
|
||||
- ``std`` : float (cluster std dev)
|
||||
|
||||
top_n : int
|
||||
Number of most-distinguishing features to include (default 5).
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
A short paragraph describing the cluster's distinguishing traits.
|
||||
|
||||
Example:
|
||||
``"低延迟 (23ms)，比全局均值高3.1个标准差; 大数据包 (1520 bytes)，比全局均值高2.3个标准差; 系统端口 (443)"``
|
||||
"""
|
||||
if not features:
|
||||
return "无显著区分特征"
|
||||
|
||||
# Sort by |distinguishing_score| descending
|
||||
ranked = sorted(
|
||||
features,
|
||||
key=lambda f: abs(f.get("distinguishing_score", 0.0)),
|
||||
reverse=True,
|
||||
)[:top_n]
|
||||
|
||||
sentences: list[str] = []
|
||||
for f in ranked:
|
||||
name = f.get("feature_name", "")
|
||||
mean = f.get("mean")
|
||||
score = abs(f.get("distinguishing_score", 0.0))
|
||||
cstd = f.get("std")
|
||||
|
||||
if mean is None or name == "":
|
||||
continue
|
||||
|
||||
desc = describe_feature(name, float(mean), std_dev=score if score >= 1.0 else None,
|
||||
cluster_std=float(cstd) if cstd is not None else None)
|
||||
sentences.append(desc)
|
||||
|
||||
if not sentences:
|
||||
return "无显著区分特征"
|
||||
|
||||
return "；".join(sentences) + "。"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _signed(value: float) -> str:
|
||||
"""Return a signed string representation, e.g. ``"+2.1"``."""
|
||||
return f"{value:+.1f}"
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Line profiling utility for the TianXuan backend.
|
||||
|
||||
Usage in any backend module::
|
||||
|
||||
from analysis.profile_util import profile
|
||||
|
||||
@profile
|
||||
def my_function(...):
|
||||
...
|
||||
|
||||
When ``TIANXUAN_PROFILE=1`` env var is set AND line_profiler is installed,
|
||||
the decorator activates real line-level profiling and prints stats at exit.
|
||||
Otherwise the decorator is a no-op.
|
||||
|
||||
To auto-profile ALL public functions in a module automatically, add at the
|
||||
bottom of the file::
|
||||
|
||||
from analysis.profile_util import auto_profile_module
|
||||
auto_profile_module(__name__)
|
||||
|
||||
Auto-profiling is also gated by ``TIANXUAN_PROFILE=1``.
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
import atexit
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_PROFILER_ACTIVE = os.environ.get('TIANXUAN_PROFILE', '') == '1'
|
||||
|
||||
if _PROFILER_ACTIVE:
|
||||
try:
|
||||
from line_profiler import profile as _line_profile
|
||||
except ImportError:
|
||||
logger.warning('TIANXUAN_PROFILE=1 but line_profiler not installed')
|
||||
_PROFILER_ACTIVE = False
|
||||
|
||||
|
||||
def profile(func):
|
||||
"""Decorate a function for line-level profiling.
|
||||
|
||||
Active only when ``TIANXUAN_PROFILE=1`` env var is set and
|
||||
line_profiler is installed.
|
||||
"""
|
||||
if _PROFILER_ACTIVE:
|
||||
return _line_profile(func)
|
||||
return func
|
||||
|
||||
|
||||
def auto_profile_module(module_name: str) -> int:
|
||||
"""Apply @profile to every public (non-underscore) function in *module_name*.
|
||||
|
||||
Must be called at module level, typically at the bottom of the file::
|
||||
|
||||
from analysis.profile_util import auto_profile_module
|
||||
auto_profile_module(__name__)
|
||||
|
||||
Active only when ``TIANXUAN_PROFILE=1`` env var is set and
|
||||
line_profiler is installed. Returns number of functions wrapped, or 0
|
||||
when inactive.
|
||||
"""
|
||||
if not _PROFILER_ACTIVE:
|
||||
return 0
|
||||
|
||||
try:
|
||||
from line_profiler import LineProfiler
|
||||
except ImportError:
|
||||
return 0
|
||||
|
||||
module = sys.modules.get(module_name)
|
||||
if module is None:
|
||||
logger.warning('auto_profile_module: module %r not found', module_name)
|
||||
return 0
|
||||
|
||||
lp = LineProfiler()
|
||||
count = 0
|
||||
for name in dir(module):
|
||||
if name.startswith('_'):
|
||||
continue
|
||||
obj = getattr(module, name)
|
||||
if callable(obj):
|
||||
try:
|
||||
lp.add_function(obj)
|
||||
count += 1
|
||||
except (TypeError, ValueError):
|
||||
pass # skip non-Python or special objects
|
||||
|
||||
atexit.register(lambda: lp.print_stats(output_unit=1e-6))
|
||||
logger.info('auto_profile: wrapped %d functions in %s', count, module_name)
|
||||
return count
|
||||
@@ -0,0 +1,2 @@
|
||||
"""Analysis services package — pure business logic, no request/response/rendering."""
|
||||
from .clustering import run_clustering_pipeline, compute_umap_embedding, save_entity_profiles
|
||||
@@ -0,0 +1,361 @@
|
||||
"""Clustering service layer: pure business logic for the clustering pipeline.
|
||||
|
||||
Extracted from analysis/views/clustering.py — no request/response, no rendering.
|
||||
"""
|
||||
import asyncio
|
||||
import traceback
|
||||
import logging
|
||||
import warnings
|
||||
|
||||
import polars as pl
|
||||
import numpy as np
|
||||
|
||||
from analysis.constants import RANDOM_SEED, UMAP_BATCH_SIZE, UMAP_TRAIN_SAMPLE, NUMERIC_DTYPES
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def run_clustering_pipeline(run, store, entity_ds_id, feature_columns, algorithm,
|
||||
min_cluster_size, run_umap=True, head=None):
|
||||
"""Pure business logic — no request/response, no rendering.
|
||||
|
||||
Unified clustering pipeline: clustering → feature extraction → UMAP → ORM save.
|
||||
|
||||
Args:
|
||||
run: AnalysisRun ORM object.
|
||||
store: SessionStore instance.
|
||||
entity_ds_id: dataset ID in SessionStore pointing to entity-aggregated data.
|
||||
feature_columns: list of feature column names (None = auto-detect).
|
||||
algorithm: 'hdbscan' or 'kmeans'.
|
||||
min_cluster_size: int for HDBSCAN.
|
||||
run_umap: bool, whether to compute and save UMAP-2D embeddings.
|
||||
head: optional int, downsample to at most this many rows (min 100).
|
||||
"""
|
||||
warnings.filterwarnings('ignore', category=RuntimeWarning, module='sklearn')
|
||||
|
||||
try:
|
||||
from analysis.tool_registry import _handle_run_clustering, _handle_extract_features
|
||||
|
||||
entry = store.get_dataset(entity_ds_id)
|
||||
if entry is None:
|
||||
run.status = 'failed'
|
||||
run.error_message = 'Entity dataset not found in session store'
|
||||
run.save(update_fields=['status', 'error_message'])
|
||||
return
|
||||
|
||||
schema = entry.get('schema', {})
|
||||
|
||||
# ── Downsampling ──
|
||||
if head is not None and head > 0:
|
||||
lf = entry['lazyframe']
|
||||
try:
|
||||
row_count = lf.select(pl.len()).collect(streaming=True).item()
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
row_count = 0
|
||||
|
||||
if row_count > head:
|
||||
head = max(head, 100) # ensure minimum rows for clustering
|
||||
run.progress_msg = f'正在采样 {head}/{row_count} 行...'
|
||||
run.save(update_fields=['progress_msg'])
|
||||
lf = lf.head(head)
|
||||
store.store_dataset(
|
||||
entity_ds_id, lf,
|
||||
schema=schema,
|
||||
metadata=entry.get('metadata', {}),
|
||||
)
|
||||
entry = store.get_dataset(entity_ds_id) # refresh
|
||||
row_count = head
|
||||
|
||||
# Resolve feature columns — use actual LazyFrame dtypes (not stored schema dict
|
||||
# which may be stale after Utf8 coercion in _background_process).
|
||||
numeric_types = (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
|
||||
pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
|
||||
pl.Float32, pl.Float64)
|
||||
|
||||
lf = entry['lazyframe']
|
||||
live_schema = lf.collect_schema()
|
||||
live_names = live_schema.names()
|
||||
live_dtypes = list(live_schema.dtypes())
|
||||
|
||||
if feature_columns:
|
||||
feature_cols = [
|
||||
c for c in feature_columns
|
||||
if c in live_names and live_dtypes[live_names.index(c)] in numeric_types
|
||||
]
|
||||
else:
|
||||
feature_cols = [
|
||||
live_names[i] for i, dt in enumerate(live_dtypes)
|
||||
if dt in numeric_types and not live_names[i].startswith('_')
|
||||
][:10]
|
||||
|
||||
# ── Step 1: Clustering ────────────────────────────────────────────────
|
||||
# If no numeric columns detected, pass empty list to let
|
||||
# _handle_run_clustering apply its Utf8→Float64 coercion fallback.
|
||||
run.status = 'clustering'
|
||||
run.progress_pct = 70
|
||||
run.progress_msg = '正在进行聚类分析...'
|
||||
run.save(update_fields=['status', 'progress_pct', 'progress_msg'])
|
||||
|
||||
result = asyncio.run(_handle_run_clustering(
|
||||
dataset_id=entity_ds_id,
|
||||
cluster_columns=feature_cols,
|
||||
algorithm=algorithm,
|
||||
params={'min_cluster_size': min_cluster_size},
|
||||
random_state=RANDOM_SEED,
|
||||
))
|
||||
|
||||
if 'error' in result:
|
||||
err = result['error']
|
||||
if 'numeric' in err.lower():
|
||||
available = [
|
||||
f"{k}({v})" for k, v in schema.items()
|
||||
if v.split('(')[0].strip() in numeric_types and not k.startswith('_')
|
||||
]
|
||||
err += f"。可用数值列: {available}"
|
||||
raise Exception(err)
|
||||
|
||||
cluster_id = result.get('cluster_result_id', '')
|
||||
run.cluster_count = result.get('n_clusters', 0)
|
||||
run.save(update_fields=['cluster_count'])
|
||||
|
||||
# ── Step 2: Feature extraction ──────────────────────────────────────
|
||||
run.status = 'extracting'
|
||||
run.progress_pct = 90
|
||||
run.progress_msg = '正在提取聚类特征...'
|
||||
run.save(update_fields=['status', 'progress_pct', 'progress_msg'])
|
||||
|
||||
feat_result = asyncio.run(_handle_extract_features(
|
||||
dataset_id=entity_ds_id,
|
||||
cluster_result_id=cluster_id,
|
||||
top_k=10, method='zscore', save_to_db=True,
|
||||
run_id=run.id,
|
||||
))
|
||||
|
||||
if 'error' in feat_result:
|
||||
raise Exception(feat_result['error'])
|
||||
|
||||
# ── Step 2b: Cluster SVD feature extraction ──────────────────────
|
||||
run.progress_msg = '正在进行聚类SVD特征提取...'
|
||||
run.save(update_fields=['progress_msg'])
|
||||
try:
|
||||
from analysis.distance import cluster_svd_extract
|
||||
lf = entry['lazyframe']
|
||||
cluster_entry = store.get_cluster_result(cluster_id)
|
||||
labels_list = cluster_entry.get('labels', []) if cluster_entry else []
|
||||
|
||||
if labels_list and feature_cols:
|
||||
svd_results = cluster_svd_extract(lf, labels_list, feature_cols)
|
||||
|
||||
# Persist SVD features to ClusterFeature (method='cluster_svd')
|
||||
from analysis.models import ClusterFeature as CF
|
||||
from analysis.models import ClusterResult as CR
|
||||
for label, svd_info in svd_results.items():
|
||||
cr = CR.objects.filter(run=run, cluster_label=label).first()
|
||||
if cr is None:
|
||||
continue
|
||||
# Delete old zscore entries for this cluster, replace with SVD
|
||||
CF.objects.filter(cluster=cr, distinguishing_method='zscore').delete()
|
||||
for feat_name in svd_info.get('features', [])[:10]:
|
||||
idx = svd_info['features'].index(feat_name) if feat_name in svd_info['features'] else -1
|
||||
strength = (svd_info.get('feature_strength', [])[idx]
|
||||
if idx >= 0 and idx < len(svd_info.get('feature_strength', []))
|
||||
else None)
|
||||
CF.objects.get_or_create(
|
||||
cluster=cr,
|
||||
feature_name=feat_name,
|
||||
defaults={
|
||||
'mean': None, 'std': None, 'median': None,
|
||||
'p25': None, 'p75': None, 'missing_rate': None,
|
||||
'distinguishing_score': float(strength) if strength is not None else None,
|
||||
'distinguishing_method': 'cluster_svd',
|
||||
},
|
||||
)
|
||||
except Exception as svd_err:
|
||||
logger.warning(f'Cluster SVD extraction skipped: {svd_err}')
|
||||
|
||||
# ── Step 3: UMAP-3D embedding (2D fallback) ───────────────────────
|
||||
if run_umap:
|
||||
try:
|
||||
lf = entry['lazyframe']
|
||||
# UMAP: sample max 10K for training, batch-transform rest in 1K batches
|
||||
MAX_UMAP_TRAIN = UMAP_TRAIN_SAMPLE
|
||||
BATCH_SIZE = UMAP_BATCH_SIZE
|
||||
try:
|
||||
n_total = lf.select(pl.len()).collect(streaming=True).item()
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
n_total = 0
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
import umap
|
||||
|
||||
live_schema = lf.collect_schema()
|
||||
num_cols = [name for name, dt in zip(live_schema.names(), live_schema.dtypes())
|
||||
if dt in (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
|
||||
pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
|
||||
pl.Float32, pl.Float64) and not name.startswith('_')]
|
||||
|
||||
if len(num_cols) >= 2:
|
||||
if n_total > MAX_UMAP_TRAIN:
|
||||
run.progress_msg = f'UMAP 采样 {MAX_UMAP_TRAIN}/{n_total} 实体训练...'
|
||||
run.save(update_fields=['progress_msg'])
|
||||
df_sample = lf.sample(n=MAX_UMAP_TRAIN, seed=RANDOM_SEED).collect(streaming=True)
|
||||
df_umap = lf.collect(streaming=True)
|
||||
else:
|
||||
df_umap = lf.collect(streaming=True)
|
||||
|
||||
coords, umap_components = compute_umap_embedding(
|
||||
df_umap, num_cols, n_total,
|
||||
max_train=MAX_UMAP_TRAIN,
|
||||
batch_size=BATCH_SIZE,
|
||||
seed=RANDOM_SEED,
|
||||
)
|
||||
|
||||
cluster_entry = store.get_cluster_result(cluster_id)
|
||||
labels_list = cluster_entry.get('labels', []) if cluster_entry else []
|
||||
|
||||
save_entity_profiles(run, df_umap, labels_list, coords, umap_components)
|
||||
except Exception as umap_err:
|
||||
logger.warning(f'UMAP embedding skipped: {umap_err}')
|
||||
|
||||
# Fallback: if entity_count not set (UMAP skipped), derive from cluster sizes
|
||||
if run.entity_count is None:
|
||||
from django.db.models import Sum
|
||||
total = run.clusters.aggregate(total=Sum('size'))['total'] or 0
|
||||
run.entity_count = total
|
||||
run.save(update_fields=['entity_count'])
|
||||
|
||||
# ── Done ──
|
||||
run.status = 'completed'
|
||||
run.progress_pct = 100
|
||||
run.progress_msg = '分析完成'
|
||||
run.save(update_fields=['status', 'progress_pct', 'progress_msg', 'entity_count'])
|
||||
|
||||
except Exception:
|
||||
tb = traceback.format_exc()
|
||||
logger.error(tb)
|
||||
try:
|
||||
run.status = 'failed'
|
||||
run.error_message = tb
|
||||
run.progress_msg = f'失败: {tb}'
|
||||
run.run_log += f'\n[ERROR] {tb}'
|
||||
run.save(update_fields=['status', 'error_message', 'progress_msg', 'run_log'])
|
||||
except Exception as e:
|
||||
logger.warning('save failed after pipeline error: %s', e)
|
||||
|
||||
|
||||
def compute_umap_embedding(df, num_cols, n_total, max_train, batch_size, seed):
|
||||
"""UMAP 3D→2D fallback, returns (coords, umap_components).
|
||||
|
||||
Args:
|
||||
df: Collected Polars DataFrame with entity-level data.
|
||||
num_cols: list of numeric column names to use for UMAP.
|
||||
n_total: total row count.
|
||||
max_train: max rows for UMAP training sample.
|
||||
batch_size: batch size for transform when dataset exceeds max_train.
|
||||
seed: random seed.
|
||||
|
||||
Returns:
|
||||
(coords, umap_components): numpy array of shape (n_total, umap_components),
|
||||
and umap_components is 2 or 3.
|
||||
"""
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
import umap
|
||||
|
||||
if len(df) <= 2 or len(num_cols) < 2:
|
||||
return None, 0
|
||||
|
||||
if n_total > max_train:
|
||||
df_sample = df.sample(n=max_train, seed=seed)
|
||||
else:
|
||||
df_sample = df
|
||||
|
||||
umap_components = 3 # try 3D first
|
||||
coords = None
|
||||
|
||||
for attempt_n in (3, 2):
|
||||
try:
|
||||
if n_total > max_train:
|
||||
# Train UMAP on sample, batch-transform full dataset
|
||||
mat_sample = df_sample.select(num_cols).to_numpy()
|
||||
scaler = StandardScaler().fit(mat_sample)
|
||||
mat_sample_scaled = np.nan_to_num(
|
||||
scaler.transform(mat_sample), nan=0.0)
|
||||
reducer = umap.UMAP(n_components=attempt_n,
|
||||
random_state=seed,
|
||||
n_neighbors=15, min_dist=0.1,
|
||||
metric='euclidean')
|
||||
reducer.fit(mat_sample_scaled)
|
||||
# Batch transform in batch_size-row batches
|
||||
coords_list = []
|
||||
for start in range(0, len(df), batch_size):
|
||||
end = min(start + batch_size, len(df))
|
||||
mat_batch = scaler.transform(
|
||||
df[start:end].select(num_cols).to_numpy())
|
||||
mat_batch = np.nan_to_num(mat_batch, nan=0.0)
|
||||
coords_list.append(reducer.transform(mat_batch))
|
||||
coords = np.vstack(coords_list)
|
||||
else:
|
||||
mat = np.nan_to_num(StandardScaler().fit_transform(
|
||||
df.select(num_cols).to_numpy()), nan=0.0)
|
||||
reducer = umap.UMAP(n_components=attempt_n,
|
||||
random_state=seed,
|
||||
n_neighbors=15, min_dist=0.1,
|
||||
metric='euclidean')
|
||||
coords = reducer.fit_transform(mat)
|
||||
umap_components = attempt_n
|
||||
break
|
||||
except Exception as e_umap:
|
||||
if attempt_n == 2:
|
||||
raise
|
||||
logger.warning(f'UMAP 3D failed ({e_umap}), falling back to 2D')
|
||||
|
||||
if coords is None:
|
||||
raise Exception('UMAP produced no output')
|
||||
|
||||
return coords, umap_components
|
||||
|
||||
|
||||
def save_entity_profiles(run, df_umap, labels_list, coords, umap_components):
|
||||
"""Persist per-row cluster labels + UMAP coords to EntityProfile.
|
||||
|
||||
Args:
|
||||
run: AnalysisRun ORM object.
|
||||
df_umap: Collected Polars DataFrame (used for entity column detection + row iteration).
|
||||
labels_list: list of cluster labels per row.
|
||||
coords: numpy array of UMAP coordinates (must not be None).
|
||||
umap_components: 2 or 3.
|
||||
"""
|
||||
from analysis.models import EntityProfile as EP
|
||||
from analysis.models import ClusterResult as CR
|
||||
|
||||
# Determine entity column (first non-numeric, non-underscore column)
|
||||
df_num_cols = [c for c, dt in zip(df_umap.columns, df_umap.dtypes)
|
||||
if dt in NUMERIC_DTYPES]
|
||||
non_num = [c for c in df_umap.columns
|
||||
if c not in df_num_cols and not c.startswith('_')]
|
||||
ent_col = non_num[0] if non_num else df_umap.columns[0]
|
||||
|
||||
profiles = []
|
||||
cr_cache = {}
|
||||
for i, row in enumerate(df_umap.iter_rows(named=True)):
|
||||
ev = str(row.get(ent_col, '')) or 'entity'
|
||||
ev = f'{ev}_{i}'
|
||||
lbl = labels_list[i] if i < len(labels_list) else -1
|
||||
cache_key = f'{run.id}_{lbl}'
|
||||
if cache_key not in cr_cache:
|
||||
cr_cache[cache_key] = CR.objects.filter(
|
||||
run=run, cluster_label=lbl).first()
|
||||
has_z = umap_components >= 3 and coords.shape[1] >= 3
|
||||
profiles.append(EP(
|
||||
entity_value=ev, run=run, cluster_label=lbl,
|
||||
cluster=cr_cache[cache_key],
|
||||
embedding_x=float(coords[i, 0]) if i < len(coords) else None,
|
||||
embedding_y=float(coords[i, 1]) if i < len(coords) else None,
|
||||
embedding_z=float(coords[i, 2]) if has_z
|
||||
and i < len(coords) else None,
|
||||
))
|
||||
if profiles:
|
||||
EP.objects.bulk_create(profiles, ignore_conflicts=True, batch_size=1000)
|
||||
run.entity_count = len(profiles)
|
||||
run.save(update_fields=['entity_count'])
|
||||
+174
-51
@@ -46,6 +46,18 @@ def _generate_id(prefix: str = 'ds') -> str:
|
||||
return f"{prefix}_{uuid.uuid4().hex[:12]}"
|
||||
|
||||
|
||||
def _estimate_row_count(lf: pl.LazyFrame) -> int:
|
||||
"""Estimate row count from a LazyFrame (fast, streaming).
|
||||
|
||||
Uses ``select(pl.len()).collect(streaming=True)`` which scans the
|
||||
underlying source efficiently without loading all columns.
|
||||
"""
|
||||
try:
|
||||
return lf.select(pl.len()).collect(streaming=True).item()
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
|
||||
def _get_store_path() -> Path:
|
||||
"""Return the persistent JSON file path for dataset metadata.
|
||||
|
||||
@@ -78,6 +90,7 @@ class SessionStore:
|
||||
cls._instance = super().__new__(cls)
|
||||
cls._instance._stores: dict[str, dict[str, Any]] = {}
|
||||
cls._instance._datalock = threading.RLock()
|
||||
cls._instance._restore_done = threading.Event()
|
||||
return cls._instance
|
||||
|
||||
# ── persistence ────────────────────────────────────────────────────
|
||||
@@ -85,8 +98,9 @@ class SessionStore:
|
||||
def _save_to_disk(self) -> None:
|
||||
"""Persist current dataset metadata to JSON file.
|
||||
|
||||
Only metadata (schema, csv_glob, row_count, …) is saved — never the
|
||||
LazyFrame itself, which is an in-memory query plan.
|
||||
Only metadata (schema, row_count, file_count, sqlite_table, …) is
|
||||
saved — never the LazyFrame itself. The resulting file is typically
|
||||
under 10 KB even with many datasets.
|
||||
"""
|
||||
with self._datalock:
|
||||
serializable = {}
|
||||
@@ -94,9 +108,16 @@ class SessionStore:
|
||||
if entry.get('type') != 'dataset':
|
||||
continue
|
||||
meta = entry.get('metadata', {})
|
||||
# Store only restore-critical fields; omit runtime noise
|
||||
slim_meta = {
|
||||
k: v for k, v in meta.items()
|
||||
if k in ('row_count', 'file_count', 'sqlite_table',
|
||||
'csv_glob', 'columns', 'memory_mb',
|
||||
'schema_overrides', 'svd_components')
|
||||
}
|
||||
serializable[ds_id] = {
|
||||
'schema': entry.get('schema'),
|
||||
'metadata': meta,
|
||||
'metadata': slim_meta,
|
||||
'csv_glob': meta.get('csv_glob'),
|
||||
}
|
||||
path = _get_store_path()
|
||||
@@ -105,17 +126,15 @@ class SessionStore:
|
||||
encoding='utf-8',
|
||||
)
|
||||
|
||||
def restore_from_disk(self) -> int:
|
||||
"""Re-register datasets from persisted metadata.
|
||||
def _restore_metadata(self) -> int:
|
||||
"""Read metadata from disk and store as lightweight *stubs*.
|
||||
|
||||
Attempts two strategies in order:
|
||||
1. **SQLite table** — when ``metadata.sqlite_table`` exists (fast,
|
||||
always available, no CSV dependency).
|
||||
2. **CSV glob** — falls back to ``load_csv_directory`` for datasets
|
||||
that were stored before the SQLite migration or whose SQLite table
|
||||
was dropped.
|
||||
A stub contains everything needed to reconstruct the LazyFrame on
|
||||
demand but performs **no I/O** beyond reading the JSON file. Real
|
||||
data loading is deferred until the first :meth:`get_dataset` call
|
||||
(or the background thread).
|
||||
|
||||
Returns the number of successfully restored datasets.
|
||||
Returns the number of stubs created.
|
||||
"""
|
||||
path = _get_store_path()
|
||||
if not path.exists():
|
||||
@@ -125,47 +144,131 @@ class SessionStore:
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return 0
|
||||
|
||||
# Lazy imports to avoid circular dependency at module level
|
||||
from analysis.data_loader import load_csv_directory, load_from_db # fmt: skip
|
||||
count = 0
|
||||
with self._datalock:
|
||||
for ds_id, info in data.items():
|
||||
# Skip if already loaded (idempotent on re-calls)
|
||||
if ds_id in self._stores:
|
||||
continue
|
||||
meta = dict(info.get('metadata', {}))
|
||||
schema = dict(info.get('schema', {}))
|
||||
csv_glob = info.get('csv_glob')
|
||||
self._stores[ds_id] = {
|
||||
'type': 'dataset',
|
||||
'_stub': {
|
||||
'sqlite_table': meta.get('sqlite_table'),
|
||||
'csv_glob': csv_glob,
|
||||
'schema_overrides': meta.get('schema_overrides'),
|
||||
},
|
||||
'schema': schema,
|
||||
'metadata': meta,
|
||||
'parent_id': None,
|
||||
}
|
||||
count += 1
|
||||
self._restore_done.set()
|
||||
return count
|
||||
|
||||
restored = 0
|
||||
for ds_id, info in data.items():
|
||||
meta = dict(info.get('metadata', {}))
|
||||
schema = dict(info.get('schema', {}))
|
||||
restored_ok = False
|
||||
def _load_pending_dataset(self, ds_id: str) -> bool:
|
||||
"""Resolve a stub entry into a real LazyFrame.
|
||||
|
||||
# Strategy 1: SQLite table (fast path)
|
||||
sqlite_table = meta.get('sqlite_table')
|
||||
if sqlite_table:
|
||||
try:
|
||||
lf = load_from_db(sqlite_table)
|
||||
if lf is not None:
|
||||
self.store_dataset(ds_id, lf, schema=schema, metadata=meta)
|
||||
restored += 1
|
||||
restored_ok = True
|
||||
except Exception:
|
||||
pass # fall through to CSV restore
|
||||
Called on the first :meth:`get_dataset` for a dataset that was
|
||||
restored via :meth:`_restore_metadata`.
|
||||
|
||||
if restored_ok:
|
||||
continue
|
||||
Attempts two strategies:
|
||||
1. **SQLite table** (fast — uses :func:`load_from_db_lazy`).
|
||||
2. **CSV glob** (legacy fallback — scans the filesystem).
|
||||
|
||||
# Strategy 2: CSV glob (legacy / fallback)
|
||||
csv_glob = info.get('csv_glob')
|
||||
if not csv_glob:
|
||||
continue
|
||||
Returns ``True`` if the dataset was successfully loaded.
|
||||
"""
|
||||
from analysis.data_loader import load_csv_directory, load_from_db_lazy # fmt: skip
|
||||
|
||||
entry = self._stores.get(ds_id)
|
||||
if not entry or '_stub' not in entry:
|
||||
return True # nothing to do
|
||||
|
||||
stub = entry.pop('_stub')
|
||||
meta = entry.get('metadata', {})
|
||||
schema = entry.get('schema', {})
|
||||
|
||||
# Strategy 1: SQLite table (fast, native Polars reader)
|
||||
sqlite_table = stub.get('sqlite_table')
|
||||
if sqlite_table:
|
||||
try:
|
||||
lf, schema_csv, row_count, file_count, memory_mb = load_csv_directory(csv_glob)
|
||||
# Preserve original metadata counts from disk — CSV re-scan may
|
||||
# detect different file/row counts if files changed on disk.
|
||||
if 'file_count' not in meta and file_count is not None:
|
||||
meta['file_count'] = file_count
|
||||
if 'row_count' not in meta and row_count is not None:
|
||||
meta['row_count'] = row_count
|
||||
self.store_dataset(ds_id, lf, schema=schema_csv, metadata=meta)
|
||||
restored += 1
|
||||
lf = load_from_db_lazy(sqlite_table, schema_overrides=stub.get('schema_overrides'))
|
||||
if lf is not None:
|
||||
entry['lazyframe'] = lf
|
||||
if 'row_count' not in meta:
|
||||
meta['row_count'] = _estimate_row_count(lf)
|
||||
return True
|
||||
except Exception:
|
||||
# File(s) no longer exist or schema changed — skip gracefully
|
||||
continue
|
||||
pass # fall through to CSV restore
|
||||
|
||||
# Strategy 2: CSV glob (legacy fallback)
|
||||
csv_glob = stub.get('csv_glob')
|
||||
if not csv_glob:
|
||||
# No recoverable data source — entry remains without LazyFrame
|
||||
return False
|
||||
try:
|
||||
lf, schema_csv, row_count, file_count, memory_mb = load_csv_directory(csv_glob)
|
||||
if 'file_count' not in meta and file_count is not None:
|
||||
meta['file_count'] = file_count
|
||||
if 'row_count' not in meta and row_count is not None:
|
||||
meta['row_count'] = row_count
|
||||
# Merge CSV-scanned schema (may have more/different columns)
|
||||
merged_schema = {**schema, **schema_csv}
|
||||
entry['lazyframe'] = lf
|
||||
entry['schema'] = merged_schema
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def start_async_restore(self) -> int:
|
||||
"""Restore dataset metadata immediately; load data in background.
|
||||
|
||||
At startup this reads the small metadata JSON and stores *stubs*
|
||||
(metadata only, no data I/O). A daemon thread then resolves each
|
||||
stub into a real LazyFrame without blocking the main thread.
|
||||
|
||||
Returns the number of datasets queued for restoration.
|
||||
"""
|
||||
count = self._restore_metadata()
|
||||
if count > 0:
|
||||
t = threading.Thread(
|
||||
target=self._background_restore,
|
||||
daemon=True,
|
||||
name='session-restore',
|
||||
)
|
||||
t.start()
|
||||
return count
|
||||
|
||||
def _background_restore(self) -> None:
|
||||
"""Pre-load all stubs in a daemon thread (best-effort)."""
|
||||
with self._datalock:
|
||||
ds_ids = [
|
||||
ds_id for ds_id, entry in self._stores.items()
|
||||
if entry.get('type') == 'dataset' and '_stub' in entry
|
||||
]
|
||||
for ds_id in ds_ids:
|
||||
try:
|
||||
self._load_pending_dataset(ds_id)
|
||||
except Exception:
|
||||
pass # best-effort; stub remains for on-demand fallback
|
||||
|
||||
def restore_from_disk(self) -> int:
|
||||
"""Legacy synchronous restore (kept for backward compatibility).
|
||||
|
||||
Prefer :meth:`start_async_restore` for non-blocking startup.
|
||||
"""
|
||||
self._restore_metadata()
|
||||
with self._datalock:
|
||||
ds_ids = [
|
||||
ds_id for ds_id, entry in self._stores.items()
|
||||
if entry.get('type') == 'dataset' and '_stub' in entry
|
||||
]
|
||||
restored = 0
|
||||
for ds_id in ds_ids:
|
||||
if self._load_pending_dataset(ds_id):
|
||||
restored += 1
|
||||
return restored
|
||||
|
||||
# ── dataset operations ──────────────────────────────────────────────
|
||||
@@ -199,22 +302,41 @@ class SessionStore:
|
||||
self._save_to_disk()
|
||||
|
||||
def get_dataset(self, dataset_id: str) -> Optional[dict]:
|
||||
"""Retrieve a stored dataset entry (or *None*)."""
|
||||
"""Retrieve a stored dataset entry (or *None*).
|
||||
|
||||
If the entry is a *stub* (restored from metadata but not yet
|
||||
materialised), this method triggers on-demand loading of the
|
||||
LazyFrame before returning.
|
||||
"""
|
||||
with self._datalock:
|
||||
entry = self._stores.get(dataset_id)
|
||||
if entry and entry.get('type') == 'dataset':
|
||||
return entry
|
||||
return None
|
||||
if not entry or entry.get('type') != 'dataset':
|
||||
return None
|
||||
needs_load = '_stub' in entry
|
||||
|
||||
if needs_load:
|
||||
self._load_pending_dataset(dataset_id)
|
||||
|
||||
with self._datalock:
|
||||
return self._stores.get(dataset_id)
|
||||
|
||||
def drop_dataset(self, dataset_id: str) -> bool:
|
||||
"""Remove a dataset from the store and force garbage collection.
|
||||
|
||||
Cascades to child datasets (e.g. filtered results) via parent_id tracking.
|
||||
Returns True if the dataset existed.
|
||||
"""
|
||||
with self._datalock:
|
||||
existed = dataset_id in self._stores and self._stores[dataset_id].get('type') == 'dataset'
|
||||
if existed:
|
||||
del self._stores[dataset_id]
|
||||
# Cascade: drop all child datasets that reference the dropped one as parent
|
||||
child_ids = [
|
||||
k for k, v in self._stores.items()
|
||||
if v.get('type') == 'dataset' and v.get('parent_id') == dataset_id
|
||||
]
|
||||
for cid in child_ids:
|
||||
del self._stores[cid]
|
||||
if existed:
|
||||
gc.collect()
|
||||
self._save_to_disk()
|
||||
@@ -255,6 +377,7 @@ class SessionStore:
|
||||
'svd_components': meta.get('svd_components', 0),
|
||||
'columns': list(entry.get('schema', {}).keys()),
|
||||
'column_count': len(entry.get('schema', {})),
|
||||
'is_stub': '_stub' in entry,
|
||||
})
|
||||
return results
|
||||
|
||||
|
||||
+4
-3329
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,103 @@
|
||||
"""Tool registry package — split from analysis/tool_registry.py.
|
||||
|
||||
Re-exports all tool metadata, dispatcher, and handlers.
|
||||
"""
|
||||
|
||||
# ── Helpers ────────────────────────────────────────────────────────────────
|
||||
from ._helpers import (
|
||||
_json_size,
|
||||
_truncate_response,
|
||||
_build_filter_expr,
|
||||
_resolve_dataset,
|
||||
_count_rows,
|
||||
_get_numeric_columns,
|
||||
NUMERIC_DTYPES,
|
||||
NUMERIC_TYPE_NAMES,
|
||||
)
|
||||
|
||||
# ── Registry + dispatch ────────────────────────────────────────────────────
|
||||
from ._registry import get_tools_meta
|
||||
from ._dispatch import handle_call
|
||||
|
||||
# ── Handlers (tool implementations) ────────────────────────────────────────
|
||||
from .load_data import _handle_load_data
|
||||
from .profile import _handle_profile_data
|
||||
from .filter import _handle_filter_data
|
||||
from .preprocess import _handle_preprocess_data
|
||||
from .clustering import _handle_run_clustering, _handle_filter_and_cluster
|
||||
from .evaluate import _handle_evaluate_clustering
|
||||
from .features import _handle_extract_features, _save_features_to_db
|
||||
from .export import _handle_export_results
|
||||
from .entities import (
|
||||
_handle_build_entity_profiles,
|
||||
_handle_compute_scores,
|
||||
_add_adaptive_scores,
|
||||
_ENTITY_KEYWORDS,
|
||||
_NUMERIC_TYPES_FOR_AGG,
|
||||
_ANOMALY_FEATURE_COLS,
|
||||
)
|
||||
from .anomalies import _handle_detect_anomalies, _handle_visualize_anomalies
|
||||
from .diagnostics import (
|
||||
_handle_validate_data,
|
||||
_handle_explore_distributions,
|
||||
_handle_find_outliers,
|
||||
_handle_diagnose_clustering,
|
||||
_handle_compare_datasets,
|
||||
_handle_export_debug_sample,
|
||||
_handle_repair_schema,
|
||||
)
|
||||
from .analysis import (
|
||||
_handle_analyze_patterns,
|
||||
_handle_analyze_temporal,
|
||||
_handle_analyze_fft,
|
||||
_handle_analyze_tls_health,
|
||||
_handle_analyze_geo_distribution,
|
||||
_handle_analyze_entity_detail,
|
||||
)
|
||||
from .data_mgmt import (
|
||||
_handle_list_datasets,
|
||||
_handle_drop_dataset,
|
||||
_handle_clone_dataset,
|
||||
)
|
||||
from .distance_matrix import _handle_compute_distance_matrix
|
||||
|
||||
__all__ = [
|
||||
# Helpers
|
||||
'_json_size', '_truncate_response', '_build_filter_expr', '_resolve_dataset',
|
||||
'_count_rows', '_get_numeric_columns', 'NUMERIC_DTYPES', 'NUMERIC_TYPE_NAMES',
|
||||
# Registry + dispatch
|
||||
'get_tools_meta', 'handle_call',
|
||||
# Load
|
||||
'_handle_load_data',
|
||||
# Profile
|
||||
'_handle_profile_data',
|
||||
# Filter
|
||||
'_handle_filter_data',
|
||||
# Preprocess
|
||||
'_handle_preprocess_data',
|
||||
# Clustering
|
||||
'_handle_run_clustering', '_handle_filter_and_cluster',
|
||||
# Evaluate
|
||||
'_handle_evaluate_clustering',
|
||||
# Features
|
||||
'_handle_extract_features', '_save_features_to_db',
|
||||
# Export
|
||||
'_handle_export_results',
|
||||
# Entities
|
||||
'_handle_build_entity_profiles', '_handle_compute_scores', '_add_adaptive_scores',
|
||||
'_ENTITY_KEYWORDS', '_NUMERIC_TYPES_FOR_AGG', '_ANOMALY_FEATURE_COLS',
|
||||
# Anomalies
|
||||
'_handle_detect_anomalies', '_handle_visualize_anomalies',
|
||||
# Diagnostics
|
||||
'_handle_validate_data', '_handle_explore_distributions', '_handle_find_outliers',
|
||||
'_handle_diagnose_clustering', '_handle_compare_datasets',
|
||||
'_handle_export_debug_sample', '_handle_repair_schema',
|
||||
# Analysis
|
||||
'_handle_analyze_patterns', '_handle_analyze_temporal', '_handle_analyze_fft',
|
||||
'_handle_analyze_tls_health', '_handle_analyze_geo_distribution',
|
||||
'_handle_analyze_entity_detail',
|
||||
# Data management
|
||||
'_handle_list_datasets', '_handle_drop_dataset', '_handle_clone_dataset',
|
||||
# Distance matrix
|
||||
'_handle_compute_distance_matrix',
|
||||
]
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Tool call dispatcher — maps tool names to handler functions."""
|
||||
import asyncio
|
||||
from ._helpers import _truncate_response
|
||||
from .load_data import _handle_load_data
|
||||
from .profile import _handle_profile_data
|
||||
from .filter import _handle_filter_data
|
||||
from .preprocess import _handle_preprocess_data
|
||||
from .clustering import _handle_run_clustering, _handle_filter_and_cluster
|
||||
from .evaluate import _handle_evaluate_clustering
|
||||
from .features import _handle_extract_features
|
||||
from .export import _handle_export_results
|
||||
from .entities import _handle_build_entity_profiles, _handle_compute_scores
|
||||
from .anomalies import _handle_detect_anomalies, _handle_visualize_anomalies
|
||||
from .diagnostics import (
|
||||
_handle_validate_data, _handle_explore_distributions, _handle_find_outliers,
|
||||
_handle_diagnose_clustering, _handle_compare_datasets,
|
||||
_handle_export_debug_sample, _handle_repair_schema,
|
||||
)
|
||||
from .analysis import (
|
||||
_handle_analyze_patterns, _handle_analyze_temporal, _handle_analyze_fft,
|
||||
_handle_analyze_tls_health, _handle_analyze_geo_distribution,
|
||||
_handle_analyze_entity_detail,
|
||||
)
|
||||
from .data_mgmt import _handle_list_datasets, _handle_drop_dataset, _handle_clone_dataset
|
||||
from .distance_matrix import _handle_compute_distance_matrix
|
||||
|
||||
|
||||
async def handle_call(name: str, arguments: dict) -> dict:
|
||||
"""Dispatch a tool call to the appropriate handler.
|
||||
|
||||
Extracts the mandatory ``_timeout`` parameter (set by the LLM) from
|
||||
*arguments* and enforces it as an ``asyncio.wait_for`` deadline.
|
||||
|
||||
Args:
|
||||
name: Tool name (must match one of the 30 tools).
|
||||
arguments: Tool-specific parameters, must include ``_timeout``.
|
||||
|
||||
Returns:
|
||||
A JSON-serialisable dict.
|
||||
|
||||
Raises:
|
||||
ValueError: If *name* is not a recognised tool.
|
||||
"""
|
||||
_handlers = {
|
||||
'load_data': _handle_load_data,
|
||||
'profile_data': _handle_profile_data,
|
||||
'filter_data': _handle_filter_data,
|
||||
'preprocess_data': _handle_preprocess_data,
|
||||
'run_clustering': _handle_run_clustering,
|
||||
'evaluate_clustering': _handle_evaluate_clustering,
|
||||
'extract_features': _handle_extract_features,
|
||||
'export_results': _handle_export_results,
|
||||
'list_datasets': _handle_list_datasets,
|
||||
'drop_dataset': _handle_drop_dataset,
|
||||
'clone_dataset': _handle_clone_dataset,
|
||||
'build_entity_profiles': _handle_build_entity_profiles,
|
||||
'compute_scores': _handle_compute_scores,
|
||||
'filter_and_cluster': _handle_filter_and_cluster,
|
||||
'detect_anomalies': _handle_detect_anomalies,
|
||||
'visualize_anomalies': _handle_visualize_anomalies,
|
||||
'validate_data': _handle_validate_data,
|
||||
'explore_distributions': _handle_explore_distributions,
|
||||
'find_outliers': _handle_find_outliers,
|
||||
'diagnose_clustering': _handle_diagnose_clustering,
|
||||
'compare_datasets': _handle_compare_datasets,
|
||||
'export_debug_sample': _handle_export_debug_sample,
|
||||
'repair_schema': _handle_repair_schema,
|
||||
'analyze_patterns': _handle_analyze_patterns,
|
||||
'analyze_temporal': _handle_analyze_temporal,
|
||||
'analyze_fft': _handle_analyze_fft,
|
||||
'analyze_tls_health': _handle_analyze_tls_health,
|
||||
'analyze_geo_distribution': _handle_analyze_geo_distribution,
|
||||
'analyze_entity_detail': _handle_analyze_entity_detail,
|
||||
'compute_distance_matrix': _handle_compute_distance_matrix,
|
||||
}
|
||||
handler = _handlers.get(name)
|
||||
if handler is None:
|
||||
raise ValueError(f"Unknown tool: '{name}'")
|
||||
|
||||
# Extract mandatory _timeout from the LLM-provided arguments
|
||||
timeout = arguments.pop('_timeout', 0)
|
||||
cleaned_args = arguments # _timeout already removed by pop
|
||||
|
||||
# Execute the handler with optional timeout
|
||||
if timeout and timeout > 0:
|
||||
try:
|
||||
result = await asyncio.wait_for(
|
||||
handler(**cleaned_args),
|
||||
timeout=timeout,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
result = {'error': f'工具 {name} 执行超时 ({timeout}秒)'}
|
||||
else:
|
||||
result = await handler(**cleaned_args)
|
||||
|
||||
# Always ensure truncated key exists
|
||||
if 'truncated' not in result:
|
||||
result['truncated'] = False
|
||||
return _truncate_response(result)
|
||||
@@ -0,0 +1,138 @@
|
||||
"""Shared utilities for tool handlers.
|
||||
|
||||
Provides:
|
||||
- NUMERIC_DTYPES / NUMERIC_TYPE_NAMES: shared constants for dtype filtering
|
||||
- _json_size, _truncate_response: LLM context-window helpers
|
||||
- _build_filter_expr: Polars filter expression builder
|
||||
- _resolve_dataset: one-liner SessionStore lookup (replaces 30+ repeated blocks)
|
||||
- _count_rows: fast streaming row count
|
||||
- _get_numeric_columns: extract numeric column names from schema
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import traceback
|
||||
from typing import Any
|
||||
|
||||
import polars as pl
|
||||
|
||||
from ..session_store import SessionStore
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# Shared constants
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
NUMERIC_DTYPES = (
|
||||
pl.Int8, pl.Int16, pl.Int32, pl.Int64,
|
||||
pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
|
||||
pl.Float32, pl.Float64,
|
||||
)
|
||||
|
||||
NUMERIC_TYPE_NAMES = {
|
||||
'Int8', 'Int16', 'Int32', 'Int64',
|
||||
'UInt8', 'UInt16', 'UInt32', 'UInt64',
|
||||
'Float32', 'Float64',
|
||||
}
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# LLM response helpers
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
_MAX_RESPONSE_KB = 64
|
||||
|
||||
|
||||
def _json_size(obj: Any) -> int:
|
||||
"""Estimate JSON byte size of *obj*."""
|
||||
return len(json.dumps(obj, default=str, ensure_ascii=False))
|
||||
|
||||
|
||||
def _truncate_response(result: dict, max_kb: int = _MAX_RESPONSE_KB) -> dict:
|
||||
"""Truncate large lists/keys in *result* to stay under *max_kb*."""
|
||||
if _json_size(result) <= max_kb * 1024:
|
||||
result.setdefault('truncated', False)
|
||||
return result
|
||||
|
||||
# Drop verbose keys first
|
||||
for costly_key in ('column_stats', 'features', 'correlation_matrix',
|
||||
'labels', 'cluster_labels'):
|
||||
if costly_key in result and isinstance(result[costly_key], dict):
|
||||
result[costly_key] = {'__truncated__': True,
|
||||
'key_count': len(result[costly_key])}
|
||||
elif costly_key in result and isinstance(result[costly_key], list):
|
||||
lst = result[costly_key]
|
||||
if len(lst) > 10:
|
||||
result[costly_key] = lst[:10] + [{'__truncated__': True,
|
||||
'omitted': len(lst) - 10}]
|
||||
result['truncated'] = True
|
||||
return result
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# Filter expression builder
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
def _build_filter_expr(column: str, op: str, value: Any) -> pl.Expr:
|
||||
"""Build a single Polars expression from a filter descriptor."""
|
||||
col = pl.col(column)
|
||||
op_map = {
|
||||
'eq': col == value,
|
||||
'neq': col != value,
|
||||
'gt': col > value,
|
||||
'gte': col >= value,
|
||||
'lt': col < value,
|
||||
'lte': col <= value,
|
||||
'contains': col.str.contains(value) if value is not None else col.cast(str).str.contains(''),
|
||||
'not_contains': ~(col.str.contains(value) if value is not None else col.cast(str).str.contains('')),
|
||||
'in': col.is_in(value) if isinstance(value, (list, tuple)) else col.is_in([value]),
|
||||
'not_in': ~(col.is_in(value) if isinstance(value, (list, tuple)) else col.is_in([value])),
|
||||
'is_null': col.is_null(),
|
||||
'is_not_null': col.is_not_null(),
|
||||
}
|
||||
expr = op_map.get(op)
|
||||
if expr is None:
|
||||
raise ValueError(f"Unsupported filter operator: {op}")
|
||||
return expr
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# SessionStore helpers
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
def _resolve_dataset(dataset_id: str) -> tuple:
|
||||
"""Resolve a dataset from SessionStore.
|
||||
|
||||
Returns:
|
||||
(entry, None) on success, where *entry* is the SessionStore entry dict.
|
||||
(None, error_dict) if the dataset is not found.
|
||||
"""
|
||||
store = SessionStore()
|
||||
entry = store.get_dataset(dataset_id)
|
||||
if entry is None:
|
||||
return None, {'error': f'Dataset not found: {dataset_id}', 'truncated': False}
|
||||
return entry, None
|
||||
|
||||
|
||||
def _count_rows(lf: pl.LazyFrame) -> int:
|
||||
"""Count rows in a LazyFrame via fast streaming count."""
|
||||
try:
|
||||
return lf.select(pl.len()).collect(streaming=True).item()
|
||||
except Exception:
|
||||
logger.error("_count_rows failed: {}".format(traceback.format_exc()))
|
||||
return 0
|
||||
|
||||
|
||||
def _get_numeric_columns(lf: pl.LazyFrame, schema: dict, max_cols: int = 10) -> list[str]:
|
||||
"""Get numeric column names from a dataset.
|
||||
|
||||
Prefers *schema* (string-keyed); falls back to LazyFrame collect_schema().
|
||||
"""
|
||||
if schema:
|
||||
return [c for c in schema
|
||||
if schema[c].split('(')[0].strip() in NUMERIC_TYPE_NAMES
|
||||
and not c.startswith('_')][:max_cols]
|
||||
# Fallback to LazyFrame schema
|
||||
available = lf.collect_schema()
|
||||
return [available.names()[i] for i, dt in enumerate(available.dtypes())
|
||||
if dt in NUMERIC_DTYPES and not available.names()[i].startswith('_')][:max_cols]
|
||||
@@ -0,0 +1,777 @@
|
||||
"""Tool metadata definitions for the MCP server.
|
||||
|
||||
All 30 Tool(...) descriptor objects and the get_tools_meta() function.
|
||||
"""
|
||||
from mcp.types import Tool
|
||||
|
||||
|
||||
def get_tools_meta() -> list[Tool]:
|
||||
"""Return the list of Tool metadata objects for MCP server registration.
|
||||
|
||||
Each tool gets a mandatory ``_timeout`` parameter so the LLM must specify
|
||||
how many seconds to allow for each tool invocation (0 = no limit).
|
||||
"""
|
||||
raw_tools = [
|
||||
Tool(
|
||||
name="load_data",
|
||||
description=(
|
||||
"加载匹配glob模式的CSV文件。自动检测每个文件的BOM，验证schema一致性，"
|
||||
"纵向合并，并将结果作为惰性数据集存储在会话存储中。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"csv_glob": {
|
||||
"type": "string",
|
||||
"description": "Glob pattern for CSV files (e.g. 'data/*.csv')",
|
||||
},
|
||||
"config_path": {
|
||||
"type": "string",
|
||||
"description": "Optional path to a YAML config file",
|
||||
},
|
||||
"encoding": {
|
||||
"type": "string",
|
||||
"description": "Fallback encoding when no BOM is found",
|
||||
"default": "utf-8",
|
||||
},
|
||||
"delimiter": {
|
||||
"type": "string",
|
||||
"description": "Column delimiter",
|
||||
"default": ",",
|
||||
},
|
||||
"schema_strict": {
|
||||
"type": "boolean",
|
||||
"description": "When True, raise on column mismatch; when False, merge leniently with nulls",
|
||||
"default": False,
|
||||
},
|
||||
"recursive": {
|
||||
"type": "boolean",
|
||||
"description": "When True, use recursive glob for **/*.csv patterns",
|
||||
"default": False,
|
||||
},
|
||||
},
|
||||
"required": ["csv_glob"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="profile_data",
|
||||
description=(
|
||||
"计算数据集的每列统计信息和数值相关矩阵。结果自动截断至约16 KB以"
|
||||
"节省LLM上下文窗口空间。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dataset_id": {
|
||||
"type": "string",
|
||||
"description": "Dataset identifier from a previous load_data call",
|
||||
},
|
||||
"sample_size": {
|
||||
"type": "integer",
|
||||
"description": "Maximum rows to profile (sampled if larger)",
|
||||
"default": 1000,
|
||||
},
|
||||
"columns": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Columns to profile (default: all)",
|
||||
},
|
||||
},
|
||||
"required": ["dataset_id"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="filter_data",
|
||||
description=(
|
||||
"对数据集应用列级过滤器，将结果存储为新的派生数据集。支持eq/neq/gt/gte/"
|
||||
"lt/lte/contains/not_contains/in/not_in/is_null/is_not_null运算符。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dataset_id": {
|
||||
"type": "string",
|
||||
"description": "Source dataset identifier",
|
||||
},
|
||||
"filters": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"column": {"type": "string"},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"eq", "neq", "gt", "gte", "lt", "lte",
|
||||
"contains", "not_contains",
|
||||
"in", "not_in",
|
||||
"is_null", "is_not_null",
|
||||
],
|
||||
},
|
||||
"value": {"description": "Operand value"},
|
||||
},
|
||||
"required": ["column", "op"],
|
||||
},
|
||||
"description": "List of filter expressions",
|
||||
},
|
||||
"logic": {
|
||||
"type": "string",
|
||||
"enum": ["and", "or"],
|
||||
"description": "Combine filters with AND or OR logic",
|
||||
"default": "and",
|
||||
},
|
||||
},
|
||||
"required": ["dataset_id", "filters"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="preprocess_data",
|
||||
description=(
|
||||
"预处理选定列：标准化（StandardScaler）、独热编码、缺失值填充和列删除。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dataset_id": {
|
||||
"type": "string",
|
||||
"description": "Source dataset identifier",
|
||||
},
|
||||
"columns": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Columns to preprocess",
|
||||
},
|
||||
"config": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"standardize": {
|
||||
"type": "boolean",
|
||||
"description": "Apply StandardScaler to numeric columns",
|
||||
"default": False,
|
||||
},
|
||||
"onehot": {
|
||||
"type": "boolean",
|
||||
"description": "One-hot encode categorical columns",
|
||||
"default": False,
|
||||
},
|
||||
"fillna": {
|
||||
"type": "string",
|
||||
"enum": ["mean", "median", "zero", "drop"],
|
||||
"description": "Missing value strategy",
|
||||
},
|
||||
"drop": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Columns to drop entirely",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"required": ["dataset_id", "columns"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="run_clustering",
|
||||
description=(
|
||||
"对选定列执行聚类。使用HDBSCAN（默认）或KMeans。特征在聚类前进行"
|
||||
"标准化缩放。返回聚类标签和质量指标。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dataset_id": {
|
||||
"type": "string",
|
||||
"description": "Dataset identifier",
|
||||
},
|
||||
"cluster_columns": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Columns to use as clustering features",
|
||||
},
|
||||
"algorithm": {
|
||||
"type": "string",
|
||||
"enum": ["agglomerative", "hdbscan", "kmeans"],
|
||||
"description": "Clustering algorithm (default: agglomerative/Ward)",
|
||||
"default": "agglomerative",
|
||||
},
|
||||
"params": {
|
||||
"type": "object",
|
||||
"description": (
|
||||
"Algorithm parameters. Agglomerative: n_clusters (5), metric ('euclidean'), "
|
||||
"linkage ('ward'). HDBSCAN: min_cluster_size (5), "
|
||||
"min_samples (None), metric ('euclidean'). "
|
||||
"KMeans: n_clusters (3), random_state (42)."
|
||||
),
|
||||
},
|
||||
"random_state": {
|
||||
"type": "integer",
|
||||
"description": "Random seed for reproducibility",
|
||||
"default": 42,
|
||||
},
|
||||
},
|
||||
"required": ["dataset_id", "cluster_columns"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="evaluate_clustering",
|
||||
description=(
|
||||
"评估数据集的聚类结果。当前支持轮廓系数（silhouette）、Davies-Bouldin指数、"
|
||||
"Calinski-Harabasz指数、噪声比（noise_ratio）和簇大小（cluster_sizes）指标。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"cluster_result_id": {
|
||||
"type": "string",
|
||||
"description": "Result ID from run_clustering",
|
||||
},
|
||||
"dataset_id": {
|
||||
"type": "string",
|
||||
"description": "Original dataset for metric computation",
|
||||
},
|
||||
"metrics": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": (
|
||||
"Metrics to compute. Options: silhouette, "
|
||||
"davies_bouldin, calinski_harabasz, noise_ratio, "
|
||||
"cluster_sizes (default: all)"
|
||||
),
|
||||
},
|
||||
},
|
||||
"required": ["cluster_result_id", "dataset_id"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="extract_features",
|
||||
description=(
|
||||
"提取每个聚类的区分特征并自动保存到Django ORM（ClusterFeature模型）。"
|
||||
"使用z-score或ANOVA对每个聚类的特征重要性进行排序。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dataset_id": {
|
||||
"type": "string",
|
||||
"description": "Dataset containing the feature columns",
|
||||
},
|
||||
"cluster_result_id": {
|
||||
"type": "string",
|
||||
"description": "Result ID from run_clustering",
|
||||
},
|
||||
"top_k": {
|
||||
"type": "integer",
|
||||
"description": "Top distinguishing features per cluster",
|
||||
"default": 10,
|
||||
},
|
||||
"method": {
|
||||
"type": "string",
|
||||
"enum": ["zscore", "anova"],
|
||||
"description": "Method to rank feature importance",
|
||||
"default": "zscore",
|
||||
},
|
||||
"save_to_db": {
|
||||
"type": "boolean",
|
||||
"description": "Save features to Django ORM if True",
|
||||
"default": True,
|
||||
},
|
||||
},
|
||||
"required": ["dataset_id", "cluster_result_id"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="export_results",
|
||||
description=(
|
||||
"将数据集或聚类结果以CSV或Parquet格式导出到磁盘。数据集物化后写入；"
|
||||
"聚类结果序列化为JSON。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"result_id": {
|
||||
"type": "string",
|
||||
"description": "Dataset ID or cluster result ID to export",
|
||||
},
|
||||
"output_path": {
|
||||
"type": "string",
|
||||
"description": "Output file path or directory",
|
||||
},
|
||||
"format": {
|
||||
"type": "string",
|
||||
"enum": ["csv", "parquet", "json"],
|
||||
"description": "Output format",
|
||||
"default": "csv",
|
||||
},
|
||||
"overwrite": {
|
||||
"type": "boolean",
|
||||
"description": "Overwrite existing file",
|
||||
"default": False,
|
||||
},
|
||||
},
|
||||
"required": ["result_id", "output_path"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="list_datasets",
|
||||
description="列出会话存储中的所有活跃数据集和结果。",
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="drop_dataset",
|
||||
description=(
|
||||
"从会话存储中移除数据集（或结果）并通过垃圾回收释放内存。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dataset_id": {
|
||||
"type": "string",
|
||||
"description": "Dataset or result identifier to remove",
|
||||
},
|
||||
},
|
||||
"required": ["dataset_id"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="clone_dataset",
|
||||
description=(
|
||||
"浅拷贝数据集。由于LazyFrame是查询计划，不会复制实际数据。适用于分支分析。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dataset_id": {
|
||||
"type": "string",
|
||||
"description": "Source dataset identifier",
|
||||
},
|
||||
},
|
||||
"required": ["dataset_id"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="filter_and_cluster",
|
||||
description=(
|
||||
"一步完成：先应用列级过滤器，再对过滤结果执行聚类。支持eq/neq/gt/gte/lt/lte/"
|
||||
"contains/not_contains/in/not_in/is_null/is_not_null运算符和HDBSCAN/KMeans聚类。"
|
||||
"自动选择数值列作为聚类特征。将过滤后的数据集和聚类结果都存储在会话中。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dataset_id": {
|
||||
"type": "string",
|
||||
"description": "Source dataset identifier",
|
||||
},
|
||||
"filters": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"column": {"type": "string"},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"eq", "neq", "gt", "gte", "lt", "lte",
|
||||
"contains", "not_contains",
|
||||
"in", "not_in",
|
||||
"is_null", "is_not_null",
|
||||
],
|
||||
},
|
||||
"value": {"description": "Operand value"},
|
||||
},
|
||||
"required": ["column", "op"],
|
||||
},
|
||||
"description": "List of filter expressions",
|
||||
},
|
||||
"logic": {
|
||||
"type": "string",
|
||||
"enum": ["and", "or"],
|
||||
"description": "Combine filters with AND or OR logic",
|
||||
"default": "and",
|
||||
},
|
||||
"algorithm": {
|
||||
"type": "string",
|
||||
"enum": ["agglomerative", "hdbscan", "kmeans"],
|
||||
"description": "Clustering algorithm (default: agglomerative/Ward)",
|
||||
"default": "agglomerative",
|
||||
},
|
||||
"params": {
|
||||
"type": "object",
|
||||
"description": (
|
||||
"Algorithm parameters. Agglomerative: n_clusters (5), metric ('euclidean'), "
|
||||
"linkage ('ward'). HDBSCAN: min_cluster_size (5), "
|
||||
"min_samples (None), metric ('euclidean'). "
|
||||
"KMeans: n_clusters (3), random_state (42)."
|
||||
),
|
||||
},
|
||||
},
|
||||
"required": ["dataset_id", "filters"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="build_entity_profiles",
|
||||
description=(
|
||||
"按实体列（源IP、SNI等）分组原始流数据。自动检测实体列（如 src_ip, "
|
||||
"dst_ip, sni, host, mac, domain），按列分组并计算每个实体的聚合特征："
|
||||
"流数量、唯一端口数、TLS版本分布、字节总量、唯一目标数。"
|
||||
"生成的数据集每个实体一行。结果存储在会话中，可供后续 compute_scores 使用。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dataset_id": {
|
||||
"type": "string",
|
||||
"description": "Dataset ID from load_data or profile_data",
|
||||
},
|
||||
"entity_column": {
|
||||
"type": "string",
|
||||
"description": "Specific entity column to group by (e.g. 'src_ip')",
|
||||
},
|
||||
"entity_columns": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Multiple entity columns to group by",
|
||||
},
|
||||
"auto_detect": {
|
||||
"type": "boolean",
|
||||
"description": "Auto-detect entity columns (IP, SNI, host, MAC, domain)",
|
||||
"default": True,
|
||||
},
|
||||
},
|
||||
"required": ["dataset_id"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="compute_scores",
|
||||
description=(
|
||||
"在实体数据上计算自适应代理/异常/威胁评分。从数据自动学习"
|
||||
"特征权重，计算加权代理评分（proxy_score）、基于IQR的风险等级"
|
||||
"（normal/watch/suspicious/critical）和威胁评分（threat_score）。"
|
||||
"返回评分摘要与阈值。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dataset_id": {
|
||||
"type": "string",
|
||||
"description": "Entity dataset ID from filter_and_cluster or a scored dataset",
|
||||
},
|
||||
},
|
||||
"required": ["dataset_id"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="detect_anomalies",
|
||||
description=(
|
||||
"对已评分的实体数据运行孤立森林（Isolation Forest）异常检测。识别偏离"
|
||||
"正常模式的实体。分块处理以应对大数据集。返回异常实体ID列表。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dataset_id": {
|
||||
"type": "string",
|
||||
"description": "Scored entity dataset ID (after compute_scores)",
|
||||
},
|
||||
"contamination": {
|
||||
"type": "number",
|
||||
"description": "Expected proportion of anomalies (0.0-0.5)",
|
||||
"default": 0.05,
|
||||
},
|
||||
},
|
||||
"required": ["dataset_id"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="visualize_anomalies",
|
||||
description=(
|
||||
"为高风险实体生成UMAP-2D嵌入和聚类可视化数据。按风险等级和聚类着色。"
|
||||
"返回用于前端渲染的散点图JSON数据。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dataset_id": {
|
||||
"type": "string",
|
||||
"description": "Scored entity dataset ID (after compute_scores)",
|
||||
},
|
||||
},
|
||||
"required": ["dataset_id"],
|
||||
},
|
||||
),
|
||||
# ── Edge-case / Diagnostic tools (unexpected situations) ────────
|
||||
Tool(
|
||||
name="validate_data",
|
||||
description=(
|
||||
"诊断工具：检查数据质量问题——文件间schema一致性、缺失值比例、列类型冲突、"
|
||||
"编码问题和空列。当流水线步骤因schema错误失败，或怀疑CSV数据损坏/格式异常时使用。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dataset_id": {"type": "string", "description": "Dataset ID from load_data"},
|
||||
"fix_issues": {"type": "boolean", "description": "Auto-fix schema inconsistencies", "default": False},
|
||||
},
|
||||
"required": ["dataset_id"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="explore_distributions",
|
||||
description=(
|
||||
"诊断工具：计算每列分布统计（唯一值计数、空值比例、最小/最大/均值/标准差、"
|
||||
"前10高频值、直方图分箱）。当聚类产生意外结果、实体检测失败，或需要在选择"
|
||||
"分析策略前了解数据时使用。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dataset_id": {"type": "string", "description": "Dataset ID from load_data or entity data"},
|
||||
"columns": {"type": "array", "items": {"type": "string"},
|
||||
"description": "Specific columns to analyze (default: auto-detect numeric)"},
|
||||
"max_sample": {"type": "integer", "description": "Max rows to sample", "default": 10000},
|
||||
},
|
||||
"required": ["dataset_id"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="find_outliers",
|
||||
description=(
|
||||
"诊断工具：使用IQR方法检测数值列中的统计异常值。返回任何数值列超过中位数3倍IQR的行。"
|
||||
"当怀疑数据质量问题或极端值偏离分析时使用。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dataset_id": {"type": "string", "description": "Dataset ID from load_data"},
|
||||
"columns": {"type": "array", "items": {"type": "string"},
|
||||
"description": "Columns to check (default: all numeric)"},
|
||||
"threshold": {"type": "number", "description": "IQR multiplier (default 3.0)", "default": 3.0},
|
||||
},
|
||||
"required": ["dataset_id"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="diagnose_clustering",
|
||||
description=(
|
||||
"诊断工具：当聚类产生太少/太多聚类、全部为噪声或轮廓系数低时——诊断原因。"
|
||||
"检查：特征方差、相关矩阵、特征降维分析、数据稀疏性，并建议参数调整"
|
||||
"（min_cluster_size、算法选择、特征选择）。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dataset_id": {"type": "string", "description": "Entity dataset ID"},
|
||||
"cluster_result_id": {"type": "string", "description": "Cluster result ID from previous run_clustering"},
|
||||
},
|
||||
"required": ["dataset_id", "cluster_result_id"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="compare_datasets",
|
||||
description=(
|
||||
"诊断工具：比较两个数据集（如处理前/后，或两次不同运行）。报告schema差异、"
|
||||
"行数变化、列值分布偏移。用于验证处理步骤未损坏数据或比较不同运行的结果。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dataset_id_a": {"type": "string", "description": "First dataset ID"},
|
||||
"dataset_id_b": {"type": "string", "description": "Second dataset ID"},
|
||||
},
|
||||
"required": ["dataset_id_a", "dataset_id_b"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="export_debug_sample",
|
||||
description=(
|
||||
"诊断工具：导出原始数据样本为JSON用于外部调试。当流水线深处发生错误且需要"
|
||||
"检查实际数据值时使用。返回前N行所有列。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dataset_id": {"type": "string", "description": "Dataset ID"},
|
||||
"max_rows": {"type": "integer", "description": "Rows to export", "default": 50},
|
||||
},
|
||||
"required": ["dataset_id"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="repair_schema",
|
||||
description=(
|
||||
"诊断工具：通过对齐列名（不区分大小写合并、下划线/连字符规范化）并用空值填充"
|
||||
"缺失列，尝试修复文件间的schema不匹配。当load_data因CSV文件具有不同列集而"
|
||||
"失败时使用。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dataset_id": {"type": "string", "description": "Failed dataset ID from load_data"},
|
||||
},
|
||||
"required": ["dataset_id"],
|
||||
},
|
||||
),
|
||||
# ── Read-only analysis tools (LLM deep-dive, no side effects) ────
|
||||
Tool(
|
||||
name="analyze_patterns",
|
||||
description=(
|
||||
"分析工具：检测流量模式——top源/目标IP、端口分布、TLS版本分布、协议混合。"
|
||||
"返回汇总统计。只读，不修改数据。用于了解高层流量结构。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dataset_id": {"type": "string", "description": "Dataset ID from load_data"},
|
||||
"top_n": {"type": "integer", "description": "Number of top items per category", "default": 10},
|
||||
},
|
||||
"required": ["dataset_id"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="analyze_temporal",
|
||||
description=(
|
||||
"分析工具：分析随时间变化的流量——小时/日流量计数、繁忙时段、空闲时段。"
|
||||
"需要时间戳列。只读。用于了解时间相关模式和周期性。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dataset_id": {"type": "string", "description": "Dataset ID from load_data"},
|
||||
"timestamp_column": {"type": "string", "description": "Column name for timestamps (auto-detected if omitted)"},
|
||||
},
|
||||
"required": ["dataset_id"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="analyze_fft",
|
||||
description=(
|
||||
"FFT频谱分析——从时间戳列提取周期模式和频率特征。适用于检测时间周期性"
|
||||
"（如每日/每周流量模式）。返回频率、幅度、top周期、频谱质心、周期性评分。"
|
||||
"只读，无副作用。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dataset_id": {
|
||||
"type": "string",
|
||||
"description": "Dataset ID from load_data",
|
||||
},
|
||||
"timestamp_column": {
|
||||
"type": "string",
|
||||
"description": "Column name containing timestamps or numeric time values",
|
||||
},
|
||||
"top_n": {
|
||||
"type": "integer",
|
||||
"description": "Number of top frequency peaks to return",
|
||||
"default": 5,
|
||||
},
|
||||
},
|
||||
"required": ["dataset_id", "timestamp_column"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="analyze_tls_health",
|
||||
description=(
|
||||
"分析工具：TLS安全态势评估——检查过时的TLS版本（1.0/1.1）、弱加密套件、"
|
||||
"证书问题、SNI异常。只读。用于评估流量的TLS安全性。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dataset_id": {"type": "string", "description": "Dataset ID from load_data"},
|
||||
},
|
||||
"required": ["dataset_id"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="analyze_geo_distribution",
|
||||
description=(
|
||||
"分析工具：流量地理分布——top源/目标国家、异常地点对（如不可能的行程）、"
|
||||
"ISP多样性。只读。需要国家或经纬度列。用于检测地理异常。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dataset_id": {"type": "string", "description": "Dataset ID from load_data"},
|
||||
},
|
||||
"required": ["dataset_id"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="analyze_entity_detail",
|
||||
description=(
|
||||
"分析工具：深入分析单个实体——其所有流、使用的TLS版本、目标多样性、时间模式、"
|
||||
"异常评分。只读。在compute_scores后调查特定可疑实体时使用。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dataset_id": {"type": "string", "description": "Entity dataset ID (after filter_and_cluster or compute_scores)"},
|
||||
"entity_value": {"type": "string", "description": "Entity value to investigate (e.g. an IP address or SNI)"},
|
||||
},
|
||||
"required": ["dataset_id", "entity_value"],
|
||||
},
|
||||
),
|
||||
Tool(
|
||||
name="compute_distance_matrix",
|
||||
description=(
|
||||
"LLM驱动工具：对数据集的每一行执行用户编写的Python函数，返回距离/相似度评分摘要。"
|
||||
"LLM提供函数体，签名格式为`def distance_fn(row: dict) -> float:`。执行器将其包装，"
|
||||
"在受限命名空间（仅math、numpy）中逐行运行，返回得分的最小/最大/均值/标准差以及"
|
||||
"评分行样本。用于现有工具无法表达的一次性度量、过滤或评分任务。"
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dataset_id": {
|
||||
"type": "string",
|
||||
"description": "Source dataset identifier",
|
||||
},
|
||||
"python_function": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Body of a Python function with signature `def distance_fn(row: dict) -> float:`. "
|
||||
"Example: \"return abs(row.get('col1', 0)) + row.get('col2', 0)\". "
|
||||
"Available namespace: math, numpy (as np). No dangerous modules."
|
||||
),
|
||||
},
|
||||
"columns": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Columns to pass to the function (default: auto-detect numeric columns)",
|
||||
},
|
||||
"max_sample": {
|
||||
"type": "integer",
|
||||
"description": "Maximum rows to evaluate (default 10000 for performance)",
|
||||
"default": 10000,
|
||||
},
|
||||
},
|
||||
"required": ["dataset_id", "python_function"],
|
||||
},
|
||||
),
|
||||
] # end of raw_tools
|
||||
|
||||
# Inject mandatory _timeout parameter into every tool
|
||||
result = []
|
||||
for tool in raw_tools:
|
||||
schema = dict(tool.inputSchema)
|
||||
props = dict(schema.get("properties", {}))
|
||||
req = list(schema.get("required", []))
|
||||
# Add _timeout as the first required property
|
||||
props["_timeout"] = {
|
||||
"type": "integer",
|
||||
"description": "工具调用超时秒数（必填，LLM决定每次调用的最长等待时间；0表示不限制）",
|
||||
"default": 120,
|
||||
}
|
||||
req.insert(0, "_timeout")
|
||||
schema["properties"] = props
|
||||
schema["required"] = req
|
||||
result.append(Tool(
|
||||
name=tool.name,
|
||||
description=tool.description,
|
||||
inputSchema=schema,
|
||||
))
|
||||
return result
|
||||
@@ -0,0 +1,335 @@
|
||||
"""Read-only analysis handlers — patterns, temporal, FFT, TLS health, geo, entity detail."""
|
||||
import traceback
|
||||
from analysis.constants import LOW_MEMORY_THRESHOLD_ROWS
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
|
||||
from ..session_store import SessionStore
|
||||
from ._helpers import _resolve_dataset, _count_rows, NUMERIC_TYPE_NAMES
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def _handle_analyze_patterns(dataset_id: str, top_n: int = 10) -> dict:
|
||||
"""Read-only: traffic pattern analysis."""
|
||||
entry, err = _resolve_dataset(dataset_id)
|
||||
if err:
|
||||
return err
|
||||
lf = entry['lazyframe']
|
||||
schema = entry.get('schema', {})
|
||||
n = _count_rows(lf)
|
||||
df = lf.head(LOW_MEMORY_THRESHOLD_ROWS).collect(streaming=True)
|
||||
|
||||
result = {'total_rows': n, 'sampled': min(n, LOW_MEMORY_THRESHOLD_ROWS)}
|
||||
|
||||
for col_name, label in [(':ips', 'src_ips'), (':ipd', 'dst_ips'),
|
||||
('snam', 'snis'), ('cnam', 'cert_names')]:
|
||||
if col_name in df.columns:
|
||||
try:
|
||||
top = df[col_name].value_counts(sort=True).head(top_n)
|
||||
result[label] = {
|
||||
'top': [{'value': str(r[col_name]), 'count': int(r['count'])}
|
||||
for r in top.iter_rows(named=True)],
|
||||
'unique': int(df[col_name].n_unique()),
|
||||
}
|
||||
except Exception:
|
||||
logger.error("_handle_analyze_patterns failed: {}".format(traceback.format_exc()))
|
||||
pass
|
||||
|
||||
for port_col in [':prs', ':prd']:
|
||||
if port_col in df.columns:
|
||||
try:
|
||||
top = df[port_col].value_counts(sort=True).head(top_n)
|
||||
result[f'{port_col}_dist'] = {
|
||||
'top': [{'port': int(r[port_col]), 'count': int(r['count'])}
|
||||
for r in top.iter_rows(named=True)],
|
||||
'unique': int(df[port_col].n_unique()),
|
||||
}
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
pass
|
||||
|
||||
for tls_col in ['0ver']:
|
||||
if tls_col in df.columns:
|
||||
try:
|
||||
vc = df[tls_col].value_counts(sort=True)
|
||||
result['tls_versions'] = [{'version': str(r[tls_col]), 'count': int(r['count'])}
|
||||
for r in vc.iter_rows(named=True)]
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
pass
|
||||
|
||||
return result
|
||||
|
||||
|
||||
async def _handle_analyze_temporal(dataset_id: str, timestamp_column: str = None) -> dict:
|
||||
"""Read-only: temporal traffic pattern analysis."""
|
||||
entry, err = _resolve_dataset(dataset_id)
|
||||
if err:
|
||||
return err
|
||||
lf = entry['lazyframe']
|
||||
schema = entry.get('schema', {})
|
||||
|
||||
if not timestamp_column:
|
||||
for cand in ['timestamp', 'time', '8dbd', '8did']:
|
||||
if cand in schema:
|
||||
timestamp_column = cand
|
||||
break
|
||||
if not timestamp_column:
|
||||
return {'error': 'No timestamp column found', 'truncated': False}
|
||||
|
||||
n = _count_rows(lf)
|
||||
return {'total_rows': n, 'timestamp_column': timestamp_column,
|
||||
'note': 'Temporal analysis requires parsed timestamps; raw values shown in explore_distributions'}
|
||||
|
||||
|
||||
async def _handle_analyze_fft(
|
||||
dataset_id: str,
|
||||
timestamp_column: str,
|
||||
top_n: int = 5,
|
||||
) -> dict:
|
||||
"""FFT spectrum analysis on a timestamp column to detect periodic patterns."""
|
||||
entry, err = _resolve_dataset(dataset_id)
|
||||
if err:
|
||||
return err
|
||||
|
||||
lf: pl.LazyFrame = entry['lazyframe']
|
||||
schema = entry.get('schema', {})
|
||||
|
||||
col = None
|
||||
if timestamp_column in schema:
|
||||
col = timestamp_column
|
||||
else:
|
||||
col_lower = timestamp_column.lower()
|
||||
for c in schema:
|
||||
if c.lower() == col_lower:
|
||||
col = c
|
||||
break
|
||||
if col is None:
|
||||
return {'error': f"Timestamp column '{timestamp_column}' not found in dataset. Available: {list(schema.keys())[:20]}",
|
||||
'truncated': False}
|
||||
|
||||
df = lf.select(pl.col(col).alias('__ts__')).collect(streaming=True)
|
||||
n_raw = len(df)
|
||||
if n_raw < 4:
|
||||
return {'error': f'Need at least 4 samples for FFT; got {n_raw}', 'truncated': False}
|
||||
|
||||
ts_series = df['__ts__']
|
||||
dtype = ts_series.dtype
|
||||
str_dtypes = (pl.Utf8, pl.String)
|
||||
|
||||
if dtype in str_dtypes:
|
||||
try:
|
||||
ts_parsed = ts_series.str.strptime(pl.Datetime, format=None, strict=False)
|
||||
ts_numeric = ts_parsed.cast(pl.Int64) / 1_000_000_000.0
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
try:
|
||||
ts_numeric = ts_series.cast(pl.Float64)
|
||||
except Exception as e:
|
||||
return {'error': f'Cannot convert string column to numeric or datetime: {e}',
|
||||
'truncated': False}
|
||||
elif dtype in (pl.Datetime, pl.Date):
|
||||
ts_numeric = ts_series.cast(pl.Int64) / 1_000_000_000.0
|
||||
else:
|
||||
try:
|
||||
ts_numeric = ts_series.cast(pl.Float64)
|
||||
except Exception as e:
|
||||
return {'error': f'Cannot cast column to float: {e}', 'truncated': False}
|
||||
|
||||
df_sorted = pl.DataFrame({'__val__': ts_numeric}).drop_nulls().sort('__val__')
|
||||
n = len(df_sorted)
|
||||
if n < 4:
|
||||
return {'error': f'After removing nulls, only {n} samples remain (need ≥4)', 'truncated': False}
|
||||
|
||||
values = df_sorted['__val__'].to_numpy().astype(np.float64)
|
||||
|
||||
values_centered = values - np.mean(values)
|
||||
fft_complex = np.fft.fft(values_centered)
|
||||
fft_magnitudes = np.abs(fft_complex)
|
||||
freqs = np.fft.fftfreq(n)
|
||||
|
||||
pos_mask = freqs > 0
|
||||
freqs_pos = freqs[pos_mask]
|
||||
mags_pos = fft_magnitudes[pos_mask]
|
||||
|
||||
if len(freqs_pos) == 0:
|
||||
return {'error': 'Insufficient positive frequencies for FFT analysis', 'truncated': False}
|
||||
|
||||
peak_indices = np.argsort(mags_pos)[::-1][:min(top_n, len(mags_pos))]
|
||||
top_periods = []
|
||||
for idx in peak_indices:
|
||||
freq = float(freqs_pos[idx])
|
||||
mag = float(mags_pos[idx])
|
||||
period = float(1.0 / freq) if freq > 1e-10 else float('inf')
|
||||
if n > 1:
|
||||
time_span = float(values[-1] - values[0])
|
||||
sample_period = time_span / max(n - 1, 1)
|
||||
time_period = period * sample_period
|
||||
else:
|
||||
time_period = float('inf')
|
||||
top_periods.append({
|
||||
'period': round(period, 2),
|
||||
'magnitude': round(mag, 4),
|
||||
'frequency': round(freq, 6),
|
||||
'time_domain_period_seconds': round(time_period, 2) if time_period != float('inf') else None,
|
||||
})
|
||||
|
||||
mag_sum = float(np.sum(mags_pos))
|
||||
if mag_sum > 0:
|
||||
spectral_centroid = float(np.sum(freqs_pos * mags_pos) / mag_sum)
|
||||
else:
|
||||
spectral_centroid = 0.0
|
||||
|
||||
mean_mag = float(np.mean(mags_pos)) if len(mags_pos) > 0 else 1.0
|
||||
max_mag = float(mags_pos[peak_indices[0]]) if len(peak_indices) > 0 else 0.0
|
||||
periodicity_score = round(max_mag / max(mean_mag, 1e-10), 4)
|
||||
|
||||
decimate_step = max(1, len(freqs_pos) // 200)
|
||||
freqs_decimated = freqs_pos[::decimate_step].tolist()
|
||||
mags_decimated = mags_pos[::decimate_step].tolist()
|
||||
|
||||
return {
|
||||
'frequencies': [round(float(f), 6) for f in freqs_decimated],
|
||||
'magnitudes': [round(float(m), 4) for m in mags_decimated],
|
||||
'top_periods': top_periods,
|
||||
'spectral_centroid': round(spectral_centroid, 6),
|
||||
'periodicity_score': periodicity_score,
|
||||
'n_samples': n,
|
||||
'n_raw': n_raw,
|
||||
'nan_dropped': n_raw - n,
|
||||
'timestamp_column': col,
|
||||
}
|
||||
|
||||
|
||||
async def _handle_analyze_tls_health(dataset_id: str) -> dict:
|
||||
"""Read-only: TLS security assessment."""
|
||||
entry, err = _resolve_dataset(dataset_id)
|
||||
if err:
|
||||
return err
|
||||
lf = entry['lazyframe']
|
||||
schema = entry.get('schema', {})
|
||||
n = _count_rows(lf)
|
||||
df = lf.head(LOW_MEMORY_THRESHOLD_ROWS).collect(streaming=True)
|
||||
|
||||
result = {'total_rows': n, 'sampled': min(n, LOW_MEMORY_THRESHOLD_ROWS)}
|
||||
|
||||
if '0ver' in df.columns:
|
||||
vc = df['0ver'].value_counts(sort=True)
|
||||
versions = {}
|
||||
for r in vc.iter_rows(named=True):
|
||||
v = str(r['0ver']).replace(' ', '')
|
||||
if v in ('0303', '0304'):
|
||||
versions['tls_1_2_plus'] = versions.get('tls_1_2_plus', 0) + int(r['count'])
|
||||
else:
|
||||
versions['tls_legacy'] = versions.get('tls_legacy', 0) + int(r['count'])
|
||||
total = sum(versions.values())
|
||||
result.update({
|
||||
'tls_1_2_plus_pct': round(versions.get('tls_1_2_plus', 0) / max(total, 1) * 100, 1),
|
||||
'tls_legacy_pct': round(versions.get('tls_legacy', 0) / max(total, 1) * 100, 1),
|
||||
'tls_legacy_versions': [{'version': str(r['0ver']), 'count': int(r['count'])}
|
||||
for r in vc.iter_rows(named=True)
|
||||
if str(r['0ver']).replace(' ', '') not in ('0303', '0304')],
|
||||
})
|
||||
|
||||
if 'snam' in df.columns:
|
||||
empty_sni = df['snam'].is_null().sum() + df['snam'].cast(pl.Utf8).is_in(['', 'null', 'None']).sum()
|
||||
result['sni_missing_pct'] = round(float(empty_sni) / max(len(df), 1) * 100, 1)
|
||||
|
||||
if 'cipher-suite' in df.columns:
|
||||
weak_ciphers = ['rc4', 'cbc', 'null', 'export']
|
||||
total_ciphers = len(df)
|
||||
weak_count = 0
|
||||
for w in weak_ciphers:
|
||||
try:
|
||||
weak_count += df['cipher-suite'].str.to_lowercase().str.contains(w).sum()
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
pass
|
||||
result['weak_cipher_pct'] = round(float(weak_count) / max(total_ciphers, 1) * 100, 1) if total_ciphers else 0
|
||||
|
||||
if result.get('tls_legacy_pct', 0) > 20:
|
||||
result['warning'] = 'High proportion of legacy TLS — possible downgrade attacks'
|
||||
if result.get('sni_missing_pct', 0) > 50:
|
||||
result['warning'] = 'Most flows lack SNI — possible IP-direct connections (proxy/VPN)'
|
||||
return result
|
||||
|
||||
|
||||
async def _handle_analyze_geo_distribution(dataset_id: str) -> dict:
|
||||
"""Read-only: geographic traffic analysis."""
|
||||
entry, err = _resolve_dataset(dataset_id)
|
||||
if err:
|
||||
return err
|
||||
lf = entry['lazyframe']
|
||||
schema = entry.get('schema', {})
|
||||
n = _count_rows(lf)
|
||||
df = lf.head(LOW_MEMORY_THRESHOLD_ROWS).collect(streaming=True)
|
||||
|
||||
result = {'total_rows': n, 'sampled': min(n, LOW_MEMORY_THRESHOLD_ROWS)}
|
||||
|
||||
for geo_col, label in [('scnt', 'src_countries'), ('dcnt', 'dst_countries')]:
|
||||
if geo_col in df.columns:
|
||||
try:
|
||||
vc = df[geo_col].value_counts(sort=True).head(15)
|
||||
result[label] = [{'country': str(r[geo_col]), 'count': int(r['count'])}
|
||||
for r in vc.iter_rows(named=True)]
|
||||
result[f'{label}_unique'] = int(df[geo_col].n_unique())
|
||||
except Exception:
|
||||
logger.error("_handle_analyze_geo_distribution failed: {}".format(traceback.format_exc()))
|
||||
pass
|
||||
|
||||
if 'scnt' in df.columns and 'dcnt' in df.columns:
|
||||
try:
|
||||
same = (df['scnt'] == df['dcnt']).sum()
|
||||
result['domestic_pct'] = round(float(same) / max(len(df), 1) * 100, 1)
|
||||
result['international_pct'] = round(100.0 - result['domestic_pct'], 1)
|
||||
except Exception:
|
||||
logger.error("_handle_analyze_geo_distribution failed: {}".format(traceback.format_exc()))
|
||||
pass
|
||||
|
||||
return result
|
||||
|
||||
|
||||
async def _handle_analyze_entity_detail(dataset_id: str, entity_value: str) -> dict:
|
||||
"""Read-only: deep-dive into a single entity."""
|
||||
entry, err = _resolve_dataset(dataset_id)
|
||||
if err:
|
||||
return err
|
||||
lf = entry['lazyframe']
|
||||
schema = entry.get('schema', {})
|
||||
|
||||
entity_col = None
|
||||
for c in schema:
|
||||
if c.startswith('_') or schema[c].split('(')[0].strip() in NUMERIC_TYPE_NAMES or '.' in c:
|
||||
continue
|
||||
entity_col = c
|
||||
break
|
||||
if not entity_col:
|
||||
entity_col = list(schema.keys())[0]
|
||||
|
||||
filtered = lf.filter(pl.col(entity_col).cast(pl.Utf8).str.contains(entity_value, literal=True))
|
||||
n = _count_rows(filtered)
|
||||
if n == 0:
|
||||
return {'error': f"Entity '{entity_value}' not found in column '{entity_col}'", 'truncated': False}
|
||||
df = filtered.collect(streaming=True)
|
||||
|
||||
result = {
|
||||
'entity_value': entity_value,
|
||||
'entity_column': entity_col,
|
||||
'flow_count': n,
|
||||
'columns': list(df.columns[:15]),
|
||||
}
|
||||
|
||||
for c in df.columns:
|
||||
if df[c].dtype in (pl.Float32, pl.Float64, pl.Int32, pl.Int64):
|
||||
try:
|
||||
result[f'{c}_stats'] = {
|
||||
'mean': round(float(df[c].mean()), 2),
|
||||
'min': float(df[c].min()) if df[c].min() is not None else None,
|
||||
'max': float(df[c].max()) if df[c].max() is not None else None,
|
||||
}
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
pass
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Anomaly detection + visualization handlers."""
|
||||
import traceback
|
||||
import numpy as np
|
||||
from analysis.constants import MAX_DISTRIBUTION_SAMPLE, RANDOM_SEED
|
||||
import polars as pl
|
||||
|
||||
from ..session_store import SessionStore
|
||||
from ._helpers import _resolve_dataset, NUMERIC_TYPE_NAMES
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def _handle_detect_anomalies(dataset_id: str, contamination: float = 0.05) -> dict:
|
||||
"""Run Isolation Forest on scored entity data."""
|
||||
from sklearn.ensemble import IsolationForest
|
||||
entry, err = _resolve_dataset(dataset_id)
|
||||
if err:
|
||||
return err
|
||||
|
||||
lf: pl.LazyFrame = entry['lazyframe']
|
||||
schema = entry.get('schema', {})
|
||||
num_cols = [c for c in schema if schema[c].split('(')[0].strip() in NUMERIC_TYPE_NAMES and not c.startswith('_')]
|
||||
if not num_cols:
|
||||
return {'error': 'No numeric columns for anomaly detection', 'truncated': False}
|
||||
|
||||
df = lf.select(num_cols).collect(streaming=True)
|
||||
mat = df.to_numpy()
|
||||
mat = np.nan_to_num(mat, nan=0.0)
|
||||
|
||||
n = len(mat)
|
||||
chunk_size = min(MAX_SAMPLE_ROWS, n)
|
||||
labels = np.zeros(n, dtype=int)
|
||||
for start in range(0, n, chunk_size):
|
||||
end = min(start + chunk_size, n)
|
||||
iso = IsolationForest(n_estimators=100, max_samples=min(256, chunk_size),
|
||||
contamination=contamination, random_state=RANDOM_SEED, n_jobs=-1)
|
||||
labels[start:end] = iso.fit_predict(mat[start:end])
|
||||
|
||||
anomaly_count = int((labels == -1).sum())
|
||||
return {
|
||||
'status': 'completed',
|
||||
'total_entities': n,
|
||||
'anomaly_count': anomaly_count,
|
||||
'anomaly_rate': round(anomaly_count / max(n, 1), 4),
|
||||
'dataset_id': dataset_id,
|
||||
}
|
||||
|
||||
|
||||
async def _handle_visualize_anomalies(dataset_id: str) -> dict:
|
||||
"""Generate UMAP-2D embedding for anomaly visualization."""
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
import umap
|
||||
entry, err = _resolve_dataset(dataset_id)
|
||||
if err:
|
||||
return err
|
||||
|
||||
lf: pl.LazyFrame = entry['lazyframe']
|
||||
schema = entry.get('schema', {})
|
||||
num_cols = [c for c in schema if schema[c].split('(')[0].strip() in NUMERIC_TYPE_NAMES and not c.startswith('_')]
|
||||
if len(num_cols) < 2:
|
||||
return {'error': 'Need at least 2 numeric columns for UMAP', 'truncated': False}
|
||||
|
||||
try:
|
||||
n_total = lf.select(pl.len()).collect(streaming=True).item()
|
||||
except Exception:
|
||||
logger.error("_handle_visualize_anomalies failed: {}".format(traceback.format_exc()))
|
||||
n_total = 0
|
||||
sample_lf = lf if n_total <= MAX_DISTRIBUTION_SAMPLE else lf.collect(streaming=True).sample(n=MAX_DISTRIBUTION_SAMPLE, seed=RANDOM_SEED).lazy()
|
||||
df = sample_lf.select(num_cols).collect(streaming=True)
|
||||
mat = StandardScaler().fit_transform(df.to_numpy())
|
||||
mat = np.nan_to_num(mat, nan=0.0)
|
||||
reducer = umap.UMAP(n_components=2, random_state=RANDOM_SEED,
|
||||
n_neighbors=15, min_dist=0.1, metric='euclidean')
|
||||
coords = reducer.fit_transform(mat)
|
||||
|
||||
risk_col = next((c for c in schema if 'risk' in c.lower()), None)
|
||||
score_col = next((c for c in schema if 'proxy_score' in c.lower() or 'threat' in c.lower()), None)
|
||||
|
||||
non_num = [c for c in df.columns if c not in num_cols and not c.startswith('_')]
|
||||
ent_col = non_num[0] if non_num else df.columns[0]
|
||||
entities = df.select(ent_col).to_series().to_list() if ent_col in df.columns else []
|
||||
|
||||
scatter = [
|
||||
{'x': float(coords[i, 0]), 'y': float(coords[i, 1]),
|
||||
'entity': str(entities[i]) if i < len(entities) else '',
|
||||
'score': round(float(mat[i].mean()), 4)}
|
||||
for i in range(len(coords))
|
||||
]
|
||||
return {
|
||||
'status': 'completed',
|
||||
'scatter_points': len(scatter),
|
||||
'scatter_data': scatter[:200],
|
||||
'dataset_id': dataset_id,
|
||||
}
|
||||
@@ -0,0 +1,458 @@
|
||||
"""Clustering handlers — HDBSCAN, KMeans, AgglomerativeClustering.
|
||||
|
||||
Contains:
|
||||
- _handle_run_clustering: cluster selected columns
|
||||
- _handle_filter_and_cluster: filter then auto-cluster
|
||||
- _cluster_core: shared clustering pipeline extracted from both handlers
|
||||
"""
|
||||
import logging
|
||||
import traceback
|
||||
from analysis.constants import LOW_MEMORY_THRESHOLD_GB, LOW_MEMORY_THRESHOLD_ROWS, MAX_CLUSTER_FEATURES, MAX_SAMPLE_ROWS, RANDOM_SEED
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
|
||||
from ..session_store import SessionStore, _generate_id
|
||||
from ._helpers import (
|
||||
NUMERIC_DTYPES,
|
||||
_build_filter_expr,
|
||||
_resolve_dataset,
|
||||
_count_rows,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_clust_logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _cluster_core(
|
||||
data: np.ndarray,
|
||||
feature_cols: list[str],
|
||||
algorithm: str,
|
||||
params: dict,
|
||||
random_state: int = RANDOM_SEED,
|
||||
):
|
||||
"""Shared clustering pipeline: scale, filter, reduce, fit, score.
|
||||
|
||||
Args:
|
||||
data: Numpy array (n_samples, n_features).
|
||||
feature_cols: Column names matching data columns.
|
||||
algorithm: 'hdbscan', 'kmeans', or 'agglomerative'.
|
||||
params: Algorithm-specific parameters dict.
|
||||
random_state: Random seed.
|
||||
|
||||
Returns:
|
||||
(labels_array, n_clusters_found, noise_count, quality_metrics, _ds_idx)
|
||||
"""
|
||||
p = dict(params)
|
||||
|
||||
# ── Low memory check ───────────────────────────────────────────────
|
||||
low_memory = False
|
||||
try:
|
||||
import psutil
|
||||
mem = psutil.virtual_memory()
|
||||
low_memory = mem.available < LOW_MEMORY_THRESHOLD_GB * 1024 ** 3 # 2 GB
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
if low_memory and len(data) > LOW_MEMORY_THRESHOLD_ROWS:
|
||||
rng = np.random.default_rng(RANDOM_SEED)
|
||||
idx = rng.choice(len(data), MAX_SAMPLE_ROWS, replace=False)
|
||||
data = data[idx]
|
||||
|
||||
# ── Downsampling to 50K max ────────────────────────────────────────
|
||||
_ds_idx = None
|
||||
if len(data) > MAX_SAMPLE_ROWS:
|
||||
rng = np.random.default_rng(RANDOM_SEED)
|
||||
_ds_idx = rng.choice(len(data), MAX_SAMPLE_ROWS, replace=False)
|
||||
data = data[_ds_idx]
|
||||
|
||||
# ── Variance filtering ─────────────────────────────────────────────
|
||||
if data.shape[1] > 1:
|
||||
variances = np.var(data, axis=0)
|
||||
keep_var = np.where(variances >= 1e-10)[0]
|
||||
if len(keep_var) == 0:
|
||||
keep_var = np.arange(data.shape[1])
|
||||
if len(keep_var) < data.shape[1]:
|
||||
data = data[:, keep_var]
|
||||
feature_cols = [feature_cols[i] for i in keep_var]
|
||||
|
||||
# Log clustering input
|
||||
_clust_logger.info(
|
||||
'[CLUSTER_INPUT] rows=%d cols=%d feature_cols=%s',
|
||||
len(data), data.shape[1] if len(data.shape) > 1 else 1, feature_cols,
|
||||
)
|
||||
|
||||
# Standard scale
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
_clust_logger.info('[CLUSTER_SCALE] Starting StandardScaler...')
|
||||
data_scaled = StandardScaler().fit_transform(data)
|
||||
_clust_logger.info('[CLUSTER_SCALE] Done. shape=%s', data_scaled.shape)
|
||||
|
||||
# ── Correlation filtering ──────────────────────────────────────────
|
||||
if data_scaled.shape[1] > 1:
|
||||
corr = np.corrcoef(data_scaled.T)
|
||||
upper = np.triu(np.abs(corr), 1)
|
||||
to_drop = [j for j in range(upper.shape[1]) if any(upper[:, j] > 0.95)]
|
||||
if to_drop:
|
||||
keep = [j for j in range(data_scaled.shape[1]) if j not in to_drop]
|
||||
data_scaled = data_scaled[:, keep]
|
||||
feature_cols = [feature_cols[j] for j in keep]
|
||||
|
||||
# ── SVD dimensionality reduction (if >50 features) ─────────────────
|
||||
n_features = data_scaled.shape[1]
|
||||
if n_features > MAX_CLUSTER_FEATURES:
|
||||
from sklearn.decomposition import TruncatedSVD
|
||||
svd = TruncatedSVD(n_components=MAX_CLUSTER_FEATURES, random_state=random_state)
|
||||
data_scaled = svd.fit_transform(data_scaled)
|
||||
|
||||
# ── Hyperparameter auto-tuning ─────────────────────────────────────
|
||||
n = len(data_scaled)
|
||||
if algorithm == 'hdbscan':
|
||||
min_cluster = p.get('min_cluster_size', max(3, min(MAX_CLUSTER_FEATURES, n // 20)))
|
||||
p['min_cluster_size'] = min_cluster
|
||||
p['min_samples'] = p.get('min_samples', min_cluster)
|
||||
p['cluster_selection_epsilon'] = p.get('cluster_selection_epsilon', 0.0)
|
||||
else:
|
||||
p['min_cluster_size'] = min(
|
||||
p.get('min_cluster_size', 5),
|
||||
max(n // 2, 2),
|
||||
)
|
||||
|
||||
# ── Model fit ──────────────────────────────────────────────────────
|
||||
from sklearn.cluster import HDBSCAN, MiniBatchKMeans, AgglomerativeClustering
|
||||
if algorithm == 'kmeans':
|
||||
n_clusters_param = p.get('n_clusters', 3)
|
||||
model = MiniBatchKMeans(
|
||||
n_clusters=n_clusters_param, random_state=random_state,
|
||||
batch_size=1024, n_init='auto',
|
||||
)
|
||||
elif algorithm in ('agglomerative', 'ward'):
|
||||
n_clusters_param = p.get('n_clusters', 5)
|
||||
model = AgglomerativeClustering(
|
||||
n_clusters=n_clusters_param,
|
||||
metric=p.get('metric', 'euclidean'),
|
||||
linkage=p.get('linkage', 'ward'),
|
||||
distance_threshold=None,
|
||||
)
|
||||
else:
|
||||
model = HDBSCAN(
|
||||
min_cluster_size=p['min_cluster_size'],
|
||||
min_samples=p.get('min_samples', None),
|
||||
metric=p.get('metric', 'euclidean'),
|
||||
cluster_selection_epsilon=p.get('cluster_selection_epsilon', 0.0),
|
||||
)
|
||||
|
||||
labels_array = model.fit_predict(data_scaled)
|
||||
n_clusters_found = int(len(set(labels_array)) - (1 if -1 in labels_array else 0))
|
||||
noise_count = int((labels_array == -1).sum()) if algorithm == 'hdbscan' else 0
|
||||
noise_ratio = round(noise_count / len(labels_array), 4) if algorithm == 'hdbscan' else 0.0
|
||||
|
||||
# ── Quality metrics ────────────────────────────────────────────────
|
||||
quality = {
|
||||
'n_clusters': n_clusters_found,
|
||||
'n_noise': noise_count,
|
||||
'noise_ratio': noise_ratio,
|
||||
'algorithm': algorithm,
|
||||
}
|
||||
|
||||
if n_clusters_found > 1:
|
||||
try:
|
||||
from sklearn.metrics import silhouette_score
|
||||
score = silhouette_score(data_scaled, labels_array, random_state=random_state)
|
||||
quality['silhouette_score'] = round(float(score), 4)
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
quality['silhouette_score'] = None
|
||||
try:
|
||||
from sklearn.metrics import davies_bouldin_score
|
||||
dbs = davies_bouldin_score(data_scaled, labels_array)
|
||||
quality['davies_bouldin_score'] = round(float(dbs), 4)
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
quality['davies_bouldin_score'] = None
|
||||
try:
|
||||
from sklearn.metrics import calinski_harabasz_score
|
||||
chs = calinski_harabasz_score(data_scaled, labels_array)
|
||||
quality['calinski_harabasz_score'] = round(float(chs), 4)
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
quality['calinski_harabasz_score'] = None
|
||||
|
||||
# Cluster sizes
|
||||
cluster_sizes = {}
|
||||
for label in np.unique(labels_array):
|
||||
cluster_sizes[int(label)] = int((labels_array == label).sum())
|
||||
quality['cluster_sizes'] = cluster_sizes
|
||||
|
||||
return labels_array, n_clusters_found, noise_count, quality, _ds_idx
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# _handle_run_clustering
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
async def _handle_run_clustering(
|
||||
dataset_id: str,
|
||||
cluster_columns: list[str],
|
||||
algorithm: str = 'agglomerative',
|
||||
params: Optional[dict] = None,
|
||||
random_state: int = RANDOM_SEED,
|
||||
) -> dict:
|
||||
"""Run HDBSCAN, KMeans, or Agglomerative clustering on selected columns."""
|
||||
entry, err = _resolve_dataset(dataset_id)
|
||||
if err:
|
||||
return err
|
||||
|
||||
lf: pl.LazyFrame = entry['lazyframe']
|
||||
p = params or {}
|
||||
|
||||
# Collect cluster features - numeric only
|
||||
available = lf.collect_schema()
|
||||
available_names = available.names()
|
||||
available_dtypes = available.dtypes()
|
||||
feature_cols = [
|
||||
c for c in cluster_columns
|
||||
if c in available_names
|
||||
and available_dtypes[available_names.index(c)] in NUMERIC_DTYPES
|
||||
]
|
||||
if not feature_cols:
|
||||
# Fall back: auto-select all numeric columns (exclude internal ones)
|
||||
feature_cols = [
|
||||
available_names[i] for i, dt in enumerate(available_dtypes)
|
||||
if dt in NUMERIC_DTYPES and not available_names[i].startswith('_')
|
||||
][:10]
|
||||
if not feature_cols:
|
||||
# Fallback: try coercing Utf8 columns to Float64
|
||||
utf8_cols = [
|
||||
available_names[i] for i, dt in enumerate(available_dtypes)
|
||||
if dt == pl.Utf8 and not available_names[i].startswith('_')
|
||||
][:10]
|
||||
if utf8_cols:
|
||||
for col in utf8_cols:
|
||||
try:
|
||||
casted = lf.select(
|
||||
pl.col(col).cast(pl.Float64, strict=False).alias(col)
|
||||
).collect(streaming=True)
|
||||
null_ratio = casted[col].is_null().sum() / max(len(casted), 1)
|
||||
if null_ratio < 0.9:
|
||||
lf = lf.with_columns(
|
||||
pl.col(col).cast(pl.Float64, strict=False)
|
||||
)
|
||||
feature_cols.append(col)
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
pass
|
||||
if not feature_cols:
|
||||
all_numeric = [f"{available_names[i]}" for i, dt in enumerate(available_dtypes)
|
||||
if dt in NUMERIC_DTYPES]
|
||||
return {
|
||||
'error': f'No numeric cluster columns found in dataset. Available: {all_numeric}',
|
||||
'truncated': False
|
||||
}
|
||||
|
||||
# Downsample before collect
|
||||
row_count = _count_rows(lf)
|
||||
if row_count > LOW_MEMORY_THRESHOLD_ROWS:
|
||||
lf = lf.collect(streaming=True).sample(n=LOW_MEMORY_THRESHOLD_ROWS, seed=RANDOM_SEED).lazy()
|
||||
|
||||
df_features = lf.select(feature_cols).collect(streaming=True)
|
||||
|
||||
# Drop all-NaN columns
|
||||
for col in df_features.columns:
|
||||
if df_features[col].is_null().all():
|
||||
df_features = df_features.drop(col)
|
||||
feature_cols = [c for c in feature_cols if c != col]
|
||||
|
||||
if df_features.width == 0:
|
||||
return {'error': 'No valid feature columns after dropping all-NaN columns',
|
||||
'truncated': False}
|
||||
|
||||
data = df_features.to_numpy()
|
||||
|
||||
# Fill remaining NaN with column mean
|
||||
if data.dtype.kind == 'f':
|
||||
col_mean = np.nanmean(data, axis=0)
|
||||
col_mean = np.nan_to_num(col_mean, nan=0.0)
|
||||
data = np.where(np.isnan(data), col_mean, data)
|
||||
|
||||
# Zero-sample guard
|
||||
if len(data) < 3:
|
||||
return {
|
||||
'cluster_result_id': _generate_id('clust'),
|
||||
'n_clusters': 0,
|
||||
'n_noise': 0,
|
||||
'quality_metrics': {'note': 'too few samples'},
|
||||
'truncated': False,
|
||||
}
|
||||
|
||||
labels_array, n_found, noise_count, quality, _ds_idx = _cluster_core(
|
||||
data, feature_cols, algorithm, p, random_state,
|
||||
)
|
||||
|
||||
|
||||
result_id = _generate_id('clust')
|
||||
store = SessionStore()
|
||||
store.store_cluster_result(
|
||||
result_id=result_id,
|
||||
labels=labels_array.tolist(),
|
||||
n_clusters=n_found,
|
||||
params={
|
||||
'algorithm': algorithm,
|
||||
'params': p,
|
||||
'random_state': random_state,
|
||||
'feature_columns': feature_cols,
|
||||
'_ds_idx': _ds_idx.tolist() if _ds_idx is not None else None,
|
||||
},
|
||||
parent_dataset_id=dataset_id,
|
||||
)
|
||||
|
||||
return {
|
||||
'cluster_result_id': result_id,
|
||||
'n_clusters': n_found,
|
||||
'n_noise': noise_count,
|
||||
'labels_sample': labels_array[:50].tolist(),
|
||||
'quality_metrics': quality,
|
||||
}
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# _handle_filter_and_cluster
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
async def _handle_filter_and_cluster(
|
||||
dataset_id: str,
|
||||
filters: list[dict],
|
||||
logic: str = 'and',
|
||||
algorithm: str = 'hdbscan',
|
||||
params: dict | None = None,
|
||||
) -> dict:
|
||||
"""Apply column-level filters then run clustering on the filtered result."""
|
||||
entry, err = _resolve_dataset(dataset_id)
|
||||
if err:
|
||||
return err
|
||||
|
||||
lf: pl.LazyFrame = entry['lazyframe']
|
||||
full_schema = entry.get('schema', {})
|
||||
|
||||
# ── Step 1: Apply filters ─────────────────────────────────────────────
|
||||
exprs = [_build_filter_expr(
|
||||
f['column'], f.get('op', 'eq'), f.get('value'),
|
||||
) for f in filters]
|
||||
|
||||
if exprs:
|
||||
if logic == 'and':
|
||||
combined = exprs[0]
|
||||
for e in exprs[1:]:
|
||||
combined = combined & e
|
||||
else:
|
||||
combined = exprs[0]
|
||||
for e in exprs[1:]:
|
||||
combined = combined | e
|
||||
filtered_lf = lf.filter(combined)
|
||||
else:
|
||||
filtered_lf = lf
|
||||
|
||||
try:
|
||||
row_count = _count_rows(filtered_lf)
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
row_count = None
|
||||
|
||||
|
||||
# ── Step 2: Auto-detect numeric feature columns ───────────────────────
|
||||
available = filtered_lf.collect_schema()
|
||||
available_names = available.names()
|
||||
available_dtypes = available.dtypes()
|
||||
feature_cols = [
|
||||
available_names[i] for i, dt in enumerate(available_dtypes)
|
||||
if dt in NUMERIC_DTYPES and not available_names[i].startswith('_')
|
||||
][:10]
|
||||
|
||||
if not feature_cols:
|
||||
return {
|
||||
'error': 'No numeric columns found for clustering after filtering',
|
||||
'filtered_dataset_id': None,
|
||||
'truncated': False,
|
||||
}
|
||||
|
||||
# ── Step 3: Store filtered dataset ────────────────────────────────────
|
||||
filtered_dataset_id = _generate_id('flt')
|
||||
store = SessionStore()
|
||||
store.store_dataset(
|
||||
dataset_id=filtered_dataset_id,
|
||||
lazyframe=filtered_lf,
|
||||
schema=full_schema,
|
||||
metadata={
|
||||
'row_count': row_count,
|
||||
'parent_dataset_id': dataset_id,
|
||||
'filter_count': len(filters),
|
||||
'logic': logic,
|
||||
},
|
||||
parent_id=dataset_id,
|
||||
)
|
||||
|
||||
# ── Step 4: Run clustering ────────────────────────────────────────────
|
||||
p = params or {}
|
||||
total_count = _count_rows(filtered_lf)
|
||||
clamp_lf = filtered_lf
|
||||
if total_count > LOW_MEMORY_THRESHOLD_ROWS:
|
||||
clamp_lf = filtered_lf.collect(streaming=True).sample(n=LOW_MEMORY_THRESHOLD_ROWS, seed=RANDOM_SEED).lazy()
|
||||
|
||||
df_features = clamp_lf.select(feature_cols).collect(streaming=True)
|
||||
|
||||
for col in df_features.columns:
|
||||
if df_features[col].is_null().all():
|
||||
df_features = df_features.drop(col)
|
||||
feature_cols = [c for c in feature_cols if c != col]
|
||||
|
||||
if df_features.width == 0:
|
||||
return {'error': 'No valid feature columns after dropping all-NaN columns',
|
||||
'filtered_dataset_id': filtered_dataset_id, 'truncated': False}
|
||||
|
||||
data = df_features.to_numpy()
|
||||
|
||||
if data.dtype.kind == 'f':
|
||||
col_mean = np.nanmean(data, axis=0)
|
||||
col_mean = np.nan_to_num(col_mean, nan=0.0)
|
||||
data = np.where(np.isnan(data), col_mean, data)
|
||||
|
||||
if len(data) < 3:
|
||||
return {
|
||||
'cluster_result_id': _generate_id('clust'),
|
||||
'n_clusters': 0,
|
||||
'n_noise': 0,
|
||||
'quality_metrics': {'note': 'too few samples'},
|
||||
'filtered_dataset_id': filtered_dataset_id,
|
||||
'truncated': False,
|
||||
}
|
||||
|
||||
labels_array, n_found, noise_count, quality, _ds_idx = _cluster_core(
|
||||
data, feature_cols, algorithm, p, RANDOM_SEED,
|
||||
)
|
||||
|
||||
|
||||
result_id = _generate_id('clust')
|
||||
store.store_cluster_result(
|
||||
result_id=result_id,
|
||||
labels=labels_array.tolist(),
|
||||
n_clusters=n_found,
|
||||
params={
|
||||
'algorithm': algorithm,
|
||||
'params': p,
|
||||
'random_state': RANDOM_SEED,
|
||||
'feature_columns': feature_cols,
|
||||
'_ds_idx': _ds_idx.tolist() if _ds_idx is not None else None,
|
||||
},
|
||||
parent_dataset_id=filtered_dataset_id,
|
||||
)
|
||||
|
||||
return {
|
||||
'cluster_result_id': result_id,
|
||||
'filtered_dataset_id': filtered_dataset_id,
|
||||
'n_clusters': n_found,
|
||||
'n_noise': noise_count,
|
||||
'labels_sample': labels_array[:50].tolist(),
|
||||
'quality_metrics': quality,
|
||||
'feature_columns': feature_cols,
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Data management handlers — list, drop, clone datasets."""
|
||||
from typing import Optional
|
||||
|
||||
from ..session_store import SessionStore
|
||||
|
||||
|
||||
async def _handle_list_datasets(
|
||||
_filter_type: Optional[str] = None,
|
||||
) -> dict:
|
||||
"""List all active datasets and results."""
|
||||
store = SessionStore()
|
||||
datasets = store.list_datasets()
|
||||
return {'datasets': datasets, 'count': len(datasets)}
|
||||
|
||||
|
||||
async def _handle_drop_dataset(dataset_id: str) -> dict:
|
||||
"""Remove a dataset/result and release memory."""
|
||||
store = SessionStore()
|
||||
|
||||
dropped = (store.drop_dataset(dataset_id)
|
||||
or store.drop_cluster_result(dataset_id)
|
||||
or store.drop_feature_result(dataset_id))
|
||||
|
||||
if dropped:
|
||||
return {'dropped': True, 'dataset_id': dataset_id}
|
||||
return {'error': f'No dataset or result found: {dataset_id}',
|
||||
'truncated': False}
|
||||
|
||||
|
||||
async def _handle_clone_dataset(dataset_id: str) -> dict:
|
||||
"""Shallow-clone a dataset."""
|
||||
store = SessionStore()
|
||||
new_id = store.clone_dataset(dataset_id)
|
||||
if new_id is None:
|
||||
return {'error': f'Dataset not found: {dataset_id}', 'truncated': False}
|
||||
return {'original_dataset_id': dataset_id, 'cloned_dataset_id': new_id}
|
||||
@@ -0,0 +1,253 @@
|
||||
"""Diagnostic tool handlers — validate, explore, diagnose, compare, repair."""
|
||||
import traceback
|
||||
from analysis.constants import LOW_MEMORY_THRESHOLD_ROWS, RANDOM_SEED
|
||||
import polars as pl
|
||||
|
||||
from ..session_store import SessionStore
|
||||
from ._helpers import _resolve_dataset, _count_rows, NUMERIC_TYPE_NAMES
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def _handle_validate_data(dataset_id: str, fix_issues: bool = False) -> dict:
|
||||
"""Check data quality: schema consistency, missing values, type conflicts."""
|
||||
entry, err = _resolve_dataset(dataset_id)
|
||||
if err:
|
||||
return err
|
||||
lf = entry['lazyframe']
|
||||
schema = entry.get('schema', {})
|
||||
issues = []
|
||||
df_sample = lf.head(1000).collect(streaming=True)
|
||||
for col in schema:
|
||||
null_count = df_sample[col].is_null().sum()
|
||||
if null_count > 500:
|
||||
issues.append(f'{col}: {null_count/10:.0f}% null')
|
||||
try:
|
||||
unique = df_sample[col].n_unique()
|
||||
if unique <= 1 and null_count == 0:
|
||||
issues.append(f'{col}: constant value (no variation)')
|
||||
except Exception:
|
||||
logger.error("_handle_validate_data failed: {}".format(traceback.format_exc()))
|
||||
pass
|
||||
total_rows = _count_rows(lf)
|
||||
return {
|
||||
'status': 'ok',
|
||||
'total_rows': total_rows,
|
||||
'columns': len(schema),
|
||||
'issues_found': len(issues),
|
||||
'issues': issues[:20],
|
||||
'recommendation': 'Try explore_distributions for detailed column analysis' if issues else 'No issues found',
|
||||
}
|
||||
|
||||
|
||||
async def _handle_explore_distributions(dataset_id: str, columns: list[str] = None,
|
||||
max_sample: int = 10000) -> dict:
|
||||
"""Per-column distribution statistics."""
|
||||
entry, err = _resolve_dataset(dataset_id)
|
||||
if err:
|
||||
return err
|
||||
lf = entry['lazyframe']
|
||||
schema = entry.get('schema', {})
|
||||
|
||||
if not columns:
|
||||
columns = [c for c in schema if schema[c].split('(')[0].strip() in NUMERIC_TYPE_NAMES][:10]
|
||||
if len(columns) > 20:
|
||||
columns = columns[:20]
|
||||
|
||||
try:
|
||||
n = _count_rows(lf)
|
||||
if n > max_sample:
|
||||
sample_lf = lf.select(columns).sample(n=max_sample, seed=RANDOM_SEED)
|
||||
else:
|
||||
sample_lf = lf.select(columns)
|
||||
df = sample_lf.collect(streaming=True)
|
||||
except Exception as exc:
|
||||
return {'error': f'explore_distributions failed: {exc}', 'truncated': False}
|
||||
|
||||
stats = {}
|
||||
for col in columns:
|
||||
try:
|
||||
s = df[col]
|
||||
stats[col] = {
|
||||
'dtype': str(s.dtype),
|
||||
'null_pct': round(float(s.is_null().mean() * 100), 2),
|
||||
'unique': int(s.n_unique()),
|
||||
'top_values': [str(v) for v in s.head(5).to_list()],
|
||||
}
|
||||
if s.dtype in (pl.Float32, pl.Float64, pl.Int32, pl.Int64):
|
||||
stats[col]['min'] = float(s.min()) if s.min() is not None else None
|
||||
stats[col]['max'] = float(s.max()) if s.max() is not None else None
|
||||
stats[col]['mean'] = round(float(s.mean()), 4) if s.mean() is not None else None
|
||||
stats[col]['std'] = round(float(s.std()), 4) if s.std() is not None else None
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
stats[col] = {'error': 'failed to compute'}
|
||||
return {'columns': stats, 'total_rows': n, 'sampled': min(n, max_sample)}
|
||||
|
||||
|
||||
async def _handle_find_outliers(dataset_id: str, columns: list[str] = None,
|
||||
threshold: float = 3.0) -> dict:
|
||||
"""Find statistical outliers using IQR method."""
|
||||
entry, err = _resolve_dataset(dataset_id)
|
||||
if err:
|
||||
return err
|
||||
lf = entry['lazyframe']
|
||||
schema = entry.get('schema', {})
|
||||
if not columns:
|
||||
columns = [c for c in schema if schema[c].split('(')[0].strip() in NUMERIC_TYPE_NAMES]
|
||||
n = _count_rows(lf)
|
||||
sample_lf = lf if n <= 100000 else lf.collect(streaming=True).sample(n=LOW_MEMORY_THRESHOLD_ROWS, seed=RANDOM_SEED).lazy()
|
||||
df = sample_lf.select(columns).collect(streaming=True)
|
||||
|
||||
outlier_counts = {}
|
||||
for col in columns:
|
||||
try:
|
||||
s = df[col]
|
||||
q1, q3 = s.quantile(0.25), s.quantile(0.75)
|
||||
iqr = q3 - q1
|
||||
lo = q1 - threshold * iqr
|
||||
hi = q3 + threshold * iqr
|
||||
n_out = ((s < lo) | (s > hi)).sum()
|
||||
if n_out > 0:
|
||||
outlier_counts[col] = int(n_out)
|
||||
except Exception:
|
||||
logger.error("_handle_find_outliers failed: {}".format(traceback.format_exc()))
|
||||
pass
|
||||
return {'outlier_counts': outlier_counts, 'total_sampled': len(df), 'threshold_iqr': threshold}
|
||||
|
||||
|
||||
async def _handle_diagnose_clustering(dataset_id: str, cluster_result_id: str) -> dict:
|
||||
"""Diagnose clustering quality issues."""
|
||||
from sklearn.decomposition import TruncatedSVD
|
||||
store = SessionStore()
|
||||
entry = store.get_dataset(dataset_id)
|
||||
cluster_entry = store.get_cluster_result(cluster_result_id)
|
||||
if entry is None:
|
||||
return {'error': f'Dataset not found: {dataset_id}', 'truncated': False}
|
||||
if cluster_entry is None:
|
||||
return {'error': f'Cluster result not found: {cluster_result_id}', 'truncated': False}
|
||||
lf = entry['lazyframe']
|
||||
schema = entry.get('schema', {})
|
||||
num_cols = [c for c in schema if schema[c].split('(')[0].strip() in NUMERIC_TYPE_NAMES and not c.startswith('_')]
|
||||
|
||||
n_clusters = cluster_entry.get('n_clusters', 0)
|
||||
n_noise = cluster_entry.get('n_noise', 0)
|
||||
quality = cluster_entry.get('quality_metrics', {})
|
||||
labels = cluster_entry.get('labels', [])
|
||||
params = cluster_entry.get('params', {})
|
||||
algo = params.get('algorithm', 'unknown')
|
||||
feature_cols = params.get('feature_columns', [])
|
||||
|
||||
diagnosis = []
|
||||
if n_clusters <= 1:
|
||||
diagnosis.append('Only 1 cluster — data may lack structure or need feature scaling')
|
||||
if n_noise > 0.5 * len(labels) if labels else False:
|
||||
diagnosis.append(f'High noise ratio ({n_noise}/{len(labels)}) — try HDBSCAN with larger min_cluster_size')
|
||||
sil = quality.get('silhouette_score')
|
||||
if sil and sil < 0.2:
|
||||
diagnosis.append(f'Low silhouette ({sil:.3f}) — clusters overlap, try KMeans or feature selection')
|
||||
if len(feature_cols) < 3:
|
||||
diagnosis.append(f'Only {len(feature_cols)} features — add more columns for meaningful clusters')
|
||||
|
||||
try:
|
||||
df_sample = lf.select(num_cols).head(5000).collect(streaming=True)
|
||||
mat = df_sample.to_numpy()
|
||||
n_comp = min(mat.shape)
|
||||
svd = TruncatedSVD(n_components=n_comp, random_state=RANDOM_SEED).fit(mat)
|
||||
var_explained = svd.explained_variance_ratio_
|
||||
if var_explained[0] > 0.95:
|
||||
diagnosis.append(f'First SVD component explains {var_explained[0]:.0%} variance — data nearly 1D')
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
pass
|
||||
|
||||
return {
|
||||
'diagnosis': diagnosis,
|
||||
'n_clusters': n_clusters,
|
||||
'n_noise': n_noise,
|
||||
'silhouette': sil,
|
||||
'algorithm': algo,
|
||||
'n_features': len(feature_cols),
|
||||
'suggestions': [
|
||||
f'Try algorithm={"kmeans" if algo == "hdbscan" else "hdbscan"}',
|
||||
'Increase min_cluster_size for fewer, larger clusters',
|
||||
'Use explore_distributions to check feature quality',
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
async def _handle_compare_datasets(dataset_id_a: str, dataset_id_b: str) -> dict:
|
||||
"""Compare two datasets."""
|
||||
store = SessionStore()
|
||||
entry_a, entry_b = store.get_dataset(dataset_id_a), store.get_dataset(dataset_id_b)
|
||||
if entry_a is None or entry_b is None:
|
||||
return {'error': 'One or both datasets not found', 'truncated': False}
|
||||
|
||||
def _get_info(entry):
|
||||
schema = entry.get('schema', {})
|
||||
lf = entry['lazyframe']
|
||||
try:
|
||||
n = _count_rows(lf)
|
||||
except Exception:
|
||||
logger.error("_get_info failed: {}".format(traceback.format_exc()))
|
||||
n = -1
|
||||
return {'schema': set(schema.keys()), 'row_count': n, 'columns': len(schema)}
|
||||
|
||||
info_a, info_b = _get_info(entry_a), _get_info(entry_b)
|
||||
common_cols = info_a['schema'] & info_b['schema']
|
||||
only_a = info_a['schema'] - info_b['schema']
|
||||
only_b = info_b['schema'] - info_a['schema']
|
||||
return {
|
||||
'dataset_a': {'id': dataset_id_a, **info_a},
|
||||
'dataset_b': {'id': dataset_id_b, **info_b},
|
||||
'common_columns': len(common_cols),
|
||||
'columns_only_in_a': list(only_a)[:10] if only_a else [],
|
||||
'columns_only_in_b': list(only_b)[:10] if only_b else [],
|
||||
'row_diff': info_a['row_count'] - info_b['row_count'],
|
||||
'identical_schema': len(only_a) == 0 and len(only_b) == 0,
|
||||
}
|
||||
|
||||
|
||||
async def _handle_export_debug_sample(dataset_id: str, max_rows: int = 50) -> dict:
|
||||
"""Export raw data sample as JSON for debugging."""
|
||||
entry, err = _resolve_dataset(dataset_id)
|
||||
if err:
|
||||
return err
|
||||
df = entry['lazyframe'].head(max_rows).collect(streaming=True)
|
||||
rows = df.to_dict(as_series=False)
|
||||
result = []
|
||||
for i in range(len(df)):
|
||||
row = {}
|
||||
for col in df.columns:
|
||||
val = rows[col][i]
|
||||
if isinstance(val, (float, int, str, bool)):
|
||||
row[col] = val
|
||||
elif val is None:
|
||||
row[col] = None
|
||||
else:
|
||||
row[col] = str(val)
|
||||
result.append(row)
|
||||
return {'rows': result, 'count': len(result), 'columns': df.columns}
|
||||
|
||||
|
||||
async def _handle_repair_schema(dataset_id: str) -> dict:
|
||||
"""Attempt to repair schema mismatches by normalising column names."""
|
||||
entry, err = _resolve_dataset(dataset_id)
|
||||
if err:
|
||||
return err
|
||||
lf = entry['lazyframe']
|
||||
schema = entry.get('schema', {})
|
||||
col_map = {}
|
||||
for c in schema:
|
||||
norm = c.lower().replace('-', '_').replace('.', '_')
|
||||
col_map[c] = norm
|
||||
if len(set(col_map.values())) < len(col_map):
|
||||
return {'status': 'conflict', 'message': 'Normalised names conflict', 'column_map': col_map}
|
||||
renames = {old: new for old, new in col_map.items() if old != new}
|
||||
if renames:
|
||||
lf = lf.rename(renames)
|
||||
new_schema = {col_map.get(c, c): dtype for c, dtype in schema.items()}
|
||||
store = SessionStore()
|
||||
store.store_dataset(dataset_id, lf, schema=new_schema, metadata=entry.get('metadata', {}))
|
||||
return {'status': 'repaired', 'renamed': len(renames), 'column_map': col_map}
|
||||
return {'status': 'no_change', 'message': 'Column names already normalised'}
|
||||
@@ -0,0 +1,146 @@
|
||||
"""compute_distance_matrix handler — LLM-driven per-row evaluation."""
|
||||
import traceback
|
||||
from analysis.constants import RANDOM_SEED
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
|
||||
from ..session_store import SessionStore
|
||||
from ._helpers import _resolve_dataset, NUMERIC_DTYPES
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _indent_function_body(body: str, indent: str = ' ') -> str:
|
||||
"""Indent each non-empty line of a code block for wrapping in a def."""
|
||||
return '\n'.join(
|
||||
indent + line if line.strip() else line
|
||||
for line in body.split('\n')
|
||||
)
|
||||
|
||||
|
||||
async def _handle_compute_distance_matrix(
|
||||
dataset_id: str,
|
||||
python_function: str,
|
||||
columns: Optional[list[str]] = None,
|
||||
max_sample: int = 10000,
|
||||
) -> dict:
|
||||
"""Evaluate a user-provided Python function on each row of a dataset."""
|
||||
entry, err = _resolve_dataset(dataset_id)
|
||||
if err:
|
||||
return err
|
||||
|
||||
lf: pl.LazyFrame = entry['lazyframe']
|
||||
df = lf.collect(streaming=True)
|
||||
|
||||
if columns is None:
|
||||
columns = [c for c in df.columns
|
||||
if df[c].dtype in NUMERIC_DTYPES]
|
||||
|
||||
available = set(df.columns)
|
||||
columns = [c for c in columns if c in available]
|
||||
if not columns:
|
||||
return {'error': 'No valid columns found in dataset',
|
||||
'truncated': False}
|
||||
|
||||
n_total = len(df)
|
||||
|
||||
if n_total > max_sample:
|
||||
df_sampled = df.sample(n=max_sample, seed=RANDOM_SEED)
|
||||
n_evaluated = max_sample
|
||||
else:
|
||||
df_sampled = df
|
||||
n_evaluated = n_total
|
||||
|
||||
rows = df_sampled.select(columns).to_dicts()
|
||||
|
||||
safe_builtins = {
|
||||
'abs': abs, 'len': len, 'range': range,
|
||||
'min': min, 'max': max, 'sum': sum, 'round': round,
|
||||
'int': int, 'float': float, 'str': str, 'bool': bool,
|
||||
'list': list, 'dict': dict, 'tuple': tuple,
|
||||
'isinstance': isinstance, 'type': type,
|
||||
'True': True, 'False': False, 'None': None,
|
||||
'enumerate': enumerate, 'zip': zip, 'map': map,
|
||||
'filter': filter, 'sorted': sorted, 'reversed': reversed,
|
||||
}
|
||||
|
||||
safe_ns = {
|
||||
'__builtins__': safe_builtins,
|
||||
'math': __import__('math'),
|
||||
'np': __import__('numpy'),
|
||||
}
|
||||
|
||||
body = python_function.strip()
|
||||
if body.startswith('def '):
|
||||
full_code = body
|
||||
else:
|
||||
indented = _indent_function_body(body)
|
||||
full_code = f'def distance_fn(row):\n{indented}'
|
||||
|
||||
try:
|
||||
exec(full_code, safe_ns)
|
||||
distance_fn = safe_ns.get('distance_fn')
|
||||
if distance_fn is None:
|
||||
for _name, _obj in safe_ns.items():
|
||||
if callable(_obj) and _name not in ('math', 'np', '__builtins__'):
|
||||
distance_fn = _obj
|
||||
break
|
||||
if distance_fn is None:
|
||||
return {'error': 'Could not locate the compiled function',
|
||||
'truncated': False}
|
||||
except SyntaxError as e:
|
||||
return {'error': f'Function syntax error: {e}', 'truncated': False}
|
||||
except Exception as e:
|
||||
return {'error': f'Function compilation failed: {e}',
|
||||
'truncated': False}
|
||||
|
||||
scores: list[float | None] = []
|
||||
errors = 0
|
||||
for row in rows:
|
||||
try:
|
||||
score = distance_fn(row)
|
||||
if isinstance(score, (int, float)):
|
||||
scores.append(float(score))
|
||||
else:
|
||||
scores.append(None)
|
||||
errors += 1
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
scores.append(None)
|
||||
errors += 1
|
||||
|
||||
valid_scores = [s for s in scores if s is not None]
|
||||
if not valid_scores:
|
||||
return {'error': 'No valid scores computed — all rows raised errors',
|
||||
'truncated': False}
|
||||
|
||||
arr = np.array(valid_scores)
|
||||
summary = {
|
||||
'min': float(np.min(arr)),
|
||||
'max': float(np.max(arr)),
|
||||
'mean': float(np.mean(arr)),
|
||||
'std': float(np.std(arr)),
|
||||
'count': len(arr),
|
||||
'errors': errors,
|
||||
'total_rows_in_sample': len(rows),
|
||||
'matrix_shape': [len(rows), len(columns)],
|
||||
}
|
||||
|
||||
scored_sample = []
|
||||
for i, row in enumerate(rows[:30]):
|
||||
if i < len(scores):
|
||||
scored_sample.append({
|
||||
'index': i,
|
||||
'values': {c: row.get(c) for c in columns[:3]},
|
||||
'score': scores[i],
|
||||
})
|
||||
|
||||
return {
|
||||
'summary': summary,
|
||||
'scored_sample': scored_sample,
|
||||
'columns_used': columns,
|
||||
'total_rows_in_dataset': n_total,
|
||||
'rows_evaluated': n_evaluated,
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
"""Entity profile building + adaptive scoring handlers."""
|
||||
import traceback
|
||||
from analysis.constants import MAX_DISTRIBUTION_SAMPLE, RANDOM_SEED
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
|
||||
from ..session_store import SessionStore, _generate_id
|
||||
from ..profile_util import profile
|
||||
from ._helpers import _resolve_dataset, _count_rows, NUMERIC_DTYPES
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# Constants
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
_ENTITY_KEYWORDS = [
|
||||
'src_ip', 'dst_ip', 'source_ip', 'destination_ip', 'src_addr', 'dst_addr',
|
||||
'sip', 'dip', 'ip_src', 'ip_dst', 'saddr', 'daddr',
|
||||
'sni', 'host', 'domain', 'uri', 'url', 'fqdn', 'server_name',
|
||||
'mac', 'mac_addr', 'src_mac', 'dst_mac',
|
||||
'user_agent', 'ua', 'asn', 'organization', 'org',
|
||||
]
|
||||
|
||||
_NUMERIC_TYPES_FOR_AGG = (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
|
||||
pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
|
||||
pl.Float32, pl.Float64)
|
||||
|
||||
_ANOMALY_FEATURE_COLS = [
|
||||
'non_std_port_rate', 'modern_tls_rate', 'sni_missing_ratio',
|
||||
'recoverable_ratio', 'flow_count', 'total_bytes_sent',
|
||||
'total_packets', 'unique_dst_ips', 'unique_dst_ports',
|
||||
'unique_protocols', 'has_sni', 'avg_sni_length',
|
||||
'countries_visited',
|
||||
]
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# _handle_build_entity_profiles
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
async def _handle_build_entity_profiles(
|
||||
dataset_id: str,
|
||||
entity_column: Optional[str] = None,
|
||||
entity_columns: Optional[list[str]] = None,
|
||||
auto_detect: bool = True,
|
||||
) -> dict:
|
||||
"""Group raw flow data by entity column(s) and compute per-entity aggregate features."""
|
||||
entry, err = _resolve_dataset(dataset_id)
|
||||
if err:
|
||||
return err
|
||||
|
||||
lf: pl.LazyFrame = entry['lazyframe']
|
||||
schema = entry.get('schema', {})
|
||||
col_names = list(schema.keys())
|
||||
|
||||
# ── Resolve entity columns ──────────────────────────────────────────
|
||||
if entity_column and entity_column in col_names:
|
||||
group_cols = [entity_column]
|
||||
elif entity_columns:
|
||||
group_cols = [c for c in entity_columns if c in col_names]
|
||||
if not group_cols:
|
||||
return {'error': f'None of entity_columns {entity_columns} exist in dataset. Available: {col_names[:20]}',
|
||||
'truncated': False}
|
||||
elif auto_detect:
|
||||
matched = []
|
||||
for kw in _ENTITY_KEYWORDS:
|
||||
found = [c for c in col_names if kw.lower() in c.lower()]
|
||||
matched.extend(found)
|
||||
seen = set()
|
||||
group_cols = [c for c in matched if not (c in seen or seen.add(c))]
|
||||
if not group_cols:
|
||||
for c in col_names:
|
||||
if c.startswith('_'):
|
||||
continue
|
||||
group_cols = [c]
|
||||
break
|
||||
group_cols = group_cols[:3]
|
||||
else:
|
||||
return {'error': 'No entity column specified and auto_detect is disabled',
|
||||
'truncated': False}
|
||||
|
||||
if not group_cols:
|
||||
return {'error': 'Could not determine entity column(s) for grouping',
|
||||
'truncated': False}
|
||||
|
||||
# ── Build aggregation expressions ──────────────────────────────────
|
||||
agg_exprs: list[pl.Expr] = [
|
||||
pl.len().alias('flow_count'),
|
||||
pl.col(group_cols[0]).n_unique().alias(f'unique_{group_cols[0]}'),
|
||||
]
|
||||
|
||||
numeric_cols = [c for c in col_names if c not in group_cols
|
||||
and schema[c].startswith(('Int', 'UInt', 'Float'))]
|
||||
for c in numeric_cols[:20]:
|
||||
agg_exprs.append(pl.sum(c).alias(f'total_{c}'))
|
||||
agg_exprs.append(pl.mean(c).alias(f'avg_{c}'))
|
||||
|
||||
str_cols = [c for c in col_names if c not in group_cols
|
||||
and schema[c].startswith(('Utf8', 'String', 'Cat'))]
|
||||
for c in str_cols[:10]:
|
||||
agg_exprs.append(pl.col(c).n_unique().alias(f'unique_{c}'))
|
||||
|
||||
for kw in ['dst_ip', 'dip', 'dst_addr', 'daddr', 'ip_dst', 'dest_ip',
|
||||
'dst_port', 'dport', 'destination_port', 'dest_port',
|
||||
'port', 'protocol', 'proto', 'sni', 'server_name',
|
||||
'cipher_suite', 'cipher', 'tls_version', 'version', '0ver',
|
||||
'country', 'scnt', 'dcnt', 'asn', 'organization']:
|
||||
found = [c for c in col_names if c not in group_cols and kw.lower() in c.lower()]
|
||||
for c in found:
|
||||
alias_name = f'unique_{c}'
|
||||
if alias_name not in [str(a.meta.output_name()) if hasattr(a, 'meta') else '' for a in agg_exprs]:
|
||||
agg_exprs.append(pl.col(c).n_unique().alias(alias_name))
|
||||
|
||||
# ── Execute grouping ───────────────────────────────────────────────
|
||||
row_count = _count_rows(lf)
|
||||
ENTITY_CHUNK_THRESHOLD = 1_000_000 # 1M rows
|
||||
ENTITY_CHUNK_SIZE = 100_000 # 100K rows per chunk
|
||||
|
||||
if row_count >= ENTITY_CHUNK_THRESHOLD:
|
||||
# Chunked path: process in 100K batches → group → merge across chunks
|
||||
all_chunk_dfs = []
|
||||
for offset in range(0, row_count, ENTITY_CHUNK_SIZE):
|
||||
chunk_df = lf.slice(offset, ENTITY_CHUNK_SIZE).collect(streaming=True)
|
||||
chunk_grouped = chunk_df.group_by(group_cols).agg(agg_exprs)
|
||||
all_chunk_dfs.append(chunk_grouped)
|
||||
merged_df = pl.concat(all_chunk_dfs)
|
||||
|
||||
# Re-aggregate across chunks (same entity may appear in multiple chunks)
|
||||
re_agg_exprs: list[pl.Expr] = []
|
||||
for col_name in merged_df.columns:
|
||||
if col_name in group_cols:
|
||||
continue
|
||||
if col_name == 'flow_count':
|
||||
re_agg_exprs.append(pl.sum('flow_count').alias('flow_count'))
|
||||
elif col_name.startswith('total_'):
|
||||
re_agg_exprs.append(pl.sum(col_name).alias(col_name))
|
||||
elif col_name.startswith('avg_'):
|
||||
# Weighted average: sum(total) / sum(flow_count)
|
||||
base = col_name[4:]
|
||||
total_col = f'total_{base}'
|
||||
if total_col in merged_df.columns:
|
||||
re_agg_exprs.append(
|
||||
(pl.sum(total_col) / pl.sum('flow_count')).alias(col_name))
|
||||
elif col_name.startswith('unique_'):
|
||||
# Approximate: max of per-chunk distinct counts
|
||||
re_agg_exprs.append(pl.max(col_name).alias(col_name))
|
||||
|
||||
entity_lf = merged_df.lazy().group_by(group_cols).agg(re_agg_exprs)
|
||||
logger.info('[ENTITY] chunked aggregation: %d rows → %d chunks → %d merged rows',
|
||||
row_count, len(all_chunk_dfs), len(merged_df))
|
||||
else:
|
||||
entity_lf = lf.group_by(group_cols).agg(agg_exprs)
|
||||
|
||||
n_entities = _count_rows(entity_lf)
|
||||
|
||||
# ── Store result ───────────────────────────────────────────────────
|
||||
new_schema = {name: str(dtype) for name, dtype in
|
||||
zip(entity_lf.collect_schema().names(), entity_lf.collect_schema().dtypes())}
|
||||
new_id = f'entity_{_generate_id("ds")}'
|
||||
store = SessionStore()
|
||||
store.store_dataset(
|
||||
dataset_id=new_id,
|
||||
lazyframe=entity_lf,
|
||||
schema=new_schema,
|
||||
metadata={
|
||||
'row_count': n_entities,
|
||||
'entity_columns': group_cols,
|
||||
'parent_dataset_id': dataset_id,
|
||||
'n_agg_features': len(agg_exprs),
|
||||
},
|
||||
parent_id=dataset_id,
|
||||
)
|
||||
|
||||
return {
|
||||
'dataset_id': new_id,
|
||||
'entity_columns': group_cols,
|
||||
'n_entities': n_entities,
|
||||
'n_features': len(agg_exprs),
|
||||
'aggregated_columns': list(new_schema.keys())[:20],
|
||||
}
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# Adaptive scoring helpers
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
@profile
|
||||
def _add_adaptive_scores(lf: pl.LazyFrame, feature_names: list[str]) -> pl.LazyFrame:
|
||||
"""Add anomaly scores using data-driven weights (SVD + IQR + percentile)."""
|
||||
_agg_cols = lf.collect_schema().names()
|
||||
avail = [c for c in _ANOMALY_FEATURE_COLS if c in _agg_cols]
|
||||
if not avail:
|
||||
return lf
|
||||
|
||||
try:
|
||||
n_rows = _count_rows(lf)
|
||||
sample_lf = lf if n_rows <= MAX_DISTRIBUTION_SAMPLE else lf.sample(n=MAX_DISTRIBUTION_SAMPLE, seed=RANDOM_SEED)
|
||||
df_sample = sample_lf.select(avail).collect(streaming=True)
|
||||
mat = df_sample.to_numpy()
|
||||
mat = np.nan_to_num(mat, nan=0.0)
|
||||
|
||||
from sklearn.decomposition import TruncatedSVD
|
||||
n_comp = min(1, min(mat.shape) - 1)
|
||||
if n_comp < 1 or mat.shape[1] < 2:
|
||||
return lf
|
||||
svd = TruncatedSVD(n_components=1, random_state=RANDOM_SEED).fit(mat)
|
||||
raw_weights = np.abs(svd.components_[0])
|
||||
weights = dict(zip(avail, raw_weights / raw_weights.sum()))
|
||||
|
||||
expr_parts = []
|
||||
for col, w in weights.items():
|
||||
if col == 'modern_tls_rate':
|
||||
expr_parts.append((1.0 - pl.col(col).fill_null(0.5)) * w)
|
||||
else:
|
||||
expr_parts.append(pl.col(col).fill_null(0.0) * w)
|
||||
if expr_parts:
|
||||
proxy_expr = expr_parts[0]
|
||||
for p in expr_parts[1:]:
|
||||
proxy_expr = proxy_expr + p
|
||||
else:
|
||||
return lf
|
||||
|
||||
lf = lf.with_columns(proxy_expr.clip(0, 1).alias('_raw_score'))
|
||||
|
||||
sample_scores = df_sample.select(
|
||||
pl.sum_horizontal([
|
||||
(1.0 - pl.col('modern_tls_rate').fill_null(0.5)) * weights.get('modern_tls_rate', 0)
|
||||
if c == 'modern_tls_rate'
|
||||
else pl.col(c).fill_null(0.0) * weights.get(c, 0)
|
||||
for c in avail
|
||||
]).clip(0, 1)
|
||||
).to_series().to_numpy()
|
||||
|
||||
q25, q50, q75 = np.percentile(sample_scores, [25, 50, 75])
|
||||
iqr = q75 - q25
|
||||
upper_fence = float(q75 + 1.5 * iqr) if iqr > 0 else 0.9
|
||||
extreme_fence = float(q75 + 3.0 * iqr) if iqr > 0 else 0.99
|
||||
|
||||
lf = lf.with_columns([
|
||||
pl.col('_raw_score').alias('proxy_score'),
|
||||
pl.when(pl.col('_raw_score') >= extreme_fence)
|
||||
.then(pl.lit('critical'))
|
||||
.when(pl.col('_raw_score') >= upper_fence)
|
||||
.then(pl.lit('suspicious'))
|
||||
.when(pl.col('_raw_score') >= q50)
|
||||
.then(pl.lit('watch'))
|
||||
.otherwise(pl.lit('normal'))
|
||||
.alias('risk_level'),
|
||||
(pl.col('_raw_score') * 0.6 + 0.4 * (pl.col('_raw_score') > q75).cast(pl.Float64))
|
||||
.alias('threat_score'),
|
||||
]).drop('_raw_score')
|
||||
|
||||
valid_counts = {c: int((mat[:, i] != 0).sum()) for i, c in enumerate(avail)}
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.info(
|
||||
'[SCORE] adaptive weights from SVD: %s | thresholds: p50=%.3f upper=%.3f extreme=%.3f | sample=%d/%d',
|
||||
{c: f'{w:.3f}' for c, w in sorted(weights.items(), key=lambda x: -x[1])},
|
||||
float(q50), upper_fence, extreme_fence,
|
||||
min(n_rows, MAX_DISTRIBUTION_SAMPLE), n_rows,
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
logging.getLogger(__name__).warning('[SCORE] adaptive scoring skipped: %s', exc)
|
||||
|
||||
return lf
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# _handle_compute_scores
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
async def _handle_compute_scores(dataset_id: str) -> dict:
|
||||
"""Compute adaptive proxy/anomaly/threat scores on entity data."""
|
||||
entry, err = _resolve_dataset(dataset_id)
|
||||
if err:
|
||||
return err
|
||||
|
||||
lf = entry['lazyframe']
|
||||
lf = _add_adaptive_scores(lf, [])
|
||||
try:
|
||||
updated_schema = lf.collect_schema()
|
||||
new_schema = {name: str(dtype) for name, dtype in
|
||||
zip(updated_schema.names(), updated_schema.dtypes())}
|
||||
except Exception:
|
||||
logger.error("_handle_compute_scores failed: {}".format(traceback.format_exc()))
|
||||
new_schema = entry.get('schema', {})
|
||||
store = SessionStore()
|
||||
store.store_dataset(dataset_id, lf, schema=new_schema,
|
||||
metadata=entry.get('metadata', {}))
|
||||
return {'status': 'scored', 'dataset_id': dataset_id}
|
||||
@@ -0,0 +1,76 @@
|
||||
"""evaluate_clustering handler — compute clustering quality metrics."""
|
||||
from typing import Optional
|
||||
from analysis.constants import RANDOM_SEED
|
||||
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
|
||||
from ..session_store import SessionStore
|
||||
|
||||
|
||||
async def _handle_evaluate_clustering(
|
||||
cluster_result_id: str,
|
||||
dataset_id: str,
|
||||
metrics: Optional[list[str]] = None,
|
||||
) -> dict:
|
||||
"""Evaluate clustering result against the dataset."""
|
||||
store = SessionStore()
|
||||
cluster_entry = store.get_cluster_result(cluster_result_id)
|
||||
if cluster_entry is None:
|
||||
return {'error': f'Cluster result not found: {cluster_result_id}',
|
||||
'truncated': False}
|
||||
|
||||
dataset_entry = store.get_dataset(dataset_id)
|
||||
if dataset_entry is None:
|
||||
return {'error': f'Dataset not found: {dataset_id}', 'truncated': False}
|
||||
|
||||
labels = cluster_entry['labels']
|
||||
default_metrics = ['silhouette', 'davies_bouldin', 'calinski_harabasz',
|
||||
'noise_ratio', 'cluster_sizes']
|
||||
requested = metrics or default_metrics
|
||||
|
||||
scores: dict = {'n_clusters': cluster_entry['n_clusters']}
|
||||
|
||||
lf: pl.LazyFrame = dataset_entry['lazyframe']
|
||||
params = cluster_entry.get('params', {})
|
||||
feature_cols = params.get('feature_columns', [])
|
||||
if not feature_cols and 'feature_columns' in params:
|
||||
pass
|
||||
elif feature_cols:
|
||||
df_features = lf.select(feature_cols).collect(streaming=True)
|
||||
data = df_features.to_numpy()
|
||||
|
||||
# Align labels with data (handle missing rows)
|
||||
valid_mask = ~np.isnan(data).any(axis=1) if data.dtype.kind == 'f' else np.ones(len(data), dtype=bool)
|
||||
data_clean = data[valid_mask]
|
||||
labels_clean = np.array(labels)[:len(data_clean)]
|
||||
|
||||
unique_labels = set(labels_clean)
|
||||
n_clusters_actual = len(unique_labels - {-1})
|
||||
|
||||
for m in requested:
|
||||
if m == 'silhouette' and n_clusters_actual >= 2:
|
||||
from sklearn.metrics import silhouette_score
|
||||
scores['silhouette'] = round(float(
|
||||
silhouette_score(data_clean, labels_clean, random_state=RANDOM_SEED)), 4)
|
||||
elif m == 'davies_bouldin' and n_clusters_actual >= 2:
|
||||
from sklearn.metrics import davies_bouldin_score
|
||||
scores['davies_bouldin'] = round(float(
|
||||
davies_bouldin_score(data_clean, labels_clean)), 4)
|
||||
elif m == 'calinski_harabasz' and n_clusters_actual >= 2:
|
||||
from sklearn.metrics import calinski_harabasz_score
|
||||
scores['calinski_harabasz'] = round(float(
|
||||
calinski_harabasz_score(data_clean, labels_clean)), 4)
|
||||
elif m == 'noise_ratio':
|
||||
noise = int((labels_clean == -1).sum())
|
||||
scores['noise_ratio'] = round(noise / len(labels_clean), 4)
|
||||
elif m == 'cluster_sizes':
|
||||
sizes = {}
|
||||
for label in unique_labels:
|
||||
sizes[int(label)] = int((labels_clean == label).sum())
|
||||
scores['cluster_sizes'] = sizes
|
||||
|
||||
scores['n_clusters'] = cluster_entry['n_clusters']
|
||||
|
||||
|
||||
return {'scores': scores}
|
||||
@@ -0,0 +1,91 @@
|
||||
"""export_results handler — write datasets/results to disk."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import polars as pl
|
||||
|
||||
from ..session_store import SessionStore
|
||||
|
||||
|
||||
async def _handle_export_results(
|
||||
result_id: str,
|
||||
output_path: str,
|
||||
format: str = 'csv',
|
||||
overwrite: bool = False,
|
||||
) -> dict:
|
||||
"""Export a dataset (or result) to disk."""
|
||||
store = SessionStore()
|
||||
|
||||
out = Path(output_path)
|
||||
file_paths: list[str] = []
|
||||
|
||||
# Check if it's a dataset
|
||||
dataset_entry = store.get_dataset(result_id)
|
||||
if dataset_entry is not None:
|
||||
lf: pl.LazyFrame = dataset_entry['lazyframe']
|
||||
df = lf.collect(streaming=True)
|
||||
if out.suffix == '':
|
||||
out = out / f"export_{result_id}.{format}"
|
||||
|
||||
if out.exists() and not overwrite:
|
||||
return {'error': f'Output exists and overwrite=False: {out}',
|
||||
'truncated': False}
|
||||
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if format == 'csv':
|
||||
df.write_csv(out)
|
||||
elif format == 'parquet':
|
||||
df.write_parquet(out)
|
||||
elif format == 'json':
|
||||
df.write_json(out)
|
||||
else:
|
||||
return {'error': f'Unsupported format: {format}',
|
||||
'truncated': False}
|
||||
|
||||
file_paths.append(str(out.resolve()))
|
||||
|
||||
return {'file_paths': file_paths, 'format': format,
|
||||
'row_count': len(df)}
|
||||
|
||||
# Check if it's a cluster result
|
||||
cluster_entry = store.get_cluster_result(result_id)
|
||||
if cluster_entry is not None:
|
||||
if out.suffix == '':
|
||||
out = out / f"cluster_{result_id}.json"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if out.exists() and not overwrite:
|
||||
return {'error': f'Output exists and overwrite=False: {out}',
|
||||
'truncated': False}
|
||||
|
||||
export_data = {
|
||||
'result_id': result_id,
|
||||
'n_clusters': cluster_entry['n_clusters'],
|
||||
'params': cluster_entry['params'],
|
||||
'labels_sample': cluster_entry['labels'][:1000],
|
||||
}
|
||||
with open(out, 'w', encoding='utf-8') as f:
|
||||
json.dump(export_data, f, default=str)
|
||||
file_paths.append(str(out.resolve()))
|
||||
|
||||
return {'file_paths': file_paths, 'format': 'json'}
|
||||
|
||||
# Check feature result
|
||||
feat_entry = store.get_feature_result(result_id)
|
||||
if feat_entry is not None:
|
||||
if out.suffix == '':
|
||||
out = out / f"features_{result_id}.json"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if out.exists() and not overwrite:
|
||||
return {'error': f'Output exists and overwrite=False: {out}',
|
||||
'truncated': False}
|
||||
|
||||
with open(out, 'w', encoding='utf-8') as f:
|
||||
json.dump(feat_entry['features'], f, default=str)
|
||||
file_paths.append(str(out.resolve()))
|
||||
return {'file_paths': file_paths, 'format': 'json'}
|
||||
|
||||
return {'error': f'No dataset or result found for ID: {result_id}',
|
||||
'truncated': False}
|
||||
@@ -0,0 +1,314 @@
|
||||
"""extract_features handler — per-cluster feature ranking + ORM persistence."""
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
|
||||
from ..session_store import SessionStore, _generate_id
|
||||
from ..profile_util import profile
|
||||
from analysis.constants import RANDOM_SEED, UMAP_BATCH_SIZE, UMAP_TRAIN_SAMPLE
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def _handle_extract_features(
|
||||
dataset_id: str,
|
||||
cluster_result_id: str,
|
||||
top_k: int = 10,
|
||||
method: str = 'zscore',
|
||||
save_to_db: bool = True,
|
||||
run_id: int | None = None,
|
||||
) -> dict:
|
||||
"""Compute per-cluster distinguishing features and optionally persist."""
|
||||
store = SessionStore()
|
||||
dataset_entry = store.get_dataset(dataset_id)
|
||||
if dataset_entry is None:
|
||||
return {'error': f'Dataset not found: {dataset_id}', 'truncated': False}
|
||||
|
||||
cluster_entry = store.get_cluster_result(cluster_result_id)
|
||||
if cluster_entry is None:
|
||||
return {'error': f'Cluster result not found: {cluster_result_id}',
|
||||
'truncated': False}
|
||||
|
||||
lf: pl.LazyFrame = dataset_entry['lazyframe']
|
||||
labels = cluster_entry['labels']
|
||||
|
||||
# D5: If clustering downsampled the data, filter matching rows
|
||||
_ds_idx = cluster_entry.get('params', {}).get('_ds_idx')
|
||||
if _ds_idx is not None:
|
||||
df = (lf.with_row_index('__ds_idx')
|
||||
.filter(pl.col('__ds_idx').is_in(_ds_idx))
|
||||
.drop('__ds_idx')
|
||||
.collect(streaming=True))
|
||||
else:
|
||||
df = lf.collect(streaming=True)
|
||||
|
||||
# Truncate labels if df was filtered but labels weren't
|
||||
if len(labels) != len(df):
|
||||
labels = labels[:len(df)]
|
||||
|
||||
# Only use numeric columns for feature extraction
|
||||
numeric_cols = [c for c in df.columns
|
||||
if df[c].dtype in (pl.Float32, pl.Float64,
|
||||
pl.Int32, pl.Int64,
|
||||
pl.UInt32, pl.UInt64)]
|
||||
if len(labels) != len(df):
|
||||
labels = labels[:len(df)]
|
||||
|
||||
# Global statistics per column
|
||||
global_stats = {}
|
||||
for col in numeric_cols:
|
||||
s = df[col].drop_nulls()
|
||||
if len(s) > 0:
|
||||
global_stats[col] = {
|
||||
'mean': float(s.mean()),
|
||||
'std': float(s.std()) if s.std() is not None else 1.0,
|
||||
}
|
||||
|
||||
unique_labels = sorted(set(labels))
|
||||
all_features = []
|
||||
|
||||
for label in unique_labels:
|
||||
mask = np.array(labels) == label
|
||||
if mask.sum() < 2:
|
||||
continue
|
||||
|
||||
cluster_df = df.filter(pl.Series('__mask__', mask))
|
||||
cluster_features = []
|
||||
|
||||
for col in numeric_cols:
|
||||
c_series = cluster_df[col].drop_nulls()
|
||||
gs = global_stats.get(col)
|
||||
if gs is None or gs['std'] == 0 or len(c_series) == 0:
|
||||
continue
|
||||
|
||||
c_mean = float(c_series.mean())
|
||||
c_std = float(c_series.std()) if c_series.std() is not None else 0.0
|
||||
|
||||
if method == 'zscore':
|
||||
score = (c_mean - gs['mean']) / max(gs['std'], 1e-10)
|
||||
else:
|
||||
score = abs(c_mean - gs['mean']) / max(c_std + gs['std'], 1e-10)
|
||||
|
||||
cluster_features.append({
|
||||
'feature_name': col,
|
||||
'cluster_label': int(label),
|
||||
'mean': round(c_mean, 4),
|
||||
'std': round(c_std, 4),
|
||||
'global_mean': round(gs['mean'], 4),
|
||||
'global_std': round(gs['std'], 4),
|
||||
'distinguishing_score': round(float(score), 4),
|
||||
'distinguishing_method': method,
|
||||
})
|
||||
|
||||
cluster_features.sort(key=lambda x: abs(x['distinguishing_score']), reverse=True)
|
||||
all_features.extend(cluster_features[:top_k])
|
||||
|
||||
# ── Compute UMAP-2D embedding ──────────────────────────────────
|
||||
if len(numeric_cols) >= 2 and len(df) > 2:
|
||||
try:
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
import umap
|
||||
MAX_UMAP_TRAIN = UMAP_TRAIN_SAMPLE
|
||||
BATCH_SIZE = UMAP_BATCH_SIZE
|
||||
data_matrix = df.select(numeric_cols).to_numpy()
|
||||
if len(df) > MAX_UMAP_TRAIN:
|
||||
idx_sample = np.random.RandomState(RANDOM_SEED).choice(
|
||||
len(df), size=MAX_UMAP_TRAIN, replace=False)
|
||||
mat_sample = data_matrix[idx_sample]
|
||||
scaler = StandardScaler().fit(mat_sample)
|
||||
mat_sample_scaled = np.nan_to_num(scaler.transform(mat_sample),
|
||||
nan=0.0, posinf=0.0, neginf=0.0)
|
||||
reducer = umap.UMAP(n_components=2, random_state=RANDOM_SEED,
|
||||
n_neighbors=15, min_dist=0.1,
|
||||
metric='euclidean')
|
||||
reducer.fit(mat_sample_scaled)
|
||||
coords_list = []
|
||||
for start in range(0, len(df), BATCH_SIZE):
|
||||
end = min(start + BATCH_SIZE, len(df))
|
||||
mat_batch = scaler.transform(data_matrix[start:end])
|
||||
mat_batch = np.nan_to_num(mat_batch, nan=0.0, posinf=0.0, neginf=0.0)
|
||||
coords_list.append(reducer.transform(mat_batch))
|
||||
coords = np.vstack(coords_list)
|
||||
else:
|
||||
data_scaled = StandardScaler().fit_transform(data_matrix)
|
||||
data_scaled = np.nan_to_num(data_scaled, nan=0.0, posinf=0.0, neginf=0.0)
|
||||
reducer = umap.UMAP(n_components=2, random_state=RANDOM_SEED,
|
||||
n_neighbors=15, min_dist=0.1,
|
||||
metric='euclidean')
|
||||
coords = reducer.fit_transform(data_scaled)
|
||||
non_numeric = [c for c in df.columns if c not in numeric_cols and not c.startswith('_')]
|
||||
entity_col_umap = non_numeric[0] if non_numeric else df.columns[0]
|
||||
labels_list = cluster_entry.get('labels', [])
|
||||
from analysis.models import EntityProfile as EP
|
||||
from analysis.models import ClusterResult as CR
|
||||
from analysis.models import AnalysisRun as AR
|
||||
from asgiref.sync import sync_to_async
|
||||
csv_glob = dataset_entry.get('metadata', {}).get('csv_glob', 'manual')
|
||||
run_obj = await sync_to_async(
|
||||
AR.objects.filter(csv_glob=csv_glob).order_by('-id').first,
|
||||
thread_sensitive=True
|
||||
)()
|
||||
if run_obj:
|
||||
run_id_for_ep = run_obj.id
|
||||
profiles = []
|
||||
cr_cache = {}
|
||||
for i, row in enumerate(df.iter_rows(named=True)):
|
||||
if i >= len(coords): break
|
||||
ev = str(row.get(entity_col_umap, ''))
|
||||
if not ev: continue
|
||||
lbl = labels_list[i] if i < len(labels_list) else -1
|
||||
cache_key = f'{run_id_for_ep}_{lbl}'
|
||||
if cache_key not in cr_cache:
|
||||
cr_cache[cache_key] = CR.objects.filter(run_id=run_id_for_ep, cluster_label=lbl).first()
|
||||
profiles.append(EP(
|
||||
entity_value=ev, run_id=run_id_for_ep, cluster_label=lbl,
|
||||
cluster=cr_cache[cache_key],
|
||||
embedding_x=float(coords[i, 0]),
|
||||
embedding_y=float(coords[i, 1]),
|
||||
))
|
||||
if profiles:
|
||||
EP.objects.bulk_create(profiles, ignore_conflicts=True, batch_size=1000)
|
||||
except Exception:
|
||||
print(f'[UMAP_ERR] {traceback.format_exc()}', file=sys.stderr)
|
||||
|
||||
# Persist to Django ORM
|
||||
db_saved = False
|
||||
if save_to_db:
|
||||
try:
|
||||
from asgiref.sync import sync_to_async
|
||||
from analysis.models import AnalysisRun as AnalysisRunModel
|
||||
ce = cluster_entry
|
||||
async def _do_save(ce_inner):
|
||||
if run_id is not None:
|
||||
run = await sync_to_async(AnalysisRunModel.objects.get)(id=run_id)
|
||||
else:
|
||||
csv_glob_fallback = dataset_entry.get('metadata', {}).get('csv_glob', 'manual')
|
||||
run = await sync_to_async(
|
||||
AnalysisRunModel.objects.filter(csv_glob=csv_glob_fallback).order_by('-id').first,
|
||||
thread_sensitive=True
|
||||
)()
|
||||
if run is None:
|
||||
run = await sync_to_async(AnalysisRunModel.objects.create)(
|
||||
csv_glob=csv_glob_fallback, status='running', total_flows=0,
|
||||
)
|
||||
run.status = 'completed'
|
||||
run.cluster_count = ce_inner['n_clusters']
|
||||
await sync_to_async(run.save)()
|
||||
return await sync_to_async(_save_features_to_db)(cluster_result_id, all_features, run_id=run.id)
|
||||
db_saved = await _do_save(ce)
|
||||
except Exception as e:
|
||||
print(f'[DB_SAVE_ERROR] {e}\n{traceback.format_exc()}', file=sys.stderr)
|
||||
db_saved = False
|
||||
|
||||
|
||||
result_id = _generate_id('feat')
|
||||
store.store_feature_result(
|
||||
result_id=result_id,
|
||||
features=all_features,
|
||||
parent_cluster_result_id=cluster_result_id,
|
||||
)
|
||||
|
||||
return {
|
||||
'feature_result_id': result_id,
|
||||
'features': all_features,
|
||||
'n_features': len(all_features),
|
||||
'db_saved': db_saved,
|
||||
}
|
||||
|
||||
|
||||
@profile
|
||||
def _save_features_to_db(
|
||||
cluster_result_id: str,
|
||||
features: list[dict],
|
||||
run_id: int | None = None,
|
||||
) -> bool:
|
||||
"""Save extracted features to the Django ORM ClusterFeature model."""
|
||||
if run_id is not None:
|
||||
from analysis.models import AnalysisRun as AnalysisRun2
|
||||
run = AnalysisRun2.objects.filter(id=run_id).first()
|
||||
if run is None:
|
||||
return False
|
||||
else:
|
||||
store = SessionStore()
|
||||
cluster_entry = store.get_cluster_result(cluster_result_id)
|
||||
if cluster_entry is None:
|
||||
return False
|
||||
|
||||
parent_ds_id = cluster_entry.get('parent_dataset_id', '')
|
||||
from analysis.models import AnalysisRun
|
||||
ds_entry = store.get_dataset(parent_ds_id)
|
||||
if ds_entry is None:
|
||||
return False
|
||||
|
||||
csv_glob = ds_entry.get('metadata', {}).get('csv_glob', '')
|
||||
unique_global = sorted(set(f['cluster_label'] for f in features))
|
||||
n_clusters_global = len([l for l in unique_global if l >= 0])
|
||||
try:
|
||||
run = AnalysisRun.objects.filter(csv_glob=csv_glob).order_by('-created_at').first()
|
||||
if run is None:
|
||||
run = AnalysisRun.objects.create(
|
||||
csv_glob=csv_glob,
|
||||
status='completed',
|
||||
cluster_count=n_clusters_global,
|
||||
)
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
return False
|
||||
|
||||
from analysis.models import ClusterResult as ClusterResultModel
|
||||
|
||||
store = SessionStore()
|
||||
cluster_entry = store.get_cluster_result(cluster_result_id)
|
||||
label_counts = {}
|
||||
if cluster_entry:
|
||||
full_labels = cluster_entry.get('labels', [])
|
||||
for lbl in full_labels:
|
||||
label_counts[lbl] = label_counts.get(lbl, 0) + 1
|
||||
|
||||
unique_labels = sorted(set(
|
||||
f['cluster_label'] for f in features
|
||||
))
|
||||
n_clusters = len([l for l in unique_labels if l >= 0])
|
||||
for label in unique_labels:
|
||||
if label == -1:
|
||||
continue
|
||||
label_features = [f for f in features if f['cluster_label'] == label]
|
||||
if not label_features:
|
||||
continue
|
||||
|
||||
cluster_db, _ = ClusterResultModel.objects.get_or_create(
|
||||
run=run,
|
||||
cluster_label=label,
|
||||
defaults={'size': label_counts.get(label, 1)},
|
||||
)
|
||||
if label_counts.get(label) is not None:
|
||||
ClusterResultModel.objects.filter(
|
||||
run=run, cluster_label=label
|
||||
).update(size=label_counts[label])
|
||||
|
||||
from analysis.models import ClusterFeature as ClusterFeatureModel
|
||||
for feat in label_features:
|
||||
try:
|
||||
ClusterFeatureModel.objects.get_or_create(
|
||||
cluster=cluster_db,
|
||||
feature_name=feat['feature_name'],
|
||||
defaults={
|
||||
'mean': feat.get('mean'),
|
||||
'std': feat.get('std'),
|
||||
'median': None,
|
||||
'p25': None,
|
||||
'p75': None,
|
||||
'missing_rate': None,
|
||||
'distinguishing_score': feat.get('distinguishing_score'),
|
||||
'distinguishing_method': feat.get('distinguishing_method', 'zscore'),
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
continue
|
||||
|
||||
run.cluster_count = n_clusters
|
||||
run.save(update_fields=['cluster_count'])
|
||||
return True
|
||||
@@ -0,0 +1,68 @@
|
||||
"""filter_data handler — apply column-level filters to a dataset."""
|
||||
import traceback
|
||||
import polars as pl
|
||||
|
||||
from ..session_store import SessionStore, _generate_id
|
||||
from ._helpers import _build_filter_expr, _resolve_dataset
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def _handle_filter_data(
|
||||
dataset_id: str,
|
||||
filters: list[dict],
|
||||
logic: str = 'and',
|
||||
) -> dict:
|
||||
"""Apply filters and store filtered dataset."""
|
||||
entry, err = _resolve_dataset(dataset_id)
|
||||
if err:
|
||||
return err
|
||||
|
||||
lf: pl.LazyFrame = entry['lazyframe']
|
||||
full_schema = entry.get('schema', {})
|
||||
|
||||
exprs = [_build_filter_expr(
|
||||
f['column'], f.get('op', 'eq'), f.get('value'),
|
||||
) for f in filters]
|
||||
|
||||
if not exprs:
|
||||
return {'error': 'No filter expressions could be built', 'truncated': False}
|
||||
|
||||
if logic == 'and':
|
||||
combined = exprs[0]
|
||||
for e in exprs[1:]:
|
||||
combined = combined & e
|
||||
else:
|
||||
combined = exprs[0]
|
||||
for e in exprs[1:]:
|
||||
combined = combined | e
|
||||
|
||||
filtered_lf = lf.filter(combined)
|
||||
|
||||
# Estimate row count (fast streaming count)
|
||||
try:
|
||||
row_count = filtered_lf.select(pl.len()).collect(streaming=True).item()
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
row_count = None
|
||||
|
||||
new_id = _generate_id('flt')
|
||||
store = SessionStore()
|
||||
store.store_dataset(
|
||||
dataset_id=new_id,
|
||||
lazyframe=filtered_lf,
|
||||
schema=full_schema,
|
||||
metadata={
|
||||
'row_count': row_count,
|
||||
'parent_dataset_id': dataset_id,
|
||||
'filter_count': len(filters),
|
||||
'logic': logic,
|
||||
},
|
||||
parent_id=dataset_id,
|
||||
)
|
||||
|
||||
return {
|
||||
'filtered_dataset_id': new_id,
|
||||
'row_count': row_count,
|
||||
'column_deltas': {'kept': list(full_schema.keys()), 'dropped': []},
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
"""load_data handler — load CSV files and store as dataset."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import polars as pl
|
||||
|
||||
from ..data_loader import load_csv_directory
|
||||
from ..session_store import SessionStore, _generate_id
|
||||
from analysis.constants import MAX_DISTRIBUTION_SAMPLE
|
||||
|
||||
|
||||
async def _handle_load_data(
|
||||
csv_glob: str,
|
||||
config_path: Optional[str] = None,
|
||||
encoding: str = 'utf-8',
|
||||
delimiter: str = ',',
|
||||
schema_strict: bool = False,
|
||||
recursive: bool = False,
|
||||
) -> dict:
|
||||
"""Load, validate, merge CSVs, and store in session."""
|
||||
try:
|
||||
lf, schema, row_count, file_count, memory_mb = load_csv_directory(
|
||||
glob_pattern=csv_glob,
|
||||
encoding=encoding,
|
||||
delimiter=delimiter,
|
||||
config_path=config_path,
|
||||
sample_rows=MAX_DISTRIBUTION_SAMPLE,
|
||||
schema_strict=schema_strict,
|
||||
recursive=recursive,
|
||||
)
|
||||
except FileNotFoundError as e:
|
||||
return {'error': f'File not found: {e}', 'truncated': False}
|
||||
except ValueError as e:
|
||||
return {'error': f'Schema validation failed: {e}', 'truncated': False}
|
||||
|
||||
store = SessionStore()
|
||||
dataset_id = _generate_id('ds')
|
||||
store.store_dataset(
|
||||
dataset_id=dataset_id,
|
||||
lazyframe=lf,
|
||||
schema=schema,
|
||||
metadata={
|
||||
'row_count': row_count,
|
||||
'file_count': file_count,
|
||||
'memory_mb': round(memory_mb, 2),
|
||||
'csv_glob': csv_glob,
|
||||
'encoding': encoding,
|
||||
},
|
||||
)
|
||||
|
||||
return {
|
||||
'dataset_id': dataset_id,
|
||||
'schema': schema,
|
||||
'row_count': row_count,
|
||||
'file_count': file_count,
|
||||
'memory_mb': round(memory_mb, 2),
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
"""preprocess_data handler — scale, encode, impute, drop columns."""
|
||||
from typing import Optional
|
||||
|
||||
import polars as pl
|
||||
|
||||
from ..session_store import SessionStore, _generate_id
|
||||
from ._helpers import _resolve_dataset
|
||||
|
||||
|
||||
async def _handle_preprocess_data(
|
||||
dataset_id: str,
|
||||
columns: list[str],
|
||||
config: Optional[dict] = None,
|
||||
) -> dict:
|
||||
"""Preprocess selected columns (scale, encode, impute, drop)."""
|
||||
entry, err = _resolve_dataset(dataset_id)
|
||||
if err:
|
||||
return err
|
||||
|
||||
lf: pl.LazyFrame = entry['lazyframe']
|
||||
cfg = config or {}
|
||||
column_mapping: dict[str, str] = {}
|
||||
|
||||
df = lf.collect(streaming=True)
|
||||
|
||||
# Resolve columns that actually exist
|
||||
available = set(df.columns)
|
||||
target_cols = [c for c in columns if c in available]
|
||||
if not target_cols:
|
||||
return {'error': 'None of the specified columns exist in the dataset',
|
||||
'truncated': False}
|
||||
|
||||
# Drop columns
|
||||
drop_cols = cfg.get('drop', [])
|
||||
if drop_cols:
|
||||
df = df.drop([c for c in drop_cols if c in df.columns])
|
||||
for c in drop_cols:
|
||||
if c in available:
|
||||
column_mapping[c] = f'<dropped:{c}>'
|
||||
|
||||
# Fill missing values
|
||||
fill_strategy = cfg.get('fillna')
|
||||
if fill_strategy and fill_strategy != 'drop':
|
||||
for col in target_cols:
|
||||
if col not in df.columns:
|
||||
continue
|
||||
if df[col].null_count() == 0:
|
||||
continue
|
||||
if fill_strategy == 'mean':
|
||||
mean_val = df[col].drop_nulls().mean()
|
||||
if mean_val is not None:
|
||||
df = df.with_columns(pl.col(col).fill_null(mean_val))
|
||||
elif fill_strategy == 'median':
|
||||
med_val = df[col].drop_nulls().median()
|
||||
if med_val is not None:
|
||||
df = df.with_columns(pl.col(col).fill_null(med_val))
|
||||
elif fill_strategy == 'zero':
|
||||
df = df.with_columns(pl.col(col).fill_null(0))
|
||||
elif fill_strategy == 'drop':
|
||||
df = df.drop_nulls(subset=target_cols)
|
||||
|
||||
# Standard scaling (z-score)
|
||||
if cfg.get('standardize'):
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
num_cols = [c for c in target_cols if c in df.columns
|
||||
and df[c].dtype in (pl.Float32, pl.Float64, pl.Int32, pl.Int64)]
|
||||
if num_cols:
|
||||
scaler = StandardScaler()
|
||||
scaled = scaler.fit_transform(df.select(num_cols).to_numpy())
|
||||
for i, col in enumerate(num_cols):
|
||||
col_scaled = f'{col}_scaled'
|
||||
df = df.with_columns(pl.Series(col_scaled, scaled[:, i]))
|
||||
column_mapping[col] = col_scaled
|
||||
|
||||
# One-hot encoding
|
||||
if cfg.get('onehot'):
|
||||
str_cols = [c for c in target_cols if c in df.columns
|
||||
and df[c].dtype in (pl.Utf8, pl.String, pl.Categorical)]
|
||||
if str_cols:
|
||||
df = df.to_dummies(columns=str_cols, drop_first=False)
|
||||
for c in str_cols:
|
||||
column_mapping[c] = f'<onehot:{c}>'
|
||||
|
||||
new_id = _generate_id('pp')
|
||||
new_lf = df.lazy()
|
||||
new_schema = {name: str(dtype) for name, dtype in
|
||||
zip(new_lf.collect_schema().names(), new_lf.collect_schema().dtypes())}
|
||||
|
||||
store = SessionStore()
|
||||
store.store_dataset(
|
||||
dataset_id=new_id,
|
||||
lazyframe=new_lf,
|
||||
schema=new_schema,
|
||||
metadata={
|
||||
'row_count': len(df),
|
||||
'parent_dataset_id': dataset_id,
|
||||
'preprocessing_config': cfg,
|
||||
},
|
||||
parent_id=dataset_id,
|
||||
)
|
||||
|
||||
return {
|
||||
'preprocessed_dataset_id': new_id,
|
||||
'column_mapping': column_mapping,
|
||||
'row_count': len(df),
|
||||
'column_count': len(new_schema),
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
"""profile_data handler — compute per-column statistics and correlations."""
|
||||
from typing import Optional
|
||||
|
||||
import polars as pl
|
||||
|
||||
from ..data_profiler import profile_dataset as _profile_dataset
|
||||
from ._helpers import _resolve_dataset
|
||||
|
||||
|
||||
async def _handle_profile_data(
|
||||
dataset_id: str,
|
||||
sample_size: int = 1000,
|
||||
columns: Optional[list[str]] = None,
|
||||
) -> dict:
|
||||
"""Profile dataset columns and correlations."""
|
||||
entry, err = _resolve_dataset(dataset_id)
|
||||
if err:
|
||||
return err
|
||||
|
||||
lf: pl.LazyFrame = entry['lazyframe']
|
||||
profile = _profile_dataset(lf, sample_size=sample_size, columns=columns)
|
||||
return profile
|
||||
+3
-415
@@ -1,415 +1,3 @@
|
||||
"""Data type classifier for column-level type inference.
|
||||
|
||||
Provides :class:`DataType` enumeration and functions to classify Polars
|
||||
columns based on name heuristics and value sampling.
|
||||
|
||||
Typical usage::
|
||||
|
||||
from analysis.type_classifier import classify_column, classify_schema, DataType
|
||||
|
||||
# Single column
|
||||
dtype = classify_column('src_ip', series, config_type='ipv4')
|
||||
|
||||
# Full schema
|
||||
type_map = classify_schema(lazyframe, config_overrides={'src_ip': 'ipv4'})
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from enum import Enum
|
||||
|
||||
import polars as pl
|
||||
|
||||
from analysis.tls_ref import get_tls_ref_dict
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TLS version hex-to-label mapping
|
||||
# (Duplicated in views.py for globe visualization — keep both in sync.)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
TLS_HEX_MAP: dict[str, str] = {
|
||||
'0303': 'TLSv1.2',
|
||||
'0304': 'TLSv1.3',
|
||||
'0302': 'TLSv1.1',
|
||||
'0301': 'TLSv1.0',
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DataType enumeration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class DataType(Enum):
|
||||
"""Enumerated column types recognised by the classifier engine."""
|
||||
|
||||
INT = 1
|
||||
FLOAT = 2
|
||||
ENUM = 3
|
||||
HEX = 4
|
||||
URL = 5
|
||||
IPv4 = 6
|
||||
LAT_LON = 7
|
||||
TIMESTAMP = 8
|
||||
BOOL_ENUM = 9
|
||||
STRING = 10
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Detection patterns (raw strings for regex)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
HEX_PATTERN = r'^[0-9a-f]{2}(?:\s[0-9a-f]{2})*$'
|
||||
"""Matches ``"0e a6 3f"``, ``"ab cd ef"`` etc. (space-separated hex pairs)."""
|
||||
|
||||
URL_PATTERN = r'^https?://'
|
||||
"""Matches URLs starting with ``http://`` or ``https://``."""
|
||||
|
||||
IPv4_PATTERN = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$'
|
||||
"""Matches ``"192.168.1.1"`` style IPv4 addresses."""
|
||||
|
||||
MAC_PATTERN = r'^([0-9A-Fa-f]{2}[:.-]){5}[0-9A-Fa-f]{2}$'
|
||||
"""Matches MAC addresses with ``:``, ``-``, or ``.`` separators."""
|
||||
|
||||
BOOL_ENUM_VALUES = (
|
||||
'', ' ', '+', '-', '0', '1',
|
||||
'true', 'false', 'yes', 'no', 't', 'f',
|
||||
)
|
||||
"""String values considered boolean / binary indicators."""
|
||||
|
||||
LAT_LON_NAMES = ('lat', 'latitude', 'lon', 'lng', 'longitude')
|
||||
|
||||
# Name-to-type heuristics (checked after config override, before values)
|
||||
_NAME_TYPE_MAP: list[tuple[re.Pattern, DataType]] = [
|
||||
# User column names - EXACT match only (no generic fuzzy patterns)
|
||||
# IPv4 columns
|
||||
(re.compile(r'^:ips$|^:ipd$|^server.ip$|^client.ip$', re.I), DataType.IPv4),
|
||||
# ENUM columns
|
||||
(re.compile(r'^:prs$|^:prd$', re.I), DataType.ENUM),
|
||||
(re.compile(r'^(?:cnrs|isrs)$', re.I), DataType.BOOL_ENUM),
|
||||
(re.compile(r'^(?:scnt|dcnt|1ipp|4dbn|4srs|eiph)$', re.I), DataType.ENUM),
|
||||
(re.compile(r'^0ver$', re.I), DataType.ENUM),
|
||||
# FLOAT columns
|
||||
(re.compile(r'^4dur$|^8ses$|^2tmo$|^0rnt$', re.I), DataType.FLOAT),
|
||||
# INT columns
|
||||
(re.compile(r'^4ksz$|^8ack$|^8ppk$|^8pak$|^8seq$|^8did$|^8byt$|^row$', re.I), DataType.INT),
|
||||
# HEX columns
|
||||
(re.compile(r'^0cph$|^0crv$', re.I), DataType.HEX),
|
||||
# TIMESTAMP columns
|
||||
(re.compile(r'^8dbd$|^time$|^timestamp$', re.I), DataType.TIMESTAMP),
|
||||
# Dot-notation suffixes for lat/lon (e.g. :ips.latd, :ipd.lond)
|
||||
(re.compile(r'\.latd$', re.I), DataType.LAT_LON),
|
||||
(re.compile(r'\.lond$', re.I), DataType.LAT_LON),
|
||||
# Dot-notation STRING suffixes (e.g. :ips.ispn, :ipd.city)
|
||||
(re.compile(r'\.(ispn|orgn|city|anon)$', re.I), DataType.STRING),
|
||||
# Standalone string columns - exact match
|
||||
(re.compile(r'^(?:cnam|snam|tabl|name|0rnd|source.node|ecdhe.named.curve|crcc|orga|orgu)$', re.I), DataType.STRING),
|
||||
# Special columns with @ prefix
|
||||
(re.compile(r'^@', re.I), DataType.STRING),
|
||||
]
|
||||
|
||||
# Mapping from config.yaml type strings to DataType
|
||||
_CONFIG_TYPE_MAP: dict[str, DataType] = {
|
||||
'int': DataType.INT,
|
||||
'float': DataType.FLOAT,
|
||||
'enum': DataType.ENUM,
|
||||
'hex': DataType.HEX,
|
||||
'url': DataType.URL,
|
||||
'ipv4': DataType.IPv4,
|
||||
'lat_lon': DataType.LAT_LON,
|
||||
'timestamp': DataType.TIMESTAMP,
|
||||
'bool_enum': DataType.BOOL_ENUM,
|
||||
'string': DataType.STRING,
|
||||
}
|
||||
|
||||
# Mapping from tls_ref_data ``data_type`` column values to DataType
|
||||
TLSDB_TYPE_MAP: dict[str, DataType] = {
|
||||
'IPv4': DataType.IPv4,
|
||||
'浮点': DataType.FLOAT,
|
||||
'字符串': DataType.STRING,
|
||||
'整数': DataType.INT,
|
||||
'2小写字母缩写': DataType.ENUM,
|
||||
'XX:YY.Z': DataType.STRING,
|
||||
'2字节': DataType.HEX,
|
||||
'2大写字母': DataType.STRING,
|
||||
'枚举': DataType.ENUM,
|
||||
'布尔': DataType.BOOL_ENUM,
|
||||
'字符串或整数': DataType.STRING,
|
||||
}
|
||||
|
||||
|
||||
def _tlsdb_type_to_datatype(tlsdb_type_str: str) -> DataType | None:
|
||||
"""Convert a TlsDB ``data_type`` string to a :class:`DataType`.
|
||||
|
||||
Returns ``None`` if the type string is unknown, allowing the
|
||||
caller to fall through to value-based detection.
|
||||
"""
|
||||
return TLSDB_TYPE_MAP.get(tlsdb_type_str)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Column classification (single column)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _name_to_type(name: str) -> DataType | None:
|
||||
"""Match column name against known heuristics.
|
||||
|
||||
Returns a :class:`DataType` if the name matches strongly, otherwise
|
||||
``None`` (fall through to value-based detection).
|
||||
"""
|
||||
for pattern, dtype in _NAME_TYPE_MAP:
|
||||
if pattern.search(name):
|
||||
return dtype
|
||||
return None
|
||||
|
||||
|
||||
def _is_port_value(v: str) -> bool:
|
||||
"""Check if *v* is an integer string in the registered port range (1-65535)."""
|
||||
try:
|
||||
n = int(v)
|
||||
return 1 <= n <= 65535
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def _can_float(v: str) -> bool:
|
||||
"""Return ``True`` when *v* can be parsed as a Python float."""
|
||||
try:
|
||||
float(v)
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def _values_to_type(series: pl.Series, col_name: str | None = None) -> DataType:
|
||||
"""Infer column type by sampling non-null values.
|
||||
|
||||
Detection order: IPv4 → MAC guard → HEX → URL → PORT → BOOL_ENUM
|
||||
→ LAT_LON → ENUM → FLOAT → STRING
|
||||
"""
|
||||
dtype = series.dtype
|
||||
|
||||
# Boolean dtype ──────────────────────────────────────────────────────
|
||||
if dtype == pl.Boolean:
|
||||
return DataType.BOOL_ENUM
|
||||
|
||||
# Numeric dtypes ─────────────────────────────────────────────────────
|
||||
if dtype in (pl.Float32, pl.Float64):
|
||||
# Check LAT_LON possibility before returning FLOAT
|
||||
non_null = series.drop_nulls()
|
||||
if len(non_null) > 0:
|
||||
sample = non_null.head(100)
|
||||
sample_strs = [str(v).strip().lower() for v in sample]
|
||||
non_blank = [v for v in sample_strs if v not in ('', ' ', '+', '-')]
|
||||
if non_blank:
|
||||
try:
|
||||
floats = [float(v) for v in non_blank]
|
||||
max_abs = max(abs(f) for f in floats)
|
||||
if (
|
||||
all(-180 <= f <= 180 for f in floats)
|
||||
and any('.' in v for v in non_blank)
|
||||
and max_abs > 10
|
||||
):
|
||||
return DataType.LAT_LON
|
||||
except ValueError:
|
||||
pass
|
||||
return DataType.FLOAT
|
||||
|
||||
if dtype in (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
|
||||
pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64):
|
||||
return DataType.INT
|
||||
|
||||
# String / categorical types ─────────────────────────────────────────
|
||||
non_null = series.drop_nulls()
|
||||
if len(non_null) == 0:
|
||||
return DataType.STRING
|
||||
|
||||
sample = non_null.head(100)
|
||||
# Convert all to string representations for pattern matching
|
||||
sample_strs = [str(v).strip().lower() for v in sample]
|
||||
|
||||
# ── 1. IPv4 detection ──────────────────────────────────────────────
|
||||
if all(re.match(IPv4_PATTERN, v) for v in sample_strs):
|
||||
if all(_valid_ip_octets(v) for v in sample_strs):
|
||||
return DataType.IPv4
|
||||
|
||||
# ── 2. MAC guard (skip HEX if values look like MAC addresses) ──────
|
||||
is_mac = all(re.match(MAC_PATTERN, v) for v in sample_strs)
|
||||
|
||||
# ── 3. HEX detection ───────────────────────────────────────────────
|
||||
if not is_mac and all(re.match(HEX_PATTERN, v) for v in sample_strs):
|
||||
return DataType.HEX
|
||||
|
||||
# ── 4. URL detection ───────────────────────────────────────────────
|
||||
if all(re.match(URL_PATTERN, v) for v in sample_strs):
|
||||
return DataType.URL
|
||||
|
||||
# ── 5. PORT detection → ENUM ───────────────────────────────────────
|
||||
if all(_is_port_value(v) for v in sample_strs):
|
||||
return DataType.ENUM
|
||||
|
||||
# ── 6. BOOL_ENUM detection ─────────────────────────────────────────
|
||||
if all(v in BOOL_ENUM_VALUES for v in sample_strs):
|
||||
return DataType.BOOL_ENUM
|
||||
|
||||
# ── 7. LAT_LON detection (value-based) ──────────────────────────────
|
||||
non_blank = [v for v in sample_strs if v not in ('', ' ', '+', '-')]
|
||||
if non_blank:
|
||||
try:
|
||||
floats = [float(v) for v in non_blank]
|
||||
max_abs = max(abs(f) for f in floats)
|
||||
if (
|
||||
all(-180 <= f <= 180 for f in floats)
|
||||
and any('.' in v for v in non_blank)
|
||||
and max_abs > 10
|
||||
):
|
||||
return DataType.LAT_LON
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# ── 8. ENUM detection (few unique values) ──────────────────────────
|
||||
unique_count = non_null.n_unique()
|
||||
if unique_count < 20:
|
||||
# Safety: if >80 % of the sample values are unique strings,
|
||||
# the column is high-cardinality — classify as STRING, not ENUM.
|
||||
sample_unique_count = len(set(sample_strs))
|
||||
if sample_unique_count > 0.8 * len(sample_strs):
|
||||
return DataType.STRING
|
||||
return DataType.ENUM
|
||||
|
||||
# ── 9. TLS version hex detection (before STRING fallback) ──────────
|
||||
if col_name is not None and col_name.lower() in ('0ver', '_tlsver', 'tls_version', 'version'):
|
||||
sample_hex = sample_strs[0].replace(' ', '').strip()
|
||||
if sample_hex.upper() in TLS_HEX_MAP:
|
||||
return DataType.ENUM
|
||||
|
||||
# ── 10. FLOAT detection (>80 % can float) ──────────────────────────
|
||||
# Exclude strings starting with '+' from FLOAT detection — these are
|
||||
# text markers (phone codes, IDs, error codes), not real numbers.
|
||||
if any(v.startswith('+') for v in sample_strs):
|
||||
return DataType.STRING
|
||||
|
||||
float_count = sum(1 for v in sample_strs if _can_float(v))
|
||||
if float_count / len(sample_strs) > 0.8:
|
||||
# Guard: values with leading "-" that happen to parse as float
|
||||
# should not force FLOAT when non-float values also exist.
|
||||
if any(v.startswith('-') for v in sample_strs):
|
||||
if float_count < len(sample_strs):
|
||||
return DataType.STRING
|
||||
return DataType.FLOAT
|
||||
|
||||
return DataType.STRING
|
||||
|
||||
|
||||
def _valid_ip_octets(ip_str: str) -> bool:
|
||||
"""Check that each octet in an IPv4 string is 0-255."""
|
||||
parts = ip_str.split('.')
|
||||
if len(parts) != 4:
|
||||
return False
|
||||
for p in parts:
|
||||
try:
|
||||
n = int(p)
|
||||
if n < 0 or n > 255:
|
||||
return False
|
||||
except ValueError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def classify_column(
|
||||
name: str,
|
||||
series: pl.Series,
|
||||
config_type: str | None = None,
|
||||
) -> DataType:
|
||||
"""Classify a single column based on its *name* and *series* values.
|
||||
|
||||
Priority
|
||||
--------
|
||||
1. **config_type** — user override from ``config.yaml``.
|
||||
If provided (e.g. ``'ipv4'``), returns the corresponding
|
||||
:class:`DataType` immediately.
|
||||
2. **tls_ref_data** — authoritative type from ``TlsDB.csv`` /
|
||||
``tls_ref_data`` table. Looked up by column name; if found,
|
||||
the declared type is used directly (no value-based guessing).
|
||||
3. **Value-based** pattern matching — samples up to 100 non-null
|
||||
values and tests against IPv4 → HEX → URL → PORT → BOOL_ENUM
|
||||
→ LAT_LON → ENUM → FLOAT in order.
|
||||
4. **Name-based** heuristics — only consulted when value-based
|
||||
detection returned ``STRING`` (i.e. no value pattern matched).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
name:
|
||||
Column name (used for name-based heuristics and tls_ref_data
|
||||
lookup).
|
||||
series:
|
||||
Polars :class:`Series` containing the column's data.
|
||||
config_type:
|
||||
Optional user override (case-insensitive, matched to
|
||||
:data:`_CONFIG_TYPE_MAP`).
|
||||
|
||||
Returns
|
||||
-------
|
||||
DataType
|
||||
"""
|
||||
# 1. Config override (highest priority)
|
||||
if config_type is not None:
|
||||
mapped = _CONFIG_TYPE_MAP.get(config_type.lower().strip())
|
||||
if mapped is not None:
|
||||
return mapped
|
||||
|
||||
# 2. tls_ref_data lookup (authoritative type from TlsDB)
|
||||
tls_ref = get_tls_ref_dict()
|
||||
if name in tls_ref:
|
||||
dtype = _tlsdb_type_to_datatype(tls_ref[name]['data_type'])
|
||||
if dtype is not None:
|
||||
return dtype
|
||||
|
||||
# 3. Value-based sampling
|
||||
result = _values_to_type(series, name)
|
||||
|
||||
# 4. Name-based heuristics (only when values returned STRING)
|
||||
if result == DataType.STRING:
|
||||
name_match = _name_to_type(name)
|
||||
if name_match is not None:
|
||||
return name_match
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Schema classification (all columns)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def classify_schema(
|
||||
lf: pl.LazyFrame,
|
||||
config_overrides: dict[str, str] | None = None,
|
||||
) -> dict[str, DataType]:
|
||||
"""Classify all columns in a :class:`LazyFrame`.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
lf:
|
||||
A Polars :class:`LazyFrame` whose columns to classify.
|
||||
config_overrides:
|
||||
Optional mapping of ``{col_name: type_string}`` from a
|
||||
``config.yaml`` ``columns:`` section. These take highest
|
||||
priority (see :func:`classify_column`).
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict[str, DataType]
|
||||
Mapping from column name to inferred :class:`DataType`.
|
||||
"""
|
||||
config_overrides = config_overrides or {}
|
||||
df_sample = lf.collect(streaming=True)
|
||||
result: dict[str, DataType] = {}
|
||||
for col_name in df_sample.columns:
|
||||
config_type = config_overrides.get(col_name)
|
||||
result[col_name] = classify_column(
|
||||
name=col_name,
|
||||
series=df_sample[col_name],
|
||||
config_type=config_type,
|
||||
)
|
||||
return result
|
||||
"""Re-export from new types module for backward compatibility."""
|
||||
from analysis.types import ColumnType, TYPE_REGISTRY, TYPE_ALIASES, get_type, classify_column, classify_schema
|
||||
TLS_HEX_MAP = {"0303": "TLSv1.2", "0304": "TLSv1.3", "0302": "TLSv1.1", "0301": "TLSv1.0"}
|
||||
|
||||
@@ -0,0 +1,300 @@
|
||||
"""天璇类型系统 — 仅8种固定类型，面向对象接口。
|
||||
|
||||
每个类型继承 :class:`ColumnType`，实现:
|
||||
- ``distance()`` — 为该类型列计算距离矩阵（追加到 LazyFrame）
|
||||
- ``display()`` — 将值格式化为可读字符串
|
||||
- ``polars_dtype`` — 映射到 Polars 数据类型
|
||||
- ``name`` — 中文名称
|
||||
|
||||
:data:`TYPE_REGISTRY` 是唯一的类型注册表，按名称索引。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# Base type — all 8 types inherit from this
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
class ColumnType(ABC):
|
||||
"""Base class for all column types.
|
||||
|
||||
Subclasses MUST define:
|
||||
- ``name`` (class attribute, Chinese)
|
||||
- ``polars_dtype`` (class attribute)
|
||||
- ``distance()``
|
||||
- ``display()``
|
||||
"""
|
||||
|
||||
name: str = ""
|
||||
polars_dtype: pl.DataType = pl.Utf8
|
||||
|
||||
@abstractmethod
|
||||
def distance(self, lf: pl.LazyFrame, columns: list[str]) -> pl.LazyFrame:
|
||||
"""Compute per-row distances for *columns* of this type.
|
||||
|
||||
Returns a new LazyFrame with distance columns appended
|
||||
(prefixed ``_dist_``).
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def display(self, value: Any) -> str:
|
||||
"""Format *value* for human-readable display."""
|
||||
...
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# 8 concrete types
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
class IPv4Type(ColumnType):
|
||||
name = "IPv4"
|
||||
polars_dtype = pl.Utf8
|
||||
|
||||
def distance(self, lf: pl.LazyFrame, columns: list[str]) -> pl.LazyFrame:
|
||||
if len(columns) < 2:
|
||||
return lf
|
||||
from analysis.distance import ip_distance
|
||||
return ip_distance(lf, columns)
|
||||
|
||||
def display(self, value: Any) -> str:
|
||||
return str(value) if value else "-"
|
||||
|
||||
|
||||
class StringType(ColumnType):
|
||||
name = "字符串"
|
||||
polars_dtype = pl.Utf8
|
||||
|
||||
def distance(self, lf: pl.LazyFrame, columns: list[str]) -> pl.LazyFrame:
|
||||
from analysis.distance import string_distance
|
||||
return string_distance(lf, columns)
|
||||
|
||||
def display(self, value: Any) -> str:
|
||||
return str(value)[:80] if value else "-"
|
||||
|
||||
|
||||
class EnumType(ColumnType):
|
||||
name = "枚举"
|
||||
polars_dtype = pl.Utf8
|
||||
|
||||
def distance(self, lf: pl.LazyFrame, columns: list[str]) -> pl.LazyFrame:
|
||||
# 0-1 match distance: compare to mode
|
||||
for col in columns:
|
||||
mode_val = lf.select(pl.col(col).mode().first()).collect(streaming=True).item()
|
||||
lf = lf.with_columns(
|
||||
(pl.col(col) != mode_val).cast(pl.Float64).alias(f'_enum_dist_{col}')
|
||||
)
|
||||
return lf
|
||||
|
||||
def display(self, value: Any) -> str:
|
||||
return str(value) if value else "-"
|
||||
|
||||
|
||||
class FloatType(ColumnType):
|
||||
name = "浮点"
|
||||
polars_dtype = pl.Float64
|
||||
|
||||
def distance(self, lf: pl.LazyFrame, columns: list[str]) -> pl.LazyFrame:
|
||||
# Normalised euclidean distance — handled in preprocessing pipeline
|
||||
for col in columns:
|
||||
lf = lf.with_columns(
|
||||
pl.col(col).cast(pl.Float64).alias(f'_raw_{col}')
|
||||
)
|
||||
return lf
|
||||
|
||||
def display(self, value: Any) -> str:
|
||||
try:
|
||||
return f"{float(value):.2f}"
|
||||
except (ValueError, TypeError):
|
||||
return str(value)
|
||||
|
||||
|
||||
class IntType(ColumnType):
|
||||
name = "整数"
|
||||
polars_dtype = pl.Int64
|
||||
|
||||
def distance(self, lf: pl.LazyFrame, columns: list[str]) -> pl.LazyFrame:
|
||||
# Normalised manhattan distance — handled in preprocessing pipeline
|
||||
for col in columns:
|
||||
lf = lf.with_columns(
|
||||
pl.col(col).cast(pl.Int64).alias(f'_raw_{col}')
|
||||
)
|
||||
return lf
|
||||
|
||||
def display(self, value: Any) -> str:
|
||||
try:
|
||||
return str(int(value))
|
||||
except (ValueError, TypeError):
|
||||
return str(value)
|
||||
|
||||
|
||||
class BoolType(ColumnType):
|
||||
name = "布尔"
|
||||
polars_dtype = pl.Boolean
|
||||
|
||||
def distance(self, lf: pl.LazyFrame, columns: list[str]) -> pl.LazyFrame:
|
||||
# 0-1 match distance
|
||||
for col in columns:
|
||||
lf = lf.with_columns(
|
||||
pl.col(col).cast(pl.Float64).alias(f'_bool_dist_{col}')
|
||||
)
|
||||
return lf
|
||||
|
||||
def display(self, value: Any) -> str:
|
||||
if value is None:
|
||||
return "-"
|
||||
return "是" if str(value).lower() in ('true', '1', '+', 't') else "否"
|
||||
|
||||
|
||||
class BytesType(ColumnType):
|
||||
name = "字节"
|
||||
polars_dtype = pl.Utf8
|
||||
|
||||
def distance(self, lf: pl.LazyFrame, columns: list[str]) -> pl.LazyFrame:
|
||||
from analysis.distance import bytes_distance
|
||||
return bytes_distance(lf, columns)
|
||||
|
||||
def display(self, value: Any) -> str:
|
||||
s = str(value).strip() if value else ""
|
||||
if len(s) > 20:
|
||||
return s[:20] + "..."
|
||||
return s or "-"
|
||||
|
||||
|
||||
class TimestampType(ColumnType):
|
||||
name = "时间戳"
|
||||
polars_dtype = pl.Float64
|
||||
|
||||
def distance(self, lf: pl.LazyFrame, columns: list[str]) -> pl.LazyFrame:
|
||||
from analysis.distance import timestamp_fft_distance
|
||||
return timestamp_fft_distance(lf, columns)
|
||||
|
||||
def display(self, value: Any) -> str:
|
||||
try:
|
||||
t = float(value)
|
||||
if t < 1e8 or t > 2e12:
|
||||
return str(value)
|
||||
from datetime import datetime
|
||||
return datetime.fromtimestamp(t).strftime("%Y-%m-%d %H:%M")
|
||||
except (ValueError, TypeError, OSError):
|
||||
return str(value) if value else "-"
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# Registry — single source of truth
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
TYPE_REGISTRY: dict[str, ColumnType] = {
|
||||
"枚举": EnumType(),
|
||||
"字节": BytesType(),
|
||||
"浮点": FloatType(),
|
||||
"整数": IntType(),
|
||||
"布尔": BoolType(),
|
||||
"字符串": StringType(),
|
||||
"IPv4": IPv4Type(),
|
||||
"时间戳": TimestampType(),
|
||||
}
|
||||
|
||||
# Alias map for config.yaml type strings
|
||||
TYPE_ALIASES: dict[str, str] = {
|
||||
'enum': '枚举', '枚举': '枚举',
|
||||
'bytes': '字节', '字节': '字节', 'hex': '字节',
|
||||
'float': '浮点', '浮点': '浮点',
|
||||
'int': '整数', '整数': '整数',
|
||||
'bool': '布尔', '布尔': '布尔', 'boolean': '布尔',
|
||||
'string': '字符串', '字符串': '字符串', 'str': '字符串',
|
||||
'ipv4': 'IPv4', 'IPv4': 'IPv4',
|
||||
'timestamp': '时间戳', '时间戳': '时间戳',
|
||||
}
|
||||
|
||||
|
||||
def get_type(name: str) -> ColumnType:
|
||||
"""Resolve a type name (Chinese or English alias) to a ColumnType."""
|
||||
resolved = TYPE_ALIASES.get(name.strip().lower() if name.isascii() else name.strip())
|
||||
if resolved is None:
|
||||
return StringType() # fallback
|
||||
return TYPE_REGISTRY[resolved]
|
||||
|
||||
|
||||
def classify_column(name: str, series: pl.Series, config_type: str | None = None) -> ColumnType:
|
||||
"""Classify a single column to a ColumnType.
|
||||
|
||||
Priority: config override → TlsDB lookup → name heuristics → value sampling.
|
||||
"""
|
||||
from analysis.tls_ref import get_tls_ref_dict
|
||||
import re
|
||||
|
||||
# 1. Config override
|
||||
if config_type is not None:
|
||||
t = get_type(config_type)
|
||||
if t.name != "字符串" or config_type.lower().strip() == 'string':
|
||||
return t
|
||||
|
||||
# 2. TlsDB lookup
|
||||
tls_ref = get_tls_ref_dict()
|
||||
if name in tls_ref:
|
||||
t = get_type(tls_ref[name]['data_type'])
|
||||
return t
|
||||
|
||||
# 3. Name heuristics
|
||||
_patterns: list[tuple[re.Pattern, str]] = [
|
||||
(re.compile(r'^(?:src_ip|dst_ip|:ips|:ipd|server.ip|client.ip)$', re.I), 'IPv4'),
|
||||
(re.compile(r'^(?::prs|:prd|scnt|dcnt|1ipp|4dbn|4srs|eiph|name|tabl|0ver|cnrs|isrs)$', re.I), '枚举'),
|
||||
(re.compile(r'^(?:0cph|0crv|cipher|ecdhe)', re.I), '字节'),
|
||||
(re.compile(r'^(?:4dur|8ses|2tmo|0rnt)$', re.I), '浮点'),
|
||||
(re.compile(r'^(?:4ksz|8ack|8ppk|8pak|8seq|8did|8byt|row)$', re.I), '整数'),
|
||||
(re.compile(r'^(?:timestamp|time|8dbd|created|updated)', re.I), '时间戳'),
|
||||
(re.compile(r'\.(ispn|orgn|city|anon|doma|latd|lond)$', re.I), '字符串'),
|
||||
(re.compile(r'^(?:cnam|snam|0rnd|source.node|crcc|orga|orgu|@)', re.I), '字符串'),
|
||||
]
|
||||
for pat, type_name in _patterns:
|
||||
if pat.search(name):
|
||||
return TYPE_REGISTRY[type_name]
|
||||
|
||||
# 4. Value-based sampling
|
||||
non_null = series.drop_nulls()
|
||||
if len(non_null) == 0:
|
||||
return TYPE_REGISTRY["字符串"]
|
||||
|
||||
strs = [str(v).strip().lower() for v in non_null.head(100)]
|
||||
ip4 = re.compile(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$')
|
||||
hex_p = re.compile(r'^[0-9a-f]{2}(?:\s[0-9a-f]{2})*$')
|
||||
|
||||
if all(ip4.match(v) for v in strs):
|
||||
return TYPE_REGISTRY["IPv4"]
|
||||
if all(hex_p.match(v) for v in strs):
|
||||
return TYPE_REGISTRY["字节"]
|
||||
if series.dtype.is_integer():
|
||||
return TYPE_REGISTRY["整数"]
|
||||
if series.dtype == pl.Float64:
|
||||
return TYPE_REGISTRY["浮点"]
|
||||
if series.dtype == pl.Boolean:
|
||||
return TYPE_REGISTRY["布尔"]
|
||||
if non_null.n_unique() < 50:
|
||||
return TYPE_REGISTRY["枚举"]
|
||||
|
||||
return TYPE_REGISTRY["字符串"]
|
||||
|
||||
|
||||
def classify_schema(
|
||||
lf: pl.LazyFrame,
|
||||
config_overrides: dict[str, str] | None = None,
|
||||
) -> dict[str, ColumnType]:
|
||||
"""Classify all columns in a LazyFrame."""
|
||||
config_overrides = config_overrides or {}
|
||||
df_sample = lf.collect(streaming=True)
|
||||
result = {}
|
||||
for col_name in df_sample.columns:
|
||||
config_type = config_overrides.get(col_name)
|
||||
result[col_name] = classify_column(
|
||||
name=col_name,
|
||||
series=df_sample[col_name],
|
||||
config_type=config_type,
|
||||
)
|
||||
return result
|
||||
+2
-1
@@ -15,7 +15,7 @@ urlpatterns = [
|
||||
path('runs/<int:display_id>/', views.run_detail, name='run_detail'),
|
||||
path('runs/<int:display_id>/status/', views.run_status_api, name='run_status'),
|
||||
path('clusters/<int:display_id>/', views.cluster_overview, name='cluster_overview'),
|
||||
path('clusters/<int:display_id>/<int:cluster_label>/', views.cluster_detail, name='cluster_detail'),
|
||||
path('clusters/<int:display_id>/<path:cluster_label>/', views.cluster_detail, name='cluster_detail'),
|
||||
path('entities/<int:entity_id>/', views.entity_profile, name='entity_profile'),
|
||||
path('config/', views.config_view, name='config'),
|
||||
path('api/llm/test/', views.llm_test, name='llm_test'),
|
||||
@@ -25,6 +25,7 @@ urlpatterns = [
|
||||
path('logs/', views.log_viewer, name='log_viewer'),
|
||||
path('globe/', views.globe_view, name='globe'),
|
||||
path('tools/run/', views.tool_lab_run, name='tool_lab_run'),
|
||||
path('tools/reload-run/', views.reload_run_data, name='reload_run_data'),
|
||||
path('tools/plan/', views.tool_plan, name='tool_plan'),
|
||||
path('tools/filter/', views.apply_filter, name='apply_filter'),
|
||||
]
|
||||
|
||||
-1935
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,13 @@
|
||||
"""Analysis views package — re-exports all public view functions."""
|
||||
from .helpers import _extract_lat, _extract_lon
|
||||
from .dashboard import dashboard, run_list, run_detail, run_status_api, start_analysis
|
||||
from .pipeline import _run_pipeline_worker, _background_process
|
||||
from .clustering import cluster_overview, cluster_detail, _run_clustering_pipeline, _get_globe_flows
|
||||
from .entity import entity_profile
|
||||
from .upload import upload_page, upload_csv, upload_csv_batch, finalize_upload, delete_upload
|
||||
from .manual import manual_page, manual_run_analysis
|
||||
from .auto import auto_page, run_llm_analysis_view
|
||||
from .globe import globe_view, _extract_flows_from_df
|
||||
from .config import config_view, llm_test
|
||||
from .log_viewer import log_viewer
|
||||
from .tools import tool_lab, tool_lab_run, tool_plan, apply_filter, reload_run_data, retry_run
|
||||
@@ -0,0 +1,348 @@
|
||||
"""LLM auto analysis views."""
|
||||
import json
|
||||
import asyncio
|
||||
import threading
|
||||
import traceback
|
||||
import logging
|
||||
|
||||
from django.shortcuts import render, get_object_or_404
|
||||
from django.http import JsonResponse
|
||||
from django.views.decorators.csrf import csrf_exempt
|
||||
|
||||
from config import get_config
|
||||
from analysis.models import AnalysisRun
|
||||
from .pipeline import _run_pipeline_worker
|
||||
from analysis.services.clustering import run_clustering_pipeline
|
||||
from .helpers import _PLANS_DIR, _add_to_auto_index
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def auto_page(request):
|
||||
"""LLM-driven auto analysis page."""
|
||||
from analysis.session_store import SessionStore
|
||||
import json as _json
|
||||
|
||||
cfg = get_config()
|
||||
|
||||
# Get datasets from SessionStore + ALL runs from DB
|
||||
store = SessionStore()
|
||||
session_datasets = store.list_datasets() if hasattr(store, 'list_datasets') else []
|
||||
session_ds_map = {ds['dataset_id']: ds for ds in session_datasets}
|
||||
|
||||
# Build dataset entries from ALL AnalysisRun records
|
||||
all_runs = AnalysisRun.objects.all().order_by('-created_at')[:100]
|
||||
datasets_with_status = []
|
||||
|
||||
for run in all_runs:
|
||||
if run.run_type == 'upload':
|
||||
ds_id = f'upload_{run.display_id}'
|
||||
else:
|
||||
ds_id = f'run_{run.display_id}'
|
||||
|
||||
session_entry = session_ds_map.get(ds_id)
|
||||
|
||||
if session_entry:
|
||||
datasets_with_status.append({
|
||||
**session_entry,
|
||||
'dataset_id': ds_id,
|
||||
'run_status': run.status,
|
||||
'total_flows': run.total_flows,
|
||||
'display_id': run.display_id,
|
||||
'needs_reload': False,
|
||||
'csv_glob': run.csv_glob or '',
|
||||
'sqlite_table': run.sqlite_table or '',
|
||||
})
|
||||
elif run.csv_glob or run.sqlite_table:
|
||||
datasets_with_status.append({
|
||||
'dataset_id': ds_id,
|
||||
'run_status': run.status,
|
||||
'display_id': run.display_id,
|
||||
'total_flows': run.total_flows,
|
||||
'row_count': run.total_flows,
|
||||
'file_count': None,
|
||||
'columns': [],
|
||||
'column_count': 0,
|
||||
'svd_components': 0,
|
||||
'needs_reload': True,
|
||||
'csv_glob': run.csv_glob or '',
|
||||
'sqlite_table': run.sqlite_table or '',
|
||||
})
|
||||
|
||||
for ds_info in datasets_with_status:
|
||||
s = ds_info['run_status']
|
||||
if s in ('completed',):
|
||||
ds_info['status_label'] = '已分析'
|
||||
ds_info['badge_class'] = 'completed'
|
||||
elif s in ('failed',):
|
||||
ds_info['status_label'] = '错误'
|
||||
ds_info['badge_class'] = 'failed'
|
||||
elif s in ('pending', 'loading', 'profiling', 'aggregating', 'clustering', 'extracting'):
|
||||
ds_info['status_label'] = '分析中'
|
||||
ds_info['badge_class'] = 'analyzing'
|
||||
else:
|
||||
ds_info['status_label'] = '待分析'
|
||||
ds_info['badge_class'] = 'ready'
|
||||
|
||||
return render(request, 'tianxuan/auto.html', {
|
||||
'llm_configured': bool(cfg.llm.base_url and cfg.llm.api_key),
|
||||
'datasets_with_status_json': _json.dumps(datasets_with_status, ensure_ascii=False),
|
||||
})
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
def run_llm_analysis_view(request):
|
||||
"""AJAX POST: run LLM-driven analysis pipeline."""
|
||||
if request.method != 'POST':
|
||||
return JsonResponse({'error': 'POST required'}, status=405)
|
||||
body = json.loads(request.body)
|
||||
display_id = body.get('run_id')
|
||||
run = get_object_or_404(AnalysisRun, display_id=display_id)
|
||||
pk = run.id
|
||||
run.run_type = 'auto'
|
||||
run.save(update_fields=['run_type'])
|
||||
head = body.get('head')
|
||||
|
||||
from analysis.session_store import SessionStore
|
||||
store = SessionStore()
|
||||
|
||||
# Determine dataset key format (consistent with manual_page)
|
||||
upload_ds_id = f'upload_{display_id}'
|
||||
run_ds_id = f'run_{display_id}'
|
||||
entry = store.get_dataset(upload_ds_id)
|
||||
if entry is None:
|
||||
entry = store.get_dataset(run_ds_id)
|
||||
|
||||
if entry is None:
|
||||
# Try primary key fallback
|
||||
upload_pk_id = f'upload_{pk}'
|
||||
entity_pk_id = f'entity_{pk}'
|
||||
entry = store.get_dataset(upload_pk_id)
|
||||
if entry is None:
|
||||
entry = store.get_dataset(entity_pk_id)
|
||||
|
||||
if entry is None:
|
||||
# Auto-reload from sqlite_table or csv_glob
|
||||
from analysis.data_loader import load_csv_directory, load_from_db
|
||||
lf = None
|
||||
schema = {}
|
||||
if run.sqlite_table:
|
||||
try:
|
||||
lf = load_from_db(run.sqlite_table)
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
pass
|
||||
if lf is None and run.csv_glob:
|
||||
try:
|
||||
lf, schema, _, _, _ = load_csv_directory(run.csv_glob)
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
pass
|
||||
if lf is None:
|
||||
return JsonResponse({'error': '数据集不存在，请先上传并等待预处理完成'})
|
||||
entry_key = upload_ds_id if run.run_type == 'upload' else run_ds_id
|
||||
store.store_dataset(entry_key, lf, schema=schema, metadata={
|
||||
'row_count': run.total_flows,
|
||||
'csv_glob': run.csv_glob or '',
|
||||
'sqlite_table': run.sqlite_table or '',
|
||||
})
|
||||
# Re-fetch entry
|
||||
entry = store.get_dataset(entry_key)
|
||||
|
||||
# Determine the active dataset_id
|
||||
for candidate in [upload_ds_id, run_ds_id, f'upload_{pk}', f'entity_{pk}']:
|
||||
if store.get_dataset(candidate):
|
||||
dataset_id = candidate
|
||||
break
|
||||
else:
|
||||
return JsonResponse({'error': '数据集不存在，请先上传并等待预处理完成'})
|
||||
|
||||
# ── Optional: apply filter before LLM pipeline ──
|
||||
filters = body.get('filters', [])
|
||||
if filters and len(filters) > 0:
|
||||
logic = body.get('logic', 'and')
|
||||
try:
|
||||
from analysis.tool_registry import _handle_filter_data
|
||||
filter_result = asyncio.run(_handle_filter_data(
|
||||
dataset_id=dataset_id,
|
||||
filters=filters,
|
||||
logic=logic,
|
||||
))
|
||||
if 'error' not in filter_result:
|
||||
filtered_id = filter_result.get('filtered_dataset_id', '')
|
||||
if filtered_id and store.get_dataset(filtered_id):
|
||||
dataset_id = filtered_id
|
||||
except Exception:
|
||||
logger.warning(f'[LLM auto] Filter failed (non-fatal): {traceback.format_exc()}')
|
||||
|
||||
def _analysis_fn(run, ctx):
|
||||
from tianxuan.llm_orchestrator import run_llm_pipeline, LLMConfig
|
||||
dataset_id = ctx['dataset_id']
|
||||
pk = ctx['pk']
|
||||
|
||||
def progress_callback(step, tool_name, result=None, meta=None):
|
||||
nonlocal run
|
||||
try:
|
||||
run = AnalysisRun.objects.get(id=pk)
|
||||
pct = min(10 + step * 6, 95)
|
||||
run.progress_pct = pct
|
||||
run.progress_msg = f'步骤 {step}: {tool_name}'
|
||||
|
||||
save_fields = ['progress_pct', 'progress_msg']
|
||||
if tool_name == 'thinking':
|
||||
# Accumulate LLM thinking text (appended to llm_thinking)
|
||||
thought = (result or '').strip()
|
||||
if not thought:
|
||||
thought = f'步骤 {step} 分析中...'
|
||||
run.llm_thinking = (run.llm_thinking or '') + f'\n[{step}] {thought}'
|
||||
save_fields = ['progress_pct', 'progress_msg', 'llm_thinking']
|
||||
elif tool_name == 'complete':
|
||||
pass # No extra tracking needed
|
||||
else:
|
||||
# Single tool call — always has result
|
||||
if isinstance(result, dict):
|
||||
summary = json.dumps(result, default=str, ensure_ascii=False)[:500]
|
||||
else:
|
||||
summary = str(result)[:500]
|
||||
run.run_log += f'[{step}] {tool_name}: {summary}...\n'
|
||||
|
||||
# One entry per tool call
|
||||
if meta and isinstance(meta, dict) and 'args' in meta:
|
||||
tool_calls = list(run.tool_calls_json or [])
|
||||
tool_calls.append({
|
||||
'step': step,
|
||||
'name': tool_name,
|
||||
'input': meta['args'],
|
||||
'output': result,
|
||||
})
|
||||
run.tool_calls_json = tool_calls
|
||||
save_fields = ['progress_pct', 'progress_msg', 'run_log', 'tool_calls_json']
|
||||
|
||||
run.save(update_fields=save_fields)
|
||||
except Exception as e:
|
||||
logger.warning('progress_callback error: %s', e)
|
||||
|
||||
try:
|
||||
from config import get_config
|
||||
cfg = get_config()
|
||||
llm_cfg = LLMConfig(cfg.llm.base_url, cfg.llm.api_key, cfg.llm.model)
|
||||
run.run_log = '启动 LLM 分析...\n'
|
||||
run.status = 'loading'
|
||||
run.progress_pct = 5
|
||||
run.progress_msg = 'LLM 初始化中...'
|
||||
run.save(update_fields=['status', 'progress_pct', 'progress_msg', 'run_log'])
|
||||
|
||||
result = run_llm_pipeline(dataset_id, config=llm_cfg, callback=progress_callback)
|
||||
|
||||
if 'error' in result:
|
||||
run.status = 'failed'
|
||||
run.progress_msg = f'失败: {result["error"]}'
|
||||
run.run_log += f'\n[ERROR] {result["error"]}'
|
||||
run.save(update_fields=['status', 'progress_pct', 'progress_msg', 'run_log'])
|
||||
else:
|
||||
run.progress_pct = 55
|
||||
run.progress_msg = f'LLM分析完成: {result.get("result", "")[:200]}'
|
||||
run.run_log += f'\n[完成] {result.get("result", "")}\n[INFO] 开始自动聚类流程...'
|
||||
run.save(update_fields=['progress_pct', 'progress_msg', 'run_log'])
|
||||
|
||||
# ── Auto-run clustering pipeline after LLM analysis ──
|
||||
entity_ds_id = None
|
||||
tool_calls = run.tool_calls_json or []
|
||||
for tc in tool_calls:
|
||||
tool_name = tc.get('name', '')
|
||||
tool_input = tc.get('input', {})
|
||||
if tool_name in ('build_entity_profiles', 'run_clustering'):
|
||||
entity_ds_id = tool_input.get('dataset_id', '')
|
||||
if entity_ds_id and store.get_dataset(entity_ds_id):
|
||||
break
|
||||
entity_ds_id = None
|
||||
|
||||
# Fallback: try common dataset ID patterns
|
||||
if not entity_ds_id:
|
||||
candidates = [
|
||||
ctx.get('dataset_id', ''),
|
||||
f'entity_{pk}',
|
||||
f'entity_{run.display_id}',
|
||||
f'upload_{run.display_id}',
|
||||
]
|
||||
for cid in candidates:
|
||||
if cid and store.get_dataset(cid):
|
||||
entity_ds_id = cid
|
||||
break
|
||||
|
||||
if entity_ds_id:
|
||||
run_clustering_pipeline(
|
||||
run=run,
|
||||
store=store,
|
||||
ds_id=entity_ds_id,
|
||||
feature_columns=None,
|
||||
algorithm='agglomerative',
|
||||
min_cluster_size=5,
|
||||
run_umap=True,
|
||||
head=ctx.get('head'),
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
f'[LLM auto] No entity dataset found for clustering. '
|
||||
f'Tool calls: {[tc.get("name") for tc in tool_calls]}. '
|
||||
f'Marking run as completed without clustering.'
|
||||
)
|
||||
run.status = 'completed'
|
||||
run.progress_pct = 100
|
||||
run.progress_msg = 'LLM分析完成（无实体数据，跳过聚类）'
|
||||
run.save(update_fields=['status', 'progress_pct', 'progress_msg'])
|
||||
|
||||
# ── Auto-save workflow as loadable plan ──
|
||||
try:
|
||||
tool_calls_data = run.tool_calls_json or []
|
||||
steps = []
|
||||
for tc in tool_calls_data:
|
||||
tc_name = tc.get('name', '')
|
||||
if tc_name == 'thinking':
|
||||
continue
|
||||
steps.append({
|
||||
'tool_name': tc_name,
|
||||
'params': tc.get('input', {}),
|
||||
})
|
||||
if steps:
|
||||
plan_data = {
|
||||
'name': f'_auto_{run.display_id}',
|
||||
'steps': steps,
|
||||
'auto': True,
|
||||
'display_id': run.display_id,
|
||||
'created_at': str(run.created_at) if run.created_at else '',
|
||||
}
|
||||
plan_path = _PLANS_DIR / f'_auto_{run.display_id}.json'
|
||||
plan_path.write_text(
|
||||
json.dumps(plan_data, ensure_ascii=False, indent=2),
|
||||
encoding='utf-8',
|
||||
)
|
||||
_add_to_auto_index(f'_auto_{run.display_id}.json')
|
||||
logger.info(
|
||||
'[LLM auto] Saved auto plan %s (%d steps)',
|
||||
plan_path.name, len(steps),
|
||||
)
|
||||
except Exception as save_err:
|
||||
logger.warning('[LLM auto] Failed to save auto plan: %s', save_err)
|
||||
|
||||
except Exception:
|
||||
tb = traceback.format_exc()
|
||||
logger.error(tb)
|
||||
try:
|
||||
run = AnalysisRun.objects.get(id=pk)
|
||||
run.status = 'failed'
|
||||
run.progress_msg = f'失败: {tb}'
|
||||
run.run_log += f'\n[ERROR] {tb}'
|
||||
run.error_message = tb
|
||||
run.save(update_fields=['status', 'progress_msg', 'run_log', 'error_message'])
|
||||
except Exception as e:
|
||||
logger.warning('save failed after pipeline error: %s', e)
|
||||
|
||||
t = threading.Thread(
|
||||
target=_run_pipeline_worker,
|
||||
args=(pk, _analysis_fn),
|
||||
kwargs=dict(dataset_id=dataset_id, head=head, pk=pk),
|
||||
daemon=True,
|
||||
)
|
||||
t.start()
|
||||
return JsonResponse({'status': 'started', 'run_id': run.display_id})
|
||||
@@ -0,0 +1,228 @@
|
||||
"""Cluster views: overview, detail, globe flows, and the clustering pipeline."""
|
||||
import asyncio
|
||||
import json
|
||||
import traceback
|
||||
import logging
|
||||
|
||||
from django.shortcuts import render, get_object_or_404
|
||||
from django.http import Http404
|
||||
|
||||
from analysis.models import AnalysisRun, ClusterResult, EntityProfile
|
||||
from analysis import nl_describe
|
||||
from analysis.constants import RANDOM_SEED, UMAP_BATCH_SIZE, UMAP_TRAIN_SAMPLE
|
||||
from analysis.profile_util import profile
|
||||
|
||||
from .helpers import _extract_lat, _extract_lon
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def cluster_overview(request, display_id):
|
||||
"""Overview of all clusters for a run, with UMAP scatter data and geo scatter data."""
|
||||
run = get_object_or_404(AnalysisRun, display_id=display_id)
|
||||
clusters = run.clusters.exclude(cluster_label=-1).order_by('-size').prefetch_related('features')
|
||||
noise = run.clusters.filter(cluster_label=-1).first()
|
||||
|
||||
# Build UMAP scatter data for Canvas / Three.js 3D
|
||||
entities = run.entities.exclude(embedding_x=None).exclude(embedding_y=None).only(
|
||||
'embedding_x', 'embedding_y', 'embedding_z', 'cluster_label', 'entity_value'
|
||||
)
|
||||
has_z = any(e.embedding_z is not None for e in entities)
|
||||
scatter_data = [
|
||||
{'x': e.embedding_x, 'y': e.embedding_y, 'z': e.embedding_z if has_z else 0,
|
||||
'label': e.cluster_label if e.cluster_label is not None else -1, 'entity': e.entity_value}
|
||||
for e in entities
|
||||
]
|
||||
|
||||
# Noise cluster: features + entity detail
|
||||
noise_features = []
|
||||
noise_summary = ''
|
||||
noise_entity_count = 0
|
||||
if noise:
|
||||
noise_features_qs = noise.features.all().order_by('-distinguishing_score')[:15]
|
||||
noise_features = [
|
||||
{'feature_name': f.feature_name, 'score': f.distinguishing_score, 'mean': f.mean,
|
||||
'std': f.std, 'p25': f.p25, 'p75': f.p75}
|
||||
for f in noise_features_qs
|
||||
]
|
||||
if noise_features:
|
||||
noise_summary = nl_describe.describe_cluster(noise_features)
|
||||
noise_entity_count = noise.size
|
||||
|
||||
# LLM workflow timeline for auto runs
|
||||
llm_timeline_json = ''
|
||||
if run.run_type == 'auto':
|
||||
llm_thinking = run.llm_thinking or ''
|
||||
tool_calls = run.tool_calls_json or []
|
||||
if tool_calls or llm_thinking:
|
||||
llm_timeline_json = json.dumps({
|
||||
'thinking': llm_thinking,
|
||||
'tool_calls': tool_calls,
|
||||
})
|
||||
|
||||
# Build geo scatter data from feature_json (lat/lon)
|
||||
geo_data = []
|
||||
geo_skipped = 0
|
||||
for e in run.entities.all().only('feature_json', 'cluster_label', 'entity_value'):
|
||||
fj = e.feature_json
|
||||
if not fj:
|
||||
continue
|
||||
lat = _extract_lat(fj.get('avg_latitude'))
|
||||
lon = _extract_lon(fj.get('avg_longitude'))
|
||||
if lat is not None and lon is not None:
|
||||
geo_data.append({
|
||||
'x': lon,
|
||||
'y': lat,
|
||||
'label': e.cluster_label if e.cluster_label is not None else -1,
|
||||
'entity': e.entity_value,
|
||||
})
|
||||
else:
|
||||
geo_skipped += 1
|
||||
|
||||
# Cluster sizes for bar chart
|
||||
cluster_sizes = [{'label': c.cluster_label, 'size': c.size, 'silhouette': c.silhouette_score}
|
||||
for c in clusters]
|
||||
|
||||
# Top features per cluster from ClusterFeature + NL summaries
|
||||
top_features = {}
|
||||
for c in clusters:
|
||||
features_qs = c.features.all().order_by('-distinguishing_score')[:10]
|
||||
feats = [
|
||||
{'feature_name': f.feature_name, 'score': f.distinguishing_score, 'mean': f.mean,
|
||||
'std': f.std}
|
||||
for f in features_qs
|
||||
]
|
||||
top_features[str(c.cluster_label)] = feats
|
||||
# Attach features + NL summary directly to cluster object for template
|
||||
c._top_features = feats
|
||||
c.nl_summary = nl_describe.describe_cluster(feats) if feats else ''
|
||||
|
||||
return render(request, 'analysis/cluster_overview.html', {
|
||||
'run': run,
|
||||
'clusters': clusters,
|
||||
'noise': noise,
|
||||
'scatter_data_json': json.dumps(scatter_data),
|
||||
'has_z': has_z,
|
||||
'geo_data_json': json.dumps(geo_data),
|
||||
'geo_skipped': geo_skipped,
|
||||
'geo_count': len(geo_data),
|
||||
'cluster_sizes_json': json.dumps(cluster_sizes),
|
||||
'top_features_json': json.dumps(top_features),
|
||||
'noise_features': noise_features,
|
||||
'noise_summary': noise_summary,
|
||||
'noise_entity_count': noise_entity_count,
|
||||
'llm_timeline_json': llm_timeline_json,
|
||||
# Globe flow data for the sidebar
|
||||
'globe_flows_json': json.dumps(_get_globe_flows(run)),
|
||||
'selected_ds_id': f'upload_{run.display_id}',
|
||||
})
|
||||
|
||||
|
||||
def _get_globe_flows(run, max_rows=500):
|
||||
"""Extract flow data for 3D globe visualization from a run."""
|
||||
from analysis.data_loader import load_csv_directory, load_from_db
|
||||
try:
|
||||
lf = None
|
||||
if run.sqlite_table:
|
||||
lf = load_from_db(run.sqlite_table)
|
||||
if lf is None and run.csv_glob:
|
||||
lf, _, _, _, _ = load_csv_directory(run.csv_glob)
|
||||
if lf is None:
|
||||
return []
|
||||
df = lf.head(max_rows).collect()
|
||||
from analysis.type_classifier import TLS_HEX_MAP
|
||||
from analysis.geoip import lookup as geo_lookup
|
||||
flows = []
|
||||
src_ip_col = next((c for c in df.columns if 'src_ip' in c.lower() or ':ips' in c.lower()), None)
|
||||
dst_ip_col = next((c for c in df.columns if 'dst_ip' in c.lower() or ':ipd' in c.lower()), None)
|
||||
tls_col = next((c for c in df.columns if c.lower() in ('0ver', 'tls_version', '_tlsver', 'version')), None)
|
||||
bytes_col = next((c for c in df.columns if c.lower() in ('8ack', '8pak', '8byt', 'bytes_sent', 'bytes')), None)
|
||||
for row in df.iter_rows(named=True):
|
||||
sip = str(row.get(src_ip_col, '')) if src_ip_col else ''
|
||||
dip = str(row.get(dst_ip_col, '')) if dst_ip_col else ''
|
||||
if not sip or not dip:
|
||||
continue
|
||||
sg = geo_lookup(sip)
|
||||
dg = geo_lookup(dip)
|
||||
if not sg or not dg:
|
||||
continue
|
||||
tls_val = str(row.get(tls_col, '') or '') if tls_col else ''
|
||||
tls_ver = TLS_HEX_MAP.get(tls_val.replace(' ', ''), tls_val)
|
||||
flows.append({
|
||||
'slat': sg['lat'], 'slon': sg['lon'],
|
||||
'dlat': dg['lat'], 'dlon': dg['lon'],
|
||||
'tls': tls_ver,
|
||||
'bytes': float(row.get(bytes_col, 0) or 0) if bytes_col else 0,
|
||||
})
|
||||
return flows
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
return []
|
||||
|
||||
|
||||
def cluster_detail(request, display_id, cluster_label):
|
||||
"""Detailed view of a single cluster with raw data rows, SVD features, and NL summary."""
|
||||
import json
|
||||
import polars as pl
|
||||
run = get_object_or_404(AnalysisRun, display_id=display_id)
|
||||
try:
|
||||
cluster_label = int(cluster_label)
|
||||
except (ValueError, TypeError):
|
||||
raise Http404("Invalid cluster label")
|
||||
cluster = get_object_or_404(ClusterResult, run=run, cluster_label=cluster_label)
|
||||
features = cluster.features.all().order_by('-distinguishing_score')[:50]
|
||||
|
||||
# Load raw data rows for this cluster from EP row indices
|
||||
row_data = []
|
||||
try:
|
||||
from analysis.data_loader import load_csv_directory, load_from_db
|
||||
lf = None
|
||||
if run.sqlite_table:
|
||||
lf = load_from_db(run.sqlite_table)
|
||||
if lf is None and run.csv_glob:
|
||||
lf, _, _, _, _ = load_csv_directory(run.csv_glob)
|
||||
if lf is not None:
|
||||
# Get row indices assigned to this cluster from EntityProfile
|
||||
eps = cluster.entities.all().order_by('entity_value')[:100]
|
||||
indices = []
|
||||
for ep in eps:
|
||||
try:
|
||||
idx = int(ep.entity_value.replace('row_', ''))
|
||||
indices.append(idx)
|
||||
except (ValueError, AttributeError):
|
||||
continue
|
||||
if indices:
|
||||
df = lf.collect(streaming=True)
|
||||
df_filtered = df[indices]
|
||||
# Convert to dict list for template (limit columns for display)
|
||||
cols = df_filtered.columns[:20] # show first 20 columns
|
||||
for row_idx, row in enumerate(df_filtered.select(cols).iter_rows(named=True)):
|
||||
entry = {'index': indices[row_idx] if row_idx < len(indices) else row_idx}
|
||||
for k, v in row.items():
|
||||
entry[k] = v
|
||||
row_data.append(entry)
|
||||
except Exception as exc:
|
||||
logger.warning(f'Failed to load raw data for cluster detail: {exc}')
|
||||
|
||||
nl_summary = ''
|
||||
feats_list = [{'feature_name': f.feature_name, 'score': f.distinguishing_score,
|
||||
'mean': f.mean, 'std': f.std} for f in features[:10]]
|
||||
if feats_list:
|
||||
nl_summary = nl_describe.describe_cluster(feats_list)
|
||||
|
||||
return render(request, 'analysis/cluster_detail.html', {
|
||||
'run': run,
|
||||
'cluster': cluster,
|
||||
'features': features,
|
||||
'row_data': row_data,
|
||||
'nl_summary': nl_summary,
|
||||
})
|
||||
|
||||
|
||||
@profile
|
||||
def _run_clustering_pipeline(run, store, ds_id, feature_columns, algorithm,
|
||||
min_cluster_size, run_umap=True, head=None):
|
||||
"""Clustering pipeline on raw rows. Delegates to service layer."""
|
||||
from analysis.services.clustering import run_clustering_pipeline
|
||||
run_clustering_pipeline(run, store, ds_id, feature_columns, algorithm,
|
||||
min_cluster_size, run_umap=run_umap, head=head)
|
||||
@@ -0,0 +1,132 @@
|
||||
"""Configuration views: config editor and LLM connectivity test."""
|
||||
import traceback
|
||||
import json
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import logging
|
||||
|
||||
from django.shortcuts import render
|
||||
from django.http import JsonResponse
|
||||
from django.views.decorators.csrf import csrf_exempt
|
||||
|
||||
from config import get_config, save_config, Config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def config_view(request):
|
||||
"""Config page: GET renders current config, POST saves updates."""
|
||||
cfg = get_config()
|
||||
|
||||
if request.method == 'POST':
|
||||
# Build a new Config from POST data (fall back to current values)
|
||||
server = Config.Server(
|
||||
host=request.POST.get('server_host', cfg.server.host),
|
||||
port=int(request.POST.get('server_port', cfg.server.port)),
|
||||
debug=request.POST.get('server_debug') == 'true',
|
||||
)
|
||||
data = Config.Data(
|
||||
schema_strict=request.POST.get('data_schema_strict') == 'true',
|
||||
recursive=request.POST.get('data_recursive') == 'true',
|
||||
)
|
||||
clustering = Config.Clustering(
|
||||
algorithm=request.POST.get('clustering_algorithm', cfg.clustering.algorithm),
|
||||
min_cluster_size=int(request.POST.get('clustering_min_cluster_size', cfg.clustering.min_cluster_size)),
|
||||
random_state=cfg.clustering.random_state,
|
||||
)
|
||||
llm = Config.LLM(
|
||||
enabled=request.POST.get('llm_enabled') == 'true',
|
||||
base_url=request.POST.get('llm_base_url', cfg.llm.base_url),
|
||||
api_key=request.POST.get('llm_api_key', cfg.llm.api_key),
|
||||
model=request.POST.get('llm_model', cfg.llm.model),
|
||||
)
|
||||
cfg = Config(server=server, data=data, clustering=clustering, llm=llm)
|
||||
save_config(cfg)
|
||||
# Re-render with saved flag
|
||||
return render(request, 'tianxuan/config.html', {'config': cfg, 'saved': True})
|
||||
|
||||
return render(request, 'tianxuan/config.html', {'config': cfg})
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
def llm_test(request):
|
||||
"""LLM connectivity test: POST JSON {base_url, api_key, model} → chat/completions."""
|
||||
if request.method != 'POST':
|
||||
return JsonResponse({'success': False, 'message': '仅支持 POST 请求', 'latency_ms': 0}, status=405)
|
||||
|
||||
try:
|
||||
body = json.loads(request.body)
|
||||
except json.JSONDecodeError:
|
||||
return JsonResponse({'success': False, 'message': '无效的 JSON 请求体', 'latency_ms': 0})
|
||||
|
||||
base_url = (body.get('base_url') or '').rstrip('/')
|
||||
api_key = body.get('api_key') or ''
|
||||
model = body.get('model') or ''
|
||||
|
||||
if not base_url:
|
||||
return JsonResponse({'success': False, 'message': 'Base URL 不能为空', 'latency_ms': 0})
|
||||
if not api_key:
|
||||
return JsonResponse({'success': False, 'message': 'API Key 不能为空', 'latency_ms': 0})
|
||||
if not model:
|
||||
return JsonResponse({'success': False, 'message': 'Model 不能为空', 'latency_ms': 0})
|
||||
|
||||
url = f'{base_url}/chat/completions'
|
||||
payload = json.dumps({
|
||||
'model': model,
|
||||
'messages': [{'role': 'user', 'content': 'respond with ok'}],
|
||||
'max_tokens': 5,
|
||||
}).encode('utf-8')
|
||||
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
data=payload,
|
||||
headers={
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': f'Bearer {api_key}',
|
||||
},
|
||||
method='POST',
|
||||
)
|
||||
|
||||
start = time.monotonic()
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=5) as resp:
|
||||
elapsed = int((time.monotonic() - start) * 1000)
|
||||
data = json.loads(resp.read().decode('utf-8'))
|
||||
if 'choices' in data and len(data['choices']) > 0:
|
||||
logger.info(f'[LLM_TEST] base_url={base_url} model={model} success=True latency={elapsed}ms')
|
||||
return JsonResponse({
|
||||
'success': True,
|
||||
'message': f'连接成功，模型 {model} 返回正常',
|
||||
'latency_ms': elapsed,
|
||||
})
|
||||
else:
|
||||
logger.info(f'[LLM_TEST] base_url={base_url} model={model} success=True latency={elapsed}ms')
|
||||
return JsonResponse({
|
||||
'success': True,
|
||||
'message': f'连接成功，但响应中无 choices（原始响应已记录）',
|
||||
'latency_ms': elapsed,
|
||||
})
|
||||
except urllib.error.HTTPError as e:
|
||||
elapsed = int((time.monotonic() - start) * 1000)
|
||||
try:
|
||||
detail = json.loads(e.read().decode('utf-8', errors='replace'))
|
||||
msg = detail.get('error', {}).get('message', str(e))
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
msg = str(e)
|
||||
logger.info(f'[LLM_TEST] base_url={base_url} model={model} success=False latency={elapsed}ms')
|
||||
return JsonResponse({'success': False, 'message': f'HTTP {e.code}: {msg}', 'latency_ms': elapsed})
|
||||
except urllib.error.URLError as e:
|
||||
elapsed = int((time.monotonic() - start) * 1000)
|
||||
reason = str(e.reason) if e.reason else '连接失败'
|
||||
logger.info(f'[LLM_TEST] base_url={base_url} model={model} success=False latency={elapsed}ms')
|
||||
return JsonResponse({'success': False, 'message': f'连接失败: {reason}', 'latency_ms': elapsed})
|
||||
except TimeoutError:
|
||||
elapsed = int((time.monotonic() - start) * 1000)
|
||||
logger.info(f'[LLM_TEST] base_url={base_url} model={model} success=False latency={elapsed}ms')
|
||||
return JsonResponse({'success': False, 'message': '请求超时（5秒）', 'latency_ms': elapsed})
|
||||
except Exception as e:
|
||||
elapsed = int((time.monotonic() - start) * 1000)
|
||||
logger.info(f'[LLM_TEST] base_url={base_url} model={model} success=False latency={elapsed}ms')
|
||||
return JsonResponse({'success': False, 'message': f'未知错误: {str(e)}', 'latency_ms': elapsed})
|
||||
@@ -0,0 +1,131 @@
|
||||
"""Dashboard views: home, run list, run detail, run status."""
|
||||
from django.shortcuts import render, get_object_or_404
|
||||
from django.http import JsonResponse
|
||||
|
||||
from analysis.models import AnalysisRun
|
||||
|
||||
|
||||
def dashboard(request):
|
||||
"""Home page: recent runs overview."""
|
||||
runs = AnalysisRun.objects.all()[:20]
|
||||
return render(request, 'analysis/dashboard.html', {'runs': runs})
|
||||
|
||||
|
||||
def run_list(request):
|
||||
"""List all analysis runs with pagination and optional ?type= filter."""
|
||||
run_type_filter = request.GET.get('type')
|
||||
page = int(request.GET.get('page', 1))
|
||||
per_page = 20
|
||||
|
||||
qs = AnalysisRun.objects.all()
|
||||
if run_type_filter in dict(AnalysisRun.RUN_TYPE_CHOICES):
|
||||
qs = qs.filter(run_type=run_type_filter)
|
||||
|
||||
total = qs.count()
|
||||
runs = qs[(page - 1) * per_page: page * per_page]
|
||||
return render(request, 'analysis/run_list.html', {
|
||||
'runs': runs,
|
||||
'page': page,
|
||||
'total_pages': (total + per_page - 1) // per_page if total else 1,
|
||||
'current_type': run_type_filter or '',
|
||||
})
|
||||
|
||||
|
||||
def run_detail(request, display_id):
|
||||
"""Details for a single analysis run."""
|
||||
import json
|
||||
run = get_object_or_404(AnalysisRun, display_id=display_id)
|
||||
clusters = run.clusters.all().order_by('-size')
|
||||
|
||||
# ── Global SVD features (compute on-demand from the stored run data) ──
|
||||
svd_features = []
|
||||
svd_explained_var = []
|
||||
try:
|
||||
from analysis.data_loader import load_csv_directory, load_from_db
|
||||
import polars as pl
|
||||
from sklearn.decomposition import TruncatedSVD
|
||||
import numpy as np
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
lf = None
|
||||
if run.sqlite_table:
|
||||
lf = load_from_db(run.sqlite_table)
|
||||
if lf is None and run.csv_glob:
|
||||
lf, _, _, _, _ = load_csv_directory(run.csv_glob)
|
||||
|
||||
if lf is not None:
|
||||
schema = lf.collect_schema()
|
||||
numeric_types = (pl.Int8, pl.Int16, pl.Int32, pl.Int64,
|
||||
pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
|
||||
pl.Float32, pl.Float64)
|
||||
num_cols = [n for n, dt in zip(schema.names(), schema.dtypes()) if dt in numeric_types and not n.startswith('_')]
|
||||
|
||||
if len(num_cols) >= 3:
|
||||
n_rows_lf = lf.select(pl.len()).collect(streaming=True).item()
|
||||
sample = min(int(n_rows_lf), 50000)
|
||||
df = lf.sample(n=sample, seed=42).select(num_cols).collect(streaming=True) if n_rows_lf > sample else lf.select(num_cols).collect(streaming=True)
|
||||
mat = np.nan_to_num(df.to_numpy(), nan=0.0)
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
mat = StandardScaler().fit_transform(mat)
|
||||
|
||||
n_comp = min(20, min(mat.shape) - 1)
|
||||
if n_comp >= 2:
|
||||
svd = TruncatedSVD(n_components=n_comp, random_state=42)
|
||||
svd.fit(mat)
|
||||
var_ratio = svd.explained_variance_ratio_.tolist()
|
||||
svd_explained_var = var_ratio
|
||||
for comp_idx in range(min(10, n_comp)):
|
||||
loadings = np.abs(svd.components_[comp_idx])
|
||||
top_idx = np.argsort(loadings)[-5:][::-1]
|
||||
features = [{'feature': num_cols[i], 'strength': round(float(loadings[i]), 4)} for i in top_idx]
|
||||
svd_features.append({
|
||||
'component': comp_idx + 1,
|
||||
'explained_var': round(var_ratio[comp_idx] * 100, 2),
|
||||
'cumulative_var': round(sum(var_ratio[:comp_idx + 1]) * 100, 2),
|
||||
'top_features': features,
|
||||
})
|
||||
except Exception as exc:
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.warning(f'SVD feature extraction failed for run {run.display_id}: {exc}')
|
||||
|
||||
# ── LLM workflow timeline ──
|
||||
llm_timeline_json = ''
|
||||
if run.run_type == 'auto':
|
||||
llm_thinking = run.llm_thinking or ''
|
||||
tool_calls = run.tool_calls_json or []
|
||||
if tool_calls or llm_thinking:
|
||||
llm_timeline_json = json.dumps({
|
||||
'thinking': llm_thinking,
|
||||
'tool_calls': tool_calls,
|
||||
})
|
||||
|
||||
return render(request, 'analysis/run_detail.html', {
|
||||
'run': run,
|
||||
'clusters': clusters,
|
||||
'svd_features': svd_features,
|
||||
'svd_explained_var': svd_explained_var,
|
||||
'llm_timeline_json': llm_timeline_json,
|
||||
})
|
||||
|
||||
|
||||
def run_status_api(request, display_id):
|
||||
"""Return current run status as JSON."""
|
||||
run = get_object_or_404(AnalysisRun, display_id=display_id)
|
||||
return JsonResponse({
|
||||
'id': run.display_id,
|
||||
'status': run.status,
|
||||
'entity_count': run.entity_count,
|
||||
'cluster_count': run.cluster_count,
|
||||
'error_message': run.error_message,
|
||||
'progress_pct': run.progress_pct,
|
||||
'progress_msg': run.progress_msg,
|
||||
'run_log': run.run_log[-2000:], # last 2000 chars
|
||||
'llm_thinking': run.llm_thinking[-50000:] if run.llm_thinking else '',
|
||||
'tool_calls': run.tool_calls_json,
|
||||
})
|
||||
|
||||
|
||||
def start_analysis(request):
|
||||
"""Placeholder: API endpoint to trigger analysis via MCP."""
|
||||
return JsonResponse({'status': 'not_implemented', 'message': 'Use MCP tools to trigger analysis'})
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Entity profile view."""
|
||||
import json
|
||||
|
||||
from django.shortcuts import render, get_object_or_404
|
||||
|
||||
from analysis.models import EntityProfile
|
||||
|
||||
|
||||
def entity_profile(request, entity_id):
|
||||
"""Detailed profile for a single entity."""
|
||||
entity = get_object_or_404(EntityProfile, id=entity_id)
|
||||
run = entity.run
|
||||
|
||||
# Compute deviation from cluster mean
|
||||
deviations = {}
|
||||
if entity.feature_json and entity.cluster:
|
||||
cluster_features = {
|
||||
f.feature_name: {'mean': f.mean, 'std': f.std}
|
||||
for f in entity.cluster.features.all()
|
||||
}
|
||||
for col, val in entity.feature_json.items():
|
||||
if col in cluster_features and cluster_features[col].get('std') and cluster_features[col]['std'] > 1e-10:
|
||||
deviations[col] = {
|
||||
'value': val,
|
||||
'cluster_mean': cluster_features[col]['mean'],
|
||||
'zscore': (val - cluster_features[col]['mean']) / cluster_features[col]['std'],
|
||||
}
|
||||
|
||||
return render(request, 'analysis/entity_profile.html', {
|
||||
'entity': entity,
|
||||
'run': run,
|
||||
'deviations': deviations,
|
||||
})
|
||||
@@ -0,0 +1,220 @@
|
||||
"""3D Globe view and flow extraction."""
|
||||
import json
|
||||
import logging
|
||||
import traceback
|
||||
|
||||
from django.shortcuts import render
|
||||
from django.views.decorators.clickjacking import xframe_options_exempt
|
||||
|
||||
from analysis.constants import GLOBE_MAX_ROWS
|
||||
from analysis.models import AnalysisRun
|
||||
from .helpers import _extract_lat, _extract_lon
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _extract_flows_from_df(df, MAX_ROWS):
|
||||
"""Extract flow data (lat/lon/TLS/bytes) from a Polars DataFrame."""
|
||||
from analysis.geoip import lookup as geo_lookup
|
||||
from analysis.type_classifier import TLS_HEX_MAP
|
||||
|
||||
src_lat_col = next((c for c in df.columns if c.lower() in (
|
||||
':ips.latd', ':ips_latd', ':ips_lat', 'src_latitude', 'src_lat', 'source_latitude', 'source_lat')), None)
|
||||
src_lon_col = next((c for c in df.columns if c.lower() in (
|
||||
':ips.lond', ':ips_lond', ':ips_lon', 'src_longitude', 'src_lon', 'source_longitude', 'source_lon')), None)
|
||||
dst_lat_col = next((c for c in df.columns if c.lower() in (
|
||||
':ipd.latd', ':ipd_latd', ':ipd_lat', 'dst_latitude', 'dst_lat', 'dest_latitude', 'dest_lat')), None)
|
||||
dst_lon_col = next((c for c in df.columns if c.lower() in (
|
||||
':ipd.lond', ':ipd_lond', ':ipd_lon', 'dst_longitude', 'dst_lon', 'dest_longitude', 'dest_lon')), None)
|
||||
src_ip_col = next((c for c in df.columns if 'src_ip' in c.lower() or ':ips' in c.lower()), None)
|
||||
dst_ip_col = next((c for c in df.columns if 'dst_ip' in c.lower() or ':ipd' in c.lower()), None)
|
||||
tls_col = next((c for c in df.columns if c.lower() in ('0ver', 'tls_version', '_tlsver', 'version')), None)
|
||||
bytes_col = next((c for c in df.columns if c.lower() in ('8ack', '8pak', '8byt', 'bytes_sent', 'bytes')), None)
|
||||
time_col = next((c for c in df.columns if c.lower() in (
|
||||
'timestamp', 'time', '_time', 'ts', 'datetime', 'epoch',
|
||||
'created_at', 'created', 'event_time')), None)
|
||||
|
||||
flows = []
|
||||
for row in df.iter_rows(named=True):
|
||||
slat = slon = dlat = dlon = None
|
||||
|
||||
if src_lat_col and src_lon_col:
|
||||
slat = _extract_lat(row.get(src_lat_col))
|
||||
slon = _extract_lon(row.get(src_lon_col))
|
||||
if dst_lat_col and dst_lon_col:
|
||||
dlat = _extract_lat(row.get(dst_lat_col))
|
||||
dlon = _extract_lon(row.get(dst_lon_col))
|
||||
|
||||
if (slat is None or slon is None) and src_ip_col:
|
||||
sg = geo_lookup(str(row.get(src_ip_col, '')))
|
||||
if sg:
|
||||
slat, slon = sg['lat'], sg['lon']
|
||||
if (dlat is None or dlon is None) and dst_ip_col:
|
||||
dg = geo_lookup(str(row.get(dst_ip_col, '')))
|
||||
if dg:
|
||||
dlat, dlon = dg['lat'], dg['lon']
|
||||
|
||||
if slat is None or slon is None or dlat is None or dlon is None:
|
||||
continue
|
||||
|
||||
tls_val = str(row.get(tls_col or '', ''))
|
||||
# Check hex format first (e.g. "03 03" or "0303")
|
||||
hex_key = tls_val.replace(' ', '').strip()
|
||||
if hex_key in TLS_HEX_MAP:
|
||||
tls_label = TLS_HEX_MAP[hex_key]
|
||||
elif '1.3' in tls_val:
|
||||
tls_label = 'TLSv1.3'
|
||||
elif '1.2' in tls_val:
|
||||
tls_label = 'TLSv1.2'
|
||||
else:
|
||||
tls_label = 'other'
|
||||
|
||||
bs = float(row.get(bytes_col or '', 0) or 0)
|
||||
|
||||
# Extract timestamp for animation ordering
|
||||
t_raw = row.get(time_col) if time_col else None
|
||||
t_val = None
|
||||
if t_raw is not None:
|
||||
if isinstance(t_raw, (int, float)):
|
||||
t_val = float(t_raw)
|
||||
elif hasattr(t_raw, 'timestamp'):
|
||||
t_val = t_raw.timestamp()
|
||||
else:
|
||||
try:
|
||||
t_val = float(t_raw)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
flows.append({
|
||||
'slat': slat, 'slon': slon,
|
||||
'dlat': dlat, 'dlon': dlon,
|
||||
'tls': tls_label,
|
||||
'bytes': bs,
|
||||
'time': t_val,
|
||||
})
|
||||
|
||||
# Normalize timestamps to 0-1 and sort flows by time
|
||||
timed = [f for f in flows if f['time'] is not None]
|
||||
if len(timed) > 1:
|
||||
timed.sort(key=lambda f: f['time'])
|
||||
t_min, t_max = timed[0]['time'], timed[-1]['time']
|
||||
t_range = max(t_max - t_min, 1)
|
||||
for f in timed:
|
||||
f['time'] = (f['time'] - t_min) / t_range
|
||||
# Untimed flows get evenly spaced after timed ones
|
||||
untimed = [f for f in flows if f['time'] is None]
|
||||
for i, f in enumerate(untimed):
|
||||
f['time'] = min(1.0, (len(timed) + i) / max(len(flows) - 1, 1))
|
||||
flows = timed + untimed
|
||||
else:
|
||||
for i, f in enumerate(flows):
|
||||
f['time'] = i / max(len(flows) - 1, 1)
|
||||
|
||||
return flows
|
||||
|
||||
|
||||
from django.views.decorators.clickjacking import xframe_options_exempt
|
||||
|
||||
|
||||
@xframe_options_exempt
|
||||
def globe_view(request):
|
||||
from analysis.geoip import lookup as geo_lookup
|
||||
from analysis.data_loader import load_csv_directory, load_from_db
|
||||
from analysis.models import AnalysisRun
|
||||
from analysis.session_store import SessionStore
|
||||
|
||||
MAX_ROWS_PER_RUN = GLOBE_MAX_ROWS # Limit per run to prevent browser overload
|
||||
|
||||
# Check for ?data=dataset_id parameter (direct data loading from session store)
|
||||
data_param = request.GET.get('data')
|
||||
embed_mode = request.GET.get('embed') == '1'
|
||||
template_name = 'tianxuan/globe_embed.html' if embed_mode else 'tianxuan/globe.html'
|
||||
|
||||
if data_param:
|
||||
store = SessionStore()
|
||||
flows = []
|
||||
all_runs = list(AnalysisRun.objects.filter(status='completed').order_by('-id').values('display_id', 'csv_glob'))
|
||||
try:
|
||||
entry = store.get_dataset(data_param)
|
||||
if entry is not None:
|
||||
lf = entry['lazyframe']
|
||||
df = lf.head(MAX_ROWS_PER_RUN).collect(streaming=True)
|
||||
flows = _extract_flows_from_df(df, MAX_ROWS_PER_RUN)
|
||||
except Exception as exc:
|
||||
logging.getLogger(__name__).warning(f'globe ?data= param failed: {exc}')
|
||||
return render(request, template_name, {
|
||||
'geo_flows': json.dumps(flows),
|
||||
'all_runs': all_runs,
|
||||
'selected_ids': [],
|
||||
})
|
||||
|
||||
# Get all completed runs for the selector
|
||||
all_runs = list(AnalysisRun.objects.filter(status='completed').order_by('-id').values('display_id', 'csv_glob'))
|
||||
|
||||
# Get selected run IDs from query params (multiple checkboxes)
|
||||
selected_ids_list = request.GET.getlist('runs')
|
||||
selected_runs = []
|
||||
if selected_ids_list:
|
||||
for sid in selected_ids_list:
|
||||
sid = sid.strip()
|
||||
if sid.isdigit():
|
||||
try:
|
||||
r = AnalysisRun.objects.get(display_id=int(sid), status='completed')
|
||||
selected_runs.append(r)
|
||||
except AnalysisRun.DoesNotExist:
|
||||
pass
|
||||
|
||||
# Check if user-selected runs have valid files; if not, fall through
|
||||
all_selected_failed = bool(selected_ids_list)
|
||||
fallback_run = None
|
||||
|
||||
if not selected_runs:
|
||||
all_selected_failed = False
|
||||
|
||||
flows = []
|
||||
|
||||
for run in selected_runs:
|
||||
try:
|
||||
lf = None
|
||||
if run.sqlite_table:
|
||||
lf = load_from_db(run.sqlite_table)
|
||||
if lf is None:
|
||||
lf, schema, rc, fc, mem = load_csv_directory(run.csv_glob)
|
||||
df = lf.head(MAX_ROWS_PER_RUN).collect(streaming=True)
|
||||
flows.extend(_extract_flows_from_df(df, MAX_ROWS_PER_RUN))
|
||||
all_selected_failed = False
|
||||
break # stop after first successful load
|
||||
except Exception as exc:
|
||||
logger.warning('[GLOBE] Failed to load run %s (%s): %s', run.display_id, run.csv_glob, exc)
|
||||
continue
|
||||
|
||||
if all_selected_failed or not flows:
|
||||
# Fallback: latest completed run with valid data (SQLite or CSV)
|
||||
for latest in AnalysisRun.objects.filter(status='completed').order_by('-id'):
|
||||
if not latest.csv_glob and not latest.sqlite_table:
|
||||
continue
|
||||
try:
|
||||
lf = None
|
||||
if latest.sqlite_table:
|
||||
lf = load_from_db(latest.sqlite_table)
|
||||
if lf is None:
|
||||
if not latest.csv_glob:
|
||||
continue
|
||||
lf, schema, rc, fc, mem = load_csv_directory(latest.csv_glob)
|
||||
df = lf.head(MAX_ROWS_PER_RUN).collect(streaming=True)
|
||||
flows.extend(_extract_flows_from_df(df, MAX_ROWS_PER_RUN))
|
||||
selected_runs = [latest]
|
||||
break
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
continue
|
||||
|
||||
resp = render(request, 'tianxuan/globe.html', {
|
||||
'geo_flows': json.dumps(flows),
|
||||
'all_runs': all_runs,
|
||||
'selected_ids': [r.display_id for r in selected_runs],
|
||||
})
|
||||
resp['Cache-Control'] = 'no-cache, no-store, must-revalidate'
|
||||
resp['Pragma'] = 'no-cache'
|
||||
resp['Expires'] = '0'
|
||||
return resp
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Geo extraction helpers and plan utilities."""
|
||||
import traceback
|
||||
from typing import Optional
|
||||
from pathlib import Path
|
||||
import json as _json
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _extract_lat(val):
|
||||
"""Try to extract a latitude from *val* (number or str)."""
|
||||
if val is None:
|
||||
return None
|
||||
try:
|
||||
v = float(val)
|
||||
return None if v < -90 or v > 90 else v
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _extract_lon(val):
|
||||
"""Try to extract a longitude from *val* (number or str)."""
|
||||
if val is None:
|
||||
return None
|
||||
try:
|
||||
v = float(val)
|
||||
return None if v < -180 or v > 180 else v
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
# ── Plan index utilities (shared by tools.py and auto.py) ─────────────
|
||||
|
||||
_PLANS_DIR = Path(__file__).resolve().parent.parent.parent / '.omo' / 'plans'
|
||||
_PLANS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
_AUTO_INDEX_PATH = _PLANS_DIR / '_auto_index.json'
|
||||
_MAX_AUTO_PLANS = 10
|
||||
|
||||
|
||||
def _read_auto_index():
|
||||
"""Read the auto index file, returning list of {file, pinned}."""
|
||||
if not _AUTO_INDEX_PATH.exists():
|
||||
return []
|
||||
try:
|
||||
data = _json.loads(_AUTO_INDEX_PATH.read_text(encoding='utf-8'))
|
||||
if not isinstance(data, list):
|
||||
return []
|
||||
return data
|
||||
except Exception:
|
||||
logger.error("_read_auto_index failed: {}".format(traceback.format_exc()))
|
||||
return []
|
||||
|
||||
|
||||
def _write_auto_index(index):
|
||||
"""Write the auto index list."""
|
||||
_AUTO_INDEX_PATH.write_text(
|
||||
_json.dumps(index, ensure_ascii=False, indent=2),
|
||||
encoding='utf-8',
|
||||
)
|
||||
|
||||
|
||||
def _add_to_auto_index(filename):
|
||||
"""Prepend filename to auto index, evict unpinned entries when >10."""
|
||||
index = _read_auto_index()
|
||||
# Remove if already present
|
||||
index = [entry for entry in index if entry.get('file') != filename]
|
||||
# Prepend
|
||||
index.insert(0, {'file': filename, 'pinned': False})
|
||||
# Evict unpinned entries beyond max
|
||||
pinned_entries = [e for e in index if e.get('pinned')]
|
||||
unpinned_entries = [e for e in index if not e.get('pinned')]
|
||||
# Keep all pinned + fill remaining slots with unpinned
|
||||
result = pinned_entries + unpinned_entries[:_MAX_AUTO_PLANS - len(pinned_entries)]
|
||||
_write_auto_index(result)
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Log viewer view."""
|
||||
from django.shortcuts import render
|
||||
from django.http import HttpResponse
|
||||
|
||||
|
||||
def log_viewer(request):
|
||||
"""Display last 200 lines of the app log (DEBUG only).
|
||||
With ?pos=N, returns text: line_count\\nnew_content for AJAX polling.
|
||||
"""
|
||||
from django.conf import settings
|
||||
if not settings.DEBUG:
|
||||
return HttpResponse('Log viewer only available in DEBUG mode', status=403)
|
||||
log_path = settings.BASE_DIR / 'logs' / 'tianxuan.log'
|
||||
pos = request.GET.get('pos')
|
||||
if pos is not None:
|
||||
# AJAX polling mode: return new content since byte position
|
||||
try:
|
||||
pos = int(pos)
|
||||
except ValueError:
|
||||
pos = 0
|
||||
if log_path.exists():
|
||||
with open(log_path, 'r', encoding='utf-8') as f:
|
||||
total = f.seek(0, 2) # seek to end
|
||||
if pos >= total:
|
||||
return HttpResponse(f'{total}\n', content_type='text/plain')
|
||||
f.seek(pos)
|
||||
new_content = f.read()
|
||||
return HttpResponse(f'{total}\n{new_content}', content_type='text/plain')
|
||||
return HttpResponse('0\n', content_type='text/plain')
|
||||
|
||||
# HTML page mode
|
||||
lines = []
|
||||
if log_path.exists():
|
||||
with open(log_path, 'r', encoding='utf-8') as f:
|
||||
lines = f.readlines()[-200:]
|
||||
return render(request, 'tianxuan/log_viewer.html', {'lines': lines})
|
||||
@@ -0,0 +1,191 @@
|
||||
"""Manual analysis page: workflow builder and run execution."""
|
||||
import json
|
||||
import threading
|
||||
import traceback
|
||||
import logging
|
||||
|
||||
from django.shortcuts import render, get_object_or_404
|
||||
from django.http import JsonResponse
|
||||
from django.views.decorators.csrf import csrf_exempt
|
||||
|
||||
from analysis.models import AnalysisRun
|
||||
from .pipeline import _run_pipeline_worker
|
||||
from analysis.services.clustering import run_clustering_pipeline
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def manual_page(request):
|
||||
"""Workflow builder: select dataset, add MCP tool steps, execute."""
|
||||
from analysis.session_store import SessionStore
|
||||
from analysis.tool_registry import get_tools_meta
|
||||
import json as _json
|
||||
|
||||
# Get all tools metadata
|
||||
tools = get_tools_meta()
|
||||
core_names = {'profile_data', 'build_entity_profiles', 'compute_scores',
|
||||
'run_clustering', 'extract_features', 'detect_anomalies',
|
||||
'visualize_anomalies'}
|
||||
diag_names = {'validate_data', 'explore_distributions', 'find_outliers',
|
||||
'diagnose_clustering', 'compare_datasets', 'export_debug_sample',
|
||||
'repair_schema'}
|
||||
analysis_names = {'analyze_patterns', 'analyze_temporal', 'analyze_tls_health',
|
||||
'analyze_geo_distribution', 'analyze_entity_detail'}
|
||||
core_tools, diag_tools, analysis_tools, other_tools = [], [], [], []
|
||||
for t in tools:
|
||||
if t.name in core_names: core_tools.append(t)
|
||||
elif t.name in diag_names: diag_tools.append(t)
|
||||
elif t.name in analysis_names: analysis_tools.append(t)
|
||||
else: other_tools.append(t)
|
||||
|
||||
meta_list = [{'name': t.name, 'description': t.description, 'inputSchema': t.inputSchema} for t in tools]
|
||||
|
||||
# Get datasets from SessionStore
|
||||
store = SessionStore()
|
||||
session_datasets = store.list_datasets() if hasattr(store, 'list_datasets') else []
|
||||
session_ds_map = {ds['dataset_id']: ds for ds in session_datasets}
|
||||
|
||||
# Build dataset entries from ALL AnalysisRun records (upload/manual/auto, any status)
|
||||
from analysis.models import AnalysisRun
|
||||
all_runs = AnalysisRun.objects.all().order_by('-created_at')[:100]
|
||||
datasets_with_status = []
|
||||
|
||||
for run in all_runs:
|
||||
# Determine how the dataset was keyed in SessionStore
|
||||
if run.run_type == 'upload':
|
||||
ds_id = f'upload_{run.display_id}'
|
||||
else:
|
||||
ds_id = f'run_{run.display_id}'
|
||||
|
||||
session_entry = session_ds_map.get(ds_id)
|
||||
|
||||
if session_entry:
|
||||
# Full data available in SessionStore — merge with run status
|
||||
datasets_with_status.append({
|
||||
**session_entry,
|
||||
'dataset_id': ds_id,
|
||||
'run_status': run.status,
|
||||
'total_flows': run.total_flows,
|
||||
'display_id': run.display_id,
|
||||
'needs_reload': False,
|
||||
})
|
||||
elif run.csv_glob or run.sqlite_table:
|
||||
# Data not in SessionStore but recoverable from disk/DB
|
||||
datasets_with_status.append({
|
||||
'dataset_id': ds_id,
|
||||
'run_status': run.status,
|
||||
'display_id': run.display_id,
|
||||
'total_flows': run.total_flows,
|
||||
'row_count': run.total_flows,
|
||||
'file_count': None,
|
||||
'column_count': 0,
|
||||
'columns': [],
|
||||
'svd_components': 0,
|
||||
'needs_reload': True,
|
||||
'csv_glob': run.csv_glob,
|
||||
'sqlite_table': run.sqlite_table or '',
|
||||
})
|
||||
|
||||
# Map statuses to user-facing labels for the filter tabs
|
||||
# ready → 待分析, completed → 已分析, failed → 错误
|
||||
# Everything else (pending/loading/profiling/aggregating/clustering/extracting) → analyzing → 分析中
|
||||
for ds_info in datasets_with_status:
|
||||
s = ds_info['run_status']
|
||||
if s in ('completed',):
|
||||
ds_info['status_label'] = '已分析'
|
||||
ds_info['badge_class'] = 'completed'
|
||||
elif s in ('failed',):
|
||||
ds_info['status_label'] = '错误'
|
||||
ds_info['badge_class'] = 'failed'
|
||||
elif s in ('pending', 'loading', 'profiling', 'aggregating', 'clustering', 'extracting'):
|
||||
ds_info['status_label'] = '分析中'
|
||||
ds_info['badge_class'] = 'analyzing'
|
||||
else:
|
||||
ds_info['status_label'] = '待分析'
|
||||
ds_info['badge_class'] = 'ready'
|
||||
|
||||
return render(request, 'tianxuan/manual.html', {
|
||||
'core_tools': core_tools,
|
||||
'diag_tools': diag_tools,
|
||||
'analysis_tools': analysis_tools,
|
||||
'other_tools': other_tools,
|
||||
'tools_meta': _json.dumps(meta_list, ensure_ascii=False),
|
||||
'datasets': datasets_with_status,
|
||||
'datasets_with_status_json': _json.dumps(datasets_with_status, ensure_ascii=False),
|
||||
})
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
def manual_run_analysis(request):
|
||||
"""AJAX POST: run clustering pipeline on the upload dataset."""
|
||||
if request.method != 'POST':
|
||||
return JsonResponse({'error': 'POST required'}, status=405)
|
||||
import json as _json
|
||||
body = _json.loads(request.body)
|
||||
display_id = body.get('run_id')
|
||||
run = get_object_or_404(AnalysisRun, display_id=display_id)
|
||||
pk = run.id
|
||||
run.run_type = 'manual'
|
||||
run.save(update_fields=['run_type'])
|
||||
feature_columns = body.get('feature_columns')
|
||||
algorithm = body.get('algorithm', 'agglomerative')
|
||||
min_cluster_size = int(body.get('min_cluster_size', 5))
|
||||
head = body.get('head')
|
||||
cluster_mode = body.get('cluster_mode', 'raw')
|
||||
|
||||
def _analysis_fn(run, ctx):
|
||||
from analysis.session_store import SessionStore
|
||||
|
||||
feature_columns = ctx['feature_columns']
|
||||
algorithm = ctx['algorithm']
|
||||
min_cluster_size = ctx['min_cluster_size']
|
||||
head = ctx.get('head')
|
||||
pk = ctx['pk']
|
||||
cluster_mode = ctx.get('cluster_mode', 'raw')
|
||||
|
||||
store = SessionStore()
|
||||
try:
|
||||
# Use display_id for dataset key (consistent with _background_process)
|
||||
display_id = ctx.get('display_id', pk)
|
||||
upload_ds_id = f'upload_{display_id}'
|
||||
entry = store.get_dataset(upload_ds_id)
|
||||
if entry is None:
|
||||
# Fallback: try PK-based key (old pipeline)
|
||||
entry = store.get_dataset(f'upload_{pk}')
|
||||
if entry is None:
|
||||
# Fallback: try entity dataset (from older pipeline)
|
||||
entry = store.get_dataset(f'entity_{pk}')
|
||||
if entry is None:
|
||||
run.error_message = '请先上传并等待预处理完成'
|
||||
run.status = 'failed'
|
||||
run.save(update_fields=['status', 'error_message'])
|
||||
return
|
||||
|
||||
ds_id = upload_ds_id if store.get_dataset(upload_ds_id) else f'upload_{pk}'
|
||||
|
||||
# Use the unified clustering pipeline (clustering → extraction → UMAP)
|
||||
run_clustering_pipeline(
|
||||
run=run, store=store, entity_ds_id=ds_id,
|
||||
feature_columns=feature_columns,
|
||||
algorithm=algorithm, min_cluster_size=min_cluster_size,
|
||||
run_umap=True, head=head,
|
||||
)
|
||||
except Exception:
|
||||
tb = traceback.format_exc()
|
||||
logger.error(tb)
|
||||
run.status = 'failed'
|
||||
run.error_message = tb
|
||||
run.progress_msg = f'失败: {tb}'
|
||||
run.run_log += f'\n[ERROR] {tb}'
|
||||
run.save(update_fields=['status', 'error_message', 'progress_msg', 'run_log'])
|
||||
|
||||
t = threading.Thread(
|
||||
target=_run_pipeline_worker,
|
||||
args=(pk, _analysis_fn),
|
||||
kwargs=dict(feature_columns=feature_columns, algorithm=algorithm,
|
||||
min_cluster_size=min_cluster_size, head=head, pk=pk,
|
||||
display_id=run.display_id, cluster_mode=cluster_mode),
|
||||
daemon=True,
|
||||
)
|
||||
t.start()
|
||||
return JsonResponse({'status': 'started', 'redirect': f'/runs/{run.display_id}/'})
|
||||
@@ -0,0 +1,287 @@
|
||||
"""Background pipeline: CSV loading, SVD, and pipeline worker."""
|
||||
import os
|
||||
import threading
|
||||
import traceback
|
||||
import logging
|
||||
|
||||
from analysis.appdata import ensure_appdata_dir
|
||||
from analysis.constants import MAX_CLUSTER_FEATURES, RANDOM_SEED
|
||||
from analysis.models import AnalysisRun
|
||||
from analysis.profile_util import profile
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _run_pipeline_worker(run_id, analysis_fn, **ctx):
|
||||
"""Unified background worker: setup / ORM / progress / error handling."""
|
||||
import django
|
||||
os.environ['DJANGO_SETTINGS_MODULE'] = 'tianxuan.settings'
|
||||
os.environ['DJANGO_ALLOW_ASYNC_UNSAFE'] = 'true'
|
||||
django.setup()
|
||||
|
||||
from analysis.db_utils import retry_on_lock
|
||||
from django.db import close_old_connections
|
||||
|
||||
close_old_connections()
|
||||
|
||||
@retry_on_lock()
|
||||
def _get_run():
|
||||
return AnalysisRun.objects.get(id=run_id)
|
||||
|
||||
run = _get_run()
|
||||
try:
|
||||
analysis_fn(run, ctx)
|
||||
except Exception:
|
||||
tb = traceback.format_exc()
|
||||
logger.error(tb)
|
||||
run.status = 'failed'
|
||||
run.error_message = tb
|
||||
run.run_log += f'\n[FATAL] {tb}'
|
||||
retry_on_lock()(lambda: run.save(update_fields=['status', 'error_message', 'run_log']))()
|
||||
|
||||
|
||||
@profile
|
||||
def _background_process(run_id, upload_dir):
|
||||
"""Background: stream-load CSVs → save to SQLite → ready for analysis."""
|
||||
import django
|
||||
os.environ['DJANGO_SETTINGS_MODULE'] = 'tianxuan.settings'
|
||||
django.setup()
|
||||
from analysis.models import AnalysisRun
|
||||
from analysis.session_store import SessionStore
|
||||
|
||||
run = AnalysisRun.objects.get(id=run_id)
|
||||
store = SessionStore()
|
||||
try:
|
||||
# Check upload directory still exists (may have been temp-cleaned)
|
||||
if not upload_dir.is_dir():
|
||||
msg = '上传数据目录已不存在，请重新上传'
|
||||
logger.error('[BACKGROUND] %s (run_id=%s, upload_dir=%s)', msg, run_id, upload_dir)
|
||||
run.status = 'failed'
|
||||
run.error_message = msg
|
||||
run.save(update_fields=['status', 'error_message'])
|
||||
return
|
||||
|
||||
import glob
|
||||
import polars as pl
|
||||
csv_paths = sorted(glob.glob(str(upload_dir / '*.csv')))
|
||||
if not csv_paths:
|
||||
msg = '没有找到 CSV 文件'
|
||||
logger.error('[BACKGROUND] %s (run_id=%s)', msg, run_id)
|
||||
run.status = 'failed'
|
||||
run.error_message = msg
|
||||
run.save(update_fields=['status', 'error_message'])
|
||||
return
|
||||
|
||||
# ── Detect lat/lon columns for schema override ──
|
||||
import csv as _csv
|
||||
_LAT_LON_KW = ('latd', 'lond', 'latitude', 'longitude')
|
||||
lat_lon_overrides = {}
|
||||
try:
|
||||
with open(csv_paths[0], 'r', encoding='utf-8') as _f:
|
||||
_reader = _csv.reader(_f)
|
||||
_headers = next(_reader, [])
|
||||
for _col in _headers:
|
||||
_norm = _col.lower().replace('-', '_').replace('.', '_').replace(' ', '_')
|
||||
if any(kw in _norm for kw in _LAT_LON_KW):
|
||||
lat_lon_overrides[_col] = pl.Utf8
|
||||
logger.info('[BACKGROUND] Forcing lat/lon column %r to Utf8 for safe parsing', _col)
|
||||
except Exception:
|
||||
logger.warning('[BACKGROUND] Failed to detect lat/lon columns', exc_info=True)
|
||||
|
||||
total_files = len(csv_paths)
|
||||
logger.info('[BACKGROUND] Found %d CSV files in %s', total_files, upload_dir)
|
||||
# Adaptive batch size: aim for ~20 batches total, cap at 500 files per batch
|
||||
batch_size = max(1, min(500, total_files // 20 or 1))
|
||||
total_batches = (total_files + batch_size - 1) // batch_size
|
||||
|
||||
run.status = 'loading'; run.progress_pct = 10
|
||||
run.progress_msg = f'正在流式加载 {total_files} 个文件（共 {total_batches} 批）...'
|
||||
run.save(update_fields=['status', 'progress_pct', 'progress_msg'])
|
||||
|
||||
from analysis.data_loader import save_to_db, drop_sqlite_table, load_from_db, _dtype_to_sqlite
|
||||
|
||||
table_name = f'_data_{run_id}'
|
||||
|
||||
# ── Phase 1: Pre-scan all files for full union column set ──
|
||||
all_columns = []
|
||||
for _path in csv_paths:
|
||||
with open(_path, 'r', encoding='utf-8') as _f:
|
||||
_reader = _csv.reader(_f)
|
||||
_headers = next(_reader, [])
|
||||
for _h in _headers:
|
||||
if _h not in all_columns:
|
||||
all_columns.append(_h)
|
||||
reference_columns = sorted(all_columns, key=str.lower)
|
||||
schema = {col: 'Utf8' for col in all_columns}
|
||||
|
||||
# ── Phase 2: Create SQLite table once with all columns as TEXT ──
|
||||
import sqlite3 as _sqlite3
|
||||
from django.conf import settings as _dj_settings
|
||||
_db_path = _dj_settings.DATABASES['default']['NAME']
|
||||
_conn = _sqlite3.connect(_db_path)
|
||||
try:
|
||||
_col_defs = [f'"{_col}" TEXT' for _col in reference_columns]
|
||||
_conn.execute(f'DROP TABLE IF EXISTS "{table_name}"')
|
||||
_conn.execute(f'CREATE TABLE "{table_name}" ({", ".join(_col_defs)})')
|
||||
_conn.commit()
|
||||
finally:
|
||||
_conn.close()
|
||||
|
||||
# ── Phase 3: Batched collection → SQLite append ──
|
||||
schema_overrides = {col: pl.Utf8 for col in reference_columns}
|
||||
completed_batches = 0
|
||||
total_rows = 0
|
||||
for i in range(0, total_files, batch_size):
|
||||
batch = csv_paths[i:i+batch_size]
|
||||
# Scan each file INDIVIDUALLY — batch scan fails when files have different schemas
|
||||
dfs = []
|
||||
for fp in batch:
|
||||
lf = pl.scan_csv(fp, infer_schema_length=0,
|
||||
schema_overrides=schema_overrides)
|
||||
df = lf.collect(streaming=True)
|
||||
# Fill missing columns with NULL
|
||||
for col in reference_columns:
|
||||
if col not in df.columns:
|
||||
df = df.with_columns(pl.lit(None).alias(col))
|
||||
# Select all reference_columns in canonical order
|
||||
df = df.select(reference_columns)
|
||||
dfs.append(df)
|
||||
df_batch = pl.concat(dfs, how='diagonal_relaxed')
|
||||
|
||||
# Append to pre-created SQLite table
|
||||
save_to_db(run.id, df_batch.lazy(), mode='append')
|
||||
|
||||
# Delete temp files after successful SQLite write
|
||||
for f in batch:
|
||||
try:
|
||||
os.unlink(f)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
completed_batches += 1
|
||||
run.progress_pct = int((completed_batches / total_batches) * 80 + 10)
|
||||
run.progress_msg = f'正在加载数据... ({completed_batches}/{total_batches} 批)'
|
||||
run.save(update_fields=['progress_pct', 'progress_msg'])
|
||||
|
||||
# ── Phase 3: Reload from SQLite with correct types ──
|
||||
lf = load_from_db(table_name, schema_overrides=schema)
|
||||
if lf is None:
|
||||
raise RuntimeError(f'SQLite 表 {table_name} 加载失败')
|
||||
|
||||
# Apply _coerce_to_float on lat/lon columns (restore Float64 from Utf8 storage)
|
||||
from analysis.data_loader import _coerce_to_float as _coerce_to_float_fn
|
||||
for lat_lon_col in lat_lon_overrides:
|
||||
if lat_lon_col in lf.columns:
|
||||
lf = lf.with_columns(
|
||||
pl.col(lat_lon_col).map_batches(_coerce_to_float_fn, return_dtype=pl.Float64).alias(lat_lon_col)
|
||||
)
|
||||
|
||||
# ── Restore column types after SQLite storage ──
|
||||
# All columns were stored as Utf8 for SQLite. Recover original types
|
||||
# from the classification system — ONLY the 8 allowed types.
|
||||
from analysis.type_classifier import classify_schema, DataType
|
||||
type_map = classify_schema(lf)
|
||||
# Apply Polars-level type casts based on classification
|
||||
dtypes = {
|
||||
DataType.FLOAT: pl.Float64,
|
||||
DataType.INT: pl.Int64,
|
||||
DataType.BOOL: pl.Boolean,
|
||||
DataType.TIMESTAMP: pl.Float64,
|
||||
}
|
||||
casts = []
|
||||
for col, dt in type_map.items():
|
||||
target = dtypes.get(dt)
|
||||
if target and col in lf.collect_schema().names():
|
||||
casts.append(pl.col(col).cast(target, strict=False))
|
||||
schema[col] = str(target)
|
||||
if casts:
|
||||
lf = lf.with_columns(casts)
|
||||
restored = [(col, dt.value) for col, dt in type_map.items() if dt in dtypes][:15]
|
||||
logger.info('[BACKGROUND] Type restoration: %s', restored)
|
||||
|
||||
# ── SVD dimensionality reduction ──
|
||||
svd_components = 0
|
||||
# Only operate on columns with known numeric types in the schema
|
||||
numeric_dtypes = {'Int64', 'Float64', 'Int32', 'Float32',
|
||||
'Int8', 'Int16', 'UInt8', 'UInt16', 'UInt32', 'UInt64'}
|
||||
numeric_cols = [c for c, dt in schema.items() if dt in numeric_dtypes]
|
||||
n_numeric = len(numeric_cols)
|
||||
|
||||
if n_numeric >= 2:
|
||||
n_components = min(MAX_CLUSTER_FEATURES, n_numeric - 1)
|
||||
try:
|
||||
from sklearn.decomposition import TruncatedSVD # fmt: skip
|
||||
import numpy as np # fmt: skip
|
||||
|
||||
# Collect numeric data for SVD (full materialise → transform → add back)
|
||||
df_full = lf.select(numeric_cols).collect(streaming=True)
|
||||
X = df_full.to_numpy()
|
||||
X = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)
|
||||
|
||||
svd = TruncatedSVD(n_components=n_components, random_state=RANDOM_SEED)
|
||||
X_svd = svd.fit_transform(X)
|
||||
svd_components = n_components
|
||||
|
||||
# Build a new LazyFrame that includes _svd_* columns
|
||||
# Strategy: collect full LazyFrame once, add SVD columns, re-wrap as lazy
|
||||
df_all = lf.collect(streaming=True)
|
||||
for i in range(n_components):
|
||||
col_name = f'_svd_{i}'
|
||||
schema[col_name] = 'Float64'
|
||||
df_all = df_all.with_columns(
|
||||
pl.Series(col_name, X_svd[:, i])
|
||||
)
|
||||
lf = df_all.lazy()
|
||||
|
||||
logger.info('[BACKGROUND] SVD: %d components from %d numeric columns '
|
||||
'(explained variance ratio sum=%.4f)',
|
||||
svd_components, n_numeric,
|
||||
float(svd.explained_variance_ratio_.sum()))
|
||||
except Exception as e:
|
||||
logger.warning('[BACKGROUND] SVD failed (non-fatal): %s', e)
|
||||
elif n_numeric == 1:
|
||||
logger.info('[BACKGROUND] SVD skipped: only 1 numeric column (%s)',
|
||||
numeric_cols[0])
|
||||
|
||||
# Get row count
|
||||
df_count = lf.select(pl.len()).collect(streaming=True)
|
||||
total_rows = df_count[0, 0] if df_count.height > 0 else 0
|
||||
|
||||
run.sqlite_table = table_name
|
||||
run.total_flows = total_rows
|
||||
run.save(update_fields=['sqlite_table', 'total_flows'])
|
||||
|
||||
ds_id = f'upload_{run.display_id}'
|
||||
store.store_dataset(ds_id, lf, schema=schema, metadata={
|
||||
'row_count': total_rows, 'file_count': total_files, 'upload_dir': str(upload_dir),
|
||||
'csv_glob': str(upload_dir / '*.csv'),
|
||||
'sqlite_table': table_name,
|
||||
'svd_components': svd_components,
|
||||
})
|
||||
|
||||
# ── Phase 3: Data ready for filtering/clustering via MCP tools ──
|
||||
run.status = 'ready'; run.progress_pct = 60; run.progress_msg = '数据准备就绪'
|
||||
run.save(update_fields=['status', 'progress_pct', 'progress_msg'])
|
||||
|
||||
# ── Phase 4: Clean up uploaded CSV files ──
|
||||
try:
|
||||
if upload_dir and upload_dir.is_dir():
|
||||
import shutil
|
||||
shutil.rmtree(upload_dir, ignore_errors=True)
|
||||
logger.info('[BACKGROUND] Cleaned up upload directory: %s', upload_dir)
|
||||
except Exception as clean_err:
|
||||
logger.warning('[BACKGROUND] Upload cleanup failed (non-fatal): %s', clean_err)
|
||||
except Exception:
|
||||
tb = traceback.format_exc()
|
||||
logger.error(tb)
|
||||
# Drop partial SQLite table on failure (idempotent)
|
||||
try:
|
||||
from analysis.data_loader import drop_sqlite_table # fmt: skip
|
||||
drop_sqlite_table(f'_data_{run_id}')
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
pass # Expected: drop_sqlite_table may fail if table already gone
|
||||
run.status = 'failed'
|
||||
run.error_message = tb
|
||||
run.run_log += f'\n[ERROR] {tb}'
|
||||
run.save(update_fields=['status', 'error_message', 'run_log'])
|
||||
@@ -0,0 +1,381 @@
|
||||
"""MCP Tool Lab, plan management, and utility views."""
|
||||
import json
|
||||
import asyncio
|
||||
import threading
|
||||
import traceback
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from django.shortcuts import render, get_object_or_404, redirect
|
||||
from django.http import JsonResponse
|
||||
from django.views.decorators.csrf import csrf_exempt
|
||||
|
||||
from analysis.appdata import ensure_appdata_dir
|
||||
from analysis.models import AnalysisRun
|
||||
from .pipeline import _background_process
|
||||
from .helpers import (
|
||||
_PLANS_DIR, _AUTO_INDEX_PATH, _MAX_AUTO_PLANS,
|
||||
_read_auto_index, _write_auto_index, _add_to_auto_index,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def tool_lab(request):
|
||||
"""List all available MCP tools with descriptions and input schemas."""
|
||||
from analysis.tool_registry import get_tools_meta
|
||||
tools = get_tools_meta()
|
||||
# Group tools by category
|
||||
core_names = {'profile_data', 'build_entity_profiles', 'compute_scores',
|
||||
'run_clustering', 'extract_features', 'detect_anomalies',
|
||||
'visualize_anomalies'}
|
||||
diag_names = {'validate_data', 'explore_distributions', 'find_outliers',
|
||||
'diagnose_clustering', 'compare_datasets', 'export_debug_sample',
|
||||
'repair_schema'}
|
||||
analysis_names = {'analyze_patterns', 'analyze_temporal', 'analyze_tls_health',
|
||||
'analyze_geo_distribution', 'analyze_entity_detail'}
|
||||
|
||||
core_tools, diag_tools, analysis_tools, other_tools = [], [], [], []
|
||||
for t in tools:
|
||||
if t.name in core_names:
|
||||
core_tools.append(t)
|
||||
elif t.name in diag_names:
|
||||
diag_tools.append(t)
|
||||
elif t.name in analysis_names:
|
||||
analysis_tools.append(t)
|
||||
else:
|
||||
other_tools.append(t)
|
||||
|
||||
# Get list of datasets for parameter autofill
|
||||
from analysis.session_store import SessionStore
|
||||
store = SessionStore()
|
||||
ds_list = store.list_datasets() if hasattr(store, 'list_datasets') else []
|
||||
|
||||
import json as _json
|
||||
# Serialize tools metadata for JS
|
||||
meta_list = []
|
||||
for t in tools:
|
||||
meta_list.append({
|
||||
'name': t.name,
|
||||
'description': t.description,
|
||||
'inputSchema': t.inputSchema,
|
||||
})
|
||||
return render(request, 'analysis/tool_lab.html', {
|
||||
'core_tools': core_tools,
|
||||
'diag_tools': diag_tools,
|
||||
'analysis_tools': analysis_tools,
|
||||
'other_tools': other_tools,
|
||||
'datasets': ds_list,
|
||||
'tools_meta': _json.dumps(meta_list, ensure_ascii=False),
|
||||
})
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
def tool_lab_run(request):
|
||||
"""Execute a single MCP tool via AJAX POST."""
|
||||
if request.method != 'POST':
|
||||
return JsonResponse({'error': 'POST required'}, status=405)
|
||||
import json as _json
|
||||
try:
|
||||
body = _json.loads(request.body)
|
||||
except Exception:
|
||||
logger.error("tool_lab_run failed: {}".format(traceback.format_exc()))
|
||||
return JsonResponse({'error': 'Invalid JSON body'}, status=400)
|
||||
|
||||
tool_name = body.get('tool')
|
||||
params = body.get('params', {})
|
||||
if not tool_name:
|
||||
return JsonResponse({'error': 'Missing tool name'}, status=400)
|
||||
|
||||
# ── Backend validation: check required params + dataset existence ──
|
||||
from analysis.tool_registry import handle_call, get_tools_meta
|
||||
from analysis.session_store import SessionStore
|
||||
tools_meta = get_tools_meta()
|
||||
tool_schema = None
|
||||
for t in tools_meta:
|
||||
if t.name == tool_name:
|
||||
tool_schema = t.inputSchema
|
||||
break
|
||||
if tool_schema:
|
||||
required_fields = tool_schema.get('required', [])
|
||||
for field in required_fields:
|
||||
val = params.get(field)
|
||||
if val is None or (isinstance(val, str) and val.strip() == ''):
|
||||
return JsonResponse({'error': f'Missing required parameter: {field}'}, status=400)
|
||||
# If field is a dataset_id, verify it exists in the store
|
||||
if 'dataset' in field.lower():
|
||||
store = SessionStore()
|
||||
if not store.get_dataset(str(val)):
|
||||
return JsonResponse({'error': f'Dataset not found: {val}'}, status=400)
|
||||
|
||||
result = asyncio.run(handle_call(tool_name, params))
|
||||
return JsonResponse({'status': 'ok', 'result': result})
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
def reload_run_data(request):
|
||||
"""Reload a run's data from csv_glob or sqlite_table into SessionStore."""
|
||||
if request.method != 'POST':
|
||||
return JsonResponse({'error': 'POST required'}, status=405)
|
||||
import json as _json
|
||||
try:
|
||||
body = _json.loads(request.body)
|
||||
except Exception:
|
||||
logger.error("reload_run_data failed: {}".format(traceback.format_exc()))
|
||||
return JsonResponse({'error': 'Invalid JSON body'}, status=400)
|
||||
|
||||
display_id = body.get('display_id')
|
||||
if not display_id:
|
||||
return JsonResponse({'error': 'Missing display_id'}, status=400)
|
||||
|
||||
from analysis.models import AnalysisRun
|
||||
from analysis.data_loader import load_csv_directory, load_from_db
|
||||
|
||||
run = get_object_or_404(AnalysisRun, display_id=display_id)
|
||||
|
||||
# Determine dataset_id (same keying as manual_page)
|
||||
ds_id = f'upload_{display_id}' if run.run_type == 'upload' else f'run_{display_id}'
|
||||
|
||||
from analysis.session_store import SessionStore
|
||||
store = SessionStore()
|
||||
# Check if already loaded
|
||||
existing = store.get_dataset(ds_id)
|
||||
if existing is not None:
|
||||
return JsonResponse({'status': 'ok', 'dataset_id': ds_id, 'reloaded': False})
|
||||
|
||||
# Try loading from sqlite_table first, then csv_glob
|
||||
lf = None
|
||||
schema = {}
|
||||
row_count = 0
|
||||
file_count = 0
|
||||
|
||||
if run.sqlite_table:
|
||||
try:
|
||||
lf = load_from_db(run.sqlite_table)
|
||||
if lf is not None:
|
||||
import polars as pl
|
||||
row_count = lf.select(pl.len()).collect(streaming=True)[0, 0]
|
||||
except Exception as exc:
|
||||
logging.getLogger(__name__).warning(f'Failed to load from sqlite_table: {exc}')
|
||||
|
||||
if lf is None and run.csv_glob:
|
||||
try:
|
||||
from pathlib import Path
|
||||
csv_path = Path(run.csv_glob)
|
||||
if csv_path.is_file() or csv_path.is_dir() or csv_path.parent.is_dir():
|
||||
lf, schema, row_count, file_count, _ = load_csv_directory(run.csv_glob)
|
||||
except Exception as exc:
|
||||
logging.getLogger(__name__).warning(f'Failed to load from csv_glob: {exc}')
|
||||
|
||||
if lf is None:
|
||||
return JsonResponse({'error': 'No data source available (upload dir or sqlite table gone)'}, status=404)
|
||||
|
||||
store.store_dataset(ds_id, lf, schema=schema, metadata={
|
||||
'row_count': row_count,
|
||||
'file_count': file_count,
|
||||
'csv_glob': run.csv_glob or '',
|
||||
'sqlite_table': run.sqlite_table or '',
|
||||
})
|
||||
|
||||
return JsonResponse({
|
||||
'status': 'ok',
|
||||
'dataset_id': ds_id,
|
||||
'reloaded': True,
|
||||
'row_count': row_count,
|
||||
'columns': list(schema.keys()),
|
||||
})
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
def tool_plan(request):
|
||||
"""Save, list, load, delete, or pin/unpin an analysis plan."""
|
||||
import json as _j
|
||||
if request.method == 'GET':
|
||||
name = request.GET.get('name')
|
||||
if name:
|
||||
# Auto plans stored with _auto_ prefix
|
||||
if name.startswith('_auto_'):
|
||||
path = _PLANS_DIR / f'{name}.json'
|
||||
else:
|
||||
path = _PLANS_DIR / f'{name}.json'
|
||||
if not path.exists():
|
||||
return JsonResponse({'error': f'Plan "{name}" not found'}, status=404)
|
||||
try:
|
||||
data = _j.loads(path.read_text(encoding='utf-8'))
|
||||
return JsonResponse(data)
|
||||
except Exception as e:
|
||||
return JsonResponse({'error': str(e)}, status=500)
|
||||
|
||||
# List all plans (regular + auto)
|
||||
auto_index = _read_auto_index()
|
||||
auto_index_map = {e.get('file', ''): e.get('pinned', False) for e in auto_index}
|
||||
auto_names = set()
|
||||
plans = []
|
||||
for p in sorted(_PLANS_DIR.glob('*.json')):
|
||||
# Skip index file and auto plans (handled separately)
|
||||
if p.name == '_auto_index.json':
|
||||
continue
|
||||
if p.name.startswith('_auto_'):
|
||||
auto_names.add(p.name)
|
||||
continue
|
||||
try:
|
||||
content = _j.loads(p.read_text(encoding='utf-8'))
|
||||
steps = len(content.get('steps', []))
|
||||
plans.append({'name': p.stem, 'steps': steps})
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
plans.append({'name': p.stem, 'steps': 0})
|
||||
|
||||
# Add auto plans with markers
|
||||
auto_plans = []
|
||||
for auto_name in sorted(auto_names):
|
||||
try:
|
||||
path = _PLANS_DIR / auto_name
|
||||
content = _j.loads(path.read_text(encoding='utf-8'))
|
||||
steps = len(content.get('steps', []))
|
||||
auto_plans.append({
|
||||
'name': auto_name.replace('.json', ''),
|
||||
'steps': steps,
|
||||
'auto': True,
|
||||
'pinned': auto_index_map.get(auto_name, False),
|
||||
})
|
||||
except Exception:
|
||||
logger.error("unknown failed: {}".format(traceback.format_exc()))
|
||||
auto_plans.append({
|
||||
'name': auto_name.replace('.json', ''),
|
||||
'steps': 0,
|
||||
'auto': True,
|
||||
'pinned': auto_index_map.get(auto_name, False),
|
||||
})
|
||||
# Interleave: auto plans first, then regular
|
||||
return JsonResponse({'plans': auto_plans + plans})
|
||||
|
||||
elif request.method == 'POST':
|
||||
try:
|
||||
body = _j.loads(request.body)
|
||||
# Pin/unpin toggle for auto plans
|
||||
if 'pin' in body and body.get('name', '').startswith('_auto_'):
|
||||
filename = body['name'].replace('.json', '') + '.json' \
|
||||
if not body['name'].endswith('.json') else body['name']
|
||||
index = _read_auto_index()
|
||||
new_pinned = bool(body['pin'])
|
||||
found = False
|
||||
for entry in index:
|
||||
if entry.get('file') == filename:
|
||||
entry['pinned'] = new_pinned
|
||||
found = True
|
||||
break
|
||||
if not found and new_pinned:
|
||||
index.append({'file': filename, 'pinned': True})
|
||||
_write_auto_index(index)
|
||||
return JsonResponse({
|
||||
'status': 'pinned' if new_pinned else 'unpinned',
|
||||
'name': body['name'],
|
||||
'pinned': new_pinned,
|
||||
})
|
||||
|
||||
# Regular save
|
||||
name = body.get('name', '').strip()
|
||||
steps = body.get('steps', [])
|
||||
if not name:
|
||||
return JsonResponse({'error': 'Missing plan name'}, status=400)
|
||||
path = _PLANS_DIR / f'{name}.json'
|
||||
path.write_text(_j.dumps({'name': name, 'steps': steps}, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
return JsonResponse({'status': 'saved', 'name': name, 'steps': len(steps)})
|
||||
except Exception as e:
|
||||
return JsonResponse({'error': str(e)}, status=500)
|
||||
|
||||
elif request.method == 'DELETE':
|
||||
name = request.GET.get('name', '').strip()
|
||||
if not name:
|
||||
return JsonResponse({'error': 'Missing plan name'}, status=400)
|
||||
path = _PLANS_DIR / f'{name}.json'
|
||||
if path.exists():
|
||||
path.unlink()
|
||||
# Also remove from auto index if applicable
|
||||
if name.startswith('_auto_'):
|
||||
filename = name + '.json' if not name.endswith('.json') else name
|
||||
index = _read_auto_index()
|
||||
index = [e for e in index if e.get('file') != filename]
|
||||
_write_auto_index(index)
|
||||
return JsonResponse({'status': 'deleted', 'name': name})
|
||||
|
||||
return JsonResponse({'error': 'Method not allowed'}, status=405)
|
||||
|
||||
|
||||
# ── Filter endpoint for manual analysis page ──────────────────────────
|
||||
|
||||
@csrf_exempt
|
||||
def apply_filter(request):
|
||||
"""POST {dataset_id, filters, logic} → apply filter_data tool → return filtered_X id."""
|
||||
if request.method != 'POST':
|
||||
return JsonResponse({'error': 'POST required'}, status=405)
|
||||
import json as _json
|
||||
body = _json.loads(request.body)
|
||||
dataset_id = body.get('dataset_id')
|
||||
filters = body.get('filters', [])
|
||||
logic = body.get('logic', 'and')
|
||||
|
||||
if not dataset_id:
|
||||
return JsonResponse({'error': 'dataset_id is required'})
|
||||
if not filters:
|
||||
return JsonResponse({'error': 'filters is required'})
|
||||
|
||||
try:
|
||||
from analysis.tool_registry import _handle_filter_data
|
||||
result = asyncio.run(_handle_filter_data(
|
||||
dataset_id=dataset_id,
|
||||
filters=filters,
|
||||
logic=logic,
|
||||
))
|
||||
if 'error' in result:
|
||||
return JsonResponse({'error': result['error']})
|
||||
|
||||
filtered_id = result.get('filtered_dataset_id', '')
|
||||
row_count = result.get('row_count', 0)
|
||||
|
||||
# Use the unique flt_xxx ID from _handle_filter_data — never rename
|
||||
# to filtered_{num} which would collide on repeated filters of the same source.
|
||||
return JsonResponse({
|
||||
'dataset_id': filtered_id,
|
||||
'row_count': row_count,
|
||||
})
|
||||
except Exception as e:
|
||||
logger.error(f'[apply_filter] {traceback.format_exc()}')
|
||||
return JsonResponse({'error': str(e)})
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
def retry_run(request, display_id):
|
||||
"""Reset a failed run and re-trigger based on run_type."""
|
||||
if request.method != 'POST':
|
||||
return JsonResponse({'error': 'POST required'}, status=405)
|
||||
|
||||
run = get_object_or_404(AnalysisRun, display_id=display_id)
|
||||
if run.status != 'failed':
|
||||
return JsonResponse({'error': '只能重试失败的分析运行'}, status=400)
|
||||
|
||||
run.status = 'pending'
|
||||
run.error_message = ''
|
||||
run.progress_pct = 0
|
||||
run.progress_msg = ''
|
||||
run.save(update_fields=['status', 'error_message', 'progress_pct', 'progress_msg'])
|
||||
|
||||
if run.run_type == 'upload' and run.csv_glob:
|
||||
upload_dir = Path(run.csv_glob).parent
|
||||
expected_base = ensure_appdata_dir() / 'data' / 'uploads'
|
||||
if not str(upload_dir.resolve()).startswith(str(expected_base.resolve())):
|
||||
return JsonResponse({'error': 'Invalid upload path'}, status=400)
|
||||
if upload_dir.is_dir():
|
||||
t = threading.Thread(
|
||||
target=_background_process,
|
||||
args=(run.id, upload_dir),
|
||||
daemon=True,
|
||||
)
|
||||
t.start()
|
||||
else:
|
||||
run.error_message = '上传数据目录已不存在，请重新上传'
|
||||
run.status = 'failed'
|
||||
run.save(update_fields=['status', 'error_message'])
|
||||
|
||||
return redirect('analysis:run_detail', display_id=run.display_id)
|
||||
@@ -0,0 +1,177 @@
|
||||
"""Upload views: CSV upload, batch, finalize, delete."""
|
||||
import threading
|
||||
import logging
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from django.shortcuts import render, get_object_or_404
|
||||
from django.http import JsonResponse
|
||||
from django.views.decorators.csrf import csrf_exempt
|
||||
|
||||
from analysis.appdata import ensure_appdata_dir
|
||||
from analysis.models import AnalysisRun
|
||||
from .pipeline import _background_process
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def upload_page(request):
|
||||
"""CSV upload page."""
|
||||
runs = AnalysisRun.objects.all().order_by('-created_at')[:20]
|
||||
return render(request, 'tianxuan/upload.html', {'runs': runs})
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
def upload_csv(request):
|
||||
"""Handle CSV upload AJAX POST."""
|
||||
if request.method != 'POST':
|
||||
return JsonResponse({'error': 'POST required'}, status=405)
|
||||
files = request.FILES.getlist('files')
|
||||
if not files:
|
||||
return JsonResponse({'error': 'No files uploaded'})
|
||||
logger.info('[UPLOAD] Received %d files', len(files))
|
||||
for f in files:
|
||||
if f.size > 500 * 1024 * 1024:
|
||||
return JsonResponse({'warning': f'文件 {f.name} 超过 500MB，建议分割后上传'})
|
||||
total_size = sum(f.size for f in files)
|
||||
if total_size > 50 * 1024 * 1024 * 1024:
|
||||
return JsonResponse({'error': '总大小超过 50GB 限制'})
|
||||
from django.utils import timezone
|
||||
ts = timezone.now().strftime('%Y%m%d_%H%M%S')
|
||||
from analysis.appdata import ensure_appdata_dir
|
||||
base = ensure_appdata_dir() / 'data' / 'uploads'
|
||||
upload_dir = base / ts
|
||||
upload_dir.mkdir(parents=True, exist_ok=True)
|
||||
saved = []
|
||||
run_ids = []
|
||||
for f in files:
|
||||
# Each file gets its own run and its own processing thread
|
||||
fpath = upload_dir / f.name
|
||||
with open(fpath, 'wb+') as dst:
|
||||
for chunk in f.chunks():
|
||||
dst.write(chunk)
|
||||
fsize = fpath.stat().st_size
|
||||
saved.append(str(fpath))
|
||||
|
||||
# Create a run for this single file
|
||||
run = AnalysisRun.objects.create(
|
||||
csv_glob=str(fpath),
|
||||
status='loading',
|
||||
run_type='upload',
|
||||
)
|
||||
run_ids.append(run.display_id)
|
||||
|
||||
# Start processing thread immediately (per-file, not batched)
|
||||
t = threading.Thread(
|
||||
target=_process_single_file,
|
||||
args=(run.id, str(fpath)),
|
||||
daemon=True,
|
||||
)
|
||||
t.start()
|
||||
|
||||
return JsonResponse({'run_ids': run_ids, 'files': saved, 'status': 'processing'})
|
||||
|
||||
|
||||
def _process_single_file(run_id, filepath):
|
||||
"""Load a single CSV into SQLite, then delete it. Runs in background thread inside Django."""
|
||||
from analysis.data_loader._sqlite import save_to_db, drop_sqlite_table
|
||||
from analysis.data_loader._csv import load_csv_directory
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
try:
|
||||
run = AnalysisRun.objects.get(id=run_id)
|
||||
lf, schema, row_count, file_count, mem = load_csv_directory(filepath)
|
||||
table_name = save_to_db(run.id, lf)
|
||||
run.sqlite_table = table_name
|
||||
run.total_flows = row_count
|
||||
run.status = 'ready'
|
||||
run.save(update_fields=['sqlite_table', 'total_flows', 'status'])
|
||||
|
||||
# Delete CSV file immediately
|
||||
Path(filepath).unlink(missing_ok=True)
|
||||
logger.info('[UPLOAD] File %s → SQLite table %s (%d rows), CSV deleted',
|
||||
filepath, table_name, row_count)
|
||||
except Exception as exc:
|
||||
logger.error('[UPLOAD] Failed to process %s: %s', filepath, exc)
|
||||
try:
|
||||
run = AnalysisRun.objects.get(id=run_id)
|
||||
run.status = 'failed'
|
||||
run.error_message = str(exc)
|
||||
run.save(update_fields=['status', 'error_message'])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
def upload_csv_batch(request, display_id):
|
||||
"""Append files to existing upload run. Does NOT re-trigger _background_process."""
|
||||
run = get_object_or_404(AnalysisRun, display_id=display_id)
|
||||
files = request.FILES.getlist('files')
|
||||
if not files:
|
||||
return JsonResponse({'error': 'No files uploaded'})
|
||||
upload_dir = Path(run.csv_glob).parent if run.csv_glob else None
|
||||
if upload_dir:
|
||||
expected_base = ensure_appdata_dir() / 'data' / 'uploads'
|
||||
if not str(upload_dir.resolve()).startswith(str(expected_base.resolve())):
|
||||
return JsonResponse({'error': 'Invalid upload path'}, status=400)
|
||||
if not upload_dir or not upload_dir.exists():
|
||||
return JsonResponse({'error': 'Upload directory not found'})
|
||||
saved = []
|
||||
for f in files:
|
||||
path = upload_dir / f.name
|
||||
with open(path, 'wb+') as dst:
|
||||
for chunk in f.chunks():
|
||||
dst.write(chunk)
|
||||
saved.append(str(path))
|
||||
# NO background process re-trigger — first batch handles it
|
||||
return JsonResponse({'run_id': run.display_id, 'files': len(saved), 'status': 'ok'})
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
def finalize_upload(request, display_id):
|
||||
"""Finalize batch upload: validate upload_dir, set status='loading', start background processing."""
|
||||
if request.method != 'POST':
|
||||
return JsonResponse({'error': 'POST required'}, status=405)
|
||||
run = get_object_or_404(AnalysisRun, display_id=display_id)
|
||||
if run.status != 'pending':
|
||||
return JsonResponse({'error': f'Run status is {run.status}, expected pending'}, status=400)
|
||||
upload_dir = Path(run.csv_glob).parent if run.csv_glob else None
|
||||
if not upload_dir or not upload_dir.exists():
|
||||
return JsonResponse({'error': 'Upload directory not found'}, status=400)
|
||||
run.status = 'loading'
|
||||
run.save(update_fields=['status'])
|
||||
t = threading.Thread(target=_background_process, args=(run.id, upload_dir), daemon=True)
|
||||
t.start()
|
||||
return JsonResponse({'run_id': run.display_id, 'status': 'loading'})
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
def delete_upload(request, display_id):
|
||||
"""Delete uploaded files, SQLite data table, and AnalysisRun.
|
||||
|
||||
Handles all run statuses (loading, profiling, clustering, completed, failed).
|
||||
Gracefully tolerates null csv_glob and missing/errored SQLite tables.
|
||||
"""
|
||||
run = get_object_or_404(AnalysisRun, display_id=display_id)
|
||||
|
||||
# 1. Drop the SQLite data table if one exists
|
||||
if run.sqlite_table:
|
||||
try:
|
||||
from analysis.data_loader import drop_sqlite_table # fmt: skip
|
||||
drop_sqlite_table(run.sqlite_table)
|
||||
except Exception as exc:
|
||||
logger.warning('[DELETE] failed to drop SQLite table %s: %s', run.sqlite_table, exc)
|
||||
|
||||
# 2. Remove the upload directory if csv_glob is set
|
||||
if run.csv_glob:
|
||||
p = Path(run.csv_glob).parent
|
||||
expected_base = ensure_appdata_dir() / 'data' / 'uploads'
|
||||
if not str(p.resolve()).startswith(str(expected_base.resolve())):
|
||||
return JsonResponse({'error': 'Invalid upload path'}, status=400)
|
||||
if p.exists():
|
||||
shutil.rmtree(p, ignore_errors=True)
|
||||
|
||||
# 3. Delete the ORM record
|
||||
run.delete()
|
||||
return JsonResponse({'deleted': True})
|
||||
@@ -1,6 +1,6 @@
|
||||
server:
|
||||
host: 0.0.0.0
|
||||
port: 80
|
||||
port: 8000
|
||||
debug: false
|
||||
data:
|
||||
schema_strict: false
|
||||
@@ -25,7 +25,7 @@ anomaly_detection:
|
||||
llm:
|
||||
enabled: true
|
||||
base_url: 'https://api.deepseek.com'
|
||||
api_key: 'sk-ecd33e5341ff4ec9a2c5a5c97a97699b'
|
||||
api_key: 'sk-your-api-key-here'
|
||||
model: deepseek-v4-flash
|
||||
entity:
|
||||
subnet_masks: [24, 28]
|
||||
+1
-1
@@ -19,7 +19,7 @@ class Config(BaseModel):
|
||||
recursive: bool = False
|
||||
|
||||
class Clustering(BaseModel):
|
||||
algorithm: str = 'hdbscan'
|
||||
algorithm: str = 'agglomerative'
|
||||
min_cluster_size: int = 5
|
||||
random_state: int = 42
|
||||
|
||||
|
||||
@@ -0,0 +1,683 @@
|
||||
{
|
||||
"8.8.8.8": {
|
||||
"lat": 37.422,
|
||||
"lon": -122.084,
|
||||
"city": "Sunnyvale",
|
||||
"country": "US"
|
||||
},
|
||||
"1.1.1.1": {
|
||||
"lat": 37.7749,
|
||||
"lon": -122.4194,
|
||||
"city": "San Francisco",
|
||||
"country": "US"
|
||||
},
|
||||
"52.0.0.1": {
|
||||
"lat": 39.0438,
|
||||
"lon": -77.4874,
|
||||
"city": "Ashburn",
|
||||
"country": "US"
|
||||
},
|
||||
"104.16.1.1": {
|
||||
"lat": 37.7749,
|
||||
"lon": -122.4194,
|
||||
"city": "San Francisco",
|
||||
"country": "US"
|
||||
},
|
||||
"47.96.1.1": {
|
||||
"lat": 31.2304,
|
||||
"lon": 121.4737,
|
||||
"city": "Shanghai",
|
||||
"country": "CN"
|
||||
},
|
||||
"218.1.1.1": null,
|
||||
"157.1.1.1": {
|
||||
"lat": 34.6937,
|
||||
"lon": 135.5023,
|
||||
"city": "Japan",
|
||||
"country": "JP"
|
||||
},
|
||||
"10.0.0.1": null,
|
||||
"192.168.1.1": null,
|
||||
"13.250.0.1": null,
|
||||
"185.234.0.1": null,
|
||||
"103.235.0.1": {
|
||||
"lat": 20.0,
|
||||
"lon": 77.0,
|
||||
"city": "",
|
||||
"country": "印度"
|
||||
},
|
||||
"anon, hosting": null,
|
||||
"-108.418446": null,
|
||||
"-63.303819": null,
|
||||
"156.494253": {
|
||||
"lat": 34.053,
|
||||
"lon": -118.2642,
|
||||
"city": "洛杉矶",
|
||||
"country": "美国"
|
||||
},
|
||||
"174.324906": {
|
||||
"lat": 43.6425,
|
||||
"lon": -79.3872,
|
||||
"city": "",
|
||||
"country": "加拿大"
|
||||
},
|
||||
"91.251045": {
|
||||
"lat": 53.55,
|
||||
"lon": 10.0,
|
||||
"city": "汉堡市",
|
||||
"country": "德国"
|
||||
},
|
||||
"136.838644": {
|
||||
"lat": 42.3223,
|
||||
"lon": -83.1763,
|
||||
"city": "迪尔伯恩",
|
||||
"country": "美国"
|
||||
},
|
||||
"-135.232683": null,
|
||||
"134.708535": {
|
||||
"lat": 45.4763,
|
||||
"lon": -122.6408,
|
||||
"city": "Portland",
|
||||
"country": "美国"
|
||||
},
|
||||
"172.779207": {
|
||||
"lat": 38.0,
|
||||
"lon": -97.0,
|
||||
"city": "",
|
||||
"country": "美国"
|
||||
},
|
||||
"76.375007": {
|
||||
"lat": 35.6507,
|
||||
"lon": -78.4564,
|
||||
"city": "Clayton",
|
||||
"country": "美国"
|
||||
},
|
||||
"-37.054377": null,
|
||||
"58.368656": {
|
||||
"lat": 35.685,
|
||||
"lon": 139.7514,
|
||||
"city": "东京",
|
||||
"country": "日本"
|
||||
},
|
||||
"-128.312338": null,
|
||||
"-86.295682": null,
|
||||
"-83.710197": null,
|
||||
"-117.654327": null,
|
||||
"-94.931072": null,
|
||||
"-31.118326": null,
|
||||
"41.94697": {
|
||||
"lat": -29.0,
|
||||
"lon": 24.0,
|
||||
"city": "",
|
||||
"country": "南非"
|
||||
},
|
||||
"-175.557379": null,
|
||||
"None": null,
|
||||
"90.484378": {
|
||||
"lat": 50.6846,
|
||||
"lon": 1.7610999999999999,
|
||||
"city": "维尔维涅",
|
||||
"country": "法国"
|
||||
},
|
||||
"91.136808": {
|
||||
"lat": 50.8934,
|
||||
"lon": 7.8856,
|
||||
"city": "Freudenberg",
|
||||
"country": "德国"
|
||||
},
|
||||
"98.398423": {
|
||||
"lat": 38.0,
|
||||
"lon": -97.0,
|
||||
"city": "",
|
||||
"country": "美国"
|
||||
},
|
||||
"-37.351993": null,
|
||||
"108.8728": {
|
||||
"lat": 33.5363,
|
||||
"lon": -117.0439,
|
||||
"city": "特曼库拉",
|
||||
"country": "美国"
|
||||
},
|
||||
"-40.490238": null,
|
||||
"-108.320482": null,
|
||||
"-42.599081": null,
|
||||
"60.847895": {
|
||||
"lat": 30.2936,
|
||||
"lon": 120.1614,
|
||||
"city": "杭州",
|
||||
"country": "中国"
|
||||
},
|
||||
"-70.344735": null,
|
||||
"-17.065978": null,
|
||||
"-17.594761": null,
|
||||
"3.748318": {
|
||||
"lat": 41.1412,
|
||||
"lon": -73.2637,
|
||||
"city": "Fairfield",
|
||||
"country": "美国"
|
||||
},
|
||||
"117.90784": {
|
||||
"lat": 21.0333,
|
||||
"lon": 105.85,
|
||||
"city": "Hanoi",
|
||||
"country": "越南"
|
||||
},
|
||||
"-178.325477": null,
|
||||
"47.034714": {
|
||||
"lat": 45.3155,
|
||||
"lon": -75.837,
|
||||
"city": "Ottawa",
|
||||
"country": "加拿大"
|
||||
},
|
||||
"12.6356": {
|
||||
"lat": 38.0,
|
||||
"lon": -97.0,
|
||||
"city": "",
|
||||
"country": "美国"
|
||||
},
|
||||
"-4.376563": null,
|
||||
"117.415791": {
|
||||
"lat": 21.0333,
|
||||
"lon": 105.85,
|
||||
"city": "Hanoi",
|
||||
"country": "越南"
|
||||
},
|
||||
"-95.943745": null,
|
||||
"115.811466": {
|
||||
"lat": 37.57,
|
||||
"lon": 126.98,
|
||||
"city": "",
|
||||
"country": "大韩民国"
|
||||
},
|
||||
"82.858229": {
|
||||
"lat": 54.5842,
|
||||
"lon": -1.5631,
|
||||
"city": "Darlington",
|
||||
"country": "英国"
|
||||
},
|
||||
"120.177108": {
|
||||
"lat": 39.8897,
|
||||
"lon": 115.275,
|
||||
"city": "Hebei",
|
||||
"country": "中国"
|
||||
},
|
||||
"12.054574": {
|
||||
"lat": 38.0,
|
||||
"lon": -97.0,
|
||||
"city": "",
|
||||
"country": "美国"
|
||||
},
|
||||
"-112.238338": null,
|
||||
"-161.091649": null,
|
||||
"24.207799": {
|
||||
"lat": 40.3698,
|
||||
"lon": -80.634,
|
||||
"city": "斯托本维尔",
|
||||
"country": "美国"
|
||||
},
|
||||
"-148.31237": null,
|
||||
"-132.223679": null,
|
||||
"76.252382": {
|
||||
"lat": 35.6513,
|
||||
"lon": -78.8336,
|
||||
"city": "Holly Springs",
|
||||
"country": "美国"
|
||||
},
|
||||
"104.839037": {
|
||||
"lat": 38.0,
|
||||
"lon": -97.0,
|
||||
"city": "",
|
||||
"country": "美国"
|
||||
},
|
||||
"-147.319447": null,
|
||||
"-137.111721": null,
|
||||
"-107.76671": null,
|
||||
"-167.802595": null,
|
||||
"168.500812": {
|
||||
"lat": 29.7176,
|
||||
"lon": -95.4188,
|
||||
"city": "休斯敦",
|
||||
"country": "美国"
|
||||
},
|
||||
"-19.640731": null,
|
||||
"-36.115066": null,
|
||||
"-31.172609": null,
|
||||
"4.746775": {
|
||||
"lat": 38.0,
|
||||
"lon": -97.0,
|
||||
"city": "",
|
||||
"country": "美国"
|
||||
},
|
||||
"-128.714202": null,
|
||||
"-95.66714": null,
|
||||
"-19.935247": null,
|
||||
"52.225763": {
|
||||
"lat": 39.5645,
|
||||
"lon": -75.597,
|
||||
"city": "威尔明顿",
|
||||
"country": "美国"
|
||||
},
|
||||
"-39.370987": null,
|
||||
"162.006633": {
|
||||
"lat": 43.7807,
|
||||
"lon": -79.2855,
|
||||
"city": "士嘉堡",
|
||||
"country": "加拿大"
|
||||
},
|
||||
"51.696746": {
|
||||
"lat": 51.5,
|
||||
"lon": -0.13,
|
||||
"city": "",
|
||||
"country": "英国"
|
||||
},
|
||||
"6.673336": {
|
||||
"lat": 31.5273,
|
||||
"lon": -110.3607,
|
||||
"city": "谢拉维斯塔",
|
||||
"country": "美国"
|
||||
},
|
||||
"102.5503": null,
|
||||
"-144.964956": null,
|
||||
"-104.457778": null,
|
||||
"83.087052": null,
|
||||
"-125.101539": null,
|
||||
"90.262005": {
|
||||
"lat": 48.9018,
|
||||
"lon": 2.4893,
|
||||
"city": "邦迪",
|
||||
"country": "法国"
|
||||
},
|
||||
"-83.715609": null,
|
||||
"29.378547": {
|
||||
"lat": 38.0,
|
||||
"lon": -97.0,
|
||||
"city": "",
|
||||
"country": "美国"
|
||||
},
|
||||
"-57.090496": null,
|
||||
"142.184372": {
|
||||
"lat": 44.7314,
|
||||
"lon": -63.6482,
|
||||
"city": "Dartmouth",
|
||||
"country": "加拿大"
|
||||
},
|
||||
"165.297813": {
|
||||
"lat": -33.9167,
|
||||
"lon": 18.4167,
|
||||
"city": "Cape Town",
|
||||
"country": "南非"
|
||||
},
|
||||
"105.875829": {
|
||||
"lat": -29.0,
|
||||
"lon": 24.0,
|
||||
"city": "",
|
||||
"country": "南非"
|
||||
},
|
||||
"160.97215": {
|
||||
"lat": 43.286,
|
||||
"lon": -77.6843,
|
||||
"city": "Rochester",
|
||||
"country": "美国"
|
||||
},
|
||||
"-76.327692": null,
|
||||
"-179.768984": null,
|
||||
"-135.649249": null,
|
||||
"-129.397466": null,
|
||||
"-159.604348": null,
|
||||
"123.957119": {
|
||||
"lat": 34.6836,
|
||||
"lon": 113.5325,
|
||||
"city": "郑州",
|
||||
"country": "中国"
|
||||
},
|
||||
"106.508142": {
|
||||
"lat": 28.55,
|
||||
"lon": 115.9333,
|
||||
"city": "南昌",
|
||||
"country": "中国"
|
||||
},
|
||||
"60.737607": {
|
||||
"lat": 45.75,
|
||||
"lon": 126.65,
|
||||
"city": "哈尔滨",
|
||||
"country": "中国"
|
||||
},
|
||||
"12.851572": {
|
||||
"lat": 38.0,
|
||||
"lon": -97.0,
|
||||
"city": "",
|
||||
"country": "美国"
|
||||
},
|
||||
"133.109702": {
|
||||
"lat": 35.69,
|
||||
"lon": 139.69,
|
||||
"city": "",
|
||||
"country": "日本"
|
||||
},
|
||||
"97.468122": {
|
||||
"lat": 38.0,
|
||||
"lon": -97.0,
|
||||
"city": "",
|
||||
"country": "美国"
|
||||
},
|
||||
"-49.099249": null,
|
||||
"31.114424": {
|
||||
"lat": 52.25,
|
||||
"lon": 21.0,
|
||||
"city": "华沙",
|
||||
"country": "波兰"
|
||||
},
|
||||
"49.613214": {
|
||||
"lat": 37.5985,
|
||||
"lon": 126.9783,
|
||||
"city": "首尔特别市",
|
||||
"country": "大韩民国"
|
||||
},
|
||||
"-76.937967": null,
|
||||
"24.350375": {
|
||||
"lat": 38.0,
|
||||
"lon": -97.0,
|
||||
"city": "",
|
||||
"country": "美国"
|
||||
},
|
||||
"-57.188778": null,
|
||||
"-129.431676": null,
|
||||
"-136.186649": null,
|
||||
"-61.694727": null,
|
||||
"-90.513611": null,
|
||||
"91.194896": {
|
||||
"lat": 49.5333,
|
||||
"lon": 11.8,
|
||||
"city": "哈恩巴赫",
|
||||
"country": "德国"
|
||||
},
|
||||
"132.327773": {
|
||||
"lat": 32.404,
|
||||
"lon": -86.2539,
|
||||
"city": "蒙哥马利",
|
||||
"country": "美国"
|
||||
},
|
||||
"-112.197533": null,
|
||||
"155.022955": null,
|
||||
"162.854691": {
|
||||
"lat": 51.5,
|
||||
"lon": -0.13,
|
||||
"city": "",
|
||||
"country": "英国"
|
||||
},
|
||||
"-52.303734": null,
|
||||
"-37.91953": null,
|
||||
"144.662392": {
|
||||
"lat": 37.3762,
|
||||
"lon": -122.1826,
|
||||
"city": "Palo Alto",
|
||||
"country": "美国"
|
||||
},
|
||||
"17.315107": {
|
||||
"lat": 37.323,
|
||||
"lon": -122.0322,
|
||||
"city": "Cupertino",
|
||||
"country": "美国"
|
||||
},
|
||||
"97.359727": {
|
||||
"lat": 38.0,
|
||||
"lon": -97.0,
|
||||
"city": "",
|
||||
"country": "美国"
|
||||
},
|
||||
"-37.814213": null,
|
||||
"-57.066252": null,
|
||||
"170.670964": {
|
||||
"lat": 47.6689,
|
||||
"lon": -117.4369,
|
||||
"city": "斯波坎",
|
||||
"country": "美国"
|
||||
},
|
||||
"19.237916": {
|
||||
"lat": 42.3223,
|
||||
"lon": -83.1763,
|
||||
"city": "迪尔伯恩",
|
||||
"country": "美国"
|
||||
},
|
||||
"-77.426219": null,
|
||||
"35.273579": {
|
||||
"lat": 42.2776,
|
||||
"lon": -83.7409,
|
||||
"city": "安娜堡",
|
||||
"country": "美国"
|
||||
},
|
||||
"-29.56673": null,
|
||||
"-173.438083": null,
|
||||
"-164.691055": null,
|
||||
"-140.463216": null,
|
||||
"-168.663276": null,
|
||||
"-138.543279": null,
|
||||
"-47.142859": null,
|
||||
"-102.389184": null,
|
||||
"24.754033": {
|
||||
"lat": 32.981,
|
||||
"lon": -80.0326,
|
||||
"city": "Goose Creek",
|
||||
"country": "美国"
|
||||
},
|
||||
"-109.476174": null,
|
||||
"92.836244": {
|
||||
"lat": 51.5,
|
||||
"lon": -0.13,
|
||||
"city": "",
|
||||
"country": "英国"
|
||||
},
|
||||
"2.089979": null,
|
||||
"82.41569": {
|
||||
"lat": 51.6578,
|
||||
"lon": -0.3954,
|
||||
"city": "沃特福德",
|
||||
"country": "英国"
|
||||
},
|
||||
"-117.765392": null,
|
||||
"-112.753899": null,
|
||||
"-147.563834": null,
|
||||
"-27.4301": null,
|
||||
"66.656057": {
|
||||
"lat": 38.0,
|
||||
"lon": -97.0,
|
||||
"city": "",
|
||||
"country": "美国"
|
||||
},
|
||||
"-52.329152": null,
|
||||
"16.783619": {
|
||||
"lat": 37.3762,
|
||||
"lon": -122.1826,
|
||||
"city": "Palo Alto",
|
||||
"country": "美国"
|
||||
},
|
||||
"-160.668335": null,
|
||||
"-155.782242": null,
|
||||
"-43.415128": null,
|
||||
"-151.682652": null,
|
||||
"33.713437": {
|
||||
"lat": 38.0,
|
||||
"lon": -97.0,
|
||||
"city": "",
|
||||
"country": "美国"
|
||||
},
|
||||
"151.170164": {
|
||||
"lat": 42.8333,
|
||||
"lon": 12.8333,
|
||||
"city": "",
|
||||
"country": "意大利"
|
||||
},
|
||||
"-2.437103": null,
|
||||
"17.096003": null,
|
||||
"4.372196": {
|
||||
"lat": 38.0,
|
||||
"lon": -97.0,
|
||||
"city": "",
|
||||
"country": "美国"
|
||||
},
|
||||
"12.775114": {
|
||||
"lat": 38.0,
|
||||
"lon": -97.0,
|
||||
"city": "",
|
||||
"country": "美国"
|
||||
},
|
||||
"-26.847855": null,
|
||||
"-2.96053": null,
|
||||
"-136.421502": null,
|
||||
"156.152711": {
|
||||
"lat": 46.2857,
|
||||
"lon": -119.2845,
|
||||
"city": "里奇兰",
|
||||
"country": "美国"
|
||||
},
|
||||
"-121.186972": null,
|
||||
"-126.924167": null,
|
||||
"-101.388131": null,
|
||||
"-101.03754": null,
|
||||
"77.828572": {
|
||||
"lat": 52.5167,
|
||||
"lon": 13.4,
|
||||
"city": "柏林",
|
||||
"country": "德国"
|
||||
},
|
||||
"-34.813058": null,
|
||||
"123.011334": {
|
||||
"lat": 23.7,
|
||||
"lon": 90.375,
|
||||
"city": "",
|
||||
"country": "孟加拉"
|
||||
},
|
||||
"11.265017": {
|
||||
"lat": 39.9612,
|
||||
"lon": -82.9988,
|
||||
"city": "哥伦布",
|
||||
"country": "美国"
|
||||
},
|
||||
"11.202764": {
|
||||
"lat": 39.9612,
|
||||
"lon": -82.9988,
|
||||
"city": "哥伦布",
|
||||
"country": "美国"
|
||||
},
|
||||
"70.262109": {
|
||||
"lat": 38.0,
|
||||
"lon": -97.0,
|
||||
"city": "",
|
||||
"country": "美国"
|
||||
},
|
||||
"-70.027646": null,
|
||||
"26.990087": {
|
||||
"lat": 38.0,
|
||||
"lon": -97.0,
|
||||
"city": "",
|
||||
"country": "美国"
|
||||
},
|
||||
"147.506751": {
|
||||
"lat": 47.5,
|
||||
"lon": 19.0833,
|
||||
"city": "布达佩斯",
|
||||
"country": "匈牙利"
|
||||
},
|
||||
"-107.81862": null,
|
||||
"141.75262": {
|
||||
"lat": 51.0,
|
||||
"lon": 9.0,
|
||||
"city": "",
|
||||
"country": "德国"
|
||||
},
|
||||
"30.674809": {
|
||||
"lat": 38.0,
|
||||
"lon": -97.0,
|
||||
"city": "",
|
||||
"country": "美国"
|
||||
},
|
||||
"120.327798": {
|
||||
"lat": 39.8897,
|
||||
"lon": 115.275,
|
||||
"city": "Hebei",
|
||||
"country": "中国"
|
||||
},
|
||||
"-158.088393": null,
|
||||
"-121.790773": null,
|
||||
"77.908193": {
|
||||
"lat": 50.3167,
|
||||
"lon": 7.8,
|
||||
"city": "Nassau",
|
||||
"country": "德国"
|
||||
},
|
||||
"57.252522": {
|
||||
"lat": 47.0,
|
||||
"lon": 8.0,
|
||||
"city": "",
|
||||
"country": ""
|
||||
},
|
||||
"-15.466886": null,
|
||||
"-136.800708": null,
|
||||
"-139.560022": null,
|
||||
"155.309838": {
|
||||
"lat": 59.3294,
|
||||
"lon": 18.0686,
|
||||
"city": "",
|
||||
"country": "瑞典"
|
||||
},
|
||||
"158.175545": {
|
||||
"lat": 31.5273,
|
||||
"lon": -110.3607,
|
||||
"city": "谢拉维斯塔",
|
||||
"country": "美国"
|
||||
},
|
||||
"-38.283752": null,
|
||||
"146.597747": {
|
||||
"lat": 42.3755,
|
||||
"lon": -83.0772,
|
||||
"city": "底特律",
|
||||
"country": "美国"
|
||||
},
|
||||
"-146.199812": null,
|
||||
"-60.696207": null,
|
||||
"141.547786": {
|
||||
"lat": 35.8869,
|
||||
"lon": 14.4025,
|
||||
"city": "Mdina",
|
||||
"country": "马耳他"
|
||||
},
|
||||
"10.542121": null,
|
||||
"169.586656": {
|
||||
"lat": 45.0059,
|
||||
"lon": -93.1059,
|
||||
"city": "圣保罗",
|
||||
"country": "美国"
|
||||
},
|
||||
"53.588162": {
|
||||
"lat": 51.0,
|
||||
"lon": 9.0,
|
||||
"city": "",
|
||||
"country": "德国"
|
||||
},
|
||||
"-48.221466": null,
|
||||
"-104.847562": null,
|
||||
"-97.617615": null,
|
||||
"145.767865": {
|
||||
"lat": 51.8425,
|
||||
"lon": 5.8528,
|
||||
"city": "奈梅亨",
|
||||
"country": "荷兰"
|
||||
},
|
||||
"-173.124365": null,
|
||||
"-158.431061": null,
|
||||
"-132.023148": null,
|
||||
"-171.873785": null,
|
||||
"-23.369992": null,
|
||||
"53.281077": {
|
||||
"lat": 51.0,
|
||||
"lon": 9.0,
|
||||
"city": "",
|
||||
"country": "德国"
|
||||
},
|
||||
"28.294006": {
|
||||
"lat": 38.0,
|
||||
"lon": -97.0,
|
||||
"city": "",
|
||||
"country": "美国"
|
||||
}
|
||||
}
|
||||
@@ -1,808 +0,0 @@
|
||||
# ===== Offline GeoIP Database — 天璇 (TianXuan) =====
|
||||
# Format: start_ip,end_ip,lat,lon,city,country
|
||||
# Layers: Private, DNS, AWS, Azure, GCP, Alibaba, Cloudflare, Akamai, Fastly, CN_ISP, US_ISP, EU_ISP, JP_ISP
|
||||
# Strategy: priority-ordered, overlap-resolved (first wins), gaps filled with city blocks
|
||||
|
||||
1.0.0.0,1.0.0.0,31.2304,121.4737,Shanghai,CN
|
||||
1.0.0.1,1.0.0.1,37.7749,-122.4194,San Francisco,US
|
||||
1.0.0.2,1.1.1.0,31.2304,121.4737,Shanghai,CN
|
||||
1.1.1.1,1.1.1.1,37.7749,-122.4194,San Francisco,US
|
||||
1.1.1.2,1.255.255.255,31.2304,121.4737,Shanghai,CN
|
||||
2.0.0.0,2.15.255.255,51.5074,-0.1278,London,GB
|
||||
2.16.0.0,2.23.255.255,42.3736,-71.1097,Cambridge,US
|
||||
2.24.0.0,2.255.255.255,51.5074,-0.1278,London,GB
|
||||
3.0.0.0,3.1.255.255,1.3521,103.8198,Singapore,SG
|
||||
3.2.0.0,3.5.255.255,31.2304,121.4737,Shanghai,CN
|
||||
3.6.0.0,3.7.255.255,19.076,72.8777,Mumbai,IN
|
||||
3.8.0.0,3.15.255.255,51.5074,-0.1278,London,GB
|
||||
3.16.0.0,3.23.255.255,39.9042,116.4074,Beijing,CN
|
||||
3.24.0.0,3.27.255.255,-33.8688,151.2093,Sydney,AU
|
||||
3.28.0.0,3.33.255.255,35.6762,139.6503,Tokyo,JP
|
||||
3.34.0.0,3.35.255.255,37.5665,126.978,Seoul,KR
|
||||
3.36.0.0,3.51.255.255,37.5665,126.978,Seoul,KR
|
||||
3.52.0.0,3.63.255.255,1.3521,103.8198,Singapore,SG
|
||||
3.64.0.0,3.79.255.255,50.1109,8.6821,Frankfurt,DE
|
||||
3.80.0.0,3.95.255.255,51.5074,-0.1278,London,GB
|
||||
3.96.0.0,3.99.255.255,45.5017,-73.5673,Montreal,CA
|
||||
3.100.0.0,3.111.255.255,50.1109,8.6821,Frankfurt,DE
|
||||
3.112.0.0,3.115.255.255,35.6762,139.6503,Tokyo,JP
|
||||
3.116.0.0,3.131.255.255,40.7128,-74.006,New York,US
|
||||
3.132.0.0,3.147.255.255,37.7749,-122.4194,San Francisco,US
|
||||
3.148.0.0,3.163.255.255,-33.8688,151.2093,Sydney,AU
|
||||
3.164.0.0,3.179.255.255,19.076,72.8777,Mumbai,IN
|
||||
3.180.0.0,3.195.255.255,48.8566,2.3522,Paris,FR
|
||||
3.196.0.0,3.207.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
3.208.0.0,3.215.255.255,39.0438,-77.4874,Ashburn,US
|
||||
3.216.0.0,3.223.255.255,-23.5505,-46.6333,Sao Paulo,BR
|
||||
3.224.0.0,3.239.255.255,39.0438,-77.4874,Ashburn,US
|
||||
3.240.0.0,4.111.255.255,55.7558,37.6173,Moscow,RU
|
||||
4.112.0.0,4.239.255.255,41.8781,-87.6298,Chicago,US
|
||||
4.240.0.0,4.255.255.255,32.7767,-96.797,Dallas,US
|
||||
5.0.0.0,5.255.255.255,51.5074,-0.1278,London,GB
|
||||
6.0.0.0,6.127.255.255,33.749,-84.388,Atlanta,US
|
||||
6.128.0.0,6.255.255.255,43.6532,-79.3832,Toronto,CA
|
||||
7.0.0.0,7.127.255.255,22.3193,114.1694,Hong Kong,HK
|
||||
7.128.0.0,7.255.255.255,25.033,121.5654,Taipei,TW
|
||||
8.0.0.0,8.8.4.3,40.4168,-3.7038,Madrid,ES
|
||||
8.8.4.4,8.8.4.4,37.3861,-122.0839,Mountain View,US
|
||||
8.8.4.5,8.8.8.7,45.4642,9.19,Milan,IT
|
||||
8.8.8.8,8.8.8.8,37.3861,-122.0839,Mountain View,US
|
||||
8.8.8.9,8.136.8.8,59.3293,18.0686,Stockholm,SE
|
||||
8.136.8.9,9.8.8.8,34.6937,135.5022,Osaka,JP
|
||||
9.8.8.9,9.136.8.8,23.1291,113.2644,Guangzhou,CN
|
||||
9.136.8.9,9.255.255.255,22.5431,114.0579,Shenzhen,CN
|
||||
10.0.0.0,10.255.255.255,39.9042,116.4074,Beijing,CN
|
||||
11.0.0.0,11.127.255.255,52.52,13.405,Berlin,DE
|
||||
11.128.0.0,11.255.255.255,25.2048,55.2708,Dubai,AE
|
||||
12.0.0.0,12.255.255.255,32.7767,-96.797,Dallas,US
|
||||
13.0.0.0,13.15.255.255,13.7563,100.5018,Bangkok,TH
|
||||
13.16.0.0,13.31.255.255,31.2304,121.4737,Shanghai,CN
|
||||
13.32.0.0,13.47.255.255,39.9042,116.4074,Beijing,CN
|
||||
13.48.0.0,13.51.255.255,59.3293,18.0686,Stockholm,SE
|
||||
13.52.0.0,13.55.255.255,37.7749,-122.4194,San Francisco,US
|
||||
13.56.0.0,13.59.255.255,37.7749,-122.4194,San Francisco,US
|
||||
13.60.0.0,13.63.255.255,35.6762,139.6503,Tokyo,JP
|
||||
13.64.0.0,13.95.255.255,37.7749,-122.4194,San Francisco,US
|
||||
13.96.0.0,13.111.255.255,37.5665,126.978,Seoul,KR
|
||||
13.112.0.0,13.115.255.255,35.6762,139.6503,Tokyo,JP
|
||||
13.116.0.0,13.123.255.255,1.3521,103.8198,Singapore,SG
|
||||
13.124.0.0,13.124.255.255,37.5665,126.978,Seoul,KR
|
||||
13.125.0.0,13.125.255.255,51.5074,-0.1278,London,GB
|
||||
13.126.0.0,13.127.255.255,19.076,72.8777,Mumbai,IN
|
||||
13.128.0.0,13.143.255.255,50.1109,8.6821,Frankfurt,DE
|
||||
13.144.0.0,13.159.255.255,40.7128,-74.006,New York,US
|
||||
13.160.0.0,13.175.255.255,37.7749,-122.4194,San Francisco,US
|
||||
13.176.0.0,13.191.255.255,-33.8688,151.2093,Sydney,AU
|
||||
13.192.0.0,13.207.255.255,19.076,72.8777,Mumbai,IN
|
||||
13.208.0.0,13.208.255.255,34.6937,135.5022,Osaka,JP
|
||||
13.209.0.0,13.209.255.255,37.5665,126.978,Seoul,KR
|
||||
13.210.0.0,13.211.255.255,-33.8688,151.2093,Sydney,AU
|
||||
13.212.0.0,13.215.255.255,1.3521,103.8198,Singapore,SG
|
||||
13.216.0.0,13.227.255.255,48.8566,2.3522,Paris,FR
|
||||
13.228.0.0,13.229.255.255,1.3521,103.8198,Singapore,SG
|
||||
13.230.0.0,13.231.255.255,35.6762,139.6503,Tokyo,JP
|
||||
13.232.0.0,13.243.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
13.244.0.0,13.245.255.255,-33.9249,18.4241,Cape Town,ZA
|
||||
13.246.0.0,13.255.255.255,-23.5505,-46.6333,Sao Paulo,BR
|
||||
14.0.0.0,14.255.255.255,39.9042,116.4074,Beijing,CN
|
||||
15.0.0.0,15.127.255.255,55.7558,37.6173,Moscow,RU
|
||||
15.128.0.0,15.151.255.255,41.8781,-87.6298,Chicago,US
|
||||
15.152.0.0,15.152.255.255,34.6937,135.5022,Osaka,JP
|
||||
15.153.0.0,15.159.255.255,32.7767,-96.797,Dallas,US
|
||||
15.160.0.0,15.160.255.255,45.4642,9.19,Milan,IT
|
||||
15.161.0.0,15.163.255.255,33.749,-84.388,Atlanta,US
|
||||
15.164.0.0,15.164.255.255,37.5665,126.978,Seoul,KR
|
||||
15.165.0.0,15.180.255.255,43.6532,-79.3832,Toronto,CA
|
||||
15.181.0.0,15.183.255.255,22.3193,114.1694,Hong Kong,HK
|
||||
15.184.0.0,15.185.255.255,26.0667,50.5577,Bahrain,BH
|
||||
15.186.0.0,15.187.255.255,25.033,121.5654,Taipei,TW
|
||||
15.188.0.0,15.188.255.255,48.8566,2.3522,Paris,FR
|
||||
15.189.0.0,15.204.255.255,40.4168,-3.7038,Madrid,ES
|
||||
15.205.0.0,15.205.255.255,45.4642,9.19,Milan,IT
|
||||
15.206.0.0,15.206.255.255,19.076,72.8777,Mumbai,IN
|
||||
15.207.0.0,15.221.255.255,59.3293,18.0686,Stockholm,SE
|
||||
15.222.0.0,15.223.255.255,45.5017,-73.5673,Montreal,CA
|
||||
15.224.0.0,15.227.255.255,34.6937,135.5022,Osaka,JP
|
||||
15.228.0.0,15.229.255.255,-23.5505,-46.6333,Sao Paulo,BR
|
||||
15.230.0.0,15.245.255.255,23.1291,113.2644,Guangzhou,CN
|
||||
15.246.0.0,16.5.255.255,22.5431,114.0579,Shenzhen,CN
|
||||
16.6.0.0,16.15.255.255,52.52,13.405,Berlin,DE
|
||||
16.16.0.0,16.16.255.255,59.3293,18.0686,Stockholm,SE
|
||||
16.17.0.0,16.144.255.255,25.2048,55.2708,Dubai,AE
|
||||
16.145.0.0,16.169.255.255,13.7563,100.5018,Bangkok,TH
|
||||
16.170.0.0,16.171.255.255,59.3293,18.0686,Stockholm,SE
|
||||
16.172.0.0,17.43.255.255,31.2304,121.4737,Shanghai,CN
|
||||
17.44.0.0,17.171.255.255,39.9042,116.4074,Beijing,CN
|
||||
17.172.0.0,18.43.255.255,35.6762,139.6503,Tokyo,JP
|
||||
18.44.0.0,18.135.255.255,37.5665,126.978,Seoul,KR
|
||||
18.136.0.0,18.139.255.255,1.3521,103.8198,Singapore,SG
|
||||
18.140.0.0,18.155.255.255,1.3521,103.8198,Singapore,SG
|
||||
18.156.0.0,18.161.255.255,51.5074,-0.1278,London,GB
|
||||
18.162.0.0,18.163.255.255,22.3193,114.1694,Hong Kong,HK
|
||||
18.164.0.0,18.167.255.255,50.1109,8.6821,Frankfurt,DE
|
||||
18.168.0.0,18.171.255.255,51.5074,-0.1278,London,GB
|
||||
18.172.0.0,18.175.255.255,40.7128,-74.006,New York,US
|
||||
18.176.0.0,18.177.255.255,35.6762,139.6503,Tokyo,JP
|
||||
18.178.0.0,18.183.255.255,37.7749,-122.4194,San Francisco,US
|
||||
18.184.0.0,18.187.255.255,50.1109,8.6821,Frankfurt,DE
|
||||
18.188.0.0,18.193.255.255,-33.8688,151.2093,Sydney,AU
|
||||
18.194.0.0,18.195.255.255,50.1109,8.6821,Frankfurt,DE
|
||||
18.196.0.0,18.211.255.255,19.076,72.8777,Mumbai,IN
|
||||
18.212.0.0,18.227.255.255,48.8566,2.3522,Paris,FR
|
||||
18.228.0.0,18.229.255.255,-23.5505,-46.6333,Sao Paulo,BR
|
||||
18.230.0.0,18.231.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
18.232.0.0,18.235.255.255,39.0438,-77.4874,Ashburn,US
|
||||
18.236.0.0,19.107.255.255,-23.5505,-46.6333,Sao Paulo,BR
|
||||
19.108.0.0,19.235.255.255,55.7558,37.6173,Moscow,RU
|
||||
19.236.0.0,19.255.255.255,41.8781,-87.6298,Chicago,US
|
||||
20.0.0.0,20.31.255.255,39.0438,-77.4874,Ashburn,US
|
||||
20.32.0.0,20.35.255.255,32.7767,-96.797,Dallas,US
|
||||
20.36.0.0,20.39.255.255,-33.8688,151.2093,Sydney,AU
|
||||
20.40.0.0,20.40.255.255,33.749,-84.388,Atlanta,US
|
||||
20.41.0.0,20.41.255.255,37.5665,126.978,Seoul,KR
|
||||
20.42.0.0,20.42.255.255,43.6532,-79.3832,Toronto,CA
|
||||
20.43.0.0,20.43.255.255,48.8566,2.3522,Paris,FR
|
||||
20.44.0.0,20.45.255.255,35.6762,139.6503,Tokyo,JP
|
||||
20.46.0.0,20.48.255.255,22.3193,114.1694,Hong Kong,HK
|
||||
20.49.0.0,20.49.255.255,51.5074,-0.1278,London,GB
|
||||
20.50.0.0,20.51.255.255,25.033,121.5654,Taipei,TW
|
||||
20.52.0.0,20.52.255.255,50.1109,8.6821,Frankfurt,DE
|
||||
20.53.0.0,20.55.255.255,40.4168,-3.7038,Madrid,ES
|
||||
20.56.0.0,20.56.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
20.57.0.0,20.72.255.255,45.4642,9.19,Milan,IT
|
||||
20.73.0.0,20.88.255.255,59.3293,18.0686,Stockholm,SE
|
||||
20.89.0.0,20.104.255.255,34.6937,135.5022,Osaka,JP
|
||||
20.105.0.0,20.120.255.255,23.1291,113.2644,Guangzhou,CN
|
||||
20.121.0.0,20.136.255.255,22.5431,114.0579,Shenzhen,CN
|
||||
20.137.0.0,20.152.255.255,52.52,13.405,Berlin,DE
|
||||
20.153.0.0,20.168.255.255,25.2048,55.2708,Dubai,AE
|
||||
20.169.0.0,20.183.255.255,13.7563,100.5018,Bangkok,TH
|
||||
20.184.0.0,20.185.255.255,1.3521,103.8198,Singapore,SG
|
||||
20.186.0.0,20.191.255.255,31.2304,121.4737,Shanghai,CN
|
||||
20.192.0.0,20.192.255.255,19.076,72.8777,Mumbai,IN
|
||||
20.193.0.0,20.195.255.255,39.9042,116.4074,Beijing,CN
|
||||
20.196.0.0,20.196.255.255,22.3193,114.1694,Hong Kong,HK
|
||||
20.197.0.0,20.197.255.255,-23.5505,-46.6333,Sao Paulo,BR
|
||||
20.198.0.0,21.69.255.255,35.6762,139.6503,Tokyo,JP
|
||||
21.70.0.0,21.197.255.255,37.5665,126.978,Seoul,KR
|
||||
21.198.0.0,22.69.255.255,1.3521,103.8198,Singapore,SG
|
||||
22.70.0.0,22.197.255.255,51.5074,-0.1278,London,GB
|
||||
22.198.0.0,22.255.255.255,50.1109,8.6821,Frankfurt,DE
|
||||
23.0.0.0,23.15.255.255,42.3736,-71.1097,Cambridge,US
|
||||
23.16.0.0,23.31.255.255,40.7128,-74.006,New York,US
|
||||
23.32.0.0,23.63.255.255,42.3736,-71.1097,Cambridge,US
|
||||
23.64.0.0,23.67.255.255,42.3736,-71.1097,Cambridge,US
|
||||
23.68.0.0,23.71.255.255,37.7749,-122.4194,San Francisco,US
|
||||
23.72.0.0,23.79.255.255,42.3736,-71.1097,Cambridge,US
|
||||
23.80.0.0,23.95.255.255,-33.8688,151.2093,Sydney,AU
|
||||
23.96.0.0,23.103.255.255,41.8781,-87.6298,Chicago,US
|
||||
23.104.0.0,23.119.255.255,19.076,72.8777,Mumbai,IN
|
||||
23.120.0.0,23.135.255.255,48.8566,2.3522,Paris,FR
|
||||
23.136.0.0,23.151.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
23.152.0.0,23.167.255.255,-23.5505,-46.6333,Sao Paulo,BR
|
||||
23.168.0.0,23.183.255.255,55.7558,37.6173,Moscow,RU
|
||||
23.184.0.0,23.191.255.255,41.8781,-87.6298,Chicago,US
|
||||
23.192.0.0,23.223.255.255,42.3736,-71.1097,Cambridge,US
|
||||
23.224.0.0,23.239.255.255,32.7767,-96.797,Dallas,US
|
||||
23.240.0.0,23.255.255.255,33.749,-84.388,Atlanta,US
|
||||
24.0.0.0,24.255.255.255,40.7128,-74.006,New York,US
|
||||
25.0.0.0,25.255.255.255,51.5074,-0.1278,London,GB
|
||||
26.0.0.0,26.127.255.255,43.6532,-79.3832,Toronto,CA
|
||||
26.128.0.0,26.255.255.255,22.3193,114.1694,Hong Kong,HK
|
||||
27.0.0.0,27.255.255.255,31.2304,121.4737,Shanghai,CN
|
||||
28.0.0.0,28.127.255.255,25.033,121.5654,Taipei,TW
|
||||
28.128.0.0,28.255.255.255,40.4168,-3.7038,Madrid,ES
|
||||
29.0.0.0,29.127.255.255,45.4642,9.19,Milan,IT
|
||||
29.128.0.0,29.255.255.255,59.3293,18.0686,Stockholm,SE
|
||||
30.0.0.0,30.127.255.255,34.6937,135.5022,Osaka,JP
|
||||
30.128.0.0,30.255.255.255,23.1291,113.2644,Guangzhou,CN
|
||||
31.0.0.0,31.255.255.255,51.5074,-0.1278,London,GB
|
||||
32.0.0.0,32.255.255.255,33.749,-84.388,Atlanta,US
|
||||
33.0.0.0,33.127.255.255,22.5431,114.0579,Shenzhen,CN
|
||||
33.128.0.0,33.255.255.255,52.52,13.405,Berlin,DE
|
||||
34.0.0.0,34.207.255.255,39.0438,-77.4874,Ashburn,US
|
||||
34.208.0.0,34.223.255.255,45.5152,-122.6784,Portland,US
|
||||
34.224.0.0,34.239.255.255,39.0438,-77.4874,Ashburn,US
|
||||
34.240.0.0,34.247.255.255,53.3498,-6.2603,Dublin,IE
|
||||
34.248.0.0,34.255.255.255,39.0438,-77.4874,Ashburn,US
|
||||
35.0.0.0,35.127.255.255,25.2048,55.2708,Dubai,AE
|
||||
35.128.0.0,35.151.255.255,13.7563,100.5018,Bangkok,TH
|
||||
35.152.0.0,35.152.255.255,45.4642,9.19,Milan,IT
|
||||
35.153.0.0,35.153.255.255,31.2304,121.4737,Shanghai,CN
|
||||
35.154.0.0,35.155.255.255,19.076,72.8777,Mumbai,IN
|
||||
35.156.0.0,35.159.255.255,50.1109,8.6821,Frankfurt,DE
|
||||
35.160.0.0,35.175.255.255,39.9042,116.4074,Beijing,CN
|
||||
35.176.0.0,35.177.255.255,51.5074,-0.1278,London,GB
|
||||
35.178.0.0,35.179.255.255,51.5074,-0.1278,London,GB
|
||||
35.180.0.0,35.183.255.255,48.8566,2.3522,Paris,FR
|
||||
35.184.0.0,35.191.255.255,35.6762,139.6503,Tokyo,JP
|
||||
35.192.0.0,35.195.255.255,25.033,121.5654,Taipei,TW
|
||||
35.196.0.0,35.198.255.255,37.5665,126.978,Seoul,KR
|
||||
35.199.0.0,35.199.255.255,-23.5505,-46.6333,Sao Paulo,BR
|
||||
35.200.0.0,35.203.255.255,35.6762,139.6503,Tokyo,JP
|
||||
35.204.0.0,35.207.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
35.208.0.0,35.215.255.255,1.3521,103.8198,Singapore,SG
|
||||
35.216.0.0,35.219.255.255,37.5665,126.978,Seoul,KR
|
||||
35.220.0.0,35.221.255.255,32.7767,-96.797,Dallas,US
|
||||
35.222.0.0,35.223.255.255,41.8781,-87.6298,Chicago,US
|
||||
35.224.0.0,35.239.255.255,39.0438,-77.4874,Ashburn,US
|
||||
35.240.0.0,35.247.255.255,51.5074,-0.1278,London,GB
|
||||
35.248.0.0,35.255.255.255,51.5074,-0.1278,London,GB
|
||||
36.0.0.0,36.255.255.255,23.1291,113.2644,Guangzhou,CN
|
||||
37.0.0.0,37.255.255.255,48.8566,2.3522,Paris,FR
|
||||
38.0.0.0,38.127.255.255,50.1109,8.6821,Frankfurt,DE
|
||||
38.128.0.0,38.255.255.255,40.7128,-74.006,New York,US
|
||||
39.0.0.0,39.95.255.255,37.7749,-122.4194,San Francisco,US
|
||||
39.96.0.0,39.103.255.255,39.9042,116.4074,Beijing,CN
|
||||
39.104.0.0,39.231.255.255,-33.8688,151.2093,Sydney,AU
|
||||
39.232.0.0,40.63.255.255,19.076,72.8777,Mumbai,IN
|
||||
40.64.0.0,40.127.255.255,39.0438,-77.4874,Ashburn,US
|
||||
40.128.0.0,40.255.255.255,48.8566,2.3522,Paris,FR
|
||||
41.0.0.0,41.127.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
41.128.0.0,41.255.255.255,-23.5505,-46.6333,Sao Paulo,BR
|
||||
42.0.0.0,42.255.255.255,39.9042,116.4074,Beijing,CN
|
||||
43.0.0.0,43.127.255.255,55.7558,37.6173,Moscow,RU
|
||||
43.128.0.0,43.255.255.255,41.8781,-87.6298,Chicago,US
|
||||
44.0.0.0,44.127.255.255,32.7767,-96.797,Dallas,US
|
||||
44.128.0.0,44.191.255.255,33.749,-84.388,Atlanta,US
|
||||
44.192.0.0,44.223.255.255,39.0438,-77.4874,Ashburn,US
|
||||
44.224.0.0,44.255.255.255,45.5152,-122.6784,Portland,US
|
||||
45.0.0.0,45.255.255.255,40.7128,-74.006,New York,US
|
||||
46.0.0.0,46.136.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
46.137.0.0,46.137.127.255,1.3521,103.8198,Singapore,SG
|
||||
46.137.128.0,46.255.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
47.0.0.0,47.15.255.255,43.6532,-79.3832,Toronto,CA
|
||||
47.16.0.0,47.31.255.255,22.3193,114.1694,Hong Kong,HK
|
||||
47.32.0.0,47.47.255.255,25.033,121.5654,Taipei,TW
|
||||
47.48.0.0,47.51.255.255,40.4168,-3.7038,Madrid,ES
|
||||
47.52.0.0,47.55.255.255,22.3193,114.1694,Hong Kong,HK
|
||||
47.56.0.0,47.57.255.255,22.3193,114.1694,Hong Kong,HK
|
||||
47.58.0.0,47.73.255.255,45.4642,9.19,Milan,IT
|
||||
47.74.0.0,47.75.255.255,1.3521,103.8198,Singapore,SG
|
||||
47.76.0.0,47.77.255.255,37.7749,-122.4194,San Francisco,US
|
||||
47.78.0.0,47.87.255.255,59.3293,18.0686,Stockholm,SE
|
||||
47.88.0.0,47.88.255.255,1.3521,103.8198,Singapore,SG
|
||||
47.89.0.0,47.89.255.255,22.3193,114.1694,Hong Kong,HK
|
||||
47.90.0.0,47.90.255.255,34.6937,135.5022,Osaka,JP
|
||||
47.91.0.0,47.91.255.255,35.6762,139.6503,Tokyo,JP
|
||||
47.92.0.0,47.95.255.255,39.9042,116.4074,Beijing,CN
|
||||
47.96.0.0,47.97.255.255,30.2741,120.1551,Hangzhou,CN
|
||||
47.98.0.0,47.99.255.255,30.2741,120.1551,Hangzhou,CN
|
||||
47.100.0.0,47.103.255.255,31.2304,121.4737,Shanghai,CN
|
||||
47.104.0.0,47.105.255.255,23.1291,113.2644,Guangzhou,CN
|
||||
47.106.0.0,47.106.255.255,22.5431,114.0579,Shenzhen,CN
|
||||
47.107.0.0,47.107.255.255,22.5431,114.0579,Shenzhen,CN
|
||||
47.108.0.0,47.109.255.255,30.5728,104.0668,Chengdu,CN
|
||||
47.110.0.0,47.111.255.255,52.52,13.405,Berlin,DE
|
||||
47.112.0.0,47.112.255.255,19.076,72.8777,Mumbai,IN
|
||||
47.113.0.0,47.113.255.255,25.2048,55.2708,Dubai,AE
|
||||
47.114.0.0,47.114.255.255,-33.8688,151.2093,Sydney,AU
|
||||
47.115.0.0,47.115.255.255,22.5431,114.0579,Shenzhen,CN
|
||||
47.116.0.0,47.116.255.255,50.1109,8.6821,Frankfurt,DE
|
||||
47.117.0.0,47.117.255.255,35.6762,139.6503,Tokyo,JP
|
||||
47.118.0.0,47.118.255.255,51.5074,-0.1278,London,GB
|
||||
47.119.0.0,47.119.255.255,25.2048,55.2708,Dubai,AE
|
||||
47.120.0.0,47.120.255.255,3.139,101.6869,Kuala Lumpur,MY
|
||||
47.121.0.0,47.121.255.255,13.7563,100.5018,Bangkok,TH
|
||||
47.122.0.0,47.122.255.255,-6.2088,106.8456,Jakarta,ID
|
||||
47.123.0.0,47.250.255.255,31.2304,121.4737,Shanghai,CN
|
||||
47.251.0.0,47.252.255.255,39.9042,116.4074,Beijing,CN
|
||||
47.253.0.0,47.253.255.255,37.7749,-122.4194,San Francisco,US
|
||||
47.254.0.0,47.254.255.255,50.1109,8.6821,Frankfurt,DE
|
||||
47.255.0.0,48.126.255.255,35.6762,139.6503,Tokyo,JP
|
||||
48.127.0.0,48.254.255.255,37.5665,126.978,Seoul,KR
|
||||
48.255.0.0,49.126.255.255,1.3521,103.8198,Singapore,SG
|
||||
49.127.0.0,49.254.255.255,51.5074,-0.1278,London,GB
|
||||
49.255.0.0,50.17.255.255,50.1109,8.6821,Frankfurt,DE
|
||||
50.18.0.0,50.19.255.255,37.7749,-122.4194,San Francisco,US
|
||||
50.20.0.0,50.35.255.255,40.7128,-74.006,New York,US
|
||||
50.36.0.0,50.51.255.255,37.7749,-122.4194,San Francisco,US
|
||||
50.52.0.0,50.67.255.255,-33.8688,151.2093,Sydney,AU
|
||||
50.68.0.0,50.83.255.255,19.076,72.8777,Mumbai,IN
|
||||
50.84.0.0,50.99.255.255,48.8566,2.3522,Paris,FR
|
||||
50.100.0.0,50.111.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
50.112.0.0,50.112.255.255,45.5152,-122.6784,Portland,US
|
||||
50.113.0.0,50.127.255.255,-23.5505,-46.6333,Sao Paulo,BR
|
||||
50.128.0.0,50.255.255.255,41.8781,-87.6298,Chicago,US
|
||||
51.0.0.0,51.102.255.255,51.5074,-0.1278,London,GB
|
||||
51.103.0.0,51.103.255.255,50.1109,8.6821,Frankfurt,DE
|
||||
51.104.0.0,51.104.255.255,51.5074,-0.1278,London,GB
|
||||
51.105.0.0,51.105.255.255,51.5074,-0.1278,London,GB
|
||||
51.106.0.0,51.139.255.255,51.5074,-0.1278,London,GB
|
||||
51.140.0.0,51.143.255.255,51.5074,-0.1278,London,GB
|
||||
51.144.0.0,51.255.255.255,51.5074,-0.1278,London,GB
|
||||
52.0.0.0,52.1.255.255,39.0438,-77.4874,Ashburn,US
|
||||
52.2.0.0,52.3.255.255,39.0438,-77.4874,Ashburn,US
|
||||
52.4.0.0,52.7.255.255,39.0438,-77.4874,Ashburn,US
|
||||
52.8.0.0,52.15.255.255,55.7558,37.6173,Moscow,RU
|
||||
52.16.0.0,52.19.255.255,53.3498,-6.2603,Dublin,IE
|
||||
52.20.0.0,52.23.255.255,39.0438,-77.4874,Ashburn,US
|
||||
52.24.0.0,52.27.255.255,41.8781,-87.6298,Chicago,US
|
||||
52.28.0.0,52.31.255.255,50.1109,8.6821,Frankfurt,DE
|
||||
52.32.0.0,52.46.255.255,32.7767,-96.797,Dallas,US
|
||||
52.47.0.0,52.47.255.255,48.8566,2.3522,Paris,FR
|
||||
52.48.0.0,52.51.255.255,53.3498,-6.2603,Dublin,IE
|
||||
52.52.0.0,52.56.255.255,33.749,-84.388,Atlanta,US
|
||||
52.57.0.0,52.57.255.255,50.1109,8.6821,Frankfurt,DE
|
||||
52.58.0.0,52.59.255.255,43.6532,-79.3832,Toronto,CA
|
||||
52.60.0.0,52.60.255.255,45.5017,-73.5673,Montreal,CA
|
||||
52.61.0.0,52.61.255.255,22.3193,114.1694,Hong Kong,HK
|
||||
52.62.0.0,52.62.255.255,-33.8688,151.2093,Sydney,AU
|
||||
52.63.0.0,52.65.255.255,25.033,121.5654,Taipei,TW
|
||||
52.66.0.0,52.66.255.255,19.076,72.8777,Mumbai,IN
|
||||
52.67.0.0,52.73.255.255,40.4168,-3.7038,Madrid,ES
|
||||
52.74.0.0,52.74.255.255,1.3521,103.8198,Singapore,SG
|
||||
52.75.0.0,52.77.255.255,45.4642,9.19,Milan,IT
|
||||
52.78.0.0,52.78.255.255,37.5665,126.978,Seoul,KR
|
||||
52.79.0.0,52.94.255.255,59.3293,18.0686,Stockholm,SE
|
||||
52.95.0.0,52.110.255.255,34.6937,135.5022,Osaka,JP
|
||||
52.111.0.0,52.126.255.255,23.1291,113.2644,Guangzhou,CN
|
||||
52.127.0.0,52.138.255.255,22.5431,114.0579,Shenzhen,CN
|
||||
52.139.0.0,52.139.255.255,1.3521,103.8198,Singapore,SG
|
||||
52.140.0.0,52.140.255.255,52.52,13.405,Berlin,DE
|
||||
52.141.0.0,52.141.255.255,37.5665,126.978,Seoul,KR
|
||||
52.142.0.0,52.144.255.255,25.2048,55.2708,Dubai,AE
|
||||
52.145.0.0,52.145.255.255,39.0438,-77.4874,Ashburn,US
|
||||
52.146.0.0,52.146.255.255,13.7563,100.5018,Bangkok,TH
|
||||
52.147.0.0,52.147.255.255,-33.8688,151.2093,Sydney,AU
|
||||
52.148.0.0,52.163.255.255,31.2304,121.4737,Shanghai,CN
|
||||
52.164.0.0,52.179.255.255,39.9042,116.4074,Beijing,CN
|
||||
52.180.0.0,52.184.255.255,35.6762,139.6503,Tokyo,JP
|
||||
52.185.0.0,52.185.255.255,35.6762,139.6503,Tokyo,JP
|
||||
52.186.0.0,52.201.255.255,37.5665,126.978,Seoul,KR
|
||||
52.202.0.0,52.217.255.255,1.3521,103.8198,Singapore,SG
|
||||
52.218.0.0,52.223.255.255,51.5074,-0.1278,London,GB
|
||||
52.224.0.0,52.255.255.255,39.0438,-77.4874,Ashburn,US
|
||||
53.0.0.0,53.127.255.255,50.1109,8.6821,Frankfurt,DE
|
||||
53.128.0.0,53.255.255.255,40.7128,-74.006,New York,US
|
||||
54.0.0.0,54.63.255.255,37.7749,-122.4194,San Francisco,US
|
||||
54.64.0.0,54.65.255.255,35.6762,139.6503,Tokyo,JP
|
||||
54.66.0.0,54.66.255.255,-33.8688,151.2093,Sydney,AU
|
||||
54.67.0.0,54.67.255.255,-33.8688,151.2093,Sydney,AU
|
||||
54.68.0.0,54.71.255.255,45.5152,-122.6784,Portland,US
|
||||
54.72.0.0,54.79.255.255,53.3498,-6.2603,Dublin,IE
|
||||
54.80.0.0,54.95.255.255,39.0438,-77.4874,Ashburn,US
|
||||
54.96.0.0,54.111.255.255,19.076,72.8777,Mumbai,IN
|
||||
54.112.0.0,54.127.255.255,48.8566,2.3522,Paris,FR
|
||||
54.128.0.0,54.143.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
54.144.0.0,54.151.255.255,-23.5505,-46.6333,Sao Paulo,BR
|
||||
54.152.0.0,54.159.255.255,39.0438,-77.4874,Ashburn,US
|
||||
54.160.0.0,54.191.255.255,39.0438,-77.4874,Ashburn,US
|
||||
54.192.0.0,54.205.255.255,55.7558,37.6173,Moscow,RU
|
||||
54.206.0.0,54.206.255.255,-33.8688,151.2093,Sydney,AU
|
||||
54.207.0.0,54.207.255.255,41.8781,-87.6298,Chicago,US
|
||||
54.208.0.0,54.209.255.255,39.0438,-77.4874,Ashburn,US
|
||||
54.210.0.0,54.223.255.255,32.7767,-96.797,Dallas,US
|
||||
54.224.0.0,54.239.255.255,39.0438,-77.4874,Ashburn,US
|
||||
54.240.0.0,54.240.255.255,33.749,-84.388,Atlanta,US
|
||||
54.241.0.0,54.241.255.255,37.7749,-122.4194,San Francisco,US
|
||||
54.242.0.0,54.243.255.255,43.6532,-79.3832,Toronto,CA
|
||||
54.244.0.0,54.245.255.255,45.5152,-122.6784,Portland,US
|
||||
54.246.0.0,55.117.255.255,22.3193,114.1694,Hong Kong,HK
|
||||
55.118.0.0,55.245.255.255,25.033,121.5654,Taipei,TW
|
||||
55.246.0.0,56.117.255.255,40.4168,-3.7038,Madrid,ES
|
||||
56.118.0.0,56.245.255.255,45.4642,9.19,Milan,IT
|
||||
56.246.0.0,57.117.255.255,59.3293,18.0686,Stockholm,SE
|
||||
57.118.0.0,57.245.255.255,34.6937,135.5022,Osaka,JP
|
||||
57.246.0.0,57.255.255.255,23.1291,113.2644,Guangzhou,CN
|
||||
58.0.0.0,58.255.255.255,22.5431,114.0579,Shenzhen,CN
|
||||
59.0.0.0,59.109.255.255,31.2304,121.4737,Shanghai,CN
|
||||
59.110.0.0,59.110.255.255,39.9042,116.4074,Beijing,CN
|
||||
59.111.0.0,59.255.255.255,31.2304,121.4737,Shanghai,CN
|
||||
60.0.0.0,60.255.255.255,39.9042,116.4074,Beijing,CN
|
||||
61.0.0.0,61.255.255.255,31.2304,121.4737,Shanghai,CN
|
||||
62.0.0.0,62.255.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
63.0.0.0,63.15.255.255,22.5431,114.0579,Shenzhen,CN
|
||||
63.16.0.0,63.31.255.255,52.52,13.405,Berlin,DE
|
||||
63.32.0.0,63.35.255.255,53.3498,-6.2603,Dublin,IE
|
||||
63.36.0.0,63.163.255.255,25.2048,55.2708,Dubai,AE
|
||||
63.164.0.0,63.255.255.255,13.7563,100.5018,Bangkok,TH
|
||||
64.0.0.0,64.255.255.255,41.8781,-87.6298,Chicago,US
|
||||
65.0.0.0,65.255.255.255,37.7749,-122.4194,San Francisco,US
|
||||
66.0.0.0,66.255.255.255,32.7767,-96.797,Dallas,US
|
||||
67.0.0.0,67.159.255.255,25.7617,-80.1918,Miami,US
|
||||
67.160.0.0,67.191.255.255,39.9526,-75.1652,Philadelphia,US
|
||||
67.192.0.0,67.255.255.255,25.7617,-80.1918,Miami,US
|
||||
68.0.0.0,68.127.255.255,31.2304,121.4737,Shanghai,CN
|
||||
68.128.0.0,68.255.255.255,39.9042,116.4074,Beijing,CN
|
||||
69.0.0.0,69.135.255.255,40.7128,-74.006,New York,US
|
||||
69.136.0.0,69.143.255.255,42.3601,-71.0589,Boston,US
|
||||
69.144.0.0,69.239.255.255,40.7128,-74.006,New York,US
|
||||
69.240.0.0,69.255.255.255,47.6062,-122.3321,Seattle,US
|
||||
70.0.0.0,70.255.255.255,41.8781,-87.6298,Chicago,US
|
||||
71.0.0.0,71.127.255.255,35.6762,139.6503,Tokyo,JP
|
||||
71.128.0.0,71.159.255.255,37.5665,126.978,Seoul,KR
|
||||
71.160.0.0,71.175.255.255,40.7128,-74.006,New York,US
|
||||
71.176.0.0,71.191.255.255,1.3521,103.8198,Singapore,SG
|
||||
71.192.0.0,71.255.255.255,37.7749,-122.4194,San Francisco,US
|
||||
72.0.0.0,72.15.255.255,51.5074,-0.1278,London,GB
|
||||
72.16.0.0,72.31.255.255,50.1109,8.6821,Frankfurt,DE
|
||||
72.32.0.0,72.47.255.255,40.7128,-74.006,New York,US
|
||||
72.48.0.0,72.63.255.255,37.7749,-122.4194,San Francisco,US
|
||||
72.64.0.0,72.127.255.255,32.7767,-96.797,Dallas,US
|
||||
72.128.0.0,72.143.255.255,-33.8688,151.2093,Sydney,AU
|
||||
72.144.0.0,72.159.255.255,19.076,72.8777,Mumbai,IN
|
||||
72.160.0.0,72.175.255.255,48.8566,2.3522,Paris,FR
|
||||
72.176.0.0,72.191.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
72.192.0.0,72.207.255.255,-23.5505,-46.6333,Sao Paulo,BR
|
||||
72.208.0.0,72.223.255.255,55.7558,37.6173,Moscow,RU
|
||||
72.224.0.0,72.239.255.255,41.8781,-87.6298,Chicago,US
|
||||
72.240.0.0,72.245.255.255,32.7767,-96.797,Dallas,US
|
||||
72.246.0.0,72.247.255.255,42.3736,-71.1097,Cambridge,US
|
||||
72.248.0.0,72.255.255.255,33.749,-84.388,Atlanta,US
|
||||
73.0.0.0,73.255.255.255,33.749,-84.388,Atlanta,US
|
||||
74.0.0.0,74.15.255.255,43.6532,-79.3832,Toronto,CA
|
||||
74.16.0.0,74.31.255.255,22.3193,114.1694,Hong Kong,HK
|
||||
74.32.0.0,74.47.255.255,25.033,121.5654,Taipei,TW
|
||||
74.48.0.0,74.63.255.255,40.4168,-3.7038,Madrid,ES
|
||||
74.64.0.0,74.79.255.255,45.4642,9.19,Milan,IT
|
||||
74.80.0.0,74.95.255.255,59.3293,18.0686,Stockholm,SE
|
||||
74.96.0.0,74.127.255.255,33.749,-84.388,Atlanta,US
|
||||
74.128.0.0,74.143.255.255,34.6937,135.5022,Osaka,JP
|
||||
74.144.0.0,74.159.255.255,23.1291,113.2644,Guangzhou,CN
|
||||
74.160.0.0,74.175.255.255,22.5431,114.0579,Shenzhen,CN
|
||||
74.176.0.0,74.191.255.255,52.52,13.405,Berlin,DE
|
||||
74.192.0.0,74.207.255.255,25.2048,55.2708,Dubai,AE
|
||||
74.208.0.0,74.223.255.255,13.7563,100.5018,Bangkok,TH
|
||||
74.224.0.0,74.239.255.255,31.2304,121.4737,Shanghai,CN
|
||||
74.240.0.0,74.255.255.255,39.9042,116.4074,Beijing,CN
|
||||
75.0.0.0,75.255.255.255,37.7749,-122.4194,San Francisco,US
|
||||
76.0.0.0,76.255.255.255,29.7604,-95.3698,Houston,US
|
||||
77.0.0.0,77.255.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
78.0.0.0,78.255.255.255,52.52,13.405,Berlin,DE
|
||||
79.0.0.0,79.191.255.255,52.52,13.405,Berlin,DE
|
||||
79.192.0.0,79.255.255.255,50.1109,8.6821,Frankfurt,DE
|
||||
80.0.0.0,80.127.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
80.128.0.0,80.159.255.255,52.52,13.405,Berlin,DE
|
||||
80.160.0.0,80.255.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
81.0.0.0,81.255.255.255,51.5074,-0.1278,London,GB
|
||||
82.0.0.0,82.255.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
83.0.0.0,83.255.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
84.0.0.0,84.255.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
85.0.0.0,85.255.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
86.0.0.0,86.255.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
87.0.0.0,87.127.255.255,52.52,13.405,Berlin,DE
|
||||
87.128.0.0,87.191.255.255,48.1351,11.582,Munich,DE
|
||||
87.192.0.0,87.255.255.255,52.52,13.405,Berlin,DE
|
||||
88.0.0.0,88.255.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
89.0.0.0,89.255.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
90.0.0.0,90.255.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
91.0.0.0,91.63.255.255,50.1109,8.6821,Frankfurt,DE
|
||||
91.64.0.0,91.191.255.255,35.6762,139.6503,Tokyo,JP
|
||||
91.192.0.0,91.255.255.255,37.5665,126.978,Seoul,KR
|
||||
92.0.0.0,92.121.255.255,55.7558,37.6173,Moscow,RU
|
||||
92.122.0.0,92.123.255.255,42.3736,-71.1097,Cambridge,US
|
||||
92.124.0.0,92.255.255.255,55.7558,37.6173,Moscow,RU
|
||||
93.0.0.0,93.127.255.255,1.3521,103.8198,Singapore,SG
|
||||
93.128.0.0,93.191.255.255,51.5074,-0.1278,London,GB
|
||||
93.192.0.0,93.255.255.255,52.52,13.405,Berlin,DE
|
||||
94.0.0.0,94.255.255.255,55.7558,37.6173,Moscow,RU
|
||||
95.0.0.0,95.99.255.255,52.52,13.405,Berlin,DE
|
||||
95.100.0.0,95.101.255.255,42.3736,-71.1097,Cambridge,US
|
||||
95.102.0.0,95.255.255.255,52.52,13.405,Berlin,DE
|
||||
96.0.0.0,96.5.255.255,50.1109,8.6821,Frankfurt,DE
|
||||
96.6.0.0,96.7.255.255,42.3736,-71.1097,Cambridge,US
|
||||
96.8.0.0,96.15.255.255,40.7128,-74.006,New York,US
|
||||
96.16.0.0,96.17.255.255,42.3736,-71.1097,Cambridge,US
|
||||
96.18.0.0,96.145.255.255,37.7749,-122.4194,San Francisco,US
|
||||
96.146.0.0,96.223.255.255,-33.8688,151.2093,Sydney,AU
|
||||
96.224.0.0,96.255.255.255,40.7128,-74.006,New York,US
|
||||
97.0.0.0,97.127.255.255,19.076,72.8777,Mumbai,IN
|
||||
97.128.0.0,97.255.255.255,48.8566,2.3522,Paris,FR
|
||||
98.0.0.0,98.127.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
98.128.0.0,98.191.255.255,-23.5505,-46.6333,Sao Paulo,BR
|
||||
98.192.0.0,98.255.255.255,41.8781,-87.6298,Chicago,US
|
||||
99.0.0.0,99.78.255.255,40.7128,-74.006,New York,US
|
||||
99.79.0.0,99.79.255.255,45.5017,-73.5673,Montreal,CA
|
||||
99.80.0.0,99.255.255.255,40.7128,-74.006,New York,US
|
||||
100.0.0.0,100.63.255.255,41.8781,-87.6298,Chicago,US
|
||||
100.64.0.0,100.191.255.255,55.7558,37.6173,Moscow,RU
|
||||
100.192.0.0,100.255.255.255,41.8781,-87.6298,Chicago,US
|
||||
101.0.0.0,101.131.255.255,39.9042,116.4074,Beijing,CN
|
||||
101.132.0.0,101.132.255.255,31.2304,121.4737,Shanghai,CN
|
||||
101.133.0.0,101.199.255.255,39.9042,116.4074,Beijing,CN
|
||||
101.200.0.0,101.201.255.255,39.9042,116.4074,Beijing,CN
|
||||
101.202.0.0,101.255.255.255,39.9042,116.4074,Beijing,CN
|
||||
102.0.0.0,102.127.255.255,32.7767,-96.797,Dallas,US
|
||||
102.128.0.0,102.255.255.255,33.749,-84.388,Atlanta,US
|
||||
103.0.0.0,103.21.243.255,43.6532,-79.3832,Toronto,CA
|
||||
103.21.244.0,103.21.247.255,37.7749,-122.4194,San Francisco,US
|
||||
103.21.248.0,103.22.199.255,22.3193,114.1694,Hong Kong,HK
|
||||
103.22.200.0,103.22.203.255,37.7749,-122.4194,San Francisco,US
|
||||
103.22.204.0,103.31.3.255,25.033,121.5654,Taipei,TW
|
||||
103.31.4.0,103.31.7.255,37.7749,-122.4194,San Francisco,US
|
||||
103.31.8.0,103.159.7.255,40.4168,-3.7038,Madrid,ES
|
||||
103.159.8.0,103.255.255.255,45.4642,9.19,Milan,IT
|
||||
104.0.0.0,104.15.255.255,32.7767,-96.797,Dallas,US
|
||||
104.16.0.0,104.23.255.255,37.7749,-122.4194,San Francisco,US
|
||||
104.24.0.0,104.27.255.255,37.7749,-122.4194,San Francisco,US
|
||||
104.28.0.0,104.39.255.255,32.7767,-96.797,Dallas,US
|
||||
104.40.0.0,104.47.255.255,39.0438,-77.4874,Ashburn,US
|
||||
104.48.0.0,104.63.255.255,32.7767,-96.797,Dallas,US
|
||||
104.64.0.0,104.127.255.255,42.3736,-71.1097,Cambridge,US
|
||||
104.128.0.0,104.151.255.255,32.7767,-96.797,Dallas,US
|
||||
104.152.0.0,104.153.255.255,50.1109,8.6821,Frankfurt,DE
|
||||
104.154.0.0,104.155.255.255,41.8781,-87.6298,Chicago,US
|
||||
104.156.0.0,104.195.255.255,32.7767,-96.797,Dallas,US
|
||||
104.196.0.0,104.199.255.255,39.0438,-77.4874,Ashburn,US
|
||||
104.200.0.0,104.210.255.255,32.7767,-96.797,Dallas,US
|
||||
104.211.0.0,104.211.255.255,19.076,72.8777,Mumbai,IN
|
||||
104.212.0.0,104.255.255.255,32.7767,-96.797,Dallas,US
|
||||
105.0.0.0,105.127.255.255,59.3293,18.0686,Stockholm,SE
|
||||
105.128.0.0,105.255.255.255,34.6937,135.5022,Osaka,JP
|
||||
106.0.0.0,106.13.255.255,23.1291,113.2644,Guangzhou,CN
|
||||
106.14.0.0,106.15.255.255,31.2304,121.4737,Shanghai,CN
|
||||
106.16.0.0,106.143.255.255,22.5431,114.0579,Shenzhen,CN
|
||||
106.144.0.0,106.255.255.255,52.52,13.405,Berlin,DE
|
||||
107.0.0.0,107.255.255.255,33.749,-84.388,Atlanta,US
|
||||
108.0.0.0,108.162.191.255,41.8781,-87.6298,Chicago,US
|
||||
108.162.192.0,108.162.255.255,37.7749,-122.4194,San Francisco,US
|
||||
108.163.0.0,108.255.255.255,41.8781,-87.6298,Chicago,US
|
||||
109.0.0.0,109.255.255.255,51.5074,-0.1278,London,GB
|
||||
110.0.0.0,110.255.255.255,23.1291,113.2644,Guangzhou,CN
|
||||
111.0.0.0,111.255.255.255,31.2304,121.4737,Shanghai,CN
|
||||
112.0.0.0,112.255.255.255,39.9042,116.4074,Beijing,CN
|
||||
113.0.0.0,113.255.255.255,31.2304,121.4737,Shanghai,CN
|
||||
114.0.0.0,114.255.255.255,39.9042,116.4074,Beijing,CN
|
||||
115.0.0.0,115.255.255.255,31.2304,121.4737,Shanghai,CN
|
||||
116.0.0.0,116.255.255.255,39.9042,116.4074,Beijing,CN
|
||||
117.0.0.0,117.255.255.255,31.2304,121.4737,Shanghai,CN
|
||||
118.0.0.0,118.255.255.255,39.9042,116.4074,Beijing,CN
|
||||
119.0.0.0,119.255.255.255,31.2304,121.4737,Shanghai,CN
|
||||
120.0.0.0,120.23.255.255,39.9042,116.4074,Beijing,CN
|
||||
120.24.0.0,120.27.255.255,39.9042,116.4074,Beijing,CN
|
||||
120.28.0.0,120.47.255.255,39.9042,116.4074,Beijing,CN
|
||||
120.48.0.0,120.49.255.255,39.9042,116.4074,Beijing,CN
|
||||
120.50.0.0,120.54.255.255,39.9042,116.4074,Beijing,CN
|
||||
120.55.0.0,120.55.255.255,31.2304,121.4737,Shanghai,CN
|
||||
120.56.0.0,120.75.255.255,39.9042,116.4074,Beijing,CN
|
||||
120.76.0.0,120.77.255.255,30.2741,120.1551,Hangzhou,CN
|
||||
120.78.0.0,120.79.255.255,22.5431,114.0579,Shenzhen,CN
|
||||
120.80.0.0,120.81.255.255,23.1291,113.2644,Guangzhou,CN
|
||||
120.82.0.0,120.255.255.255,39.9042,116.4074,Beijing,CN
|
||||
121.0.0.0,121.39.255.255,31.2304,121.4737,Shanghai,CN
|
||||
121.40.0.0,121.43.255.255,30.2741,120.1551,Hangzhou,CN
|
||||
121.44.0.0,121.195.255.255,31.2304,121.4737,Shanghai,CN
|
||||
121.196.0.0,121.199.255.255,30.2741,120.1551,Hangzhou,CN
|
||||
121.200.0.0,121.255.255.255,31.2304,121.4737,Shanghai,CN
|
||||
122.0.0.0,122.255.255.255,39.9042,116.4074,Beijing,CN
|
||||
123.0.0.0,123.255.255.255,31.2304,121.4737,Shanghai,CN
|
||||
124.0.0.0,124.255.255.255,39.9042,116.4074,Beijing,CN
|
||||
125.0.0.0,125.255.255.255,31.2304,121.4737,Shanghai,CN
|
||||
126.0.0.0,126.255.255.255,35.6762,139.6503,Tokyo,JP
|
||||
127.0.0.0,127.127.255.255,25.2048,55.2708,Dubai,AE
|
||||
127.128.0.0,127.255.255.255,13.7563,100.5018,Bangkok,TH
|
||||
128.0.0.0,128.255.255.255,48.8566,2.3522,Paris,FR
|
||||
129.0.0.0,129.255.255.255,48.8566,2.3522,Paris,FR
|
||||
130.0.0.0,130.255.255.255,52.52,13.405,Berlin,DE
|
||||
131.0.0.0,131.0.71.255,59.3293,18.0686,Stockholm,SE
|
||||
131.0.72.0,131.0.75.255,37.7749,-122.4194,San Francisco,US
|
||||
131.0.76.0,131.255.255.255,59.3293,18.0686,Stockholm,SE
|
||||
132.0.0.0,132.255.255.255,45.4642,9.19,Milan,IT
|
||||
133.0.0.0,133.255.255.255,35.6762,139.6503,Tokyo,JP
|
||||
134.0.0.0,134.255.255.255,52.52,13.405,Berlin,DE
|
||||
135.0.0.0,135.127.255.255,31.2304,121.4737,Shanghai,CN
|
||||
135.128.0.0,135.255.255.255,39.9042,116.4074,Beijing,CN
|
||||
136.0.0.0,136.127.255.255,35.6762,139.6503,Tokyo,JP
|
||||
136.128.0.0,136.255.255.255,37.5665,126.978,Seoul,KR
|
||||
137.0.0.0,137.255.255.255,48.8566,2.3522,Paris,FR
|
||||
138.0.0.0,138.90.255.255,45.4642,9.19,Milan,IT
|
||||
138.91.0.0,138.91.255.255,37.7749,-122.4194,San Francisco,US
|
||||
138.92.0.0,138.255.255.255,45.4642,9.19,Milan,IT
|
||||
139.0.0.0,139.195.255.255,23.1291,113.2644,Guangzhou,CN
|
||||
139.196.0.0,139.199.255.255,31.2304,121.4737,Shanghai,CN
|
||||
139.200.0.0,139.255.255.255,23.1291,113.2644,Guangzhou,CN
|
||||
140.0.0.0,140.127.255.255,1.3521,103.8198,Singapore,SG
|
||||
140.128.0.0,140.255.255.255,51.5074,-0.1278,London,GB
|
||||
141.0.0.0,141.101.63.255,52.52,13.405,Berlin,DE
|
||||
141.101.64.0,141.101.127.255,37.7749,-122.4194,San Francisco,US
|
||||
141.101.128.0,141.143.255.255,52.52,13.405,Berlin,DE
|
||||
141.144.0.0,141.159.255.255,32.7767,-96.797,Dallas,US
|
||||
141.160.0.0,141.255.255.255,52.52,13.405,Berlin,DE
|
||||
142.0.0.0,142.127.255.255,50.1109,8.6821,Frankfurt,DE
|
||||
142.128.0.0,142.255.255.255,40.7128,-74.006,New York,US
|
||||
143.0.0.0,143.255.255.255,45.4642,9.19,Milan,IT
|
||||
144.0.0.0,144.127.255.255,37.7749,-122.4194,San Francisco,US
|
||||
144.128.0.0,144.255.255.255,-33.8688,151.2093,Sydney,AU
|
||||
145.0.0.0,145.255.255.255,48.8566,2.3522,Paris,FR
|
||||
146.0.0.0,146.255.255.255,51.5074,-0.1278,London,GB
|
||||
147.0.0.0,147.255.255.255,48.8566,2.3522,Paris,FR
|
||||
148.0.0.0,148.127.255.255,19.076,72.8777,Mumbai,IN
|
||||
148.128.0.0,148.255.255.255,48.8566,2.3522,Paris,FR
|
||||
149.0.0.0,149.128.255.255,52.52,13.405,Berlin,DE
|
||||
149.129.0.0,149.129.255.255,1.3521,103.8198,Singapore,SG
|
||||
149.130.0.0,149.255.255.255,52.52,13.405,Berlin,DE
|
||||
150.0.0.0,150.191.255.255,35.6762,139.6503,Tokyo,JP
|
||||
150.192.0.0,150.255.255.255,40.4168,-3.7038,Madrid,ES
|
||||
151.0.0.0,151.100.255.255,45.4642,9.19,Milan,IT
|
||||
151.101.0.0,151.101.255.255,37.7749,-122.4194,San Francisco,US
|
||||
151.102.0.0,151.255.255.255,45.4642,9.19,Milan,IT
|
||||
152.0.0.0,152.127.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
152.128.0.0,152.255.255.255,-23.5505,-46.6333,Sao Paulo,BR
|
||||
153.0.0.0,153.127.255.255,55.7558,37.6173,Moscow,RU
|
||||
153.128.0.0,153.191.255.255,34.6937,135.5022,Osaka,JP
|
||||
153.192.0.0,154.63.255.255,41.8781,-87.6298,Chicago,US
|
||||
154.64.0.0,154.191.255.255,32.7767,-96.797,Dallas,US
|
||||
154.192.0.0,154.255.255.255,33.749,-84.388,Atlanta,US
|
||||
155.0.0.0,155.255.255.255,59.3293,18.0686,Stockholm,SE
|
||||
156.0.0.0,156.255.255.255,48.8566,2.3522,Paris,FR
|
||||
157.0.0.0,157.52.63.255,35.6762,139.6503,Tokyo,JP
|
||||
157.52.64.0,157.52.127.255,37.7749,-122.4194,San Francisco,US
|
||||
157.52.128.0,157.52.255.255,37.7749,-122.4194,San Francisco,US
|
||||
157.53.0.0,157.174.255.255,35.6762,139.6503,Tokyo,JP
|
||||
157.175.0.0,157.175.255.255,26.0667,50.5577,Bahrain,BH
|
||||
157.176.0.0,157.255.255.255,35.6762,139.6503,Tokyo,JP
|
||||
158.0.0.0,158.255.255.255,59.3293,18.0686,Stockholm,SE
|
||||
159.0.0.0,159.255.255.255,48.8566,2.3522,Paris,FR
|
||||
160.0.0.0,160.255.255.255,34.6937,135.5022,Osaka,JP
|
||||
161.0.0.0,161.255.255.255,48.8566,2.3522,Paris,FR
|
||||
162.0.0.0,162.127.255.255,43.6532,-79.3832,Toronto,CA
|
||||
162.128.0.0,162.157.255.255,22.3193,114.1694,Hong Kong,HK
|
||||
162.158.0.0,162.159.255.255,37.7749,-122.4194,San Francisco,US
|
||||
162.160.0.0,162.175.255.255,25.033,121.5654,Taipei,TW
|
||||
162.176.0.0,162.191.255.255,40.4168,-3.7038,Madrid,ES
|
||||
162.192.0.0,162.207.255.255,45.4642,9.19,Milan,IT
|
||||
162.208.0.0,162.223.255.255,59.3293,18.0686,Stockholm,SE
|
||||
162.224.0.0,162.239.255.255,34.6937,135.5022,Osaka,JP
|
||||
162.240.0.0,162.255.255.255,23.1291,113.2644,Guangzhou,CN
|
||||
163.0.0.0,163.255.255.255,35.6762,139.6503,Tokyo,JP
|
||||
164.0.0.0,164.255.255.255,34.6937,135.5022,Osaka,JP
|
||||
165.0.0.0,165.127.255.255,22.5431,114.0579,Shenzhen,CN
|
||||
165.128.0.0,165.255.255.255,52.52,13.405,Berlin,DE
|
||||
166.0.0.0,166.127.255.255,25.2048,55.2708,Dubai,AE
|
||||
166.128.0.0,166.255.255.255,13.7563,100.5018,Bangkok,TH
|
||||
167.0.0.0,167.81.255.255,31.2304,121.4737,Shanghai,CN
|
||||
167.82.0.0,167.82.127.255,37.7749,-122.4194,San Francisco,US
|
||||
167.82.128.0,167.82.191.255,37.7749,-122.4194,San Francisco,US
|
||||
167.82.192.0,167.210.191.255,39.9042,116.4074,Beijing,CN
|
||||
167.210.192.0,168.82.191.255,35.6762,139.6503,Tokyo,JP
|
||||
168.82.192.0,168.210.191.255,37.5665,126.978,Seoul,KR
|
||||
168.210.192.0,169.82.191.255,1.3521,103.8198,Singapore,SG
|
||||
169.82.192.0,169.210.191.255,51.5074,-0.1278,London,GB
|
||||
169.210.192.0,170.82.191.255,50.1109,8.6821,Frankfurt,DE
|
||||
170.82.192.0,170.210.191.255,40.7128,-74.006,New York,US
|
||||
170.210.192.0,170.255.255.255,37.7749,-122.4194,San Francisco,US
|
||||
171.0.0.0,171.255.255.255,59.3293,18.0686,Stockholm,SE
|
||||
172.0.0.0,172.15.255.255,-33.8688,151.2093,Sydney,AU
|
||||
172.16.0.0,172.31.255.255,31.2304,121.4737,Shanghai,CN
|
||||
172.32.0.0,172.47.255.255,19.076,72.8777,Mumbai,IN
|
||||
172.48.0.0,172.63.255.255,48.8566,2.3522,Paris,FR
|
||||
172.64.0.0,172.71.255.255,37.7749,-122.4194,San Francisco,US
|
||||
172.72.0.0,172.87.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
172.88.0.0,172.103.255.255,-23.5505,-46.6333,Sao Paulo,BR
|
||||
172.104.0.0,172.111.63.255,55.7558,37.6173,Moscow,RU
|
||||
172.111.64.0,172.111.127.255,37.7749,-122.4194,San Francisco,US
|
||||
172.111.128.0,172.239.127.255,41.8781,-87.6298,Chicago,US
|
||||
172.239.128.0,173.47.255.255,32.7767,-96.797,Dallas,US
|
||||
173.48.0.0,173.63.255.255,25.7617,-80.1918,Miami,US
|
||||
173.64.0.0,173.191.255.255,33.749,-84.388,Atlanta,US
|
||||
173.192.0.0,173.245.47.255,43.6532,-79.3832,Toronto,CA
|
||||
173.245.48.0,173.245.63.255,37.7749,-122.4194,San Francisco,US
|
||||
173.245.64.0,174.117.63.255,22.3193,114.1694,Hong Kong,HK
|
||||
174.117.64.0,174.245.63.255,25.033,121.5654,Taipei,TW
|
||||
174.245.64.0,175.117.63.255,40.4168,-3.7038,Madrid,ES
|
||||
175.117.64.0,175.245.63.255,45.4642,9.19,Milan,IT
|
||||
175.245.64.0,175.255.255.255,59.3293,18.0686,Stockholm,SE
|
||||
176.0.0.0,176.255.255.255,51.5074,-0.1278,London,GB
|
||||
177.0.0.0,177.127.255.255,34.6937,135.5022,Osaka,JP
|
||||
177.128.0.0,177.255.255.255,23.1291,113.2644,Guangzhou,CN
|
||||
178.0.0.0,178.127.255.255,22.5431,114.0579,Shenzhen,CN
|
||||
178.128.0.0,178.255.255.255,52.52,13.405,Berlin,DE
|
||||
179.0.0.0,179.127.255.255,25.2048,55.2708,Dubai,AE
|
||||
179.128.0.0,179.255.255.255,13.7563,100.5018,Bangkok,TH
|
||||
180.0.0.0,180.255.255.255,31.2304,121.4737,Shanghai,CN
|
||||
181.0.0.0,181.127.255.255,31.2304,121.4737,Shanghai,CN
|
||||
181.128.0.0,181.255.255.255,39.9042,116.4074,Beijing,CN
|
||||
182.0.0.0,182.91.255.255,39.9042,116.4074,Beijing,CN
|
||||
182.92.0.0,182.92.255.255,39.9042,116.4074,Beijing,CN
|
||||
182.93.0.0,182.255.255.255,39.9042,116.4074,Beijing,CN
|
||||
183.0.0.0,183.255.255.255,31.2304,121.4737,Shanghai,CN
|
||||
184.0.0.0,184.15.255.255,35.6762,139.6503,Tokyo,JP
|
||||
184.16.0.0,184.23.255.255,37.5665,126.978,Seoul,KR
|
||||
184.24.0.0,184.31.255.255,42.3736,-71.1097,Cambridge,US
|
||||
184.32.0.0,184.47.255.255,1.3521,103.8198,Singapore,SG
|
||||
184.48.0.0,184.63.255.255,51.5074,-0.1278,London,GB
|
||||
184.64.0.0,184.71.255.255,50.1109,8.6821,Frankfurt,DE
|
||||
184.72.0.0,184.73.255.255,37.7749,-122.4194,San Francisco,US
|
||||
184.74.0.0,184.83.255.255,40.7128,-74.006,New York,US
|
||||
184.84.0.0,184.87.255.255,42.3736,-71.1097,Cambridge,US
|
||||
184.88.0.0,184.215.255.255,37.7749,-122.4194,San Francisco,US
|
||||
184.216.0.0,184.255.255.255,-33.8688,151.2093,Sydney,AU
|
||||
185.0.0.0,185.31.15.255,51.5074,-0.1278,London,GB
|
||||
185.31.16.0,185.31.19.255,37.7749,-122.4194,San Francisco,US
|
||||
185.31.20.0,185.255.255.255,51.5074,-0.1278,London,GB
|
||||
186.0.0.0,186.127.255.255,19.076,72.8777,Mumbai,IN
|
||||
186.128.0.0,186.255.255.255,48.8566,2.3522,Paris,FR
|
||||
187.0.0.0,187.127.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
187.128.0.0,187.255.255.255,-23.5505,-46.6333,Sao Paulo,BR
|
||||
188.0.0.0,188.114.95.255,51.5074,-0.1278,London,GB
|
||||
188.114.96.0,188.114.111.255,37.7749,-122.4194,San Francisco,US
|
||||
188.114.112.0,188.255.255.255,51.5074,-0.1278,London,GB
|
||||
189.0.0.0,189.127.255.255,55.7558,37.6173,Moscow,RU
|
||||
189.128.0.0,189.255.255.255,41.8781,-87.6298,Chicago,US
|
||||
190.0.0.0,190.93.239.255,32.7767,-96.797,Dallas,US
|
||||
190.93.240.0,190.93.255.255,37.7749,-122.4194,San Francisco,US
|
||||
190.94.0.0,190.221.255.255,33.749,-84.388,Atlanta,US
|
||||
190.222.0.0,191.93.255.255,43.6532,-79.3832,Toronto,CA
|
||||
191.94.0.0,191.221.255.255,22.3193,114.1694,Hong Kong,HK
|
||||
191.222.0.0,191.231.255.255,25.033,121.5654,Taipei,TW
|
||||
191.232.0.0,191.233.255.255,-23.5505,-46.6333,Sao Paulo,BR
|
||||
191.234.0.0,191.249.255.255,40.4168,-3.7038,Madrid,ES
|
||||
191.250.0.0,192.9.255.255,45.4642,9.19,Milan,IT
|
||||
192.10.0.0,192.25.255.255,59.3293,18.0686,Stockholm,SE
|
||||
192.26.0.0,192.41.255.255,34.6937,135.5022,Osaka,JP
|
||||
192.42.0.0,192.49.255.255,23.1291,113.2644,Guangzhou,CN
|
||||
192.50.0.0,192.50.255.255,35.1815,136.9066,Nagoya,JP
|
||||
192.51.0.0,192.66.255.255,22.5431,114.0579,Shenzhen,CN
|
||||
192.67.0.0,192.82.255.255,52.52,13.405,Berlin,DE
|
||||
192.83.0.0,192.98.255.255,25.2048,55.2708,Dubai,AE
|
||||
192.99.0.0,192.114.255.255,13.7563,100.5018,Bangkok,TH
|
||||
192.115.0.0,192.130.255.255,31.2304,121.4737,Shanghai,CN
|
||||
192.131.0.0,192.146.255.255,39.9042,116.4074,Beijing,CN
|
||||
192.147.0.0,192.162.255.255,35.6762,139.6503,Tokyo,JP
|
||||
192.163.0.0,192.167.255.255,37.5665,126.978,Seoul,KR
|
||||
192.168.0.0,192.168.255.255,23.1291,113.2644,Guangzhou,CN
|
||||
192.169.0.0,192.184.255.255,1.3521,103.8198,Singapore,SG
|
||||
192.185.0.0,192.200.255.255,51.5074,-0.1278,London,GB
|
||||
192.201.0.0,192.216.255.255,50.1109,8.6821,Frankfurt,DE
|
||||
192.217.0.0,192.232.255.255,40.7128,-74.006,New York,US
|
||||
192.233.0.0,192.248.255.255,37.7749,-122.4194,San Francisco,US
|
||||
192.249.0.0,192.255.255.255,-33.8688,151.2093,Sydney,AU
|
||||
193.0.0.0,193.255.255.255,52.52,13.405,Berlin,DE
|
||||
194.0.0.0,194.255.255.255,48.8566,2.3522,Paris,FR
|
||||
195.0.0.0,195.255.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
196.0.0.0,196.127.255.255,19.076,72.8777,Mumbai,IN
|
||||
196.128.0.0,196.255.255.255,48.8566,2.3522,Paris,FR
|
||||
197.0.0.0,197.127.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
197.128.0.0,197.234.239.255,-23.5505,-46.6333,Sao Paulo,BR
|
||||
197.234.240.0,197.234.243.255,37.7749,-122.4194,San Francisco,US
|
||||
197.234.244.0,197.250.243.255,55.7558,37.6173,Moscow,RU
|
||||
197.250.244.0,198.10.243.255,41.8781,-87.6298,Chicago,US
|
||||
198.10.244.0,198.26.243.255,32.7767,-96.797,Dallas,US
|
||||
198.26.244.0,198.41.127.255,33.749,-84.388,Atlanta,US
|
||||
198.41.128.0,198.41.255.255,37.7749,-122.4194,San Francisco,US
|
||||
198.42.0.0,198.169.255.255,43.6532,-79.3832,Toronto,CA
|
||||
198.170.0.0,199.27.71.255,22.3193,114.1694,Hong Kong,HK
|
||||
199.27.72.0,199.27.79.255,37.7749,-122.4194,San Francisco,US
|
||||
199.27.80.0,199.155.79.255,25.033,121.5654,Taipei,TW
|
||||
199.155.80.0,199.231.255.255,40.4168,-3.7038,Madrid,ES
|
||||
199.232.0.0,199.232.255.255,37.7749,-122.4194,San Francisco,US
|
||||
199.233.0.0,200.104.255.255,45.4642,9.19,Milan,IT
|
||||
200.105.0.0,200.232.255.255,59.3293,18.0686,Stockholm,SE
|
||||
200.233.0.0,201.104.255.255,34.6937,135.5022,Osaka,JP
|
||||
201.105.0.0,201.232.255.255,23.1291,113.2644,Guangzhou,CN
|
||||
201.233.0.0,201.255.255.255,22.5431,114.0579,Shenzhen,CN
|
||||
202.0.0.0,202.255.255.255,39.9042,116.4074,Beijing,CN
|
||||
203.0.0.0,203.255.255.255,39.9042,116.4074,Beijing,CN
|
||||
204.0.0.0,204.127.255.255,52.52,13.405,Berlin,DE
|
||||
204.128.0.0,204.255.255.255,25.2048,55.2708,Dubai,AE
|
||||
205.0.0.0,205.127.255.255,13.7563,100.5018,Bangkok,TH
|
||||
205.128.0.0,205.255.255.255,31.2304,121.4737,Shanghai,CN
|
||||
206.0.0.0,206.127.255.255,39.9042,116.4074,Beijing,CN
|
||||
206.128.0.0,206.255.255.255,35.6762,139.6503,Tokyo,JP
|
||||
207.0.0.0,207.127.255.255,37.5665,126.978,Seoul,KR
|
||||
207.128.0.0,207.255.255.255,1.3521,103.8198,Singapore,SG
|
||||
208.0.0.0,208.127.255.255,51.5074,-0.1278,London,GB
|
||||
208.128.0.0,208.255.255.255,50.1109,8.6821,Frankfurt,DE
|
||||
209.0.0.0,209.127.255.255,40.7128,-74.006,New York,US
|
||||
209.128.0.0,209.255.255.255,37.7749,-122.4194,San Francisco,US
|
||||
210.0.0.0,210.255.255.255,31.2304,121.4737,Shanghai,CN
|
||||
211.0.0.0,211.255.255.255,39.9042,116.4074,Beijing,CN
|
||||
212.0.0.0,212.255.255.255,51.5074,-0.1278,London,GB
|
||||
213.0.0.0,213.255.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
214.0.0.0,214.127.255.255,-33.8688,151.2093,Sydney,AU
|
||||
214.128.0.0,214.255.255.255,19.076,72.8777,Mumbai,IN
|
||||
215.0.0.0,215.127.255.255,48.8566,2.3522,Paris,FR
|
||||
215.128.0.0,215.255.255.255,52.3676,4.9041,Amsterdam,NL
|
||||
216.0.0.0,216.127.255.255,-23.5505,-46.6333,Sao Paulo,BR
|
||||
216.128.0.0,216.255.255.255,55.7558,37.6173,Moscow,RU
|
||||
217.0.0.0,217.79.255.255,52.52,13.405,Berlin,DE
|
||||
217.80.0.0,217.95.255.255,48.1351,11.582,Munich,DE
|
||||
217.96.0.0,217.223.255.255,52.52,13.405,Berlin,DE
|
||||
217.224.0.0,217.255.255.255,50.1109,8.6821,Frankfurt,DE
|
||||
218.0.0.0,218.255.255.255,31.2304,121.4737,Shanghai,CN
|
||||
219.0.0.0,219.255.255.255,39.9042,116.4074,Beijing,CN
|
||||
220.0.0.0,220.255.255.255,23.1291,113.2644,Guangzhou,CN
|
||||
221.0.0.0,221.255.255.255,31.2304,121.4737,Shanghai,CN
|
||||
222.0.0.0,222.255.255.255,39.9042,116.4074,Beijing,CN
|
||||
223.0.0.0,223.255.255.255,39.9042,116.4074,Beijing,CN
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
:ipd.lond,timestamp,4dbn,:prd,client-ip,0cph,:ips.anon,0crv,:ipd.doma,:ipd.ispn,8dbd,1ipp,2tmo,0ver,4dur,8did,scnt,cnam,:ips.ispn,tabl,cipher-suite,:ips,4srs,:ips.doma,8ack,orga,source-node,8byt,snam,server-ip,time,:ips.orgn,orgu,8seq,8ppk,:ipd.orgn,0rnt,crcc,:prs,4ksz,ecdhe-named-curve,dcnt,name,8pak,:ips.latd,:ipd.latd,:ips.city,:ipd.city,:ipd.anon,8ses,:ipd,:ips.lond,0rnd
|
||||
-108.418446,1780243200.0,51,22,110.69.236.47,c0 13,"anon, hosting",,,China Mobile,165,26,821.8026,03 04,9.03,5339,ru,,BT Group,TlsS,,110.69.236.47,83579,,140121,Samsung,suricata,407047,mail.example.com,201.154.234.165,2026-06-01 00:00:00.000000,Alibaba Inc.,,1280.3595,139,Amazon.com Inc.,77 33 c2 8e,,443,384,,ru,wan-link,335,-33.763443,-56.815684,New York,New York,"anon, hosting",1369.5439,214.247.241.63,1.927904,8e 03 51 d8 ae 8e 4f 6e ac 34 2f c2 31 b7 b0 87 16 eb 3f c1 28 96 b9 62 23 17 74 94
|
||||
-63.303819,1780243200.000001,55,993,103.30.84.98,c0 14,"anon, hosting",,,BT Group,119,7,673.3645,03 04,16.31,2235,il,,Telstra,TlsC,TLS_AES_256_GCM_SHA384,103.30.84.98,82132,,149795,,packet_capture,453099,cdn.example.com,179.237.25.143,2026-06-01 00:00:00.000001,Samsung,,189.5573,414,Tencent,03 24 1e 75,,443,,,jp,edge-01,498,33.435177,-59.931372,Seoul,New York,"anon, hosting",709.2408,44.42.255.217,10.927322,9d 7a 1d 7b 28 2b f8 23 40 41 f3 54 87 d8 6c 66 9f cc bf e0 e7 3d 7e 73 20 ad 0a 75
|
||||
156.494253,1780243200.000002,6,8080,61.83.90.226,c0 14,"anon, hosting",,,Telstra,54,35,883.1064,03 03,6.57,2684,fi,,,TlsC,,122.86.43.73,83385,,72630,,ssl_logs,58854,,127.166.93.125,2026-06-01 00:00:00.000002,Tencent,,324.0024,327,Meta Platforms,4f 8b 90 6b,,993,,,kr,gw-09,684,-9.69301,2.733926,New York,Dublin,"anon, hosting",1411.6292,86.164.53.220,30.122024,ca 43 99 ea a1 25 04 ea 33 25 6d 87 43 b2 23 7d bd 91 50 e0 9a 04 99 35 44 87 3b 36
|
||||
174.324906,1780243200.000003,99,143,76.111.29.149,13 01,"anon, hosting",,,NTT,200,37,455.4162,03 02,79.83,3881,se,,Comcast,TlsC,TLS_ECDHE_RSA_AES128_SHA,76.111.29.149,77889,,24580,,ssl_logs,445707,,60.135.101.109,2026-06-01 00:00:00.000003,Samsung,,399.0607,159,Apple Inc.,a4 ee e2 e2,,443,,,sg,edge-01,1539,-18.517547,-7.632734,Mumbai,Mumbai,"anon, hosting",490.4402,171.157.23.57,103.114144,16 df 00 64 ba dc 23 a9 a0 3f 99 9e d1 a7 ce 97 41 62 d7 c2 59 9a cf 00 9b 92 6b dc
|
||||
91.251045,1780243200.000004,49,8080,80.192.203.242,13 02,"anon, hosting",,,SK Telecom,184,39,766.8981,03 03,93.86,4978,au,,AT&T,TlsC,,80.192.203.242,49529,,197817,,packet_capture,244011,,222.81.167.240,2026-06-01 00:00:00.000004,Apple Inc.,,477.8513,11,Baidu Inc.,c2 8e d7 81,,995,512,,au,core-02,507,-14.354277,-17.788944,Dublin,New York,"anon, hosting",229.6645,192.46.222.251,-177.87741,70 f9 70 8b df f8 0e c7 ac cf 54 ef 41 0d c9 0d 2a db 45 ec 5d 19 85 c2 a7 6c e8 a7
|
||||
136.838644,1780243200.000005,20,53,84.28.117.211,c0 27,"anon, hosting",,,Comcast,145,17,340.7059,03 03,85.11,2395,hk,,Verizon,TlsS,,84.28.117.211,25525,,12031,,zeek,246299,,179.151.65.50,2026-06-01 00:00:00.000005,,,1737.0539,484,Microsoft Corp.,5e 62 6e f7,,12345,,,no,wan-link,1134,-28.997711,11.538601,Beijing,Moscow,"anon, hosting",1745.5668,201.154.234.165,-79.322944,7b d0 fa 7b f3 fb e5 08 2f 96 71 cf 7c 9c bc f2 b0 d9 a9 b4 e8 8a 9c 80 76 3d 62 a1
|
||||
-135.232683,1780243200.000006,15,443,104.173.55.64,c0 2c,"anon, hosting",,,Orange,31,2,880.8466,03 04,0.64,3522,fr,,NTT,TlsC,,104.173.55.64,48254,mail.example.com,121204,,packet_capture,486567,cdn.example.com,10.155.252.30,2026-06-01 00:00:00.000006,Apple Inc.,,1251.1921,319,Amazon.com Inc.,fd 93 cd 12,,22,,,au,core-02,385,29.874927,-25.366729,Paris,Dublin,"anon, hosting",407.9129,51.168.117.98,24.914538,fc e0 29 14 dd a5 80 0d 2e 75 0a 89 14 59 f0 e2 8e 5c df fb 2e f0 b2 d1 aa a4 35 52
|
||||
134.708535,1780243200.000007,31,143,184.101.38.152,00 35,"anon, hosting",,,Deutsche Telekom,92,20,945.2356,03 03,74.56,5837,ru,,KDDI,TlsS,,184.101.38.152,96275,,152971,,netflow,83024,,195.18.232.23,2026-06-01 00:00:00.000007,Google LLC,,19.7855,236,Meta Platforms,96 b7 bf dc,,443,,,au,gw-09,945,0.274938,44.364038,Berlin,Sydney,"anon, hosting",1802.5176,72.16.189.56,32.464955,49 d1 4f 26 f0 87 ad cb 29 a8 c2 a2 f9 12 23 78 93 74 2e de 32 33 e3 55 99 0e 17 a6
|
||||
172.779207,1780243200.000008,17,22,153.34.120.104,cc a9,"anon, hosting",,graph.facebook.com,NTT,53,8,398.1396,03 04,74.07,4394,de,,,TlsS,,109.141.22.1,40539,,73432,Netflix Inc.,suricata,338274,,179.115.153.176,2026-06-01 00:00:00.000008,Amazon.com Inc.,,1711.0449,228,Baidu Inc.,45 c6 e8 bd,,443,,,br,wan-link,759,55.980025,26.906005,London,Berlin,"anon, hosting",1371.6749,35.32.234.10,129.650615,ca 07 87 3f e8 bc 86 c3 be 37 77 f1 0c a7 71 20 ed 9a d1 3b 47 17 13 9b fc 3b 31 78
|
||||
76.375007,1780243200.000009,44,80,131.59.197.231,c0 27,"anon, hosting",,,Singtel,47,42,818.4269,03 03,69.56,7566,ua,,BT Group,TlsC,,131.59.197.231,92991,,172259,,netflow,294307,,98.117.126.118,2026-06-01 00:00:00.000009,Netflix Inc.,,230.6557,433,Baidu Inc.,a8 3e d2 b5,,443,,,ca,backbone-03,1736,-44.857009,10.243763,Seattle,Sydney,"anon, hosting",453.092,19.30.80.113,44.754298,dc 32 3a 6e dc e7 74 d3 ad e8 cc d4 30 a0 da a0 82 bf 4e f2 22 2e 2b 2f dd 31 be 42
|
||||
-37.054377,1780243200.00001,8,465,9.169.36.132,c0 23,"anon, hosting",,,Orange,74,41,439.7628,03 03,114.18,2479,br,,Telstra,TlsS,,9.169.36.132,76462,mail.example.com,121722,Netflix Inc.,netflow,329042,graph.facebook.com,120.62.136.116,2026-06-01 00:00:00.000010,Samsung,,1129.4191,349,Baidu Inc.,63 b2 b3 4b,,995,,,in,gw-09,1557,7.046828,-17.499939,Seattle,London,"anon, hosting",1249.419,81.130.191.245,60.497342,27 a0 db cf d5 94 3a cf 0a a6 57 eb b9 2d df 36 7c df cd 28 ca 9e ad 71 aa 56 27 3a
|
||||
58.368656,1780243200.000011,11,443,60.51.194.72,c0 14,"anon, hosting",00 18,,NTT,70,4,604.6634,03 04,94.23,9410,ca,,Verizon,TlsS,TLS_AES_128_GCM_SHA256,60.51.194.72,67340,mail.example.com,50960,,packet_capture,428486,,215.255.8.162,2026-06-01 00:00:00.000011,,,564.2617,226,Meta Platforms,16 d2 ba 79,,8443,,,in,gw-09,1940,50.009575,18.726566,London,Dublin,"anon, hosting",1055.3921,223.73.205.18,-68.316574,c0 a8 41 2f d8 b9 09 b9 9e 5c 6d ae f8 62 73 46 4f 27 97 33 13 ac 43 c0 4e 53 5c 54
|
||||
-128.312338,1780243200.000012,56,53,210.235.179.79,c0 14,"anon, hosting",,,Singtel,47,12,956.4787,03 04,57.07,6262,in,,AT&T,TlsS,,210.235.179.79,52981,,169681,,suricata,389907,*.s3.amazonaws.com,49.185.255.161,2026-06-01 00:00:00.000012,Meta Platforms,,664.5174,25,Tencent,ea 12 96 67,,3389,,,jp,edge-01,732,-58.027758,-30.967424,Seattle,Shanghai,"anon, hosting",736.6703,184.149.139.107,5.535263,d5 5f 2b b8 22 0a c7 f0 16 c6 bf 81 08 b6 22 b0 7b 35 aa 44 16 b4 ad 59 ed f5 5d 45
|
||||
-86.295682,1780243200.000013,11,8080,86.66.134.42,c0 2c,"anon, hosting",,,Singtel,163,11,124.1611,03 03,77.26,5496,no,,BT Group,TlsC,,86.66.134.42,49691,,178386,,ssl_logs,455392,,62.40.43.188,2026-06-01 00:00:00.000013,Meta Platforms,,1175.9842,400,Netflix Inc.,2c 21 9e cb,,12345,,,gb,gw-09,263,-13.561514,-28.586799,Dublin,Beijing,"anon, hosting",258.9369,84.87.235.213,22.202176,fb 66 3e 45 25 e7 58 e3 2c a3 b1 21 94 99 50 59 b9 72 3e 66 47 79 fc 0d b8 bc ef 42
|
||||
-83.710197,1780243200.000014,42,50000,83.250.9.29,00 9f,"anon, hosting",,,,82,3,401.3871,03 03,91.85,5902,hk,,AT&T,TlsC,,83.250.9.29,76494,,163479,,suricata,104563,cdn.example.com,14.99.185.142,2026-06-01 00:00:00.000014,Netflix Inc.,,1589.9996,50,Baidu Inc.,87 bb ce bb,,21,,x25519,fr,gw-09,244,7.398146,-4.023292,Paris,Seattle,"anon, hosting",1877.9582,180.47.216.131,-65.118281,c4 d6 7f 51 a7 a0 61 51 ff ef ff 9d fe 0b 2e c9 ea 7b 6e b4 18 19 90 fd f0 92 04 37
|
||||
-117.654327,1780243200.000015,23,21,63.44.142.226,00 9f,"anon, hosting",,,KDDI,46,42,963.914,03 04,49.89,1595,il,,China Telecom,TlsC,,28.152.101.40,62034,login.live.com,26457,,ssl_logs,341269,,162.118.34.182,2026-06-01 00:00:00.000015,Meta Platforms,,316.7242,282,Apple Inc.,6d 67 8a 8b,,995,,,fi,wan-link,282,19.467216,-22.349688,Mumbai,Dublin,"anon, hosting",1761.4143,208.96.164.155,9.171775,87 f7 62 7e 8e 98 73 98 93 6a fa a2 f5 b2 8c 93 3e c2 ca b0 4a 94 15 93 28 b1 e2 83
|
||||
-94.931072,1780243200.000016,11,993,71.125.114.36,13 02,"anon, hosting",,,Deutsche Telekom,170,50,408.8669,03 03,47.82,3178,gb,,China Telecom,TlsS,,200.208.79.237,55582,api.example.com,195608,,zeek,247701,stream.example.org,88.73.160.221,2026-06-01 00:00:00.000016,Apple Inc.,,1082.7858,177,Tencent,fa 2e e4 1c,,8443,,,ua,gw-09,150,+,-16.853749,Berlin,Seoul,"anon, hosting",1788.5427,101.234.123.56,-85.840866,9f 3c 97 bc 71 70 44 f4 4e e8 bf d4 f1 6f 7e 29 e4 b9 27 39 1f 67 4c 54 a7 e2 3b 69
|
||||
-31.118326,1780243200.000017,34,80,104.173.55.64,c0 14,"anon, hosting",,,NTT,128,14,242.5932,03 01,87.05,8692,gb,,BT Group,TlsS,,104.173.55.64,51973,login.live.com,125770,,netflow,395686,,94.203.160.167,2026-06-01 00:00:00.000017,Netflix Inc.,,424.4037,490,Microsoft Corp.,e6 bd 2f 61,,995,,,il,core-02,1858,45.150999,2.550127,London,Singapore,"anon, hosting",1483.2089,221.24.169.70,174.850402,59 da fd 18 b0 c3 a3 d5 d1 4c 99 c0 5e f2 7b 73 99 49 ed 1d d3 d5 44 c6 7c 82 68 a9
|
||||
41.94697,1780243200.000018,70,8080,59.148.40.219,c0 2f,"anon, hosting",00 17,,Telstra,194,10,15.6581,03 04,110.34,2284,cn,,KDDI,TlsS,,59.148.40.219,55332,,181579,,suricata,487141,auth.example.net,131.228.28.117,2026-06-01 00:00:00.000018,Microsoft Corp.,,1851.244,474,Samsung,f3 1c 04 57,,80,,,us,wan-link,974,-15.215917,20.010568,London,New York,"anon, hosting",1127.4562,12.180.173.209,42.420685,d2 06 08 8c 92 08 dc 5b 36 31 4c 7b 62 81 b5 88 cb 28 bf cf eb 7c 73 99 29 10 2f cf
|
||||
-175.557379,1780243200.000019,53,465,49.43.123.185,c0 23,"anon, hosting",,www.google.com,Singtel,36,50,997.2649,03 04,48.09,4965,kr,,Verizon,TlsC,,49.43.123.185,30203,login.live.com,192742,,ssl_logs,295820,,9.228.24.63,2026-06-01 00:00:00.000019,Baidu Inc.,,752.7345,401,Meta Platforms,aa c2 a8 e1,,8443,,,de,edge-01,1006,11.630947,+,Seattle,Sydney,"anon, hosting",786.4773,14.64.215.135,102.397941,7c 56 c2 56 47 89 9a 89 fc 4a 20 55 de 8d d7 99 f7 27 b8 80 7e fd 64 ea 36 45 9b 03
|
||||
+,1780243200.00002,48,443,193.120.158.58,c0 14,"anon, hosting",,,Deutsche Telekom,131,44,845.5178,03 04,104.7,3258,ru,,NTT,TlsS,,193.120.158.58,65448,,85554,,ssl_logs,60090,,28.122.124.13,2026-06-01 00:00:00.000020,Meta Platforms,,1931.8669,442,Netflix Inc.,65 e6 db c7,,443,256,,hk,backbone-03,234,-47.454077,55.407344,London,San Francisco,"anon, hosting",777.8899,184.218.85.177,-30.248846,e9 69 f7 b2 f3 31 e0 e7 a3 22 99 16 3a 0b af 37 54 7c 59 51 a9 da ec 76 cf 5e 5f dd
|
||||
90.484378,1780243200.000021,71,50000,149.188.242.142,c0 13,"anon, hosting",,cdn.example.com,,145,35,697.289,03 03,48.64,6631,se,*.s3.amazonaws.com,BT Group,TlsS,,149.188.242.142,16345,,182212,,ssl_logs,240559,*.s3.amazonaws.com,17.237.154.167,2026-06-01 00:00:00.000021,Tencent,,5.9437,250,Amazon.com Inc.,a4 7e c9 fa,,443,,,il,core-02,756,58.773087,39.608486,Paris,Mumbai,"anon, hosting",1465.7153,174.121.227.157,-143.130082,b2 2c e0 3c 79 6b b3 db 54 47 69 69 1e b3 8f 56 a5 95 94 88 30 45 d2 1e 8d 40 43 7f
|
||||
91.136808,1780243200.000022,79,443,180.131.188.43,13 02,"anon, hosting",,,Verizon,22,33,60.4689,03 03,38.32,9377,sg,,China Mobile,TlsS,TLS_ECDHE_RSA_AES256_GCM_SHA384,180.131.188.43,30365,cdn.example.com,149097,,zeek,113311,api.example.com,21.68.178.31,2026-06-01 00:00:00.000022,Google LLC,,1974.7452,391,Alibaba Inc.,1b 7a fd c5,,443,,,jp,backbone-03,1119,45.718679,32.815258,Shanghai,Singapore,"anon, hosting",1175.1396,47.109.174.243,-130.633454,04 54 e1 b3 6d 4d d2 e2 6f 2c 33 47 3f c4 b3 db a1 47 7e 8d 2b 7f 91 0d 9a 69 60 c8
|
||||
98.398423,1780243200.000023,19,22,207.81.216.88,c0 2f,"anon, hosting",00 19,auth.example.net,Verizon,24,43,950.9903,03 04,116.64,4792,il,,NTT,TlsS,,207.81.216.88,78125,,51328,,suricata,363773,,165.89.237.252,2026-06-01 00:00:00.000023,Netflix Inc.,,1864.8627,126,Amazon.com Inc.,9f a7 2c e9,,21,,,kr,gw-09,1120,43.123174,-41.122802,Paris,Mumbai,"anon, hosting",1083.5489,117.174.80.95,-119.922699,01 16 06 7d 17 f1 ba c4 4c 5b 12 d6 72 a4 7f d5 a3 8a 27 be 3d 1a 5b 72 17 cd 23 ee
|
||||
-37.351993,1780243200.000024,73,12345,61.142.248.55,00 9f,"anon, hosting",,,Orange,100,16,642.6525,03 04,33.36,1427,br,,AT&T,TlsC,TLS_RSA_WITH_AES_256_GCM_SHA384,61.142.248.55,38707,,39062,,netflow,72151,auth.example.net,99.247.54.61,2026-06-01 00:00:00.000024,Alibaba Inc.,,824.5724,332,Google LLC,ec 19 56 dd,,3389,,,jp,edge-01,1269,-42.201144,56.208001,Dublin,Moscow,"anon, hosting",122.8972,87.67.192.40,76.016663,59 0f cf e7 7f 30 ec 34 47 3e 06 1f 71 42 65 cd be 2b 84 26 0b 21 65 e3 41 2f a9 3e
|
||||
108.8728,1780243200.000025,29,993,146.154.207.141,c0 2b,"anon, hosting",,,China Mobile,19,25,166.3384,03 04,85.22,1547,ua,,China Mobile,TlsS,,146.154.207.141,51774,,88114,,zeek,384285,,99.247.54.61,2026-06-01 00:00:00.000025,Netflix Inc.,48294,514.0592,355,Baidu Inc.,71 35 fe 48,,3389,128,secp256r1,cn,backbone-03,544,36.77345,-38.704815,Sydney,Mumbai,"anon, hosting",3.561,32.188.223.228,-57.326744,48 38 ba 25 bf ca 33 ac 9a a5 45 50 df f9 a2 59 b6 72 59 c1 9d 96 41 5a 00 c8 10 5d
|
||||
-40.490238,1780243200.000026,53,50000,84.28.117.211,c0 2f,"anon, hosting",,,China Telecom,168,10,419.5938,03 04,31.32,8750,fr,,AT&T,TlsC,,84.28.117.211,46099,auth.example.net,17666,,packet_capture,308259,*.cloudfront.net,9.118.147.181,2026-06-01 00:00:00.000026,Microsoft Corp.,,1039.8567,167,Baidu Inc.,55 6c 62 40,,443,,,jp,wan-link,1140,47.385298,17.023966,Seattle,Paris,"anon, hosting",25.7152,180.58.110.111,-84.598417,16 86 e1 ee 86 73 89 1f 4d 31 2b b0 d3 74 1f c6 d6 f3 78 f3 99 28 ca 11 4b 3d e5 58
|
||||
-108.320482,1780243200.000027,8,8080,50.35.23.170,c0 14,"anon, hosting",,,KDDI,154,41,63.8371,02 00,81.42,6236,in,,China Telecom,TlsS,,50.35.23.170,13336,,104211,,netflow,359029,auth.example.net,35.244.78.117,2026-06-01 00:00:00.000027,Meta Platforms,,1678.9175,340,Google LLC,04 72 81 9c,,993,,,il,gw-09,16,+,-16.75499,Sydney,Mumbai,"anon, hosting",1187.9513,135.10.199.224,-161.798659,3f 0f 7b be fa e4 5a f2 b3 b0 53 84 2e 91 0e c5 19 53 6d 71 73 69 89 d1 0a 04 f2 43
|
||||
-42.599081,1780243200.000028,28,110,67.104.160.62,c0 2b,"anon, hosting",,,SK Telecom,151,32,133.7894,03 03,37.8,3017,br,,,TlsS,,67.104.160.62,95267,,132046,,zeek,465754,www.google.com,74.251.102.32,2026-06-01 00:00:00.000028,Baidu Inc.,,582.3648,24,Alibaba Inc.,13 66 bf 07,,25,256,x448,no,core-02,343,46.731127,44.099731,New York,Paris,"anon, hosting",1193.1435,182.224.224.135,0.171855,46 66 af d1 ea 49 a1 5c 29 f9 aa 5a a0 1d 02 e7 8c 6b 56 51 fa 2d 42 de db ce db f4
|
||||
60.847895,1780243200.000029,2,443,198.10.194.123,c0 27,"anon, hosting",,,AT&T,174,9,838.5365,03 03,7.51,4669,fr,,Singtel,TlsS,,198.10.194.123,97797,,41884,,zeek,452438,,141.103.76.43,2026-06-01 00:00:00.000029,Tencent,,876.8874,411,Apple Inc.,cd cc e9 8b,ER,22,,secp521r1,nl,edge-01,1867,20.595976,40.22786,Singapore,Sydney,"anon, hosting",1558.5494,192.212.89.52,146.476816,33 83 76 e9 97 87 ed 19 33 59 14 94 b8 a0 da 3b 30 17 04 46 57 a8 b7 e0 89 2f be af
|
||||
-70.344735,1780243200.00003,54,12345,149.135.19.28,c0 23,"anon, hosting",,,Singtel,178,12,910.1815,03 03,107.06,5482,sg,,Singtel,TlsC,,149.135.19.28,54939,,154585,,suricata,266712,graph.facebook.com,196.172.52.24,2026-06-01 00:00:00.000030,Meta Platforms,,98.0946,3,Apple Inc.,c9 6e ea 61,,50000,,,jp,wan-link,276,-49.092534,14.287938,Berlin,Moscow,"anon, hosting",1677.7934,81.129.165.30,37.513179,ae ba 1b 72 e8 a9 f2 ba e7 3a 31 71 01 ae b5 95 bf 36 17 54 fc 47 d1 36 82 64 60 3d
|
||||
-17.065978,1780243200.000031,80,143,207.101.75.7,00 35,"anon, hosting",,,Verizon,144,6,415.9312,03 03,111.92,1144,gb,,SK Telecom,TlsS,,207.101.75.7,55351,*.s3.amazonaws.com,173610,,netflow,412425,cdn.example.com,46.138.247.120,2026-06-01 00:00:00.000031,Alibaba Inc.,,1415.6554,184,Apple Inc.,87 42 52 82,,443,,,cn,core-02,1618,-35.519892,-16.958108,Beijing,Dublin,"anon, hosting",1914.0184,58.58.236.243,14.821951,72 c7 79 93 18 c3 c6 d5 a0 1b 06 80 67 e2 73 bc 64 63 96 e5 5a 26 5f 58 3d c1 14 db
|
||||
-17.594761,1780243200.000032,41,8080,9.169.36.132,c0 2f,"anon, hosting",,www.google.com,Singtel,85,38,250.6876,03 03,74.22,6008,gb,,Orange,TlsS,,9.169.36.132,29974,graph.facebook.com,30147,,ssl_logs,152585,,53.189.160.106,2026-06-01 00:00:00.000032,Tencent,,1864.8074,28,Meta Platforms,c9 2b 9c 6b,,21,384,,us,gw-09,1638,38.594318,-11.084067,Moscow,San Francisco,"anon, hosting",923.0691,10.97.146.92,147.955167,3c 03 20 17 25 b7 36 94 9a 2c 90 e0 c3 ca 0f f2 50 70 45 c8 97 49 98 bf 05 48 3f 15
|
||||
3.748318,1780243200.000033,83,80,177.2.54.200,00 35,"anon, hosting",,,Verizon,6,15,717.3027,03 03,104.19,3095,se,,BT Group,TlsS,,122.251.229.203,31119,www.google.com,126482,,zeek,82700,auth.example.net,25.49.220.91,2026-06-01 00:00:00.000033,Netflix Inc.,,460.7515,72,Microsoft Corp.,59 c0 c1 bb,,995,,,br,gw-09,14,27.987219,-4.235863,Berlin,San Francisco,"anon, hosting",562.3382,35.37.231.45,-75.240302,54 47 e4 51 5b 4a f7 b7 14 70 f9 76 20 87 bd 77 15 69 b8 c6 f0 e8 17 14 a5 34 8c 86
|
||||
117.90784,1780243200.000034,82,443,56.119.13.144,c0 14,"anon, hosting",,,NTT,24,33,949.2452,03 03,21.86,8727,au,,Verizon,TlsS,,56.119.13.144,67065,,126914,,suricata,234344,graph.facebook.com,207.100.186.111,2026-06-01 00:00:00.000034,Netflix Inc.,,1471.8291,397,Amazon.com Inc.,da 81 85 d8,,443,,,gb,gw-09,629,-2.131088,12.416958,San Francisco,Dublin,"anon, hosting",1318.9064,34.187.139.213,-133.081617,d7 bd 16 db 75 f6 c3 b3 32 d9 44 82 f4 77 2a 91 d1 83 60 02 03 3c 03 c0 69 a1 d1 b9
|
||||
-178.325477,1780243200.000035,36,443,61.83.90.226,c0 13,"anon, hosting",,,SK Telecom,196,49,663.4241,03 04,60.13,5591,us,,AT&T,TlsC,,61.83.90.226,85584,,192888,,netflow,41350,stream.example.org,99.247.54.61,2026-06-01 00:00:00.000035,,,1672.8331,104,Apple Inc.,af 26 5b f3,,995,,,kr,edge-01,544,55.566857,-11.525224,Moscow,Sydney,"anon, hosting",1558.098,67.228.109.157,-16.286757,96 ae 3b 83 80 e2 c1 f7 7c ad ea eb 0e f6 3b 84 22 c3 70 75 a1 d3 b4 04 86 7f 93 2f
|
||||
47.034714,1780243200.000036,61,443,139.102.159.103,c0 23,"anon, hosting",,,Deutsche Telekom,7,47,183.8862,03 04,41.56,9584,ru,,Orange,TlsC,,139.102.159.103,20911,,171283,,packet_capture,236280,,221.24.169.70,2026-06-01 00:00:00.000036,Amazon.com Inc.,,890.3519,203,Amazon.com Inc.,c3 92 ed 4a,,465,,,jp,gw-09,548,-9.41752,47.192449,Sydney,Mumbai,"anon, hosting",726.9665,18.120.187.93,-131.590194,80 18 cb 32 53 1e 6e 82 f3 31 e6 ab 85 b8 b8 83 b0 0a d2 3e b3 65 59 91 b5 a6 f9 e4
|
||||
12.6356,1780243200.000037,33,50000,68.202.67.172,c0 2c,"anon, hosting",,,Comcast,4,10,80.2231,03 03,15.44,5078,se,mail.example.com,,TlsC,,68.202.67.172,68059,,2250,,packet_capture,415134,,65.233.154.51,2026-06-01 00:00:00.000037,,32862,623.8418,112,Amazon.com Inc.,e9 e7 8b fb,,110,128,,ru,gw-09,309,+,-25.58853,Beijing,Tokyo,"anon, hosting",1995.9289,63.104.203.235,-130.931836,99 04 dc 95 76 03 bf 60 d2 df ca 7f 53 c7 c1 70 86 28 d8 7a 8b 91 89 d3 60 23 5d 45
|
||||
-4.376563,1780243200.000038,10,443,98.49.183.217,00 9e,"anon, hosting",,,KDDI,54,3,107.0552,03 03,82.15,2310,gb,,Singtel,TlsS,,98.49.183.217,93744,,40504,,suricata,458161,mail.example.com,65.233.154.51,2026-06-01 00:00:00.000038,Baidu Inc.,,1046.8779,165,Microsoft Corp.,c2 5e 09 ba,,50000,,,nl,edge-01,661,-41.529788,-47.113001,Berlin,Shanghai,"anon, hosting",1670.5515,72.127.58.7,13.087197,b1 e0 62 0a c1 b7 44 80 14 f8 e0 96 fa 61 5f b2 a5 ef 8c 52 23 4f e6 97 86 bb e8 00
|
||||
117.415791,1780243200.000039,73,443,62.40.43.188,c0 14,"anon, hosting",,,BT Group,129,8,879.7925,03 03,58.85,3509,ca,"*.['live', 'com']",Telstra,TlsC,,62.40.43.188,61495,,184347,,zeek,404374,graph.facebook.com,138.4.55.111,2026-06-01 00:00:00.000039,Samsung,,1461.8479,329,Baidu Inc.,7f 77 83 99,,993,,,il,backbone-03,1859,41.382482,15.988899,Paris,Tokyo,"anon, hosting",157.8763,83.127.3.67,-31.556415,c0 82 b8 e7 e5 cb 00 11 91 08 45 9a 03 99 71 6d a1 0b 8d 4f 30 13 eb b6 4f 46 87 bb
|
||||
-95.943745,1780243200.00004,39,110,8.126.102.215,00 35,"anon, hosting",,,Verizon,129,34,21.4885,03 04,106.07,2015,nl,,BT Group,TlsS,,8.126.102.215,60747,,99898,,ssl_logs,294161,,63.207.152.117,2026-06-01 00:00:00.000040,Alibaba Inc.,,1194.6157,162,Apple Inc.,8a 1b ee 04,,21,,,fr,gw-09,1454,14.781859,50.905585,Singapore,Sydney,"anon, hosting",398.5515,208.96.164.155,-21.433167,2b 39 1b 7c 4b 93 07 af 38 9c e6 f1 0b a9 6b 6c 68 84 f7 48 09 c0 16 82 57 1e 91 66
|
||||
115.811466,1780243200.000041,7,993,5.47.150.57,13 01,"anon, hosting",,,SK Telecom,113,3,451.6096,03 04,48.88,5989,sg,,BT Group,TlsC,,5.47.150.57,47475,,192862,,packet_capture,20391,,179.249.79.49,2026-06-01 00:00:00.000041,Apple Inc.,,1144.297,314,Netflix Inc.,71 67 68 21,,443,,,kr,backbone-03,1475,55.343542,29.473168,Moscow,New York,"anon, hosting",1283.1463,178.203.41.219,-70.794682,60 36 43 5e 0d 88 76 76 76 0d b7 28 f4 44 f0 bf 5e c6 fb f2 46 53 62 73 10 d1 07 8a
|
||||
82.858229,1780243200.000042,94,443,12.125.243.157,13 01,"anon, hosting",,,KDDI,136,18,470.8168,03 03,67.68,8924,fi,,NTT,TlsS,,12.125.243.157,89269,auth.example.net,196533,,ssl_logs,288805,cdn.example.com,141.80.208.168,2026-06-01 00:00:00.000042,Amazon.com Inc.,28831,737.7703,394,Samsung,98 a9 2c 19,,143,256,,de,backbone-03,1908,32.298816,-23.032535,Sydney,Seattle,"anon, hosting",136.2657,79.13.242.68,-161.528224,43 67 01 81 06 ea fe ff 02 fc 79 6f f3 e8 d1 e2 28 03 7d 8e 6a 19 5e 4b 1c b2 12 4a
|
||||
120.177108,1780243200.000043,40,22,19.30.117.18,c0 2b,"anon, hosting",,,AT&T,176,44,822.1385,03 03,109.65,2426,kr,"*.['google', 'com']",SK Telecom,TlsC,,19.30.117.18,16047,,6636,,zeek,403629,*.cloudfront.net,58.62.102.203,2026-06-01 00:00:00.000043,Microsoft Corp.,,967.4604,98,Meta Platforms,2c fe 79 be,,3389,384,,in,core-02,1854,55.213539,-36.71813,Sydney,Seoul,"anon, hosting",621.4071,155.205.20.12,119.535397,12 45 f6 4f eb ab ec d3 1c 99 a1 09 6a a9 b5 21 4c 8f 0f ad c0 54 c7 5f 3b 9d a1 10
|
||||
12.054574,1780243200.000044,40,8443,207.100.186.111,c0 14,"anon, hosting",,,Telstra,18,37,77.8944,03 03,65.98,7208,ca,,SK Telecom,TlsS,,207.100.186.111,94540,,100715,,zeek,90946,,171.4.238.231,2026-06-01 00:00:00.000044,Amazon.com Inc.,,1960.4302,303,Microsoft Corp.,41 34 cf 36,--,80,,,cn,wan-link,427,38.093037,20.504432,Singapore,Berlin,"anon, hosting",685.0595,223.170.155.245,76.643527,43 2f ef 31 e9 9e ee f0 76 72 49 84 ba c8 5d a9 a6 e8 f9 0f 21 f3 39 b3 f5 ec 57 90
|
||||
-112.238338,1780243200.000045,94,110,164.57.12.190,c0 23,"anon, hosting",,stream.example.org,BT Group,91,40,539.1972,03 04,87.89,5827,jp,,SK Telecom,TlsC,,164.57.12.190,14630,www.google.com,196860,,packet_capture,41137,*.cloudfront.net,43.157.147.113,2026-06-01 00:00:00.000045,Microsoft Corp.,,177.8043,268,Netflix Inc.,89 d2 bb 11,,443,,,in,gw-09,953,5.414255,-3.01543,San Francisco,Tokyo,"anon, hosting",1483.6305,209.8.188.85,58.495522,4e 93 40 44 05 24 08 49 f9 df 45 2b 46 d4 e9 d9 5c e8 d9 e6 c4 b5 73 5f d9 03 ff 79
|
||||
-161.091649,1780243200.000046,36,443,210.153.53.241,c0 27,"anon, hosting",,api.example.com,China Mobile,178,12,647.9058,03 03,74.52,6298,gb,,Deutsche Telekom,TlsC,,210.153.53.241,19698,,155849,,zeek,344339,www.google.com,131.228.28.117,2026-06-01 00:00:00.000046,Google LLC,,1416.4421,307,Amazon.com Inc.,05 04 22 06,ER,465,,,ua,core-02,1712,-53.480906,+,Shanghai,Seattle,"anon, hosting",1713.0252,127.147.205.209,-84.817621,77 c2 c7 f2 86 4a de 06 93 e5 c3 a1 15 31 b5 dd 8b ab 8a 88 32 82 c8 ec 7b 63 ac 5c
|
||||
24.207799,1780243200.000047,14,443,90.35.125.95,00 9f,"anon, hosting",,www.google.com,BT Group,153,41,648.7864,03 03,63.01,4613,us,,SK Telecom,TlsC,,90.35.125.95,36560,,140756,Baidu Inc.,packet_capture,24213,,189.52.44.152,2026-06-01 00:00:00.000047,Google LLC,,284.0394,37,Netflix Inc.,9d ff e6 08,,143,,,il,gw-09,1187,54.498642,-2.152039,Shanghai,Seoul,"anon, hosting",942.9681,4.108.97.247,114.471597,cd 1c 2e bb ae 29 eb 94 67 5a 14 c4 61 99 c7 36 b7 f9 2b e0 61 47 12 62 0c 7a fa c7
|
||||
-148.31237,1780243200.000048,17,21,60.196.78.181,c0 27,"anon, hosting",,,KDDI,146,12,216.1652,03 03,81.38,3236,ua,,Telstra,TlsS,,60.196.78.181,71018,stream.example.org,28970,,ssl_logs,467072,,208.96.164.155,2026-06-01 00:00:00.000048,,,751.6892,212,Google LLC,5a 50 0e f8,,3389,,,gb,gw-09,201,11.552636,13.445978,Dublin,,"anon, hosting",942.5095,182.73.237.235,78.191552,78 62 3c 30 b9 70 ed 6a 8d 1b 63 39 4c f2 fe 5a 51 16 1b ba 91 6b de 80 ab e8 16 35
|
||||
-132.223679,1780243200.000049,87,80,112.137.165.219,c0 2f,"anon, hosting",00 19,api.example.com,BT Group,17,33,372.8138,03 04,78.06,8501,au,,Singtel,TlsC,,112.137.165.219,61207,,197304,,suricata,145503,,84.44.205.172,2026-06-01 00:00:00.000049,,,451.649,386,Baidu Inc.,14 bb bb 59,,443,,,il,edge-01,1965,-58.509241,53.992067,Sydney,Seoul,"anon, hosting",535.4483,102.253.95.240,83.443062,f5 9c 52 eb 0c e8 ed 46 cf 5e bd ed 00 24 c5 aa 10 97 da 8a 6a 29 ab 1c ba 1a 38 c4
|
||||
76.252382,1780243200.00005,51,3389,179.237.25.143,c0 27,"anon, hosting",,,,82,13,781.2834,03 03,57.74,9381,no,,Deutsche Telekom,TlsC,,179.237.25.143,97394,*.cloudfront.net,177835,,zeek,306086,,82.37.230.120,2026-06-01 00:00:00.000050,Apple Inc.,,892.4994,317,Samsung,a9 97 f2 94,,12345,,,gb,wan-link,631,-5.288816,50.895797,New York,Paris,"anon, hosting",111.0122,47.109.174.243,-120.109294,4e 52 79 3b 15 a2 40 e1 85 bd 5e 69 11 a2 b0 4b be e5 53 9a 3e 0c cf ff 4a ea a3 4f
|
||||
104.839037,1780243200.000051,25,443,201.232.146.109,c0 2c,"anon, hosting",,,Comcast,144,24,12.8977,03 04,11.79,9612,hk,,BT Group,TlsS,,201.232.146.109,82235,api.example.com,142630,Tencent,ssl_logs,184555,,200.185.55.116,2026-06-01 00:00:00.000051,Meta Platforms,15131,543.6465,351,Samsung,8b b8 d6 b6,,993,,,no,edge-01,648,-0.707942,-33.057573,Singapore,London,"anon, hosting",725.6828,122.77.31.116,86.348168,f1 3f 93 d5 5b fb 7d 0e 3d 8d fc 22 b9 f6 df 84 f3 3d bf 08 84 53 1d 4a a3 ec 45 b6
|
||||
-147.319447,1780243200.000052,14,80,21.240.9.192,c0 2b,"anon, hosting",,cdn.example.com,KDDI,44,26,674.1589,03 03,82.6,4261,ua,,Telstra,TlsS,,21.240.9.192,87982,*.s3.amazonaws.com,59311,,packet_capture,94203,,132.123.73.75,2026-06-01 00:00:00.000052,Tencent,,140.1487,220,Netflix Inc.,38 45 fd b8,,443,128,,jp,backbone-03,596,48.140984,-17.81596,San Francisco,Seoul,"anon, hosting",951.8779,172.43.184.18,-113.478543,b2 b4 52 7b 22 32 07 6f 62 90 57 02 32 6d f3 bb 5e e1 36 5c b5 c2 d9 6d 74 d0 39 a0
|
||||
-137.111721,1780243200.000053,87,50000,175.36.87.137,c0 2c,"anon, hosting",,,Deutsche Telekom,128,2,20.4023,03 03,37.8,6563,hk,graph.facebook.com,NTT,TlsS,,175.36.87.137,49258,,180452,,netflow,472798,graph.facebook.com,67.228.109.157,2026-06-01 00:00:00.000053,Netflix Inc.,,652.9076,87,Google LLC,b2 0d ae c2,,443,,,fr,edge-01,625,31.675789,45.615081,Shanghai,Singapore,"anon, hosting",1726.8737,222.81.167.240,176.656164,12 10 5a 9b 54 fb dc 44 9f c7 f9 38 fb f0 f3 50 1c c8 86 fd c1 b4 91 a4 1f 0e 17 41
|
||||
-107.76671,1780243200.000054,75,993,8.126.102.215,00 9f,"anon, hosting",,www.google.com,Singtel,192,34,828.9816,03 03,8.1,7928,us,,Comcast,TlsS,,8.126.102.215,24000,stream.example.org,95372,,ssl_logs,63451,www.google.com,177.185.45.112,2026-06-01 00:00:00.000054,Meta Platforms,,1685.6578,483,Samsung,84 8e 5a 15,,443,,,il,edge-01,1814,52.156312,-52.814201,Paris,Mumbai,"anon, hosting",1142.3758,16.45.118.137,81.581425,99 de d2 a3 20 68 d3 88 92 52 83 78 cf 7a 3a 4c 62 0e 7d 9e ed f8 df d8 12 67 51 8b
|
||||
-167.802595,1780243200.000055,56,143,50.97.229.36,00 35,"anon, hosting",,*.s3.amazonaws.com,Orange,117,17,401.0349,03 01,113.4,9096,us,,BT Group,TlsC,,50.97.229.36,36626,,58947,,ssl_logs,446804,cdn.example.com,147.213.95.171,2026-06-01 00:00:00.000055,Microsoft Corp.,,458.1858,332,Google LLC,a3 e7 f4 44,,12345,,,ca,wan-link,48,45.53607,-11.245342,New York,Beijing,"anon, hosting",925.3873,170.214.8.62,96.026724,f1 d3 aa d8 33 52 85 21 b4 d7 a7 e8 0e 1a 5e e3 45 01 cd 66 9c a0 2a 38 50 7f 42 d2
|
||||
168.500812,1780243200.000056,10,50000,85.238.212.16,00 35,"anon, hosting",,,China Mobile,128,38,272.7916,03 04,43.1,6384,nl,,AT&T,TlsC,,142.150.185.148,99300,www.google.com,127292,,packet_capture,393382,graph.facebook.com,46.138.247.120,2026-06-01 00:00:00.000056,Samsung,,699.859,444,Microsoft Corp.,1b 75 5d 12,,53,,,fi,backbone-03,231,-23.699117,-15.491854,Beijing,Beijing,"anon, hosting",568.9488,136.110.192.108,-37.909147,fc ad ef 43 2d a2 dd 48 85 3b a0 16 41 ff 8f b8 dc 90 86 87 a0 df 54 a1 fb 4d e1 1b
|
||||
-19.640731,1780243200.000057,2,143,61.142.248.55,00 9f,"anon, hosting",,,Telstra,123,12,871.3448,03 04,55.81,7637,in,,Singtel,TlsC,,61.142.248.55,88890,,103111,,packet_capture,179157,,114.227.120.219,2026-06-01 00:00:00.000057,Amazon.com Inc.,62799,1448.4653,153,Google LLC,37 d9 d3 30,,143,,,gb,backbone-03,853,-48.46662,-36.015449,Beijing,Berlin,"anon, hosting",900.4459,136.210.92.44,-47.715542,2e d8 bc 5f de da 2f 14 ae ed 72 4f fd 2a 94 ef df 28 f4 99 4c 41 99 7f da 97 18 27
|
||||
-36.115066,1780243200.000058,44,995,153.222.176.187,00 9e,"anon, hosting",,,China Mobile,197,3,827.9487,03 03,24.34,5382,in,,Telstra,TlsC,,153.222.176.187,77552,,26551,,suricata,355105,*.s3.amazonaws.com,76.98.89.157,2026-06-01 00:00:00.000058,Netflix Inc.,,1114.1042,156,Alibaba Inc.,c7 d5 b8 f1,,53,,,no,gw-09,1932,27.729426,-55.557934,Beijing,Seoul,"anon, hosting",718.2147,100.60.134.199,179.649492,f9 76 1f 09 d8 a1 20 89 d5 f2 3b fd b1 e5 df 0a 52 93 62 f0 b0 e0 76 70 ad ef 43 96
|
||||
-31.172609,1780243200.000059,16,25,71.125.114.36,00 35,"anon, hosting",,,Orange,147,8,726.5092,03 03,119.03,8424,in,,NTT,TlsC,,71.125.114.36,40945,,24432,,suricata,436527,cdn.example.com,114.217.248.156,2026-06-01 00:00:00.000059,Microsoft Corp.,,47.8201,388,Netflix Inc.,e2 b0 cf 65,,50000,128,,il,gw-09,640,-9.393335,36.212241,Moscow,Moscow,"anon, hosting",1080.3254,166.183.31.102,-97.854479,b9 0f e2 4a 05 23 2b ad a1 cd 54 de 1f c9 12 4d 1d 97 d6 cd 9b a0 b0 04 8f 4b ce e4
|
||||
4.746775,1780243200.00006,91,22,193.120.158.58,c0 27,"anon, hosting",,,China Mobile,166,43,395.3971,03 03,72.88,2395,no,,NTT,TlsS,,193.120.158.58,91094,,132420,,ssl_logs,203222,,60.196.78.181,2026-06-01 00:00:00.000060,Apple Inc.,,1456.0004,230,Baidu Inc.,b2 22 16 d5,,143,,,ua,gw-09,1560,-1.828272,56.163817,Berlin,Moscow,"anon, hosting",1249.9161,208.57.169.42,-15.223392,10 50 bd 3c d1 38 3d bd 94 14 4e 3c be 9c bf 9b 96 f8 0e 49 48 d7 18 9c 91 6e 5d fa
|
||||
-128.714202,1780243200.000061,56,465,177.2.54.200,00 9e,"anon, hosting",,graph.facebook.com,Comcast,130,35,281.3113,03 02,74.68,6215,de,login.live.com,AT&T,TlsC,,177.2.54.200,54909,,43978,,packet_capture,306991,,65.233.154.51,2026-06-01 00:00:00.000061,Meta Platforms,,1679.136,140,Amazon.com Inc.,48 bf 56 7d,--,3389,,,br,gw-09,1912,25.596075,-48.150694,Beijing,Beijing,"anon, hosting",703.4844,150.73.118.119,-72.711239,3d 1a 67 7a 79 fc c5 3a 89 1f a2 ed d0 21 f7 0e d0 77 09 89 33 7e 40 9b d0 e4 2a 68
|
||||
-95.66714,1780243200.000062,43,8443,122.251.229.203,c0 2b,"anon, hosting",,,BT Group,61,38,479.9664,03 03,40.5,8184,hk,,Singtel,TlsS,,122.251.229.203,91965,,6751,,zeek,328034,,199.178.179.187,2026-06-01 00:00:00.000062,Samsung,,1834.0734,217,Netflix Inc.,ee fd b9 16,,443,256,,nl,gw-09,463,13.613125,26.137144,Shanghai,Beijing,"anon, hosting",659.12,141.241.111.196,-70.855728,6b 03 64 ca 38 8e fc 57 63 59 21 41 21 41 84 74 8c 62 0b 81 e8 01 da 1a a5 dd d0 8c
|
||||
-19.935247,1780243200.000063,93,25,151.142.3.195,c0 2f,"anon, hosting",00 17,,Orange,142,49,194.4034,03 04,25.97,4211,no,,China Mobile,TlsS,,151.142.3.195,44313,,87004,,ssl_logs,74859,,32.232.188.173,2026-06-01 00:00:00.000063,Google LLC,,14.4441,95,Meta Platforms,d6 a8 53 35,,443,,,fr,wan-link,770,-29.245668,26.954764,Tokyo,Singapore,"anon, hosting",716.4154,42.92.84.185,-28.437479,9f df b7 9f 4a 4b f0 9e af 95 bc 04 4a 1d 79 1b d8 b0 64 66 67 23 ae 60 f3 ef a9 ab
|
||||
52.225763,1780243200.000064,34,25,103.124.75.168,c0 13,"anon, hosting",,,Verizon,77,20,196.6463,02 00,32.24,6679,ru,,,TlsS,,103.124.75.168,62928,,21705,,ssl_logs,286474,,141.228.81.191,2026-06-01 00:00:00.000064,Google LLC,,553.5735,330,Apple Inc.,ee ac da e6,,465,,,cn,core-02,1028,-52.215235,30.156965,Tokyo,Moscow,"anon, hosting",238.0572,180.58.110.111,3.10097,a4 bf f7 fd f2 89 cd 12 6a 70 67 e7 78 ca 6f 9c d9 c1 50 bf 82 2b 44 2b 97 e3 b4 75
|
||||
-39.370987,1780243200.000065,90,443,10.240.113.52,13 01,"anon, hosting",00 18,,Singtel,77,30,910.3025,03 04,72.64,3869,in,,KDDI,TlsS,,10.240.113.52,33537,,154107,,packet_capture,129615,www.google.com,175.127.52.91,2026-06-01 00:00:00.000065,Google LLC,61577,1982.1791,142,Amazon.com Inc.,d6 f4 23 c7,,143,256,,sg,backbone-03,1468,44.033371,-35.495446,Sydney,Sydney,"anon, hosting",1649.3407,73.198.209.199,172.861962,e7 9b 2f 57 35 5a 10 fa 0d 5f 65 23 c5 91 69 77 95 23 e8 1d bc 05 57 a7 d2 c8 50 38
|
||||
162.006633,1780243200.000066,29,143,110.69.236.47,c0 2f,"anon, hosting",,,KDDI,72,7,427.8803,03 03,115.07,7045,sg,,,TlsS,,110.69.236.47,38647,*.s3.amazonaws.com,174192,,zeek,284208,*.s3.amazonaws.com,38.131.101.45,2026-06-01 00:00:00.000066,Alibaba Inc.,,654.9558,263,Tencent,7f bf 7a 66,,12345,512,,ca,gw-09,343,-51.811539,+,Seattle,London,"anon, hosting",379.9055,122.233.76.156,17.434275,57 09 33 5a 96 3f c8 56 27 05 8b 4e e1 b7 5b 6e 11 6e e2 18 34 c7 9d e3 1a 02 18 c1
|
||||
51.696746,1780243200.000067,72,80,64.38.226.207,13 02,"anon, hosting",,,Singtel,19,20,427.3973,03 03,103.56,8688,gb,,,TlsC,,64.38.226.207,34854,,94362,,netflow,10586,www.google.com,98.49.183.217,2026-06-01 00:00:00.000067,Samsung,,1470.1213,217,Netflix Inc.,ef 9f 54 65,--,8080,,,nl,backbone-03,344,28.710676,-26.624827,Mumbai,Mumbai,"anon, hosting",1639.388,40.155.84.42,-61.671653,93 0c 75 a9 af 7c 69 1b 7f 71 c3 a4 5d e2 03 21 fb 23 62 1a 48 68 40 2e 58 68 78 28
|
||||
6.673336,1780243200.000068,35,22,122.58.111.120,c0 14,"anon, hosting",,,AT&T,122,1,9.5066,03 03,53.17,8036,fi,,Verizon,TlsC,,122.58.111.120,99372,,28724,,packet_capture,352943,,5.47.150.57,2026-06-01 00:00:00.000068,Alibaba Inc.,,554.2476,44,Tencent,5d 38 e6 6c,,465,,,hk,wan-link,341,+,48.119478,Mumbai,Beijing,"anon, hosting",1171.7072,14.129.244.30,31.017754,81 9a c3 9d d2 2d 37 b0 49 89 6b cb 9d 03 56 fb 00 31 23 d5 d5 93 dc 57 5a bf 5f 44
|
||||
102.5503,1780243200.000069,59,50000,8.59.133.46,c0 27,"anon, hosting",,,,76,24,979.8495,03 01,89.0,9189,ca,stream.example.org,KDDI,TlsC,,8.59.133.46,29323,login.live.com,195297,,suricata,99740,stream.example.org,136.210.92.44,2026-06-01 00:00:00.000069,Alibaba Inc.,,761.01,437,Netflix Inc.,21 af c1 c3,--,443,,x25519,fi,backbone-03,1174,25.433037,45.148081,Dublin,Paris,"anon, hosting",622.287,117.189.214.151,133.14914,2c 9a ef 56 e4 3b f6 7b 7c 6c 57 2d 36 de 0e 52 a7 c4 8f 70 83 f3 3f 45 fb 2e aa 72
|
||||
-144.964956,1780243200.00007,83,143,83.250.9.29,c0 23,"anon, hosting",,login.live.com,China Mobile,77,38,908.4785,03 03,118.94,2629,sg,,Telstra,TlsS,,83.250.9.29,80452,,23591,,packet_capture,256691,,49.43.123.185,2026-06-01 00:00:00.000070,Netflix Inc.,,1306.7378,196,Alibaba Inc.,ce 7d 11 3c,,443,,,au,gw-09,1772,50.767654,5.761204,Berlin,Mumbai,"anon, hosting",240.8808,223.251.52.224,72.399121,1b 9b ea 53 46 e2 f7 95 4a 6b 6f 64 04 b9 34 4b c0 da 56 1c 31 b6 f2 5b 1a 3c ad de
|
||||
-104.457778,1780243200.000071,7,53,153.34.120.104,c0 27,"anon, hosting",,,SK Telecom,193,20,175.7314,03 04,77.0,6655,in,login.live.com,BT Group,TlsS,TLS_AES_128_GCM_SHA256,125.35.64.33,98842,,87529,,suricata,410577,login.live.com,161.81.216.153,2026-06-01 00:00:00.000071,Amazon.com Inc.,,196.7958,116,Amazon.com Inc.,a1 de c2 0d,,443,256,,cn,wan-link,300,58.322343,4.585138,Moscow,Mumbai,"anon, hosting",445.2289,134.81.186.96,-53.237835,60 48 99 6e 44 6e b7 8e 41 d8 26 82 85 c3 53 cc e9 2a 82 d9 49 68 24 89 62 be 53 d6
|
||||
83.087052,1780243200.000072,61,8080,62.40.43.188,c0 2f,"anon, hosting",,,Singtel,77,6,101.3378,03 04,4.73,3984,kr,,Verizon,TlsC,,106.12.91.189,45321,stream.example.org,162039,,zeek,210999,*.s3.amazonaws.com,57.230.178.239,2026-06-01 00:00:00.000072,Google LLC,,597.4029,316,Amazon.com Inc.,96 87 6d 01,,25,512,,kr,edge-01,118,55.45996,1.041883,Moscow,Beijing,"anon, hosting",700.0937,192.148.19.3,-27.365895,3f cf 07 61 03 d9 b0 7d 15 e5 56 73 59 26 bb 44 b4 1a d8 fd c2 57 bf 39 ad 7f 3c 94
|
||||
-125.101539,1780243200.000073,31,443,17.197.195.153,c0 14,"anon, hosting",,,Comcast,54,9,13.6616,03 03,5.39,2429,us,,Orange,TlsS,,17.197.195.153,27102,,195332,,netflow,58673,*.s3.amazonaws.com,142.7.208.24,2026-06-01 00:00:00.000073,,,862.3841,466,Netflix Inc.,1f da 1a 9b,,110,,,ru,gw-09,585,-1.688491,-30.270324,Moscow,Seattle,"anon, hosting",983.0814,26.51.227.43,93.923963,86 7e db 65 f2 34 5c 60 4c 75 3d 43 cd 13 8c 62 4a da 09 66 2e 1a c2 d0 9c 76 71 ac
|
||||
90.262005,1780243200.000074,51,993,54.220.164.120,c0 2b,"anon, hosting",,,BT Group,60,41,381.981,03 02,98.0,8109,fi,,,TlsC,,54.220.164.120,72784,*.cloudfront.net,57582,,netflow,379051,*.s3.amazonaws.com,223.119.168.199,2026-06-01 00:00:00.000074,Baidu Inc.,,1314.9779,498,Amazon.com Inc.,b1 bc a6 2a,,443,,,br,edge-01,573,44.430686,-40.137814,Sydney,Shanghai,"anon, hosting",1878.0948,25.94.71.123,169.75383,8b 69 5e 01 51 6f 04 e9 84 9b 70 97 89 3c 57 72 17 79 95 14 4b 8e 44 8a 1a 7f 6b 3b
|
||||
-83.715609,1780243200.000075,99,443,98.138.112.176,c0 27,"anon, hosting",00 1e,,Comcast,82,31,390.5996,03 03,12.92,8885,jp,,Verizon,TlsC,,174.207.84.216,64256,api.example.com,173568,,netflow,77054,,35.135.187.196,2026-06-01 00:00:00.000075,Samsung,,1577.9234,365,Google LLC,4d 5b db ab,,443,512,x25519,se,wan-link,544,-2.530994,5.223173,Singapore,Tokyo,"anon, hosting",1956.1208,58.62.102.203,-92.228811,97 34 d4 48 00 72 0c 74 da c5 4b b4 92 73 c4 39 fb 56 38 72 34 90 b6 27 aa 3c b8 2b
|
||||
29.378547,1780243200.000076,31,53,64.62.233.35,00 9f,"anon, hosting",,,,139,28,54.6717,03 03,48.35,4471,fr,,KDDI,TlsC,,64.62.233.35,50516,,102936,,packet_capture,433919,mail.example.com,214.247.241.63,2026-06-01 00:00:00.000076,Samsung,,1633.3011,310,Meta Platforms,e8 84 9c 74,,21,,,ru,gw-09,1528,-9.034951,-47.27882,Paris,London,"anon, hosting",627.0661,44.42.255.217,-160.529996,4b bb 48 28 7d 27 06 d5 4c 5d f0 47 99 f0 d2 ac e3 06 92 38 4f 1e 10 58 52 05 ca d8
|
||||
-57.090496,1780243200.000077,29,465,196.172.52.24,13 02,"anon, hosting",,,Orange,119,36,790.5906,03 04,15.91,6716,hk,,Verizon,TlsC,,196.172.52.24,94211,*.s3.amazonaws.com,163843,Google LLC,netflow,337771,,179.237.25.143,2026-06-01 00:00:00.000077,Alibaba Inc.,,526.392,212,Netflix Inc.,f2 83 e2 47,ER,993,512,,kr,core-02,766,27.446477,51.627264,Moscow,Shanghai,"anon, hosting",1365.4286,153.150.232.130,54.502169,dc 14 c6 7a 6a 26 4e fe 31 07 49 9e 2c ff 30 e8 a0 56 97 52 22 8f ba b4 2d 24 5b eb
|
||||
142.184372,1780243200.000078,49,22,156.58.83.247,c0 2c,"anon, hosting",,,KDDI,169,22,425.8492,03 03,114.63,8167,de,,China Mobile,TlsC,,156.58.83.247,91313,*.cloudfront.net,95466,Tencent,netflow,184860,graph.facebook.com,183.227.37.231,2026-06-01 00:00:00.000078,Alibaba Inc.,,1398.935,420,Netflix Inc.,31 12 6f 60,,22,,,kr,core-02,1837,47.168807,-53.550926,Shanghai,Tokyo,"anon, hosting",912.539,134.81.186.96,-113.731816,65 b2 d6 f6 f3 52 df b5 c7 05 65 27 4a a5 a2 07 4d ea 4b ff a2 3a d0 86 01 e6 40 e6
|
||||
165.297813,1780243200.000079,81,22,114.226.109.131,c0 23,"anon, hosting",,,China Mobile,176,32,894.6048,03 01,92.56,6137,in,,KDDI,TlsS,,114.226.109.131,69097,graph.facebook.com,72180,,zeek,74562,,116.124.237.146,2026-06-01 00:00:00.000079,Meta Platforms,,1734.0112,327,Netflix Inc.,0c 42 e8 b7,OK,22,,,au,core-02,295,38.404127,-19.789207,Beijing,Moscow,"anon, hosting",1854.4709,183.227.37.231,35.980443,62 c9 1a db 41 c7 58 ce a2 51 df 76 32 40 3c c8 85 51 a5 68 2f 2f 0d df 05 4e 0e 28
|
||||
105.875829,1780243200.00008,58,3389,9.161.205.69,13 02,"anon, hosting",,*.cloudfront.net,Verizon,84,13,435.2056,03 03,8.42,4873,hk,,BT Group,TlsC,,9.161.205.69,47360,,55720,,zeek,221318,stream.example.org,78.73.113.106,2026-06-01 00:00:00.000080,Apple Inc.,,1766.8369,367,Baidu Inc.,be 4a a0 3e,,443,,,ua,wan-link,211,10.037791,-22.609784,Beijing,Shanghai,"anon, hosting",150.5736,29.145.249.135,95.921592,d8 8b a3 37 c4 db 66 b4 bf 20 e6 55 11 81 d1 96 b4 cb 0f 3a 5c a5 db 5f b7 7d 25 6e
|
||||
160.97215,1780243200.000081,96,465,109.210.239.222,00 9f,"anon, hosting",,,Verizon,5,18,780.0211,03 03,103.09,4064,jp,,Verizon,TlsS,TLS_RSA_WITH_AES_128_CBC_SHA,109.210.239.222,72421,*.cloudfront.net,81843,,suricata,325509,mail.example.com,78.73.113.106,2026-06-01 00:00:00.000081,Apple Inc.,,260.991,259,Samsung,1e 6c 06 6e,--,443,512,,sg,core-02,1343,-28.837477,-46.703568,Shanghai,New York,"anon, hosting",771.7659,45.38.80.1,102.579853,c2 8a 57 46 eb 3d 40 a3 4b d3 e4 3e 7b f4 fa 8f a4 f7 73 2f 87 97 2a b4 7c 69 27 26
|
||||
-76.327692,1780243200.000082,32,22,60.51.194.72,00 9f,"anon, hosting",,,,179,41,269.9374,03 03,65.71,6888,ru,,SK Telecom,TlsC,TLS_RSA_WITH_AES_256_GCM_SHA384,60.51.194.72,21701,,66563,,ssl_logs,312541,,192.191.212.180,2026-06-01 00:00:00.000082,Apple Inc.,,1782.8535,155,Meta Platforms,08 94 74 55,,8443,128,,fi,core-02,28,-35.570942,24.702402,New York,Paris,"anon, hosting",1714.0554,76.103.22.203,49.283717,f4 9c 7a 57 a1 23 55 5f aa e9 48 06 f2 db ab 0a 5a 87 27 57 04 b4 f2 00 fa 7e f1 38
|
||||
-179.768984,1780243200.000083,17,3389,8.199.172.172,c0 2b,"anon, hosting",,,Singtel,2,16,140.5856,03 03,53.31,9487,ua,,Deutsche Telekom,TlsS,,8.199.172.172,17123,*.s3.amazonaws.com,179711,,suricata,220882,api.example.com,99.119.210.12,2026-06-01 00:00:00.000083,Baidu Inc.,,821.107,239,Netflix Inc.,9a 9c 2b 98,,993,,,in,core-02,960,-23.30142,19.230398,Moscow,Moscow,"anon, hosting",1118.2593,184.218.85.177,-25.083782,4f 2c 8a 49 64 dd 29 94 70 10 f2 b5 7f c5 d8 cc 32 85 9a 9f 58 df a5 47 66 80 45 d5
|
||||
-135.649249,1780243200.000084,34,443,9.169.36.132,c0 23,"anon, hosting",,api.example.com,Comcast,83,25,916.16,03 03,47.18,6703,il,,,TlsS,,9.169.36.132,62923,,172045,,suricata,484367,,48.7.172.203,2026-06-01 00:00:00.000084,,,1717.4691,174,Google LLC,b5 47 ef ea,,50000,,secp384r1,hk,backbone-03,1344,13.531207,18.979118,Mumbai,Mumbai,"anon, hosting",1570.1369,142.7.208.24,-21.913254,7b 57 66 f3 f5 94 94 33 fc bc 8c c8 c5 36 ed d5 d9 4b 0e 91 ea 3d c7 05 9b 11 e3 15
|
||||
-129.397466,1780243200.000085,5,50000,19.30.117.18,c0 13,"anon, hosting",,,Comcast,19,4,329.6092,03 03,83.85,2320,hk,,SK Telecom,TlsC,TLS_RSA_WITH_AES_128_CBC_SHA,19.30.117.18,72753,,123564,,suricata,287031,,217.51.19.170,2026-06-01 00:00:00.000085,Google LLC,,522.5013,17,Baidu Inc.,b7 22 50 93,,22,,,in,edge-01,539,55.179881,-56.011297,Seattle,Tokyo,"anon, hosting",1529.7071,113.17.186.188,-145.365799,24 ae 5e fe 61 6a f2 b4 b6 46 22 e0 da c0 24 28 41 98 33 3c ff 76 1a a9 0b 72 94 80
|
||||
-159.604348,1780243200.000086,78,53,210.153.53.241,00 35,"anon, hosting",,auth.example.net,Singtel,87,43,224.4485,03 03,3.04,5301,ru,,Telstra,TlsS,,210.153.53.241,87458,,70957,,netflow,201874,mail.example.com,200.208.79.237,2026-06-01 00:00:00.000086,Alibaba Inc.,,558.9059,495,Microsoft Corp.,76 9c 6b 3e,,80,256,,kr,backbone-03,787,44.790697,-53.173961,Tokyo,Singapore,"anon, hosting",991.2196,182.43.158.144,-90.027936,7d 95 42 cb f5 3e 27 27 80 69 fd b6 70 e3 9a bb 95 7d 88 a3 35 1b b1 d2 f8 8f 28 68
|
||||
123.957119,1780243200.000087,75,443,1.199.135.238,c0 2c,"anon, hosting",,,KDDI,56,38,44.6106,03 03,77.21,1534,us,,Orange,TlsC,,1.199.135.238,29670,stream.example.org,37607,,netflow,16535,mail.example.com,175.176.65.225,2026-06-01 00:00:00.000087,Tencent,,1794.8871,172,Alibaba Inc.,b3 37 27 e3,,22,128,x25519,jp,wan-link,1244,+,-2.375051,Tokyo,Singapore,"anon, hosting",1885.0166,75.196.214.64,167.718777,8d a0 20 e1 5c 84 8d 84 fd 41 74 59 4b 19 77 3a 13 5e e1 a2 15 74 1f 84 83 08 b4 16
|
||||
106.508142,1780243200.000088,75,8080,155.107.175.53,13 02,"anon, hosting",,,KDDI,110,47,320.4369,03 04,91.04,7292,nl,,Verizon,TlsS,,155.107.175.53,33385,,192367,,zeek,340675,,182.223.0.134,2026-06-01 00:00:00.000088,Meta Platforms,26576,168.4952,420,Microsoft Corp.,ac 37 17 25,,443,,,ua,wan-link,1569,20.892615,12.66548,Paris,San Francisco,"anon, hosting",1187.513,114.179.228.11,59.724955,4b fd 65 5c e8 07 55 3d d6 86 f0 33 40 11 dc 0c 55 25 43 c6 d8 b5 41 cd 7b 96 3c ac
|
||||
60.737607,1780243200.000089,41,3389,151.142.3.195,c0 2c,"anon, hosting",,,Telstra,89,49,295.4172,03 04,28.55,3098,no,,Deutsche Telekom,TlsS,,151.142.3.195,73404,,139157,,netflow,187437,*.cloudfront.net,15.211.213.143,2026-06-01 00:00:00.000089,Apple Inc.,,584.9397,431,Microsoft Corp.,2d 6d 01 de,,995,,,au,edge-01,369,-49.57634,-26.585616,New York,San Francisco,"anon, hosting",1593.6956,30.207.10.248,-130.745507,71 26 a7 50 ab 8d e8 25 a1 13 dd 33 a2 61 12 04 7d 46 c0 70 64 87 f0 4d f8 5f fe 90
|
||||
12.851572,1780243200.00009,4,443,120.128.5.175,cc a9,"anon, hosting",,,Comcast,60,2,533.2039,03 03,3.3,8811,us,,Orange,TlsS,,120.128.5.175,94255,,53227,,suricata,149319,,99.247.54.61,2026-06-01 00:00:00.000090,Alibaba Inc.,,859.6396,242,Netflix Inc.,d4 0a 07 4d,,443,512,,ca,edge-01,1986,-16.54827,-41.160851,Tokyo,Singapore,"anon, hosting",813.2551,93.188.239.206,-108.561667,bf a9 d3 6e 7f 06 12 0a 34 cb f8 15 50 90 9a 0c 75 7e 11 8b 3f c1 cd 43 8a 2b 27 25
|
||||
133.109702,1780243200.000091,14,12345,112.137.165.219,c0 2f,"anon, hosting",,,Telstra,134,35,403.2684,03 03,19.2,7274,ua,,Orange,TlsC,TLS_ECDHE_RSA_AES256_GCM_SHA384,195.194.142.193,87616,,146679,,packet_capture,192757,,1.199.135.238,2026-06-01 00:00:00.000091,Google LLC,,569.091,476,Netflix Inc.,f0 22 02 9f,,25,,,il,wan-link,707,2.392682,-36.315887,Seoul,Beijing,"anon, hosting",852.9051,191.79.212.168,-136.268033,f9 47 d6 cc 04 97 3c de 96 8a a5 be 54 48 94 68 be 80 f1 ec 6b a9 45 54 f2 05 a2 d5
|
||||
97.468122,1780243200.000092,25,995,187.125.83.119,c0 27,"anon, hosting",,,NTT,129,38,897.7365,03 04,14.77,5368,de,,China Telecom,TlsS,,187.125.83.119,68117,,186377,,zeek,444916,graph.facebook.com,21.164.75.17,2026-06-01 00:00:00.000092,Meta Platforms,,389.1706,143,Alibaba Inc.,f2 35 13 bd,,443,,,br,backbone-03,200,-38.638511,42.644678,San Francisco,Mumbai,"anon, hosting",34.3322,97.94.242.62,131.809838,61 c3 bf 5b 05 64 e2 b8 63 02 9e a8 30 bc 81 06 8b 11 8c f5 cb 16 81 8c 1d 0f f1 e8
|
||||
-49.099249,1780243200.000093,66,993,142.150.185.148,13 02,"anon, hosting",,,BT Group,77,15,40.129,03 03,22.75,9586,br,,Deutsche Telekom,TlsC,,142.150.185.148,84447,,161741,,netflow,355958,stream.example.org,67.104.160.62,2026-06-01 00:00:00.000093,Samsung,,1668.3587,375,Samsung,b7 04 51 d7,,3389,,,gb,edge-01,1180,-7.248201,-17.381628,Seoul,Seoul,"anon, hosting",747.0563,223.158.173.57,-146.731659,57 12 c5 4f 6b de b1 93 6a af 89 2e 6b 84 6f ef 85 10 3b 5f f3 47 ef a5 be c1 11 a5
|
||||
31.114424,1780243200.000094,50,8443,148.97.130.12,00 35,"anon, hosting",,*.cloudfront.net,China Mobile,21,46,502.0616,03 03,61.08,6523,ca,,Comcast,TlsS,,148.97.130.12,87352,,3137,,packet_capture,420898,www.google.com,165.135.107.197,2026-06-01 00:00:00.000094,Samsung,,606.6061,217,Netflix Inc.,36 ee 2a b1,,53,,,gb,backbone-03,1966,-51.401184,28.851494,Paris,Sydney,"anon, hosting",1268.6002,173.56.213.150,-4.208642,bc b9 ae 00 cb 27 75 46 ff 60 16 6c 9a de 2f ad c4 5c d9 4d 91 9f c6 f6 11 85 d7 5b
|
||||
49.613214,1780243200.000095,74,21,185.134.91.130,00 9f,"anon, hosting",,api.example.com,KDDI,162,30,346.3663,03 03,39.29,6657,br,,KDDI,TlsS,,185.134.91.130,79414,,75553,,ssl_logs,180004,,206.240.163.235,2026-06-01 00:00:00.000095,Tencent,,1190.8663,302,Baidu Inc.,6a 09 d4 42,,50000,,,nl,core-02,1248,-15.264561,57.2212,Paris,,"anon, hosting",951.7302,222.14.41.12,-127.169839,28 ac 9a d8 cd 0b 5e 83 28 dd 3b 62 85 9b 7c 19 bf f2 6e df 8c 5f 8d 17 cc 5c 91 8b
|
||||
-76.937967,1780243200.000096,92,22,10.240.113.52,00 9e,"anon, hosting",,,SK Telecom,166,34,32.3226,02 00,53.52,1388,sg,,Singtel,TlsS,,10.240.113.52,17836,,46467,,zeek,36817,graph.facebook.com,221.243.230.67,2026-06-01 00:00:00.000096,Tencent,,663.1872,412,Alibaba Inc.,7a 59 f1 66,,443,384,,nl,edge-01,1025,-32.988095,30.819382,Paris,Singapore,"anon, hosting",319.3739,206.111.245.54,-34.572251,cc 93 01 ab ed 58 32 c4 d6 d6 d2 60 0e 72 46 29 d1 0b a6 7f 55 f1 2e bd 0b c7 62 52
|
||||
24.350375,1780243200.000097,89,22,8.59.133.46,c0 13,"anon, hosting",,,China Telecom,63,35,507.4165,03 03,55.39,8472,no,,BT Group,TlsC,,8.59.133.46,89840,*.s3.amazonaws.com,3298,,suricata,52020,,210.235.179.79,2026-06-01 00:00:00.000097,Google LLC,,457.5143,404,Netflix Inc.,21 af 73 9e,,12345,,,br,wan-link,1733,25.656376,25.900402,Shanghai,Seattle,"anon, hosting",1328.8239,37.15.46.192,-69.601417,4f 41 4b 95 c4 83 74 0e 6c 9b 4b 06 84 ba cf f4 57 b0 4f a0 bf c1 f8 56 f6 59 f7 c2
|
||||
-57.188778,1780243200.000098,91,12345,58.32.173.6,13 01,"anon, hosting",,,Comcast,170,26,762.6952,03 03,87.31,5829,us,,Orange,TlsC,,58.32.173.6,13103,,69005,,ssl_logs,223178,,64.41.228.136,2026-06-01 00:00:00.000098,Alibaba Inc.,,668.1106,190,Amazon.com Inc.,62 ca 57 50,,443,,,br,edge-01,1754,17.433557,3.862803,San Francisco,Singapore,"anon, hosting",1490.6827,74.251.102.32,-84.888035,39 5d f9 03 a3 f0 fd 53 5b 21 dd 9b 30 e2 af 5d 63 37 76 6d 9d 9c a3 f1 34 99 6b 4f
|
||||
-129.431676,1780243200.000099,80,465,91.107.136.180,c0 27,"anon, hosting",,,KDDI,5,22,73.0258,03 04,10.98,6281,gb,,Orange,TlsS,,91.107.136.180,43928,,160016,,zeek,79999,,41.89.185.131,2026-06-01 00:00:00.000099,Netflix Inc.,,1635.6938,393,Amazon.com Inc.,fb 90 aa 22,,21,,,no,backbone-03,391,6.800932,22.748415,Seoul,Tokyo,"anon, hosting",1778.8222,47.139.158.236,114.231597,a0 b8 e4 6d 5e a4 ef 74 eb b9 aa 65 84 82 0b 47 3d d2 da ca f9 18 1e bf 08 df 10 a7
|
||||
|
@@ -0,0 +1,201 @@
|
||||
:ips.city,1ipp,8pak,orga,:ips.lond,client-ip,:ipd.anon,server-ip,:ips.ispn,ecdhe-named-curve,orgu,:ipd.orgn,8dbd,:ips,0crv,:ipd.doma,source-node,:ipd.latd,time,cnam,2tmo,:ipd.city,:ips.anon,dcnt,snam,scnt,:ips.orgn,4srs,tabl,0rnd,name,:prd,:ipd.ispn,4dbn,cipher-suite,8ppk,4ksz,8seq,:ipd,0rnt,4dur,0cph,8ses,:ips.latd,crcc,8ack,timestamp,0ver,:ipd.lond,8byt,8did,:ips.doma,:prs
|
||||
Berlin,10,1889,,-29.049656,61.83.90.226,"anon, hosting",150.180.24.13,Comcast,,,Apple Inc.,112,61.83.90.226,,,suricata,8.19111,2026-06-01 00:00:00.000200,,962.0666,Shanghai,"anon, hosting",us,,ca,Netflix Inc.,61021,TlsS,bd 2f 6e bf 19 00 27 37 a9 95 c8 38 f3 b0 1e 18 6e 12 92 82 22 1f 90 1f 8b 7b 20 b2,wan-link,995,Verizon,62,TLS_ECDHE_RSA_AES256_GCM_SHA384,477,,564.3826,199.147.121.78,ae 26 6d a9,6.47,c0 2c,1681.7491,-15.543401,,167652,1780243200.0002,03 03,120.219634,137701,6993,,8080
|
||||
San Francisco,6,1800,Baidu Inc.,-61.109684,109.93.142.119,"anon, hosting",164.207.186.132,KDDI,,,Netflix Inc.,56,109.93.142.119,,,ssl_logs,37.726072,2026-06-01 00:00:00.000201,,602.5391,London,"anon, hosting",de,,il,,96330,TlsC,02 67 03 4d ae 68 50 ec 5b f5 c4 dc 71 82 a2 23 bd 1f 54 a3 40 bd 99 da 36 7a 02 ef,wan-link,443,Telstra,87,,150,,257.4151,71.97.62.243,03 f1 40 04,54.98,13 01,1305.294,-52.0157,,198193,1780243200.000201,03 03,154.450995,104228,1849,,443
|
||||
Beijing,28,794,,174.266758,93.157.122.15,"anon, hosting",155.78.38.46,Verizon,,,Meta Platforms,185,93.157.122.15,,,ssl_logs,24.410451,2026-06-01 00:00:00.000202,,138.7091,London,"anon, hosting",nl,www.google.com,ca,Tencent,15163,TlsS,d7 19 3a ab fc 3a 2a a8 13 31 16 23 cb 4d d6 04 0f 10 44 cd b2 6a 85 ef ac e9 68 70,core-02,465,China Mobile,76,,348,,1624.8138,139.30.17.215,69 b7 fb 2d,116.45,00 9f,416.1292,-28.995273,,108409,1780243200.000202,03 04,-14.88431,498120,3105,,3389
|
||||
Seattle,43,1766,,-178.721459,19.30.117.18,"anon, hosting",167.226.142.47,Verizon,secp256r1,,Tencent,125,19.30.117.18,,,zeek,52.86149,2026-06-01 00:00:00.000203,,124.792,San Francisco,"anon, hosting",cn,,cn,Meta Platforms,72746,TlsS,76 ab 8e 21 ab 5c 19 75 19 86 70 e3 d5 d9 2d 7c d5 c9 57 0c 2a 47 46 db 22 b2 9b ea,backbone-03,8080,Verizon,94,,255,256,1209.7159,141.241.111.196,83 ea 01 b1,66.04,c0 14,581.1416,-46.325033,,194527,1780243200.000203,03 03,127.298846,292595,2610,,443
|
||||
Berlin,11,290,,-8.416371,179.237.25.143,"anon, hosting",146.125.255.141,Deutsche Telekom,,,Netflix Inc.,54,179.237.25.143,,,suricata,-4.217618,2026-06-01 00:00:00.000204,,625.5641,Seattle,"anon, hosting",us,api.example.com,fi,Tencent,15485,TlsC,0c 60 bd 0c a2 0f 25 94 3f e1 37 c6 a8 f7 60 23 47 b4 ca b3 2f e1 29 3a 9d db 14 1c,edge-01,443,Verizon,25,,10,,1462.4417,206.240.163.235,b8 16 02 5e,119.09,00 35,1267.6066,-19.032192,,168543,1780243200.000204,03 04,-23.819721,266106,1256,,443
|
||||
Shanghai,10,105,,134.639427,146.154.207.141,"anon, hosting",114.131.245.55,AT&T,,,Meta Platforms,17,146.154.207.141,,,netflow,-36.164802,2026-06-01 00:00:00.000205,,914.6783,San Francisco,"anon, hosting",br,graph.facebook.com,in,Alibaba Inc.,56027,TlsS,,gw-09,110,BT Group,72,,144,,1971.6374,84.44.205.172,ac 10 b6 0c,32.66,c0 2f,393.1059,-17.5277,,192203,1780243200.000205,03 03,89.113798,245273,6489,,995
|
||||
Tokyo,16,1077,,10.270697,62.40.43.188,"anon, hosting",212.232.204.107,,,,Apple Inc.,9,62.40.43.188,00 18,,suricata,6.480865,2026-06-01 00:00:00.000206,,83.8363,Paris,"anon, hosting",br,,hk,Google LLC,26389,TlsC,,core-02,993,Orange,8,,500,,884.7088,78.140.28.91,47 2c a2 f3,12.63,c0 23,1178.7217,-0.490778,,7635,1780243200.000206,03 04,63.682947,222443,9844,,110
|
||||
Paris,4,925,,-149.37088,217.26.47.163,"anon, hosting",30.207.10.248,SK Telecom,,,Samsung,55,217.26.47.163,,mail.example.com,zeek,42.100744,2026-06-01 00:00:00.000207,,56.4797,Tokyo,"anon, hosting",hk,mail.example.com,ca,Google LLC,17246,TlsC,73 2b 13 50 f7 f7 3c 36 a4 08 d0 4b 3a ef 40 c7 ad 20 e7 6d 52 39 da 48 2a 1b fa 75,core-02,12345,Deutsche Telekom,72,,227,,693.1679,215.90.18.215,4f 50 32 7e,92.86,00 9f,1237.8821,-36.710099,,148179,1780243200.000207,03 03,-106.875621,195843,6073,,110
|
||||
Beijing,29,532,,-152.352949,168.255.202.227,"anon, hosting",113.17.186.188,Orange,,,Microsoft Corp.,103,168.255.202.227,,,suricata,30.27131,2026-06-01 00:00:00.000208,,427.7861,Seoul,"anon, hosting",us,mail.example.com,gb,Apple Inc.,60027,TlsS,d4 04 79 2c f5 7d c2 06 ee 9c f8 db f6 5e 17 15 0b b5 42 29 83 33 19 f2 04 e9 3a de,wan-link,993,Orange,31,TLS_ECDHE_ECDSA_CHACHA20_POLY1305,216,512,284.3944,48.254.254.95,ba 0e a4 35,10.12,13 01,677.5246,22.216959,,168549,1780243200.000208,03 04,-40.775189,314793,7581,,3389
|
||||
New York,40,1737,,-170.203264,17.108.161.55,"anon, hosting",84.87.235.213,China Telecom,,,Google LLC,140,17.108.161.55,,,netflow,-29.969768,2026-06-01 00:00:00.000209,,477.0643,Shanghai,"anon, hosting",nl,auth.example.net,jp,Meta Platforms,56654,TlsS,f7 0b 6e c1 76 01 0e 8e 70 f9 74 cd 74 28 b0 5c 4a 3a 55 f7 1d c4 a8 72 8f 20 16 87,backbone-03,993,Comcast,16,,172,,1052.1147,116.143.60.33,4a 30 29 e7,83.73,c0 14,1673.1858,-28.568175,,105083,1780243200.000209,03 03,+,132343,6500,,3389
|
||||
Seoul,37,1777,,-2.249415,139.67.242.63,"anon, hosting",76.98.89.157,SK Telecom,,,Netflix Inc.,56,139.67.242.63,,,ssl_logs,-37.376984,2026-06-01 00:00:00.000210,,696.2135,Mumbai,"anon, hosting",cn,,no,Meta Platforms,71642,TlsS,63 d7 3d 7b 27 f0 a8 89 12 17 9e 1c 7c f3 53 b6 97 be aa 73 ae d7 78 3f 4e f6 8e 6f,gw-09,443,Singtel,13,,92,384,213.4811,165.89.237.252,4e 4a a4 af,8.48,00 9f,365.2121,-8.877811,,66700,1780243200.00021,03 01,65.976405,425448,4970,,443
|
||||
Singapore,12,1838,,-118.535038,207.100.186.111,"anon, hosting",30.32.243.115,Verizon,,,Apple Inc.,192,207.100.186.111,,,packet_capture,42.118077,2026-06-01 00:00:00.000211,,55.8025,Dublin,"anon, hosting",ua,stream.example.org,us,Meta Platforms,32872,TlsS,75 1f d2 7d 5e 78 de a0 9d 07 30 33 9e 94 89 56 97 8a 4f 0f 50 8f 7d bc 03 9d e0 d2,wan-link,12345,Telstra,79,,20,,1661.8052,194.70.49.232,2e 3c da 21,42.51,00 9f,1677.7026,-39.401898,,162640,1780243200.000211,03 04,-82.665781,359010,3874,,21
|
||||
Paris,2,773,,2.782924,193.121.85.105,"anon, hosting",73.229.119.137,China Mobile,,,Tencent,135,133.231.61.64,,,netflow,33.125532,2026-06-01 00:00:00.000212,,700.2801,Mumbai,"anon, hosting",fi,,se,Baidu Inc.,61404,TlsS,5a c4 44 a8 af 0a e6 89 68 ae 0b b1 53 bf 8b f5 04 7f da d8 91 a7 6b a5 31 f0 cf a8,backbone-03,443,NTT,98,,223,,1178.4967,34.187.139.213,ea 3f 67 02,34.53,cc a9,517.17,27.798568,,169197,1780243200.000212,03 03,-49.690323,367619,8340,,3389
|
||||
London,48,850,,31.617856,200.248.122.71,"anon, hosting",198.206.0.169,Verizon,,,Tencent,23,200.248.122.71,,,packet_capture,47.94104,2026-06-01 00:00:00.000213,,508.67,Tokyo,"anon, hosting",in,,sg,Tencent,38814,TlsC,a9 1c 7b a5 fb db 58 c0 9a 24 d2 7f 93 7d d0 1a c5 64 a5 b3 71 09 2b 0b 28 48 e0 c1,core-02,50000,Verizon,57,TLS_AES_128_GCM_SHA256,485,384,845.5016,25.94.71.123,2b 6a 00 96,91.2,c0 2c,1360.3256,24.360949,,56784,1780243200.000213,03 04,108.553426,312221,1783,,53
|
||||
London,33,864,,-28.349846,133.231.61.64,"anon, hosting",136.140.146.5,Telstra,,,Meta Platforms,24,133.231.61.64,00 1d,,netflow,-17.437022,2026-06-01 00:00:00.000214,cdn.example.com,101.8023,Singapore,"anon, hosting",ru,,us,Netflix Inc.,97677,TlsC,a2 f1 6a 6a 1e 1d 10 17 8e 11 d1 be cb b6 4c 7b 94 86 af 7d aa d8 bb 7d af 5c 66 42,edge-01,22,NTT,61,,308,,1196.1103,197.248.98.58,dc 10 06 12,37.27,c0 27,1471.461,43.989166,,174602,1780243200.000214,03 03,-69.074863,42326,7690,api.example.com,12345
|
||||
Paris,34,1917,,132.341349,8.199.172.172,"anon, hosting",132.123.73.75,China Mobile,,,Apple Inc.,89,8.199.172.172,,cdn.example.com,suricata,-34.414317,2026-06-01 00:00:00.000215,,619.8767,Berlin,"anon, hosting",kr,mail.example.com,gb,,36939,TlsS,ad 55 85 e9 99 9a 41 01 60 a7 86 90 34 08 55 73 f6 73 b6 2a d0 d3 9e fe f5 9a 30 68,wan-link,993,Singtel,15,TLS_ECDHE_ECDSA_AES128_SHA,276,,336.4073,52.21.161.239,68 bb 27 60,64.37,c0 2f,1101.2043,-23.096601,,55160,1780243200.000215,03 03,32.68108,459908,3913,,21
|
||||
Beijing,32,272,,125.630958,153.34.120.104,"anon, hosting",99.247.54.61,SK Telecom,secp256r1,,Apple Inc.,50,132.171.47.210,,,zeek,54.272389,2026-06-01 00:00:00.000216,,687.0141,Seoul,"anon, hosting",in,login.live.com,fr,Amazon.com Inc.,21318,TlsS,05 e6 e1 2d 33 18 d0 b8 9b a0 a3 e9 d0 dd 79 9c a6 9b 3d 5e 32 48 d0 88 ae 4e 1c d5,edge-01,50000,SK Telecom,19,TLS_AES_256_GCM_SHA384,129,,1480.3668,74.159.180.27,2a 34 0b d6,106.64,c0 2f,283.6493,-16.506815,,190602,1780243200.000216,03 04,-64.538001,485195,9135,,465
|
||||
New York,34,1543,,119.431271,193.120.158.58,"anon, hosting",180.58.110.111,Comcast,,,Amazon.com Inc.,145,166.174.6.218,,,suricata,-28.542311,2026-06-01 00:00:00.000217,,730.2763,Seattle,"anon, hosting",us,,il,Meta Platforms,67737,TlsS,e6 cc ae 15 0b 83 2a a9 df c8 84 68 12 30 c7 a8 b5 dc 44 70 fb e2 c3 b1 d9 79 25 69,core-02,25,China Telecom,16,,452,,1155.2583,141.241.111.196,1a d5 41 46,1.59,00 35,1018.3705,34.95539,,154534,1780243200.000217,03 03,-29.336748,63771,6925,,80
|
||||
Singapore,4,652,,65.343008,62.40.43.188,"anon, hosting",182.43.158.144,Singtel,,,Tencent,189,62.40.43.188,00 1e,,ssl_logs,39.816377,2026-06-01 00:00:00.000218,,935.3653,Seattle,"anon, hosting",il,,jp,Amazon.com Inc.,30346,TlsC,cc 8e a5 70 f5 89 8f a6 f3 ab 4b 28 76 b6 66 29 f4 a5 b0 a8 51 fa 8c ea 8d 2c 20 95,gw-09,443,Telstra,5,,184,,872.2247,113.160.214.39,c5 50 db e3,40.49,c0 2b,1223.1433,27.29917,,106598,1780243200.000218,03 03,-5.414951,314170,8788,,443
|
||||
London,44,1701,,-22.529364,12.125.243.157,"anon, hosting",144.54.163.62,Comcast,,,Baidu Inc.,71,12.125.243.157,,,netflow,42.695047,2026-06-01 00:00:00.000219,,370.3253,Moscow,"anon, hosting",nl,,jp,Meta Platforms,98206,TlsS,fd 45 18 aa 69 ca 99 20 36 4e b2 d1 e3 4e ba 15 57 2b b6 a4 6f 7d 0a fe e8 ae a3 47,gw-09,50000,AT&T,13,,342,,1125.5905,18.240.226.162,51 c5 5d d5,67.56,c0 2b,201.0098,13.806511,,5965,1780243200.000219,03 03,155.302195,239319,3913,stream.example.org,143
|
||||
Singapore,7,538,,-89.185991,142.150.185.148,"anon, hosting",10.70.170.206,Orange,,,Samsung,126,142.150.185.148,,,zeek,+,2026-06-01 00:00:00.000220,,1.881,Shanghai,"anon, hosting",jp,api.example.com,in,Tencent,26316,TlsS,24 86 f7 44 58 b4 bb ef ac cd 82 35 72 af 8a f9 9e 30 c9 a8 bf ef 6d a0 4d 79 4a bb,edge-01,53,Deutsche Telekom,64,,61,,821.6787,82.50.171.203,b2 09 b0 f8,19.07,c0 27,1483.5854,-20.036893,,139273,1780243200.00022,03 04,-97.662173,334739,5023,,143
|
||||
Shanghai,22,935,,179.357172,166.153.234.81,"anon, hosting",91.135.247.50,BT Group,,,Tencent,130,166.153.234.81,,,zeek,34.870406,2026-06-01 00:00:00.000221,,513.6662,Seattle,"anon, hosting",jp,*.s3.amazonaws.com,gb,Meta Platforms,86450,TlsC,d0 17 fa 1d e8 41 44 6d 8a 2b 6d 76 b0 36 63 03 26 8b 61 18 ef 9e 54 fc 9e 74 dd cf,edge-01,143,SK Telecom,9,,117,,1548.5846,74.159.180.27,73 a0 f5 d3,116.82,00 9e,730.1501,-30.001599,,174214,1780243200.000221,03 04,-4.749465,204584,9570,,8443
|
||||
Paris,34,1517,,-173.415492,76.111.29.149,"anon, hosting",136.216.108.238,China Telecom,,,Tencent,172,76.111.29.149,,,netflow,-19.42471,2026-06-01 00:00:00.000222,,41.7966,Singapore,"anon, hosting",jp,,il,Netflix Inc.,62508,TlsS,6e e3 db 9a 33 8f fe 29 85 c3 44 54 0e 26 85 7c 65 7c a5 88 1e 31 38 5e e2 56 6a c7,backbone-03,8443,SK Telecom,6,,2,512,1308.7572,21.164.75.17,00 f6 ca 95,13.45,00 9f,132.2952,1.345128,,122005,1780243200.000222,03 04,-159.142072,49383,5872,,8080
|
||||
Dublin,32,1301,,144.478163,168.255.202.227,"anon, hosting",48.7.172.203,KDDI,,,Apple Inc.,65,168.255.202.227,,,zeek,27.435058,2026-06-01 00:00:00.000223,,928.9471,San Francisco,"anon, hosting",no,,gb,Apple Inc.,22218,TlsC,47 59 39 d8 0c e9 bb 8d 05 48 76 65 bc 76 98 62 2a 02 d1 b9 e5 7a a7 2b f5 9f ad 81,core-02,443,Orange,39,,247,,24.857,74.117.180.57,6f b0 3e 0d,23.13,c0 2c,1616.2442,-30.123814,,96475,1780243200.000223,03 03,101.145187,238119,7258,*.cloudfront.net,143
|
||||
Seattle,42,22,Google LLC,-57.49709,49.43.123.185,"anon, hosting",179.245.93.208,BT Group,,,Samsung,114,49.43.123.185,00 19,,ssl_logs,33.537793,2026-06-01 00:00:00.000224,,590.1328,San Francisco,"anon, hosting",de,*.s3.amazonaws.com,ru,Amazon.com Inc.,59097,TlsS,9b e0 6f 3b 2f b0 c6 0a ae 94 53 f7 1a e8 b8 99 f4 d3 3c 35 37 1a f7 f2 c8 9d e6 b9,wan-link,53,Comcast,80,,133,512,761.0709,26.250.208.241,b3 d6 ae ec,113.22,00 9f,1483.5443,59.345384,,198860,1780243200.000224,03 03,-71.209917,227674,8474,,80
|
||||
Beijing,36,469,,40.309944,139.41.70.39,"anon, hosting",170.41.80.62,KDDI,,,Microsoft Corp.,199,139.41.70.39,,,suricata,18.523898,2026-06-01 00:00:00.000225,,968.8388,Seattle,"anon, hosting",ca,,br,Netflix Inc.,19999,TlsS,47 9c 2e 45 af 7c f6 18 10 c3 87 81 ac 93 f5 ae 7d 36 3b 05 83 55 b9 b3 9f 49 03 a5,edge-01,22,Comcast,61,,487,,245.5642,168.172.131.196,e3 d2 9a 43,53.87,c0 2b,1834.1893,-59.723266,,178026,1780243200.000225,03 04,-111.337492,471709,9509,mail.example.com,53
|
||||
Singapore,9,1735,,-132.621106,98.138.112.176,"anon, hosting",103.225.197.87,AT&T,secp256r1,,Samsung,194,98.138.112.176,,,suricata,-4.023896,2026-06-01 00:00:00.000226,,549.7087,Beijing,"anon, hosting",nl,mail.example.com,in,Google LLC,87776,TlsC,e5 81 8d 57 1a 75 cb 1c f6 ae 95 60 67 06 be fc 48 79 6e cb 05 da 19 2d 84 ab e4 4e,core-02,110,China Mobile,83,TLS_ECDHE_RSA_AES128_SHA,343,,1235.2329,36.106.166.170,cd f8 26 89,86.82,c0 27,1143.3431,-11.305111,,130215,1780243200.000226,03 03,54.084223,444998,6578,,50000
|
||||
Singapore,22,68,,137.472661,193.120.158.58,"anon, hosting",58.62.102.203,Orange,,,Microsoft Corp.,67,193.120.158.58,,,packet_capture,-29.054043,2026-06-01 00:00:00.000227,,58.6474,Berlin,"anon, hosting",de,*.cloudfront.net,kr,Microsoft Corp.,11670,TlsC,9b ec fd 54 c0 b9 f3 38 ac cf 9d 72 16 50 18 73 8a 09 97 40 96 f1 d6 19 0f 19 92 04,edge-01,53,AT&T,40,,401,,107.0496,51.168.117.98,d6 b5 f7 74,17.04,00 35,1435.947,-2.398025,,155787,1780243200.000227,03 02,-111.143034,435455,6378,,443
|
||||
New York,11,472,,-33.631476,182.223.0.134,"anon, hosting",127.147.205.209,Deutsche Telekom,,,Microsoft Corp.,84,182.223.0.134,,,packet_capture,43.759833,2026-06-01 00:00:00.000228,,190.8774,Shanghai,"anon, hosting",us,mail.example.com,sg,Amazon.com Inc.,21766,TlsC,a6 a2 85 1a 91 5a 7f 3f cd fc 49 1f dd e2 ba 89 73 64 9c 76 cc 09 8d a3 6c 5d b1 59,core-02,143,Singtel,32,,52,,305.9015,91.135.247.50,71 ab e9 65,75.67,c0 14,1866.8759,41.091224,,54676,1780243200.000228,03 03,52.171635,299073,1464,api.example.com,53
|
||||
San Francisco,26,197,,155.45531,109.210.239.222,"anon, hosting",132.171.47.210,Verizon,,30090,Apple Inc.,91,109.210.239.222,,,suricata,-8.130876,2026-06-01 00:00:00.000229,login.live.com,38.2649,Moscow,"anon, hosting",kr,,br,Tencent,88103,TlsC,8e d1 7d 08 dc 3d d2 44 26 ba dd 79 a1 43 04 00 ea df 25 a0 15 94 4e 1c a0 c5 b7 d7,edge-01,443,NTT,91,,376,512,298.4502,91.135.247.50,eb 27 9c 61,5.77,cc a9,1049.3656,-45.446822,,197837,1780243200.000229,03 03,-96.978922,282239,6787,graph.facebook.com,3389
|
||||
Beijing,19,330,,-151.404915,190.226.218.144,"anon, hosting",194.226.218.213,China Mobile,,,Google LLC,185,200.208.79.237,,,packet_capture,-13.496613,2026-06-01 00:00:00.000230,,138.4957,Singapore,"anon, hosting",no,,fi,Microsoft Corp.,75008,TlsS,d5 e6 be 23 ac fb 33 2b d9 9b b5 dc 17 b3 bb cd 6a 9b 7f 2d 18 e0 6b 4b 08 10 44 b2,gw-09,443,China Telecom,65,TLS_ECDHE_ECDSA_CHACHA20_POLY1305,56,,1095.4643,93.188.239.206,b1 db a9 6d,59.67,c0 2c,26.2179,-4.625928,,28481,1780243200.00023,03 03,-85.16323,137889,4420,,12345
|
||||
New York,3,392,,-133.32986,58.32.173.6,"anon, hosting",74.159.180.27,Singtel,,,Meta Platforms,172,58.32.173.6,,,suricata,47.59675,2026-06-01 00:00:00.000231,,871.4875,Sydney,"anon, hosting",ca,mail.example.com,fi,Netflix Inc.,13212,TlsC,34 5e b8 28 bc 16 86 34 54 06 ac 23 d3 ce dc 16 3a 5c 6f 41 f9 96 8d f8 11 c3 63 d5,edge-01,443,China Mobile,78,,126,,470.1762,2.254.220.45,47 f2 c8 a5,66.86,00 35,734.6156,-30.022596,,142763,1780243200.000231,03 03,-76.690812,38826,1104,,8443
|
||||
Shanghai,6,35,,-30.934619,68.202.67.172,"anon, hosting",149.78.87.169,NTT,,,Apple Inc.,58,68.202.67.172,,,suricata,8.702956,2026-06-01 00:00:00.000232,,925.4597,Mumbai,"anon, hosting",de,,ru,Apple Inc.,12794,TlsC,33 97 ea 6c 23 0b 65 35 86 b5 c7 8b a1 f2 40 4e ea a7 f1 89 1b e8 89 0d 71 6d e6 87,wan-link,8080,SK Telecom,13,TLS_RSA_WITH_AES_128_GCM_SHA256,219,,902.3896,31.94.98.56,e6 01 75 7e,48.84,c0 14,131.6699,0.279367,,1229,1780243200.000232,03 03,-81.039427,24041,7365,,25
|
||||
Moscow,25,402,,-83.502171,196.172.52.24,"anon, hosting",97.94.242.62,AT&T,,,Meta Platforms,196,196.172.52.24,,,packet_capture,-24.753041,2026-06-01 00:00:00.000233,,153.4338,Tokyo,"anon, hosting",kr,,jp,Alibaba Inc.,14374,TlsS,e5 8f 74 76 77 9b d0 af bf 52 75 37 f5 0d 6c 77 b8 7c 00 1c 9f 5c c4 ce b2 6d c5 39,wan-link,8443,BT Group,26,,416,,698.643,47.109.174.243,f4 e9 78 35,61.03,c0 23,1715.7874,50.629637,ER,40007,1780243200.000233,03 04,160.132744,127917,7398,,443
|
||||
Singapore,47,1485,,126.808058,104.173.55.64,"anon, hosting",34.46.151.84,KDDI,,,Amazon.com Inc.,40,104.173.55.64,00 1e,,zeek,-53.428746,2026-06-01 00:00:00.000234,,213.3229,Moscow,"anon, hosting",ua,,gb,Baidu Inc.,92285,TlsS,18 29 38 1c 2a a7 d9 25 d4 24 eb 62 84 81 29 47 2c 76 00 18 47 62 9b c0 1a cf f4 41,gw-09,22,Comcast,29,,311,,1975.6638,41.89.185.131,6f 25 1f 87,119.03,c0 23,1061.0823,12.520409,,95206,1780243200.000234,03 02,-171.412713,73447,5289,login.live.com,53
|
||||
Dublin,49,1163,,82.672566,221.243.230.67,"anon, hosting",114.226.109.131,SK Telecom,,,Apple Inc.,173,221.243.230.67,,,ssl_logs,-7.799248,2026-06-01 00:00:00.000235,,488.8648,Berlin,"anon, hosting",de,auth.example.net,sg,Amazon.com Inc.,69775,TlsS,71 71 17 da 83 0c 0c a4 ff f5 2b aa a2 b2 1c c6 55 f6 f5 1a 57 b6 32 4d 08 88 c7 78,wan-link,993,Verizon,10,,240,128,1443.2464,72.115.219.217,3b d3 5b cb,21.44,c0 2c,1593.5143,32.009271,,172562,1780243200.000235,03 03,-139.332377,435662,7414,,993
|
||||
Seattle,39,1797,,-131.855644,180.131.188.43,"anon, hosting",114.217.248.156,Verizon,,,Apple Inc.,143,180.131.188.43,,,ssl_logs,-52.31359,2026-06-01 00:00:00.000236,,849.4207,Tokyo,"anon, hosting",kr,login.live.com,fr,Samsung,37045,TlsC,81 23 df 52 fa dc 6b ca a7 21 50 a7 b4 a9 2e 09 09 ba 29 10 3a 9b 83 41 41 db 58 6a,edge-01,443,Verizon,79,TLS_ECDHE_ECDSA_CHACHA20_POLY1305,141,256,914.3364,180.56.97.5,06 6b d6 05,84.79,c0 23,1149.057,-59.679675,--,71328,1780243200.000236,03 03,-18.291918,402601,6006,,443
|
||||
Sydney,47,1821,,18.962813,106.12.91.189,"anon, hosting",166.183.31.102,Telstra,,,Meta Platforms,127,106.12.91.189,,,packet_capture,-54.240696,2026-06-01 00:00:00.000237,,692.1365,New York,"anon, hosting",fr,,jp,Samsung,34689,TlsS,9b 78 b3 8a 1d 9c fe cf 59 ae 65 ba 17 36 2c 3b a8 51 a6 a3 de 5c ed e4 5a cd 14 71,backbone-03,21,Verizon,57,,159,,263.3614,213.93.66.111,e3 8d 09 79,56.77,cc a9,1785.8543,35.181999,,113284,1780243200.000237,03 03,1.157161,368521,2446,,443
|
||||
Dublin,47,790,,-177.468312,3.57.38.242,"anon, hosting",206.240.163.235,SK Telecom,x448,,Samsung,146,3.57.38.242,00 1e,,suricata,58.606718,2026-06-01 00:00:00.000238,graph.facebook.com,390.9992,Paris,"anon, hosting",br,graph.facebook.com,fr,Apple Inc.,85012,TlsS,7e f5 40 ee 35 a8 c9 18 70 6c f8 b4 4a bb cb 35 84 78 ef ca 7c 1a 04 02 f0 79 fe 91,core-02,443,Verizon,81,,475,,1308.7301,43.157.147.113,98 8d a0 ec,69.99,13 01,904.3972,26.582147,,195087,1780243200.000238,03 04,-1.49686,298019,8570,auth.example.net,443
|
||||
Seattle,43,1996,,-32.831919,164.57.12.190,"anon, hosting",177.76.18.214,Verizon,,,Meta Platforms,40,164.57.12.190,,,packet_capture,10.544232,2026-06-01 00:00:00.000239,stream.example.org,551.0123,Singapore,"anon, hosting",ca,,fr,,80912,TlsC,5f 72 2c 46 c9 a1 64 51 ea 79 a0 a9 3c 91 e6 4f 86 1b 60 47 41 7d 89 f6 f9 26 95 d9,core-02,443,AT&T,52,,480,,721.5419,213.241.71.195,a8 11 7c b6,49.62,00 9f,940.7444,48.502798,,19405,1780243200.000239,03 03,-57.575679,401789,3448,,443
|
||||
Singapore,21,1884,,22.610107,85.238.212.16,"anon, hosting",17.197.195.153,BT Group,,,Tencent,38,85.238.212.16,,,zeek,-37.950996,2026-06-01 00:00:00.000240,,181.8311,Moscow,"anon, hosting",us,cdn.example.com,cn,Microsoft Corp.,96538,TlsS,ac 0b cd 35 ea 13 dd ec 0c 22 ae 37 58 13 1c 6a 6a 81 9e 10 ea df f3 53 3a c3 5d 25,backbone-03,21,KDDI,22,,444,,1118.892,43.157.147.113,63 79 76 25,40.79,13 02,1022.3048,46.775038,,22392,1780243200.00024,03 04,66.834689,233231,4170,,22
|
||||
Sydney,36,1403,Meta Platforms,99.157021,182.223.0.134,"anon, hosting",58.32.173.6,Orange,,,Samsung,27,182.223.0.134,,,packet_capture,-53.65707,2026-06-01 00:00:00.000241,,616.8049,New York,"anon, hosting",se,*.cloudfront.net,ua,Samsung,95047,TlsC,e8 4e 9c 4f cc 1f 45 4b cd 24 c1 83 3d e0 0d 00 ae 64 fc 26 d9 11 ab c6 cb 05 67 26,backbone-03,50000,NTT,88,,192,,540.8439,122.233.76.156,cd c1 9c f1,21.32,00 35,1583.8147,-3.827674,,68915,1780243200.000241,03 03,-23.768521,316833,1645,api.example.com,22
|
||||
Paris,22,1639,,-14.143538,141.228.81.191,"anon, hosting",104.139.252.113,China Mobile,,,Alibaba Inc.,148,141.228.81.191,,cdn.example.com,packet_capture,23.138252,2026-06-01 00:00:00.000242,,211.4389,New York,"anon, hosting",de,www.google.com,fi,Apple Inc.,59005,TlsS,d8 a4 e7 9e 82 55 b1 4d c4 84 54 1c 90 2a cf 91 39 b2 6c e1 95 14 bb 2f 60 7a ea 75,core-02,53,China Telecom,4,,153,,782.58,96.52.63.73,d8 ca e5 c2,97.1,13 01,550.7899,-27.716351,,157048,1780243200.000242,03 02,58.900022,18476,4425,,993
|
||||
Mumbai,45,675,,-120.972463,162.99.196.127,"anon, hosting",35.32.234.10,Deutsche Telekom,,,Meta Platforms,170,162.99.196.127,,,packet_capture,8.933559,2026-06-01 00:00:00.000243,,136.1926,London,"anon, hosting",au,,gb,Tencent,42662,TlsC,34 48 20 90 f0 82 8f e4 9c c5 02 96 59 7e e9 35 f6 81 f2 5c 29 86 dc 28 ea 5d ff ac,edge-01,465,Comcast,90,,453,,590.5271,31.94.98.56,8a b0 49 5c,63.21,c0 27,555.9922,5.597872,ER,86776,1780243200.000243,03 04,133.106092,320213,4420,,80
|
||||
,30,1419,,-77.663227,72.79.110.246,"anon, hosting",56.76.216.29,BT Group,,,Google LLC,76,72.79.110.246,,,ssl_logs,-5.211808,2026-06-01 00:00:00.000244,,814.4913,San Francisco,"anon, hosting",ru,,in,,72984,TlsC,35 42 27 6b 07 fb 5f f9 18 ca d5 27 a3 98 b1 a9 cd c0 76 b5 dd a6 97 28 0e 54 5a ad,core-02,12345,NTT,49,,147,,72.9707,82.242.195.99,57 c3 dd 15,91.46,00 9e,485.8353,17.84402,,131565,1780243200.000244,03 04,134.761595,440463,8250,,465
|
||||
Berlin,16,145,,-13.578417,169.243.84.68,"anon, hosting",214.247.241.63,China Telecom,,61081,Tencent,106,169.243.84.68,,api.example.com,suricata,29.455687,2026-06-01 00:00:00.000245,,849.015,New York,"anon, hosting",cn,,ua,Apple Inc.,21159,TlsS,e5 0b 9f 53 82 15 4e a9 af ac 94 d2 72 e6 ea 13 96 35 35 d4 f4 23 b8 d7 92 cc eb 77,gw-09,12345,Orange,36,TLS_ECDHE_ECDSA_CHACHA20_POLY1305,84,,1204.973,194.70.49.232,af c2 05 98,27.78,c0 13,931.0247,-44.900746,,46844,1780243200.000245,03 03,60.909847,15408,6292,,465
|
||||
Moscow,16,1567,,29.291776,101.13.42.165,"anon, hosting",12.113.74.5,China Telecom,,,Netflix Inc.,198,101.13.42.165,,,netflow,47.756873,2026-06-01 00:00:00.000246,,728.7818,Mumbai,"anon, hosting",il,,de,Samsung,12890,TlsC,bd bc 0d af a5 7a c6 44 2a e2 3a 99 ba 92 26 72 b0 52 a0 43 bb 49 1f e8 53 26 c9 a0,wan-link,50000,SK Telecom,23,,135,,52.4156,120.62.136.116,22 5e 2f 99,94.84,c0 23,1094.3035,23.620511,,124247,1780243200.000246,03 03,-52.756839,29918,5744,,12345
|
||||
Sydney,2,1154,Samsung,-2.315226,104.173.55.64,"anon, hosting",130.37.158.119,BT Group,,,Baidu Inc.,133,104.173.55.64,,,netflow,-25.157504,2026-06-01 00:00:00.000247,,722.177,Sydney,"anon, hosting",kr,,ua,Google LLC,75002,TlsC,2c 68 2a 72 14 2f 94 9c 11 53 19 23 c2 d3 9a b8 50 60 3b 37 12 4e 23 6f 62 a4 c8 69,gw-09,110,Verizon,10,,59,,1586.5738,38.66.128.205,4d 77 dc cc,1.86,c0 14,72.4937,-49.537892,,181184,1780243200.000247,03 04,-133.768026,16019,7403,,443
|
||||
Seattle,20,401,,-65.42079,165.174.57.76,"anon, hosting",213.241.71.195,Verizon,,,Alibaba Inc.,60,76.111.29.149,00 19,,netflow,38.255064,2026-06-01 00:00:00.000248,,255.3362,Seattle,"anon, hosting",fi,,ca,Microsoft Corp.,62471,TlsS,18 d7 65 7f 5d 6e 98 6e 0f cf ac fb 96 92 62 57 7c ef ab 02 fc 5f 48 a6 48 bf ac 8c,edge-01,143,Deutsche Telekom,35,,289,,385.8245,130.37.158.119,39 6c 3a d7,65.35,13 02,1064.2463,29.062736,,143703,1780243200.000248,03 02,101.286323,364505,2447,,443
|
||||
London,2,175,,152.963767,200.208.79.237,"anon, hosting",221.243.230.67,Deutsche Telekom,,,Tencent,9,200.208.79.237,00 19,,netflow,53.170236,2026-06-01 00:00:00.000249,,891.5896,Dublin,"anon, hosting",sg,auth.example.net,sg,Microsoft Corp.,73833,TlsC,88 0d 65 94 8b a1 8f ad b5 d2 d9 0e 04 76 03 2c 2f d3 46 4d fe 4c 70 c0 f1 88 40 32,gw-09,25,KDDI,61,,471,,67.2595,28.122.124.13,e3 6a c7 76,70.46,c0 2c,378.7628,30.367151,,187830,1780243200.000249,03 02,169.544451,414765,7880,,443
|
||||
Seoul,1,1493,,5.179328,189.31.160.15,"anon, hosting",51.140.64.161,Telstra,,,Meta Platforms,167,189.31.160.15,00 1e,,zeek,-1.630398,2026-06-01 00:00:00.000250,,277.5952,Moscow,"anon, hosting",ru,cdn.example.com,de,Apple Inc.,76170,TlsS,e3 06 ee 34 40 91 ac e5 cf 26 7a 14 8a 9a 90 2e 93 58 7d a0 b3 4c 67 6d f7 56 f1 b7,backbone-03,443,Comcast,33,,34,,1730.744,132.71.167.239,fb 3f 6a 18,99.44,00 9f,1919.612,49.133619,,146997,1780243200.00025,03 04,-147.205299,273742,2688,,25
|
||||
Dublin,3,875,,78.812806,59.148.40.219,"anon, hosting",73.80.224.214,Comcast,,,Samsung,58,207.81.216.88,,,ssl_logs,-35.607788,2026-06-01 00:00:00.000251,,476.629,Seattle,"anon, hosting",ua,cdn.example.com,ca,,60979,TlsC,20 d1 9d ce 44 37 00 07 60 f1 04 0a e4 17 7d f9 1f d5 de 73 54 12 ff 11 78 8b 8a 4f,gw-09,993,China Telecom,23,,185,,180.5347,109.40.220.155,de fb 9b e1,32.65,00 9e,321.1592,-16.729707,,83111,1780243200.000251,03 04,177.390294,307421,3048,,993
|
||||
Tokyo,23,1542,,70.974736,58.32.173.6,"anon, hosting",149.135.19.28,KDDI,,,Amazon.com Inc.,172,151.142.3.195,,,ssl_logs,-40.879095,2026-06-01 00:00:00.000252,cdn.example.com,295.7924,Seattle,"anon, hosting",au,,ca,Google LLC,37034,TlsS,70 fe 5a 5f ac 6a 34 12 77 75 f1 1b 03 9f 48 6a 80 11 68 e4 05 88 65 61 77 64 3f 6b,gw-09,995,NTT,29,,347,,1899.0056,43.157.147.113,bb cc 2a 41,71.3,c0 23,771.8991,53.241185,ER,168322,1780243200.000252,03 03,+,342323,5740,,110
|
||||
Paris,46,1653,,176.568117,217.26.47.163,"anon, hosting",223.251.52.224,,,,Amazon.com Inc.,17,217.26.47.163,,,ssl_logs,-0.694525,2026-06-01 00:00:00.000253,,794.7295,San Francisco,"anon, hosting",sg,,gb,,22587,TlsC,db 7e 48 b7 2d 00 76 ab 5e 9e 4b f0 c3 56 4e 34 a6 b4 cd 7b e2 97 69 d4 74 83 ac 93,core-02,443,China Telecom,28,,482,384,1343.2905,188.234.131.184,b2 a4 92 d4,88.73,00 9f,1486.0672,-5.070882,,173688,1780243200.000253,03 03,-10.42875,385987,7413,,8080
|
||||
Dublin,3,831,,-2.247279,17.108.161.55,"anon, hosting",32.232.188.173,Verizon,,,Google LLC,16,17.108.161.55,,,zeek,-34.391035,2026-06-01 00:00:00.000254,,63.4977,Moscow,"anon, hosting",se,,nl,Meta Platforms,46202,TlsC,1d a6 42 0f a9 c7 df b0 2c 61 e2 9b e6 e2 2e 79 e3 8c 7d 87 b5 16 ce a3 6d 14 ca 50,gw-09,3389,BT Group,39,,91,,374.1427,53.189.160.106,1d 4c 7a 17,36.13,c0 27,711.9829,-29.407008,,94033,1780243200.000254,03 03,165.744539,123478,7592,,21
|
||||
Dublin,24,1785,,138.339919,164.57.12.190,"anon, hosting",136.176.217.191,China Mobile,,,Apple Inc.,47,164.57.12.190,00 19,,zeek,+,2026-06-01 00:00:00.000255,,935.6338,Tokyo,"anon, hosting",sg,,sg,Amazon.com Inc.,95153,TlsS,2f bf 1c c5 2e 97 3b 61 cd bf 12 b6 ac 3d 30 71 d2 e0 29 dd fc 27 f7 03 05 9f 4b 68,edge-01,443,Comcast,9,,396,128,83.6558,179.248.62.8,bb 08 fb 33,33.89,c0 14,1984.0605,58.982178,,35428,1780243200.000255,03 03,128.00418,457789,3151,,53
|
||||
Beijing,29,998,,17.126256,189.31.160.15,"anon, hosting",122.58.111.120,Verizon,x25519,,Meta Platforms,155,189.31.160.15,,api.example.com,ssl_logs,56.026383,2026-06-01 00:00:00.000256,,929.9127,Seattle,"anon, hosting",se,api.example.com,au,Tencent,22547,TlsS,19 6f d8 03 3e cf 8c 34 06 81 82 d0 15 26 ff 34 83 13 78 8b d9 a6 86 8f b4 0f ce 05,core-02,443,Deutsche Telekom,52,,312,,157.2699,181.185.39.235,aa 11 d3 43,65.79,c0 2f,1237.7069,-11.452638,,149914,1780243200.000256,03 04,-50.648327,325414,6658,,25
|
||||
Beijing,35,1186,,-48.987748,200.208.79.237,"anon, hosting",195.194.142.193,Comcast,,,Alibaba Inc.,140,200.208.79.237,,,ssl_logs,52.170809,2026-06-01 00:00:00.000257,,281.0374,Singapore,"anon, hosting",ua,stream.example.org,au,Apple Inc.,29475,TlsS,4b 8c 5b 67 ae 4d ae c7 11 00 a4 9e c7 20 e7 5c 37 b9 1c ad 4c 04 b8 74 a9 65 2c 60,backbone-03,443,KDDI,62,,133,,453.2154,152.89.218.143,8b be d3 08,1.29,c0 23,1628.9566,-52.451341,,38604,1780243200.000257,03 03,162.664385,336098,5976,,443
|
||||
Dublin,6,1342,,50.182047,41.55.195.224,"anon, hosting",33.140.166.98,Orange,,,Amazon.com Inc.,80,41.55.195.224,,,suricata,-35.688108,2026-06-01 00:00:00.000258,,121.7812,Beijing,"anon, hosting",se,stream.example.org,il,,88008,TlsS,5f 0c a9 17 7f 80 ac bb 75 5f f1 51 03 86 e1 cb f2 de c2 c1 16 41 fd 99 a4 e4 28 42,gw-09,443,Orange,46,,431,,1797.0573,102.7.135.138,f1 a0 6a 24,90.13,c0 27,1580.9142,-37.71847,,33757,1780243200.000258,03 04,-137.968162,2188,4354,stream.example.org,3389
|
||||
Dublin,20,1963,,30.013739,142.150.185.148,"anon, hosting",106.12.91.189,NTT,,,Apple Inc.,102,142.150.185.148,,,zeek,-54.595654,2026-06-01 00:00:00.000259,,197.5614,London,"anon, hosting",no,,nl,Apple Inc.,40357,TlsS,a4 87 14 e0 07 94 e6 69 3c 9f 6f ca ed 9e eb f1 fe 82 03 eb 09 71 ce c5 d2 b4 cd 06,backbone-03,8443,Deutsche Telekom,52,,247,256,523.9167,164.247.249.5,e6 c1 47 52,69.93,13 02,853.8967,48.362679,--,132075,1780243200.000259,03 04,+,113180,5475,,465
|
||||
Sydney,17,153,,-110.368349,62.40.43.188,"anon, hosting",140.154.4.171,SK Telecom,,,Tencent,170,140.154.4.171,,,suricata,22.721697,2026-06-01 00:00:00.000260,api.example.com,777.3374,London,"anon, hosting",nl,,se,Google LLC,85011,TlsS,fc f3 f9 bb 6b b8 d9 24 6a 9e 2d f8 83 25 b1 70 0c 98 5b dd fb 42 62 66 06 fa cf 3a,backbone-03,110,China Telecom,62,,300,,174.9748,94.203.160.167,0e 10 30 26,49.82,13 02,1049.6244,42.380739,,162229,1780243200.00026,03 03,-58.8766,419749,7249,,110
|
||||
Beijing,2,1015,,67.629579,73.80.224.214,"anon, hosting",149.21.230.233,Verizon,x25519,,Microsoft Corp.,96,73.80.224.214,,,netflow,44.098052,2026-06-01 00:00:00.000261,,641.6828,San Francisco,"anon, hosting",ua,graph.facebook.com,hk,Apple Inc.,79950,TlsC,af f2 66 b1 c1 f5 96 6c 63 d2 8c 8b fe 1a 56 c3 66 4c fa ca fc d9 b3 c9 87 ab e3 c6,edge-01,8443,Deutsche Telekom,62,,332,,1450.2251,25.190.66.143,84 87 e2 28,76.8,c0 23,1496.3263,1.006335,,95663,1780243200.000261,03 03,67.200173,165256,4537,,443
|
||||
Tokyo,35,65,,162.591608,182.223.0.134,"anon, hosting",189.95.255.133,Orange,,,Meta Platforms,33,153.222.176.187,,login.live.com,ssl_logs,48.411474,2026-06-01 00:00:00.000262,,636.5089,Beijing,"anon, hosting",fr,mail.example.com,fr,Apple Inc.,88410,TlsC,ab 4b 1b 91 23 1b 2d 0a 5b fa df 16 f7 38 a7 3a c8 99 86 ec f3 f0 67 64 78 0b da 2c,core-02,21,Orange,28,,240,,807.5719,160.172.21.212,6e a4 e4 be,80.41,c0 23,668.9062,42.091747,,179998,1780243200.000262,03 03,-109.236235,31899,8564,,443
|
||||
London,8,61,,-113.506999,151.117.112.2,"anon, hosting",213.250.153.122,China Telecom,,,Baidu Inc.,21,151.117.112.2,,,packet_capture,-53.069313,2026-06-01 00:00:00.000263,,208.4804,Paris,"anon, hosting",il,,br,Baidu Inc.,15109,TlsC,7d c7 83 7d 2a db fc a6 88 85 88 15 a1 6e 0c 66 5f 11 ec be df 02 af 4f d8 1b 3f b9,backbone-03,53,China Mobile,38,TLS_RSA_WITH_AES_128_GCM_SHA256,330,,1606.6885,76.188.113.57,13 42 cd c3,48.25,cc a9,1589.9055,-44.484455,,943,1780243200.000263,03 03,-176.439276,94572,9711,,443
|
||||
Singapore,14,676,,-48.553308,72.35.142.90,"anon, hosting",36.126.134.192,Comcast,,,Amazon.com Inc.,127,98.138.112.176,,,suricata,-3.955449,2026-06-01 00:00:00.000264,,879.384,New York,"anon, hosting",fr,graph.facebook.com,gb,Alibaba Inc.,57734,TlsS,ac ad 15 7c 1b 3f d8 a8 9b 86 e8 c6 34 ad 07 96 90 3a a6 0f 45 82 93 d0 94 15 79 aa,core-02,50000,Singtel,79,,394,,1255.5432,80.203.242.65,d3 b8 b1 b4,85.56,13 01,543.7473,44.498727,,117183,1780243200.000264,03 01,-29.231023,94529,5669,,993
|
||||
Singapore,49,861,,114.884621,122.58.111.120,"anon, hosting",164.207.186.132,China Telecom,,,Alibaba Inc.,9,122.58.111.120,,,packet_capture,-53.29414,2026-06-01 00:00:00.000265,,33.4119,Paris,"anon, hosting",kr,api.example.com,fr,Samsung,41111,TlsC,b4 b6 ac ba de 5f 3e d9 6d 68 be f7 d4 d4 1b b6 03 9a 2e 0f d5 1a 96 66 28 f2 2d bd,backbone-03,110,Orange,52,,295,,1766.8228,129.191.120.99,f1 a4 51 2f,74.12,00 9f,1093.097,51.466663,,18074,1780243200.000265,03 01,-78.149636,13015,4110,,143
|
||||
Tokyo,18,1291,,-33.672572,72.35.142.90,"anon, hosting",175.176.65.225,AT&T,secp384r1,,Samsung,134,72.35.142.90,,,packet_capture,-6.918804,2026-06-01 00:00:00.000266,,485.9704,Beijing,"anon, hosting",au,cdn.example.com,gb,Meta Platforms,16744,TlsC,f2 e3 9a 8b 56 9f c6 e4 2d 9c 83 c4 6f ab 7f 2f a7 a9 df ce 9f 49 64 8e 05 4e c1 05,edge-01,22,Comcast,91,,331,,768.0336,155.220.50.204,c0 89 c6 c7,93.89,c0 27,372.2919,38.451715,,174422,1780243200.000266,03 03,102.562409,245414,9122,,50000
|
||||
San Francisco,14,833,,-47.282884,72.35.142.90,"anon, hosting",127.147.205.209,Verizon,x25519,,Alibaba Inc.,190,72.35.142.90,,,packet_capture,-53.914899,2026-06-01 00:00:00.000267,,957.4784,Dublin,"anon, hosting",de,stream.example.org,br,Google LLC,66112,TlsS,b1 0b ae d1 bf 33 78 72 a2 a3 19 2c 86 60 5f 5f f3 d8 d2 68 5f fa 18 8e 41 6a 68 31,backbone-03,22,Comcast,62,,307,256,1694.6421,78.108.70.196,14 27 fb 16,93.53,00 9e,1406.8777,-48.029761,,152242,1780243200.000267,03 03,-62.713035,325487,3023,,995
|
||||
Mumbai,3,1402,,-66.929234,51.214.112.115,"anon, hosting",148.160.97.196,China Telecom,,,Microsoft Corp.,107,51.214.112.115,,,ssl_logs,-59.074044,2026-06-01 00:00:00.000268,,710.0949,Sydney,"anon, hosting",sg,*.s3.amazonaws.com,hk,Alibaba Inc.,73175,TlsS,df f9 15 da 01 a3 ec 26 24 5b 2a 2e c4 e8 67 5c 7b ba 60 75 76 08 bd 8b 90 a3 29 d6,wan-link,22,NTT,32,,57,,286.2263,166.88.20.64,1c e2 97 60,49.77,13 01,138.5792,-5.436646,,87742,1780243200.000268,03 03,140.133364,475162,4901,,465
|
||||
Shanghai,39,1910,,94.850514,177.2.54.200,"anon, hosting",141.80.208.168,Telstra,,,Samsung,18,177.2.54.200,,,suricata,-51.697274,2026-06-01 00:00:00.000269,,739.5038,Singapore,"anon, hosting",hk,,jp,Samsung,35730,TlsC,2a c1 60 7e 25 f9 6c c0 93 ff 0e 83 99 16 c5 b3 97 b5 4c ab 67 bc ef 0a fe c3 bc 03,edge-01,443,Singtel,1,,337,,1479.4941,177.185.45.112,c8 5c 06 f0,98.34,cc a9,243.4408,-31.322427,,199352,1780243200.000269,03 03,-159.406099,451476,1802,api.example.com,143
|
||||
Moscow,36,230,,58.642424,141.228.81.191,"anon, hosting",11.89.240.133,Comcast,,,Netflix Inc.,61,141.228.81.191,,,zeek,35.649669,2026-06-01 00:00:00.000270,"*.['google', 'com']",245.3157,San Francisco,"anon, hosting",gb,,de,Amazon.com Inc.,32500,TlsC,18 fd b6 36 d9 ef 88 d5 f2 60 b9 3d 42 39 fa fe 86 e6 90 ff e6 7a bf aa 9d 6e 78 be,wan-link,443,China Mobile,62,,292,,1170.951,25.201.191.204,f0 3f 5a b3,77.03,c0 2f,739.7523,52.050107,,100114,1780243200.00027,03 04,83.01852,111814,6431,,8443
|
||||
Paris,41,1075,,-90.851372,166.153.234.81,"anon, hosting",178.153.14.12,Verizon,,,Meta Platforms,23,166.153.234.81,00 18,,netflow,50.999094,2026-06-01 00:00:00.000271,,155.6236,Seoul,"anon, hosting",no,,ua,Amazon.com Inc.,34727,TlsC,48 eb 50 19 8b 11 60 c2 bf e6 b4 d5 5f 83 64 66 33 d0 c1 eb 02 a1 a6 08 cc 8b 74 8b,gw-09,50000,SK Telecom,82,,259,,1281.1089,47.139.158.236,36 1e 22 08,70.35,13 01,1787.4072,-35.232499,,466,1780243200.000271,03 03,-152.46309,59771,9183,,50000
|
||||
Seoul,42,1124,,9.176212,187.27.50.16,"anon, hosting",14.99.185.142,AT&T,,,Microsoft Corp.,56,187.27.50.16,00 1d,,suricata,58.233905,2026-06-01 00:00:00.000272,,116.1432,Mumbai,"anon, hosting",fi,mail.example.com,cn,Baidu Inc.,62005,TlsS,61 ac 44 26 84 2b 5a 00 57 54 95 36 f9 ad cc 22 c4 15 ac df 81 2b ab 8a 3e 62 6f 64,core-02,143,China Mobile,1,,31,,356.1805,129.113.22.187,b8 06 47 ae,77.79,00 9e,1185.7048,51.160538,,97934,1780243200.000272,03 03,-161.470317,427201,6928,,143
|
||||
Seoul,45,920,,-144.58538,8.126.102.215,"anon, hosting",48.254.254.95,Singtel,secp256r1,,Alibaba Inc.,17,8.126.102.215,,,zeek,-52.603642,2026-06-01 00:00:00.000273,,370.3709,Tokyo,"anon, hosting",in,,au,Amazon.com Inc.,37180,TlsS,57 78 94 5d 80 06 40 4b 80 ca 59 c9 f4 29 7c 50 05 47 a0 7e 81 51 7b d9 e1 14 e5 f2,core-02,8443,KDDI,56,TLS_AES_256_GCM_SHA384,92,,1685.8327,87.179.3.47,9f 44 65 fc,107.53,cc a9,482.9629,47.091645,,35251,1780243200.000273,03 04,-66.727653,18967,9713,,443
|
||||
Mumbai,1,204,,+,104.173.55.64,"anon, hosting",217.26.47.163,SK Telecom,x448,,Alibaba Inc.,35,104.173.55.64,,,zeek,-22.504628,2026-06-01 00:00:00.000274,cdn.example.com,756.3098,Tokyo,"anon, hosting",nl,cdn.example.com,fi,Alibaba Inc.,71095,TlsS,01 fd f5 bf 03 98 90 cc 65 b1 e9 c1 d5 d4 40 24 79 05 a6 b7 25 83 8f b7 8e 57 53 bd,wan-link,443,SK Telecom,19,TLS_ECDHE_RSA_AES256_GCM_SHA384,211,,685.9072,24.240.178.105,ac 33 64 60,42.85,00 9f,1954.6766,18.09608,,184649,1780243200.000274,03 04,-11.671464,225060,8976,,443
|
||||
Tokyo,19,1484,,113.908059,214.0.155.74,"anon, hosting",207.100.186.111,Comcast,,,Netflix Inc.,185,214.0.155.74,,,netflow,27.840395,2026-06-01 00:00:00.000275,,1.9443,Tokyo,"anon, hosting",in,,kr,Alibaba Inc.,26764,TlsC,0d 0d 78 30 09 7c d5 98 be 2f 13 16 42 22 4e 9b 26 81 0c 2c e5 17 44 86 d8 20 d9 f0,core-02,8080,China Mobile,39,,157,,87.732,136.68.198.63,62 5e 20 90,22.93,c0 2f,1741.7064,-31.111463,,130012,1780243200.000275,03 03,+,154460,3404,,995
|
||||
Seattle,49,1145,,-32.439883,193.126.141.197,"anon, hosting",76.156.228.155,Telstra,,,Netflix Inc.,135,193.126.141.197,,,suricata,1.176714,2026-06-01 00:00:00.000276,stream.example.org,319.6105,Shanghai,"anon, hosting",cn,,de,Microsoft Corp.,27750,TlsC,72 8f 3a 0f ff ce 9f 14 7e 36 b2 f8 a3 a9 62 36 a6 7a c8 47 10 b9 8e 9c 61 78 5d e5,edge-01,12345,SK Telecom,90,,96,,1790.7589,144.167.113.166,a1 80 0f d2,115.64,c0 2c,1298.1129,28.711664,,93191,1780243200.000276,03 03,118.452871,396436,3699,,443
|
||||
Moscow,10,1613,,109.873581,10.240.113.52,"anon, hosting",100.60.134.199,China Mobile,x448,,Samsung,98,10.240.113.52,,,zeek,57.559448,2026-06-01 00:00:00.000277,,460.2942,Singapore,"anon, hosting",gb,graph.facebook.com,au,Baidu Inc.,98048,TlsS,a7 ed 18 92 96 72 0e ed 7d 44 9d c0 d8 f7 83 1e 6d 1d 85 74 33 ed e2 49 85 ce 52 91,edge-01,465,China Telecom,89,,238,,162.2668,196.106.110.209,90 3e 35 62,43.19,c0 27,114.6821,-50.140292,,97508,1780243200.000277,03 01,0.749043,127094,2497,cdn.example.com,22
|
||||
Shanghai,20,542,,12.217836,104.124.156.170,"anon, hosting",51.140.64.161,Comcast,,,Netflix Inc.,144,104.124.156.170,,,zeek,42.12222,2026-06-01 00:00:00.000278,,511.0341,Beijing,"anon, hosting",jp,cdn.example.com,au,Meta Platforms,59786,TlsS,a2 4e cd 41 bc 1e 36 8a 9f 10 5d 62 44 63 b1 bd 75 b2 1c 69 aa 57 0f ac 0e fd a1 17,core-02,8443,AT&T,60,,126,,854.8489,73.198.209.199,2d e6 6c b8,58.15,00 9f,98.5689,24.21318,,71789,1780243200.000278,03 04,-61.39785,298152,5755,auth.example.net,8443
|
||||
Dublin,44,1960,,66.844463,190.226.218.144,"anon, hosting",157.194.172.8,Singtel,,,Netflix Inc.,70,190.226.218.144,,,suricata,+,2026-06-01 00:00:00.000279,,476.8279,Shanghai,"anon, hosting",ru,*.s3.amazonaws.com,in,Google LLC,44733,TlsS,d4 86 ac 9c b5 b5 e9 3b 58 e5 62 37 fc 93 49 e0 36 75 43 4a 8f 0c 52 6d 6a c9 42 f3,backbone-03,443,Singtel,54,,9,,1875.5089,69.72.36.16,ec d1 8a 0f,10.15,c0 13,1609.5093,-17.10149,,60761,1780243200.000279,03 03,-147.310066,316398,4443,,53
|
||||
Berlin,5,913,,175.731662,176.135.250.65,"anon, hosting",144.181.36.102,KDDI,,,Samsung,140,176.135.250.65,,login.live.com,netflow,+,2026-06-01 00:00:00.000280,,3.0252,Singapore,"anon, hosting",nl,auth.example.net,us,Microsoft Corp.,73516,TlsS,74 35 42 f3 62 8c 2b 9f ae 1a 51 e8 bf 0d 4c d2 09 00 1e f1 4d 58 0e f4 2b 50 3e 8e,gw-09,8443,Deutsche Telekom,57,,16,256,394.0277,22.82.139.116,7d a5 d3 13,86.73,13 02,1102.0458,37.187995,,30243,1780243200.00028,03 04,44.170844,416676,6727,,53
|
||||
Beijing,43,765,,151.722766,125.246.109.222,"anon, hosting",112.169.48.230,China Mobile,,,Tencent,19,89.135.22.187,,,netflow,25.783432,2026-06-01 00:00:00.000281,stream.example.org,777.4342,Seattle,"anon, hosting",jp,stream.example.org,gb,Amazon.com Inc.,82426,TlsC,d6 b8 a1 e4 8a 26 2c 1b ca df ea 4d db 94 2e e5 78 e6 a2 6e e8 f2 1a 25 51 8e 6c 38,edge-01,143,BT Group,62,,297,,864.7869,74.37.197.130,88 b6 4b 84,24.51,c0 2c,1179.3882,34.093393,OK,133521,1780243200.000281,03 04,139.194815,437288,5082,,22
|
||||
Tokyo,11,1452,,-0.460918,133.231.61.64,"anon, hosting",133.231.61.64,SK Telecom,,,Microsoft Corp.,2,133.231.61.64,,,packet_capture,52.636963,2026-06-01 00:00:00.000282,,524.1529,Berlin,"anon, hosting",au,,sg,Tencent,43890,TlsC,10 1a 8e 28 f9 9e 91 64 c6 e0 2a c3 7a ea df 5f 28 b7 f4 fa 9a 03 a6 82 bf df db 3e,gw-09,443,Singtel,18,,293,,1266.3528,86.193.233.239,94 ff 9d 22,72.72,c0 14,1427.8005,-54.722807,,45482,1780243200.000282,02 00,-144.247262,307029,5474,,443
|
||||
Mumbai,23,230,,-113.38364,204.20.183.54,"anon, hosting",118.63.193.21,China Mobile,,,Netflix Inc.,182,204.20.183.54,,,netflow,-1.078627,2026-06-01 00:00:00.000283,,371.5307,Berlin,"anon, hosting",us,,gb,,69384,TlsC,44 f9 49 ef b8 99 8d 48 55 02 dc 27 b9 14 d4 57 ef 76 dd 8d 11 1f c2 f5 47 9c d4 dd,backbone-03,110,AT&T,33,,261,128,43.883,170.41.80.62,13 7d 53 2c,29.4,00 9e,1004.0332,-21.521367,,20804,1780243200.000283,03 01,-157.671701,274439,4459,,443
|
||||
Mumbai,11,1103,,-24.61889,70.171.163.229,"anon, hosting",131.59.197.231,KDDI,,,Google LLC,4,70.171.163.229,,,suricata,43.200705,2026-06-01 00:00:00.000284,,283.818,San Francisco,"anon, hosting",au,,br,Microsoft Corp.,42342,TlsC,f6 18 2e 02 43 fd 27 1d a0 41 04 60 e8 51 33 41 3f bb 26 15 8c b4 c3 57 9c 8a d5 5d,core-02,465,China Telecom,63,,142,,108.2548,184.149.139.107,30 c8 ac d9,87.95,00 9f,627.4251,-25.92513,,84781,1780243200.000284,03 03,121.14358,396937,4479,,21
|
||||
London,17,364,,66.002867,8.59.133.46,"anon, hosting",40.155.84.42,Orange,,,Samsung,65,8.59.133.46,,mail.example.com,netflow,9.611727,2026-06-01 00:00:00.000285,*.cloudfront.net,834.1602,Shanghai,"anon, hosting",sg,,sg,Apple Inc.,70737,TlsC,5b ec 1a 47 23 b1 1e 32 6a 82 71 7f 19 df 50 f1 a9 5c 93 e4 f2 60 6e 23 43 16 71 e6,edge-01,53,Comcast,54,,475,,1712.8935,66.148.167.31,38 4b e2 b1,94.88,c0 27,651.0799,-19.249723,,176612,1780243200.000285,03 03,107.509374,2221,1122,login.live.com,50000
|
||||
Mumbai,19,544,Netflix Inc.,-102.024675,90.35.125.95,"anon, hosting",114.227.120.219,Orange,,,Microsoft Corp.,125,90.35.125.95,,,packet_capture,-37.94274,2026-06-01 00:00:00.000286,,413.2127,Paris,"anon, hosting",au,,se,Meta Platforms,91255,TlsC,d3 91 21 ab 97 d2 66 aa 3d ee a0 fd 68 a0 22 bb 37 ff ca c5 b7 b8 d1 a7 d2 06 15 75,edge-01,110,BT Group,10,,246,,1268.1613,183.227.37.231,e5 a1 4e 5d,22.49,c0 2c,396.7223,-9.461114,,85576,1780243200.000286,03 04,-76.664859,120708,3338,,143
|
||||
Seattle,17,1667,,-122.843119,68.202.67.172,"anon, hosting",218.63.153.130,NTT,,,Tencent,23,68.202.67.172,,,suricata,21.873178,2026-06-01 00:00:00.000287,,250.5802,Dublin,"anon, hosting",ca,stream.example.org,ca,Google LLC,83589,TlsC,5c ab da 54 f4 5e c1 c2 4a 6b 49 a7 e5 51 f9 7d 9b d3 54 f2 1c 8a ef 5a 62 7b 81 80,core-02,21,Verizon,91,,388,,1040.3869,40.242.34.24,dc d5 cb 35,117.65,c0 23,942.2798,-10.983037,,103871,1780243200.000287,03 04,-51.84684,89019,5910,,143
|
||||
Shanghai,47,289,,-178.24655,109.141.22.1,"anon, hosting",109.141.22.1,AT&T,,,Baidu Inc.,175,109.141.22.1,00 17,,suricata,6.397583,2026-06-01 00:00:00.000288,auth.example.net,9.6703,Shanghai,"anon, hosting",ru,,gb,Meta Platforms,38140,TlsC,da f7 5f 11 39 5d b2 09 7d 32 e9 f8 a3 cc 07 03 02 35 28 ed c0 af 6c 59 96 9e 7a 83,gw-09,443,Comcast,15,,96,,1687.6884,99.247.54.61,f6 3a be 25,86.01,c0 2f,1246.8659,-25.069346,,2929,1780243200.000288,03 03,-151.362185,384385,7138,,443
|
||||
Moscow,16,1849,Netflix Inc.,-118.572448,116.124.237.146,"anon, hosting",2.254.220.45,China Mobile,,,Samsung,51,116.124.237.146,,,packet_capture,-55.104495,2026-06-01 00:00:00.000289,,117.8048,Beijing,"anon, hosting",us,,fi,Microsoft Corp.,90996,TlsC,ad 6a 0b 90 06 b8 65 27 8f 26 1f af 0e e5 a5 a3 01 84 a3 53 95 d3 fa e4 1f 77 fe 80,wan-link,465,SK Telecom,4,TLS_ECDHE_RSA_AES256_GCM_SHA384,86,,1190.291,80.203.242.65,fb 19 03 74,38.97,c0 13,1859.7649,33.565637,,2139,1780243200.000289,03 03,26.414028,98450,9751,,21
|
||||
Dublin,28,1485,,-30.206558,156.58.83.247,"anon, hosting",176.135.250.65,NTT,,,Meta Platforms,64,156.58.83.247,,,netflow,-51.452799,2026-06-01 00:00:00.000290,,575.3262,Berlin,"anon, hosting",jp,*.s3.amazonaws.com,il,Amazon.com Inc.,61498,TlsC,31 f7 13 4e 30 e6 d8 9c a9 26 f7 ee 14 42 4e 69 02 80 b3 75 9c e7 53 e1 eb 45 a5 61,backbone-03,443,Comcast,18,TLS_ECDHE_ECDSA_AES256_GCM_SHA384,242,,1254.4048,86.164.53.220,1e b4 5d 89,20.98,00 35,1802.3821,-4.233348,,49953,1780243200.00029,03 03,129.784217,427797,4809,cdn.example.com,53
|
||||
Beijing,18,1168,,-49.29488,136.216.108.238,"anon, hosting",83.250.9.29,Comcast,,,Microsoft Corp.,84,136.216.108.238,,,ssl_logs,25.992211,2026-06-01 00:00:00.000291,,796.9375,Sydney,"anon, hosting",de,mail.example.com,ca,,75951,TlsS,94 4b 2b 27 35 ef ad 87 87 96 aa f4 93 55 29 99 5b 99 c1 79 38 bb 26 52 38 3e d0 8b,backbone-03,995,SK Telecom,72,,185,512,5.1981,44.42.255.217,58 80 c6 93,74.39,13 02,1619.2326,-28.038659,,98314,1780243200.000291,03 03,86.470749,495059,2248,,21
|
||||
Berlin,32,1063,,144.141087,60.51.194.72,"anon, hosting",35.135.187.196,SK Telecom,secp256r1,52484,Google LLC,54,60.51.194.72,00 19,,netflow,-21.789468,2026-06-01 00:00:00.000292,,436.8168,Seattle,"anon, hosting",ca,,cn,Microsoft Corp.,50879,TlsC,82 1a 5f f6 54 d4 74 44 39 58 1e c8 28 87 97 48 be 12 ec ca c2 e7 85 53 10 1f 22 92,backbone-03,443,AT&T,77,,432,512,1198.3496,96.129.8.92,8a 45 83 c8,8.59,00 35,547.2479,-47.71497,,151832,1780243200.000292,03 03,-130.034467,346071,8119,,50000
|
||||
New York,10,1559,,9.502908,179.151.65.50,"anon, hosting",89.153.217.177,Singtel,,,Netflix Inc.,88,179.151.65.50,00 17,cdn.example.com,netflow,45.173895,2026-06-01 00:00:00.000293,,935.5702,London,"anon, hosting",cn,,kr,Google LLC,47076,TlsC,b2 77 a4 cf bf 7b ac 50 8d 7d f0 e8 a9 13 e4 d5 b9 a5 ea 72 f8 81 b4 4f a9 26 57 57,core-02,993,China Mobile,34,,220,,325.0549,87.179.3.47,94 d6 ea 1a,76.33,13 01,795.4196,49.785158,,71862,1780243200.000293,03 03,-132.963925,458126,2345,,8080
|
||||
Paris,47,734,Microsoft Corp.,40.669587,146.154.207.141,"anon, hosting",195.194.142.193,Verizon,x448,,Alibaba Inc.,18,146.154.207.141,,,ssl_logs,-2.284425,2026-06-01 00:00:00.000294,mail.example.com,822.5853,Tokyo,"anon, hosting",de,,il,Alibaba Inc.,13734,TlsC,76 1f bf 95 90 2c 9a 7f d6 dc 1b 5c 08 65 2c 71 f1 18 51 22 10 7b d4 f8 d3 ff ea ed,core-02,465,Comcast,87,,370,,1803.5835,74.117.180.57,d5 af e9 c1,13.74,c0 14,1248.8995,25.427807,,182502,1780243200.000294,03 03,109.601395,194954,9520,,12345
|
||||
San Francisco,31,1004,,-45.273796,218.63.153.130,"anon, hosting",148.108.79.169,Deutsche Telekom,,,Apple Inc.,4,218.63.153.130,,,zeek,-38.311598,2026-06-01 00:00:00.000295,,309.5874,Tokyo,"anon, hosting",cn,*.s3.amazonaws.com,de,Google LLC,21508,TlsC,99 b7 2a c4 aa 22 8d a0 c9 96 98 dd 80 a4 49 79 03 0f 84 25 bd 56 9b 4f c0 fd bb a9,edge-01,110,Singtel,29,,329,,1807.4916,53.189.160.106,04 23 f0 f9,89.96,cc a9,1815.2657,-46.601207,,197605,1780243200.000295,03 04,-84.791674,30239,3089,,993
|
||||
Dublin,30,831,,-31.204562,109.93.142.119,"anon, hosting",35.215.191.229,BT Group,,,Microsoft Corp.,4,70.171.163.229,,,netflow,43.634235,2026-06-01 00:00:00.000296,,502.4095,Beijing,"anon, hosting",ru,www.google.com,no,Alibaba Inc.,37011,TlsC,5f 43 85 be 26 4c bf ca ce 22 41 29 4a db e6 9f c6 f8 9b f6 d1 65 2b d9 72 1d b1 52,backbone-03,80,NTT,20,,368,256,222.0819,136.68.198.63,1a 85 68 c7,28.59,c0 23,1946.0802,-45.669774,,35264,1780243200.000296,03 03,-154.16953,47635,6706,mail.example.com,3389
|
||||
New York,33,918,,-130.05346,177.2.54.200,"anon, hosting",113.174.94.248,BT Group,,,Samsung,160,86.210.127.69,,,packet_capture,-51.947118,2026-06-01 00:00:00.000297,,249.5248,Shanghai,"anon, hosting",gb,,il,Alibaba Inc.,75601,TlsS,34 30 53 28 ee f8 0f 6b c8 d9 c5 c9 75 c8 9b f5 59 25 7e 51 d4 a7 8f ac 05 62 66 fb,backbone-03,443,Singtel,20,,169,,1661.1042,86.164.53.220,58 8e 0f 73,85.4,c0 23,685.2649,-10.430544,,33167,1780243200.000297,03 03,-59.457305,434485,9877,,8443
|
||||
Beijing,17,1554,,-32.659479,113.39.146.61,"anon, hosting",153.34.120.104,Deutsche Telekom,,,Microsoft Corp.,104,167.35.20.194,,,zeek,-12.589452,2026-06-01 00:00:00.000298,,443.7983,,"anon, hosting",in,,no,Google LLC,47617,TlsC,af ae 08 11 2e bc b1 cc e3 60 48 6a ba 60 c4 07 3b 91 2c 4e eb 0f d4 c4 36 a7 75 d1,edge-01,465,Comcast,90,,317,256,1705.0019,181.185.39.235,fc 15 e0 45,112.81,13 02,440.4592,37.054516,,17582,1780243200.000298,03 04,79.858728,9349,6340,,443
|
||||
Dublin,40,403,,125.789791,139.41.70.39,"anon, hosting",160.172.21.212,China Mobile,,,Google LLC,100,139.41.70.39,,,ssl_logs,33.549661,2026-06-01 00:00:00.000299,,313.5136,Tokyo,"anon, hosting",gb,,au,Microsoft Corp.,60756,TlsC,06 14 0d 24 1c 11 e6 6b a3 83 c8 e1 76 9e fe 90 7c cd 76 10 8d 35 6b 36 a2 c1 fe 22,core-02,21,Telstra,54,,308,,1496.2302,31.238.157.41,49 37 66 06,75.82,c0 13,332.9382,-23.297395,,152910,1780243200.000299,03 03,-41.00206,33258,1594,auth.example.net,993
|
||||
Tokyo,39,1473,,-18.332506,207.100.186.111,"anon, hosting",139.7.47.238,China Telecom,,,Netflix Inc.,111,207.100.186.111,,,netflow,41.065926,2026-06-01 00:00:00.000300,,419.261,Shanghai,"anon, hosting",kr,*.cloudfront.net,se,Apple Inc.,81230,TlsC,45 11 74 1e 4f e3 a3 e0 8a 11 b7 5f 79 06 6b d5 06 c1 e0 fa f6 38 56 88 f1 59 c1 a5,wan-link,3389,China Telecom,15,,414,,1621.9396,186.152.250.222,ee 20 60 cf,27.85,c0 2f,1064.1554,-7.474362,,46682,1780243200.0003,03 03,-2.414538,169617,5090,mail.example.com,110
|
||||
Seattle,1,739,,-177.279747,31.126.20.159,"anon, hosting",223.170.155.245,Comcast,,,Alibaba Inc.,49,31.126.20.159,,,ssl_logs,16.780345,2026-06-01 00:00:00.000301,,239.0365,London,"anon, hosting",sg,,br,Tencent,11189,TlsC,4a ed 2d c1 38 c7 52 c1 ec 80 f6 d5 5c 6e f0 0c 6b b0 8e f0 e3 5e d3 31 60 3e 4f 5b,core-02,465,Telstra,20,,189,,1686.7816,133.145.40.65,8f 2e 59 72,69.78,cc a9,637.2769,4.249737,,24949,1780243200.000301,03 03,109.475391,465671,5960,,110
|
||||
Paris,33,1516,,87.919122,195.194.142.193,"anon, hosting",220.207.218.132,China Telecom,,,Netflix Inc.,126,195.194.142.193,,,ssl_logs,56.093677,2026-06-01 00:00:00.000302,,612.4832,Dublin,"anon, hosting",fr,,hk,Google LLC,71687,TlsC,cf 9f 19 91 ff 90 29 2e dd 7a 82 71 11 75 30 1f 32 c6 d2 78 c0 dc 5b 49 8c b9 a2 9c,edge-01,143,SK Telecom,54,,31,384,779.5426,161.148.154.124,01 a9 e8 c1,10.04,13 02,838.2592,-3.709263,,108777,1780243200.000302,03 03,-125.190404,54675,5716,,3389
|
||||
Shanghai,2,987,,18.865169,140.154.4.171,"anon, hosting",96.220.74.63,AT&T,,,Samsung,132,140.154.4.171,,,netflow,-40.099872,2026-06-01 00:00:00.000303,,524.5451,Beijing,"anon, hosting",ua,,jp,Amazon.com Inc.,17500,TlsS,5c 24 b6 5c 18 b5 47 38 e1 c8 01 3d 50 d5 dc 55 ab 17 a7 1d f4 74 45 8e 20 76 97 7a,gw-09,995,AT&T,76,,182,,1192.6522,117.189.214.151,56 bb 17 56,35.83,cc a9,309.5653,35.956884,--,178140,1780243200.000303,03 04,91.347482,397356,1333,,53
|
||||
Singapore,26,1700,,-159.10165,218.37.233.107,"anon, hosting",67.228.109.157,Deutsche Telekom,x25519,33875,Baidu Inc.,75,218.37.233.107,00 1d,,netflow,-24.937863,2026-06-01 00:00:00.000304,,366.6949,Beijing,"anon, hosting",ua,cdn.example.com,fr,Samsung,43286,TlsC,36 00 62 df 6b 4e b6 43 90 02 9c 6e 7e ba 3b cd 09 46 ca a8 f7 e1 bf 43 dc d6 94 1a,backbone-03,50000,Orange,60,,264,,1226.9396,145.229.165.222,32 da 64 cc,29.67,c0 23,1912.6806,-14.586533,,70258,1780243200.000304,03 04,137.392575,187840,1494,,25
|
||||
New York,27,1473,,140.546567,207.100.186.111,"anon, hosting",162.122.81.80,Telstra,,,Alibaba Inc.,160,207.100.186.111,,,netflow,-56.19636,2026-06-01 00:00:00.000305,,278.1003,Seattle,"anon, hosting",jp,login.live.com,no,Google LLC,22514,TlsC,e9 e8 22 d9 33 c6 a3 ec 5c 76 d2 7e 9d 23 3c 10 87 a9 e1 d6 e6 38 4a c5 a7 f4 51 22,wan-link,443,Verizon,16,,347,,1556.9953,207.151.173.206,49 3c 55 fb,105.56,c0 2c,387.4088,16.695265,,138991,1780243200.000305,03 04,127.975342,32554,5182,,143
|
||||
Beijing,39,960,,3.142489,218.63.153.130,"anon, hosting",100.214.112.125,China Mobile,,,Amazon.com Inc.,43,200.248.122.71,,,suricata,40.297434,2026-06-01 00:00:00.000306,,458.5192,New York,"anon, hosting",jp,stream.example.org,nl,Microsoft Corp.,55579,TlsC,7c f7 22 1e a5 60 49 b5 1c dc fd eb 3d a5 63 cd 01 36 f3 94 72 0a 83 1f 50 b3 c1 1e,gw-09,465,,60,,221,,1915.0125,53.250.46.231,3d 66 32 d1,19.11,00 35,1746.8185,52.757822,,185337,1780243200.000306,03 04,121.073508,93758,8182,,443
|
||||
Dublin,12,1822,,56.154299,177.124.52.179,"anon, hosting",16.168.62.106,AT&T,,,Meta Platforms,48,177.124.52.179,,,ssl_logs,-21.081379,2026-06-01 00:00:00.000307,*.cloudfront.net,546.1628,San Francisco,"anon, hosting",gb,,kr,Apple Inc.,21454,TlsS,3a d2 8e 71 ab 72 55 de 25 ee 06 bb 22 7a 10 aa e7 72 5f 3b 72 c1 e3 c5 97 b6 aa be,backbone-03,53,KDDI,5,,149,,1545.108,133.145.40.65,38 a1 3d 0f,113.58,00 9f,1347.5868,-21.243404,,104558,1780243200.000307,03 04,84.918069,446298,6666,,50000
|
||||
Dublin,28,1594,,-141.404001,193.37.4.118,"anon, hosting",103.218.55.126,Deutsche Telekom,,,Amazon.com Inc.,135,193.37.4.118,,,ssl_logs,3.363058,2026-06-01 00:00:00.000308,,912.5901,Beijing,"anon, hosting",gb,,ru,Netflix Inc.,16759,TlsS,8b 16 80 7e e5 9a 3d 45 cd 38 f9 7f 04 07 09 a5 06 a8 15 3d 89 08 62 6e 90 0c ae 11,edge-01,25,Singtel,94,,36,,1403.609,184.249.18.159,39 18 ad c0,71.16,c0 2c,836.6213,58.747683,,48433,1780243200.000308,03 04,-141.069428,333841,7496,,21
|
||||
London,14,291,,135.229739,8.199.172.172,"anon, hosting",55.135.67.239,China Telecom,,84512,Samsung,35,8.199.172.172,,,zeek,24.773099,2026-06-01 00:00:00.000309,,905.7749,Paris,"anon, hosting",il,login.live.com,ru,Netflix Inc.,69552,TlsS,41 73 0b f1 4b 8a fa c5 82 30 5f d3 32 6f ba b9 fa 80 a4 cb 3c 06 a5 66 7d 9f 12 70,gw-09,3389,Verizon,5,,224,,278.5738,114.212.139.56,8e 48 f2 b4,21.74,00 9e,472.6767,-13.725691,,156836,1780243200.000309,03 03,22.874036,246621,1290,,80
|
||||
San Francisco,38,1002,,-89.883795,210.235.179.79,"anon, hosting",120.65.13.233,Verizon,secp256r1,,Microsoft Corp.,185,210.235.179.79,00 1d,,suricata,29.001511,2026-06-01 00:00:00.000310,stream.example.org,900.2155,Tokyo,"anon, hosting",ua,stream.example.org,ua,Samsung,41633,TlsC,e4 f0 f3 b1 71 60 90 48 44 a6 6c 73 42 44 87 41 b8 fc f4 e6 5c e9 e9 81 b9 e4 ad 18,gw-09,22,Verizon,50,,259,,353.5156,134.81.186.96,36 b0 5f 2f,28.02,00 9f,1516.4825,-32.344661,,186616,1780243200.00031,03 04,124.626381,149811,3910,,50000
|
||||
San Francisco,34,358,,-27.892871,73.80.224.214,"anon, hosting",73.229.119.137,Comcast,,,Meta Platforms,47,60.51.194.72,,,netflow,-11.876595,2026-06-01 00:00:00.000311,,659.23,Berlin,"anon, hosting",de,*.s3.amazonaws.com,hk,Amazon.com Inc.,40803,TlsC,b4 95 fc ed 2b f5 4e 45 0a 16 3f 0b 0f 31 82 5b d0 d5 1d ca 5d a3 2b a4 fa a5 ae 37,wan-link,25,Deutsche Telekom,96,,345,,586.7034,117.74.196.49,15 cb e3 32,83.51,c0 2b,1384.8365,-1.510114,,71638,1780243200.000311,03 02,-109.187256,222256,5194,,443
|
||||
San Francisco,46,1561,,-118.699815,55.32.212.105,"anon, hosting",131.228.28.117,Orange,,,Meta Platforms,183,55.32.212.105,,,packet_capture,32.165583,2026-06-01 00:00:00.000312,,393.8656,Mumbai,"anon, hosting",ca,,ca,Google LLC,75723,TlsC,19 6f 4e b2 64 c6 be 49 b9 a6 e4 c2 fa 69 a3 05 f2 2f 1c ff 5a 72 98 ec c5 89 d4 24,wan-link,22,Singtel,95,,164,256,1210.7062,31.213.115.199,c1 2d 85 68,47.64,00 9e,1777.4593,30.08384,,58548,1780243200.000312,03 03,-47.429168,8855,1908,,50000
|
||||
Sydney,43,899,,-133.671329,218.63.153.130,"anon, hosting",118.39.160.148,NTT,,,Meta Platforms,63,85.238.212.16,,,netflow,1.11774,2026-06-01 00:00:00.000313,,943.2208,Tokyo,"anon, hosting",se,,in,Meta Platforms,26109,TlsS,7f de 93 26 b8 ff f6 2a 45 22 88 58 bf 20 b4 d3 2b f2 df ba ac 50 4f 70 bc b9 42 a5,edge-01,25,Verizon,20,,255,,1452.6731,211.250.251.118,0d df 63 02,95.79,00 9e,1605.8578,-34.542625,,37775,1780243200.000313,03 03,-109.787111,215127,1160,,8443
|
||||
San Francisco,23,1476,Netflix Inc.,141.899285,95.75.220.33,"anon, hosting",97.168.65.172,Comcast,,,Apple Inc.,186,95.75.220.33,,,packet_capture,-42.252357,2026-06-01 00:00:00.000314,,747.508,New York,"anon, hosting",ru,,sg,Alibaba Inc.,67034,TlsS,3a 26 b0 00 5c 6f f8 94 69 ff 7d 93 a0 19 a5 54 3b 94 04 a3 01 81 f2 42 d5 0d f4 da,edge-01,143,Comcast,34,,224,,1228.3812,220.207.218.132,26 8a b5 f9,72.98,c0 27,1054.7792,-2.458597,,197068,1780243200.000314,03 04,105.556949,422901,7394,www.google.com,3389
|
||||
Shanghai,3,1167,Microsoft Corp.,-80.190335,160.51.37.138,"anon, hosting",105.59.71.12,Deutsche Telekom,,,Meta Platforms,176,160.51.37.138,,,ssl_logs,51.066999,2026-06-01 00:00:00.000315,,277.4414,San Francisco,"anon, hosting",nl,,in,Microsoft Corp.,33850,TlsS,34 44 60 77 7e 1c d0 ac 79 67 be 63 bc f1 a0 0a ca 50 5f 3d ce 00 fa fa a6 f0 4a f3,core-02,25,China Telecom,99,TLS_ECDHE_ECDSA_CHACHA20_POLY1305,258,,1859.936,74.251.102.32,ee d9 7e 18,101.54,c0 13,1582.1355,8.598311,,149317,1780243200.000315,03 03,-75.438047,359923,8232,*.s3.amazonaws.com,443
|
||||
Berlin,48,332,,86.673575,105.250.123.177,"anon, hosting",76.98.89.157,NTT,,,Apple Inc.,169,105.250.123.177,,,packet_capture,56.849465,2026-06-01 00:00:00.000316,,355.7264,Tokyo,"anon, hosting",fr,www.google.com,nl,Meta Platforms,33440,TlsS,d4 e5 5d 21 41 36 a2 c6 7c e6 0e 5d f7 aa 72 e6 4f 39 93 57 9e 2b 72 f2 34 f5 7b 01,gw-09,443,SK Telecom,57,,203,,1333.7502,192.212.89.52,da 62 12 be,93.51,13 01,798.782,32.330819,,32506,1780243200.000316,03 04,-54.287744,489902,1195,cdn.example.com,443
|
||||
Sydney,47,508,,-51.734195,109.16.15.24,"anon, hosting",25.201.191.204,,,,Baidu Inc.,14,109.16.15.24,00 17,,netflow,-54.948937,2026-06-01 00:00:00.000317,,273.2252,New York,"anon, hosting",kr,,jp,Alibaba Inc.,37624,TlsS,46 42 f4 0b cf 13 c4 9e 9d 7b aa df b2 14 73 c9 6c 6f ce 95 f9 47 54 9b b9 a6 31 f8,edge-01,21,NTT,15,,19,,1459.5328,188.53.170.35,71 78 b2 21,73.7,c0 23,424.226,19.627808,,106785,1780243200.000317,03 04,133.80525,372473,9224,login.live.com,8443
|
||||
Dublin,46,388,,-158.42624,60.196.78.181,"anon, hosting",95.75.220.33,Verizon,,,Meta Platforms,44,60.196.78.181,,,packet_capture,7.688276,2026-06-01 00:00:00.000318,,475.2225,Sydney,"anon, hosting",de,,de,,35071,TlsS,70 2d ac fd 30 d9 12 8e 58 c3 2a 87 81 e5 2c d5 aa 12 87 86 35 bd 5e dc 85 91 b0 49,gw-09,143,China Mobile,35,,414,,493.8145,32.24.122.108,d4 92 bc 83,60.96,00 9f,651.2002,-55.545026,,35499,1780243200.000318,03 03,2.020241,471510,3031,,12345
|
||||
Shanghai,26,1875,,-10.849589,91.107.136.180,"anon, hosting",104.139.252.113,NTT,,,Meta Platforms,57,91.107.136.180,,,packet_capture,-5.84376,2026-06-01 00:00:00.000319,,95.8359,New York,"anon, hosting",in,,fr,Microsoft Corp.,19432,TlsS,db e0 95 de ab 3f 95 c5 e4 ea 96 4d 94 89 63 25 67 d5 fa b1 fd 50 ce d1 85 02 a9 75,gw-09,995,AT&T,16,,349,,379.6338,18.120.147.59,d8 e0 6b 5f,54.3,c0 14,638.4771,-41.826461,,63886,1780243200.000319,03 03,-92.216604,493763,6986,cdn.example.com,143
|
||||
London,12,1829,,-7.175695,140.154.4.171,"anon, hosting",38.70.19.74,Telstra,x448,,Tencent,84,140.154.4.171,,,ssl_logs,13.625953,2026-06-01 00:00:00.000320,,117.2042,San Francisco,"anon, hosting",jp,,au,Alibaba Inc.,97918,TlsC,20 13 f4 b5 5e d8 ca d6 bf 19 6d d6 2f c8 53 3a 94 15 e2 73 f1 64 1b e8 2a 8f 65 09,backbone-03,465,Deutsche Telekom,23,,320,512,1068.9879,142.7.208.24,05 4f 0d 1a,112.32,13 01,344.3955,-21.230619,,134754,1780243200.00032,03 02,29.496399,242340,2837,www.google.com,443
|
||||
Seoul,41,131,,-20.231742,140.154.4.171,"anon, hosting",150.223.251.249,Verizon,,,Baidu Inc.,106,140.154.4.171,,,suricata,-42.622737,2026-06-01 00:00:00.000321,,762.4098,Tokyo,"anon, hosting",nl,,ua,Google LLC,30617,TlsS,01 90 62 6b 79 38 bf d7 d7 87 09 3b 18 2a 9f cf 0f 2e 66 6a a7 5f 03 de 55 59 c4 1b,wan-link,443,BT Group,97,TLS_ECDHE_RSA_AES128_SHA256,354,,1204.5288,103.225.197.87,6a 27 c6 8d,108.54,00 9f,633.555,+,,49986,1780243200.000321,03 04,46.853747,131297,4813,,53
|
||||
Dublin,6,826,,-66.272955,86.66.134.42,"anon, hosting",81.130.191.245,China Telecom,,,Alibaba Inc.,8,86.66.134.42,,,ssl_logs,-4.249216,2026-06-01 00:00:00.000322,,213.2172,Paris,"anon, hosting",fr,,sg,Amazon.com Inc.,95191,TlsC,3b cd 7f 32 f4 36 a2 3a 2f 22 84 53 86 3b 50 95 9f ba 9c d5 e1 6f 26 68 e3 26 f0 87,edge-01,50000,Comcast,55,,46,,774.5181,65.26.246.224,00 d1 cd a4,1.39,00 9e,1753.7994,58.203489,,23892,1780243200.000322,03 03,148.566271,352575,2045,,8080
|
||||
Beijing,28,570,,-58.341598,59.148.40.219,"anon, hosting",84.28.150.92,Singtel,,,Samsung,20,59.148.40.219,,,zeek,-42.143042,2026-06-01 00:00:00.000323,,856.5521,London,"anon, hosting",kr,,gb,Apple Inc.,51912,TlsS,fd 92 d9 83 a6 60 77 32 dd 8a a1 04 dc 11 a5 93 34 31 6d f3 35 72 94 d9 7b 35 06 fe,wan-link,3389,AT&T,47,,251,512,997.1202,19.30.80.113,17 f7 db a4,13.22,c0 23,1404.5666,15.349624,,166298,1780243200.000323,03 03,-79.532038,343289,8924,login.live.com,21
|
||||
Seoul,30,208,Alibaba Inc.,40.109091,198.10.194.123,"anon, hosting",139.41.70.39,Singtel,,,Microsoft Corp.,35,198.10.194.123,,api.example.com,netflow,1.262699,2026-06-01 00:00:00.000324,,111.9333,San Francisco,"anon, hosting",au,,kr,Netflix Inc.,94016,TlsC,67 2d 3e 4c 3c eb aa 47 4c 52 6b 7b 29 5e a8 c7 f8 ff 46 b1 01 6d b0 db 3f d9 59 eb,backbone-03,3389,Comcast,43,,159,,24.719,165.135.107.197,85 2f 61 8e,84.09,c0 2c,70.3376,47.032754,,47259,1780243200.000324,03 04,11.786723,470162,6406,,993
|
||||
Tokyo,1,1625,,-114.710445,105.250.123.177,"anon, hosting",155.107.175.53,AT&T,,,Samsung,142,105.250.123.177,,,netflow,42.73225,2026-06-01 00:00:00.000325,,856.3218,London,"anon, hosting",ca,auth.example.net,br,Apple Inc.,13595,TlsC,a4 f2 1e 9b 6a 35 7f f5 06 b4 e1 69 f5 56 3f 73 39 d9 5f 47 c0 1b 03 6b e4 22 be 36,gw-09,993,BT Group,27,,32,,1150.1509,35.135.187.196,26 e5 9a 8a,88.98,00 35,613.0918,27.514174,,32422,1780243200.000325,03 03,146.285574,161988,3036,,80
|
||||
Paris,43,1875,,+,139.26.179.58,"anon, hosting",192.212.89.52,Telstra,,,Baidu Inc.,107,139.26.179.58,,,netflow,-30.611651,2026-06-01 00:00:00.000326,,621.3983,Sydney,"anon, hosting",kr,www.google.com,ru,,67073,TlsC,75 f9 fc 27 ee 04 6e f8 6c 0e d4 31 46 1a 07 92 bd 42 17 f4 49 31 c8 5d 8a e5 98 41,core-02,995,Singtel,78,TLS_ECDHE_RSA_AES256_GCM_SHA384,320,384,259.6418,157.163.83.101,8a f4 7c 4b,3.38,00 9f,1973.2368,29.227591,,57468,1780243200.000326,03 02,-105.997811,392387,5309,,143
|
||||
Shanghai,5,1396,,-154.022074,187.125.83.119,"anon, hosting",135.10.199.224,Orange,,,Microsoft Corp.,170,122.58.111.120,,,zeek,-29.418851,2026-06-01 00:00:00.000327,*.cloudfront.net,21.5343,Paris,"anon, hosting",ru,,kr,Baidu Inc.,23306,TlsS,02 19 06 14 96 13 0b 95 d3 82 b7 80 ec c5 6c 97 e1 b3 57 b8 d0 09 90 48 90 ca d9 44,gw-09,443,Comcast,99,,31,,235.8859,93.50.188.140,d8 1e 2a 62,107.15,c0 2b,1203.7231,-57.707417,,100778,1780243200.000327,03 03,102.670963,57268,4091,,80
|
||||
,23,1718,,147.416004,120.65.13.233,"anon, hosting",74.159.180.27,China Mobile,,,Baidu Inc.,188,120.65.13.233,,stream.example.org,packet_capture,9.030516,2026-06-01 00:00:00.000328,,846.8319,Moscow,"anon, hosting",fi,,ua,,96713,TlsS,ad 82 cf 78 4f 94 fd 09 f4 7c f4 cb f4 d6 82 f0 69 e7 17 a4 7f 8c 4b b3 39 33 e8 3e,edge-01,22,AT&T,94,,176,,1297.6379,83.96.122.147,e4 9c e2 de,14.9,00 9f,897.2006,56.569093,,95725,1780243200.000328,03 03,-69.930215,260360,7810,stream.example.org,22
|
||||
San Francisco,44,570,,-82.264657,125.246.109.222,"anon, hosting",73.253.240.74,Telstra,x25519,,Google LLC,122,125.246.109.222,,,ssl_logs,-37.950705,2026-06-01 00:00:00.000329,auth.example.net,333.6729,Seoul,"anon, hosting",nl,auth.example.net,br,Netflix Inc.,28602,TlsS,ae 6e 05 e6 52 f0 f0 42 49 f3 c7 a7 fe 1e 8b 50 da ce 0b 78 d1 d8 0f 5f ed de 9c 87,core-02,443,Deutsche Telekom,45,,216,,1128.9383,2.210.6.5,92 64 b8 17,104.55,00 9e,224.2809,-19.408322,,137857,1780243200.000329,03 04,75.089637,356901,8870,,22
|
||||
London,16,1341,Samsung,-150.440762,31.126.20.159,"anon, hosting",102.7.135.138,BT Group,,,Microsoft Corp.,66,31.126.20.159,,,ssl_logs,-42.589776,2026-06-01 00:00:00.000330,,707.4004,Shanghai,"anon, hosting",kr,login.live.com,ru,Tencent,90815,TlsS,1e bc c5 dd dd 81 4e 6f 94 60 6a f6 b6 92 51 99 c5 7c f2 f5 7b 9c a6 1b 7c 04 5b e5,backbone-03,80,KDDI,65,,346,512,1022.5423,182.224.224.135,45 28 e8 68,53.5,c0 14,288.0224,-48.493414,,72955,1780243200.00033,03 04,111.499427,377003,9820,mail.example.com,22
|
||||
Seattle,18,553,,-139.031117,165.204.169.241,"anon, hosting",47.109.174.243,SK Telecom,,,Microsoft Corp.,22,165.204.169.241,,,suricata,-21.204049,2026-06-01 00:00:00.000331,,341.3915,San Francisco,"anon, hosting",ca,auth.example.net,fi,Netflix Inc.,24746,TlsS,2e 70 f4 1d 17 21 ec 8c 61 2f e6 b8 d4 0e ff 77 03 5e 35 e3 6d e8 6d 78 f6 16 2a 02,wan-link,3389,BT Group,39,,304,,806.3171,179.245.93.208,94 57 d3 50,0.51,c0 2f,1301.0058,-56.357941,,137520,1780243200.000331,03 03,-71.22857,498333,4971,,8443
|
||||
London,15,783,,103.945127,116.124.237.146,"anon, hosting",121.22.198.252,NTT,,,Amazon.com Inc.,83,80.192.203.242,,,packet_capture,-14.078105,2026-06-01 00:00:00.000332,api.example.com,472.4078,New York,"anon, hosting",sg,api.example.com,ua,Samsung,20528,TlsS,a9 b5 fa 15 57 cd fb c3 a6 17 86 8a 94 2e 99 8e 46 85 4a cf fc 09 1f b1 16 21 a8 3f,edge-01,8443,China Telecom,59,,41,512,22.2087,32.24.122.108,d4 5a d2 81,14.41,c0 27,156.5401,6.969015,,1562,1780243200.000332,03 03,-47.126158,223513,9098,,21
|
||||
Seattle,25,1173,,-79.060265,157.194.172.8,"anon, hosting",4.108.97.247,SK Telecom,secp384r1,,Microsoft Corp.,141,101.13.42.165,,,packet_capture,22.573966,2026-06-01 00:00:00.000333,,132.7537,Beijing,"anon, hosting",de,,nl,Apple Inc.,45157,TlsS,17 b0 89 92 ee c8 0e 2e 5a 54 af cf 42 1f 6e d4 31 07 13 4c b7 fb 13 23 ce 56 18 cb,core-02,8443,Comcast,47,,413,,1083.8334,183.227.44.208,ea 1f 73 be,90.52,c0 27,1747.0708,49.391023,,108457,1780243200.000333,03 03,142.310598,448928,1321,,465
|
||||
Moscow,8,240,,39.8164,145.21.177.137,"anon, hosting",211.92.150.83,China Mobile,,,Amazon.com Inc.,24,145.21.177.137,,,packet_capture,-53.887334,2026-06-01 00:00:00.000334,,345.5265,Seattle,"anon, hosting",ca,mail.example.com,cn,Google LLC,10424,TlsC,86 f0 07 40 fa 0c 51 e9 d9 86 bc 3a 60 02 18 a4 19 3c a7 fb a8 98 f6 6b 00 90 5c f8,backbone-03,50000,KDDI,4,,81,512,162.1608,52.21.161.239,c3 51 54 a6,64.58,13 02,1106.1626,-3.902579,,97039,1780243200.000334,03 04,118.92308,399062,2668,www.google.com,443
|
||||
,30,1462,,-67.342413,151.117.112.2,"anon, hosting",36.121.252.7,Telstra,,,Baidu Inc.,108,151.117.112.2,,,zeek,58.615789,2026-06-01 00:00:00.000335,,379.5343,Tokyo,"anon, hosting",gb,,kr,Baidu Inc.,13793,TlsC,61 dc 98 6a da b3 63 2b 9c a1 8f be d7 3c bc 10 a3 37 96 c3 77 37 e5 f5 d2 74 7a 17,edge-01,12345,BT Group,94,,254,,12.5297,149.21.230.233,04 8f 4f 6b,0.99,13 02,491.0726,-43.571576,,7202,1780243200.000335,03 04,-154.973331,480923,4318,,80
|
||||
Seoul,34,1889,,-163.098102,195.194.142.193,"anon, hosting",117.252.68.218,Singtel,,,Meta Platforms,6,195.194.142.193,,,ssl_logs,27.229459,2026-06-01 00:00:00.000336,,841.8641,Singapore,"anon, hosting",no,api.example.com,se,Alibaba Inc.,51947,TlsC,0b d8 26 e6 44 50 92 33 e9 e2 04 6f 3f 15 1f 6c 87 02 b1 84 83 e9 16 8b 7e 62 82 d5,gw-09,21,BT Group,68,,453,256,1271.1928,184.163.176.250,a2 c1 ea 45,50.99,c0 23,1281.0502,-1.638765,,78125,1780243200.000336,03 03,-121.165015,380203,2315,,8080
|
||||
Sydney,50,124,,58.393456,165.204.169.241,"anon, hosting",10.155.252.30,China Telecom,,,Tencent,39,165.204.169.241,,,ssl_logs,41.09992,2026-06-01 00:00:00.000337,,79.2021,London,"anon, hosting",us,www.google.com,br,Netflix Inc.,61165,TlsC,99 4c 86 ab d8 cd e8 59 5b 1c ed 52 45 d7 f1 8a 16 dd 0c 32 a0 85 35 74 53 e0 77 d0,edge-01,143,China Mobile,7,TLS_ECDHE_RSA_AES256_GCM_SHA384,206,512,538.1016,202.34.176.62,6f 79 45 67,50.84,c0 23,1119.5611,-1.706525,,113013,1780243200.000337,03 03,-56.539926,257422,7759,auth.example.net,22
|
||||
San Francisco,21,1021,,-45.534109,131.59.197.231,"anon, hosting",91.107.136.180,Deutsche Telekom,secp384r1,,Microsoft Corp.,71,166.153.234.81,,,ssl_logs,32.895342,2026-06-01 00:00:00.000338,cdn.example.com,69.7526,Mumbai,"anon, hosting",no,,se,Amazon.com Inc.,42256,TlsC,ce 21 3c fd 2f 6d 81 00 4b b7 11 0b d0 8a 5a 4b ea d7 83 7d 9e 9c d3 ad 47 dc b6 1f,backbone-03,25,Deutsche Telekom,52,,307,384,1122.3189,99.247.54.61,62 91 ff fb,105.33,13 02,1127.9963,+,,144130,1780243200.000338,03 04,-36.049086,31308,1335,,465
|
||||
Dublin,6,1416,,-43.462704,189.52.44.152,"anon, hosting",169.98.204.85,BT Group,,,Microsoft Corp.,117,189.52.44.152,,,zeek,-28.839193,2026-06-01 00:00:00.000339,,94.6611,Berlin,"anon, hosting",cn,,us,Tencent,13704,TlsS,f0 19 5c 91 24 00 af ba f0 2d b9 85 75 87 b3 c1 b3 3b 4b 72 92 68 87 8c 1c ef 0d 0a,gw-09,8080,Orange,60,,64,,327.9126,222.46.196.119,6e 24 7a a8,102.9,c0 2c,1588.5021,-30.284343,,194722,1780243200.000339,03 04,-136.403337,161978,1954,,12345
|
||||
Tokyo,9,1358,,110.339573,195.194.142.193,"anon, hosting",38.131.101.45,SK Telecom,,,Amazon.com Inc.,85,195.194.142.193,,,packet_capture,5.441683,2026-06-01 00:00:00.000340,,4.9986,San Francisco,"anon, hosting",au,,fi,Tencent,60210,TlsS,4e 54 cc 4b 09 33 d6 b4 35 a1 a0 54 f6 91 e4 fc 32 b8 a9 72 62 68 24 e4 74 24 6d 98,core-02,443,Comcast,79,,289,,1633.0025,163.97.128.174,a8 c4 20 d8,94.41,00 35,1830.5771,43.963885,--,138602,1780243200.00034,03 04,-99.168869,243503,9125,login.live.com,443
|
||||
London,47,762,,-177.716748,213.215.129.214,"anon, hosting",172.115.119.14,Deutsche Telekom,,,Tencent,84,213.215.129.214,,,zeek,8.963925,2026-06-01 00:00:00.000341,stream.example.org,746.0495,Dublin,"anon, hosting",no,,no,Tencent,17145,TlsC,26 b5 e6 75 d3 a9 02 e3 1e a4 38 4f b2 2d 3c 0f 62 2d de e7 42 09 f4 b5 35 8c 8e bf,backbone-03,8443,SK Telecom,1,,194,,923.4144,80.203.245.135,27 d0 ce eb,0.68,00 9e,1285.154,6.343661,,48693,1780243200.000341,03 03,155.588574,486801,3488,,22
|
||||
San Francisco,5,399,,-14.356383,50.35.23.170,"anon, hosting",93.50.188.140,SK Telecom,,,Tencent,165,50.35.23.170,,,suricata,40.782372,2026-06-01 00:00:00.000342,,446.0778,Shanghai,"anon, hosting",ca,www.google.com,au,Meta Platforms,68901,TlsS,,wan-link,80,Orange,40,,411,,769.9041,57.230.178.239,e4 81 5f 77,71.05,c0 23,1453.9197,-45.790383,,30419,1780243200.000342,03 03,50.32275,438132,2950,www.google.com,8443
|
||||
Seoul,7,190,,131.534368,85.238.212.16,"anon, hosting",64.72.49.13,Telstra,,,Microsoft Corp.,147,85.238.212.16,,,netflow,15.116833,2026-06-01 00:00:00.000343,,419.3449,Seoul,"anon, hosting",kr,graph.facebook.com,se,Apple Inc.,62889,TlsS,,backbone-03,110,Telstra,47,,302,,870.9724,38.66.128.205,05 82 f5 f8,10.48,c0 14,1372.8329,-48.825191,,136395,1780243200.000343,03 04,-153.847959,332424,1164,login.live.com,80
|
||||
Dublin,9,1570,,92.913985,168.255.202.227,"anon, hosting",187.27.50.16,Telstra,,,Meta Platforms,6,168.255.202.227,,,ssl_logs,13.175232,2026-06-01 00:00:00.000344,,60.4874,Tokyo,"anon, hosting",fi,auth.example.net,fr,Samsung,27783,TlsC,4c 70 4b 8e 78 80 4a 31 68 b2 f7 dd a6 df 54 03 fb 65 72 bf 09 06 d7 84 a6 a6 16 9c,edge-01,8443,Deutsche Telekom,59,,386,,1279.9495,73.253.240.74,57 95 15 f3,69.4,cc a9,1903.5403,-58.329035,--,196486,1780243200.000344,03 03,-92.803219,199706,7063,api.example.com,110
|
||||
Seoul,1,1870,,45.454874,198.154.61.204,"anon, hosting",4.108.97.247,Deutsche Telekom,x25519,,Alibaba Inc.,53,198.154.61.204,,,packet_capture,51.211383,2026-06-01 00:00:00.000345,,546.2985,Seattle,"anon, hosting",sg,stream.example.org,ca,Samsung,64263,TlsS,69 8f d0 83 89 43 c5 3e 6b 1d e5 26 c0 16 dd 67 86 ae 26 12 c4 66 4d 48 24 c3 4a c1,wan-link,443,Deutsche Telekom,77,,321,,1162.9598,184.249.18.159,5e f8 bf fd,72.02,13 02,147.5631,-46.347289,,170627,1780243200.000345,03 03,-124.312628,88338,1420,*.cloudfront.net,53
|
||||
Beijing,5,1841,,-120.192601,207.100.186.111,"anon, hosting",188.53.170.35,BT Group,secp256r1,,Netflix Inc.,80,207.100.186.111,,,ssl_logs,-49.020471,2026-06-01 00:00:00.000346,,194.9684,Dublin,"anon, hosting",il,graph.facebook.com,ca,Microsoft Corp.,60792,TlsC,c5 b6 e1 9e c6 03 6f 1d 43 6c 36 44 24 fa 3a e5 08 9d 5a 7d 47 fc a1 2f 28 a2 55 0c,edge-01,25,BT Group,12,,236,512,1677.9845,218.224.17.33,0c 85 3b c3,41.62,c0 13,129.8067,-26.487178,,11246,1780243200.000346,03 04,163.31973,334293,5542,login.live.com,50000
|
||||
Mumbai,14,1384,,117.730269,196.172.52.24,"anon, hosting",208.96.164.155,BT Group,secp384r1,,Microsoft Corp.,5,196.172.52.24,,,netflow,-42.020437,2026-06-01 00:00:00.000347,"*.['google', 'com']",914.8574,Seoul,"anon, hosting",jp,,sg,Meta Platforms,55751,TlsC,ad d6 d4 ac 7a 79 19 71 c4 73 b9 f6 8e 66 71 33 a3 09 05 0e 98 34 c8 48 c1 f9 b8 16,wan-link,8080,Comcast,1,,487,384,1720.8356,127.24.176.242,9e dc 7b 3e,108.53,c0 14,323.5119,40.188639,,94669,1780243200.000347,03 03,171.167057,55539,2836,,110
|
||||
Mumbai,12,1607,,166.976464,206.237.162.244,"anon, hosting",116.124.237.146,Orange,,,Alibaba Inc.,25,206.237.162.244,,,suricata,43.258906,2026-06-01 00:00:00.000348,,743.0027,Mumbai,"anon, hosting",ca,stream.example.org,ru,Netflix Inc.,56005,TlsC,97 ac 6f 20 97 ba bc b5 07 1a 19 4a 09 da b3 9c f0 64 a1 d9 82 45 e5 47 82 68 d3 2f,backbone-03,443,KDDI,35,,86,,340.4296,21.191.114.8,0c 51 85 a5,50.49,c0 23,1831.6025,-42.881255,,29550,1780243200.000348,03 03,-11.154475,42280,8358,,8080
|
||||
Paris,2,673,Google LLC,-11.337448,50.35.23.170,"anon, hosting",188.53.170.35,Comcast,,,Samsung,142,176.135.250.65,00 18,,packet_capture,-58.12636,2026-06-01 00:00:00.000349,,348.3497,San Francisco,"anon, hosting",se,,cn,Tencent,93065,TlsS,49 b0 ab 36 80 7a 4e a5 aa 48 05 7e 5f d1 ac 3c 3e 12 b2 5f 69 15 ad 63 07 dd fa 70,backbone-03,25,KDDI,56,,251,,1157.7004,86.86.39.131,73 1b 74 94,8.29,c0 2f,708.637,-41.370873,,32348,1780243200.000349,03 04,28.844279,477982,5025,www.google.com,993
|
||||
Seoul,39,201,,-10.515148,195.194.142.193,"anon, hosting",182.73.237.235,SK Telecom,,,Tencent,27,195.194.142.193,,,packet_capture,-59.107939,2026-06-01 00:00:00.000350,,868.177,Seattle,"anon, hosting",il,,no,Amazon.com Inc.,79404,TlsS,e2 80 6e ec 7f 17 22 6f 26 86 62 93 26 60 b0 f6 3d df 49 d3 3f 21 37 92 ec c3 57 0c,core-02,465,Telstra,65,,263,384,1185.3145,186.57.16.208,82 ec f6 75,104.56,c0 2f,463.557,9.961577,,50954,1780243200.00035,03 04,65.054939,10074,8103,*.cloudfront.net,53
|
||||
Sydney,41,63,,-35.337229,139.41.70.39,"anon, hosting",104.187.23.103,Telstra,,,Tencent,25,139.41.70.39,,,ssl_logs,-1.190014,2026-06-01 00:00:00.000351,,601.7373,New York,"anon, hosting",in,,us,Microsoft Corp.,51510,TlsC,d0 31 68 77 f3 43 52 f9 34 26 ba b0 6a 5e ba 7f 11 59 2d 66 cb f9 20 d0 1a 7a 7d 9f,wan-link,443,Comcast,83,,46,,1074.5166,90.140.147.32,f2 e8 ce 06,30.48,00 35,1021.9465,18.543848,,1096,1780243200.000351,03 03,-88.17417,319796,6707,cdn.example.com,995
|
||||
Seoul,40,848,,-61.298485,83.250.9.29,"anon, hosting",194.115.203.248,Verizon,,,Tencent,16,194.226.218.213,,,netflow,-37.701662,2026-06-01 00:00:00.000352,,903.9721,Beijing,"anon, hosting",hk,,in,Google LLC,32683,TlsS,23 f5 71 42 e7 eb d0 0c d4 60 a9 21 7a d0 19 da b8 f2 95 26 d3 80 70 d4 cf 7c 7e f3,wan-link,22,Singtel,50,,499,,1609.5271,182.73.237.235,be d5 2b 8b,82.72,c0 13,384.5109,-23.091527,,89018,1780243200.000352,03 03,113.860289,284011,8854,,443
|
||||
Paris,26,174,,-47.828887,60.196.78.181,"anon, hosting",109.112.90.206,Singtel,,,Baidu Inc.,135,60.196.78.181,,,netflow,-25.088245,2026-06-01 00:00:00.000353,,846.9571,Berlin,"anon, hosting",fr,mail.example.com,il,Meta Platforms,87177,TlsC,e5 ba ba a3 f3 9a c1 6c 4c 24 05 fd 01 ae c9 1e ac a1 f5 6b 91 3b 1a cf 03 0e dc c2,edge-01,50000,China Mobile,25,TLS_AES_128_GCM_SHA256,118,,1792.2768,62.158.240.232,d0 22 af 63,45.5,13 02,1733.7509,-3.486176,,154731,1780243200.000353,03 04,-50.772342,172240,6083,*.s3.amazonaws.com,443
|
||||
Beijing,26,49,,-165.268744,51.214.112.115,"anon, hosting",87.179.3.47,Deutsche Telekom,,,Netflix Inc.,155,51.214.112.115,,,zeek,-37.735339,2026-06-01 00:00:00.000354,,377.0677,Seattle,"anon, hosting",kr,www.google.com,fr,Netflix Inc.,11069,TlsS,43 7f 1d c3 6b d6 b2 e4 65 e2 25 bd 37 1a 91 32 78 cc f8 e9 f7 c2 f9 b8 67 41 43 8a,wan-link,3389,Singtel,55,,276,,1766.9193,58.62.102.203,c4 f5 6f ef,4.1,00 9f,596.9795,-56.333408,,87782,1780243200.000354,03 03,-35.6129,386477,7403,,3389
|
||||
Paris,18,1057,,-30.994163,177.76.18.214,"anon, hosting",71.97.62.243,China Telecom,,,Tencent,139,202.160.223.156,,,netflow,53.956883,2026-06-01 00:00:00.000355,,312.8019,New York,"anon, hosting",fi,,ru,Baidu Inc.,79604,TlsC,be fd f5 eb 00 f0 b7 0f 36 61 6e 58 a5 c9 e7 f4 9b c9 7d 2f f5 f6 22 ce 5c d0 d0 e1,gw-09,443,KDDI,49,TLS_ECDHE_RSA_AES128_SHA256,14,512,805.3977,124.114.55.90,d2 23 ce 77,96.03,13 02,1650.2901,-32.593473,,179626,1780243200.000355,03 04,39.35205,237444,2249,,8080
|
||||
Paris,14,534,,-108.954788,132.171.47.210,"anon, hosting",57.70.252.24,NTT,,,Google LLC,137,132.171.47.210,,,netflow,-31.018944,2026-06-01 00:00:00.000356,"*.['google', 'com']",114.5237,San Francisco,"anon, hosting",au,,hk,Meta Platforms,94740,TlsS,4f 0a 2f 89 83 a4 e5 c9 c1 2a d4 59 ba 1b d4 90 21 07 6d 27 9d 20 f9 7d 5c d0 bc 47,core-02,465,Singtel,56,,27,128,1204.3838,155.205.20.12,db bd b9 90,88.62,00 9e,1026.5124,29.131449,,177156,1780243200.000356,03 04,-93.951723,267657,7927,login.live.com,22
|
||||
New York,12,1234,,12.652166,51.214.112.115,"anon, hosting",157.163.83.101,Deutsche Telekom,,,Microsoft Corp.,4,51.214.112.115,,*.s3.amazonaws.com,ssl_logs,-14.569845,2026-06-01 00:00:00.000357,,444.9199,London,"anon, hosting",fi,auth.example.net,jp,Baidu Inc.,83722,TlsC,41 08 a4 82 08 1f 54 6b 37 b5 18 f1 fe 7c 07 af 60 0d 13 7a 8b 6b 5f 51 c6 48 0c a9,backbone-03,8443,Telstra,58,,149,,59.9392,222.81.167.240,bc b1 c3 4b,83.13,00 9e,159.8153,-45.739997,,60245,1780243200.000357,03 03,-97.955875,266781,2166,,21
|
||||
New York,32,24,,-86.507338,72.79.110.246,"anon, hosting",93.120.31.171,Verizon,,,Tencent,7,72.79.110.246,00 18,,zeek,3.956036,2026-06-01 00:00:00.000358,,934.8963,San Francisco,"anon, hosting",br,www.google.com,cn,Samsung,96958,TlsS,98 82 89 06 f4 78 33 f7 29 00 3d 22 5c 1d 64 cb fa 5c 24 ca 81 40 dd 2a c8 d4 d7 8c,wan-link,465,,5,,39,,161.0277,21.191.114.8,0e fa 70 b1,114.47,c0 2b,443.1916,-12.13196,OK,199821,1780243200.000358,03 04,+,83214,3430,,25
|
||||
Seattle,25,214,,33.323789,93.235.78.112,"anon, hosting",48.7.172.203,China Telecom,,,Meta Platforms,164,93.235.78.112,,,suricata,46.367972,2026-06-01 00:00:00.000359,,474.7001,San Francisco,"anon, hosting",nl,,cn,Baidu Inc.,77570,TlsC,a3 8f fa 2a 0a 6f f7 90 cd a1 53 6f 16 02 93 a1 6d 63 87 6c 0c 87 59 0d ed 10 01 0c,core-02,12345,Singtel,34,,420,,1972.4293,142.7.208.24,ef b2 ac 86,113.84,00 9e,385.7257,-16.240773,,36669,1780243200.000359,03 04,110.000202,265991,1213,,8443
|
||||
Singapore,15,310,,119.328484,10.240.113.52,"anon, hosting",4.145.240.123,KDDI,,,Google LLC,189,10.240.113.52,,,suricata,-47.898017,2026-06-01 00:00:00.000360,,684.725,Berlin,"anon, hosting",kr,,jp,Apple Inc.,63559,TlsS,da ab 62 ee 95 5f af 58 a2 eb e3 c2 fb f6 71 bf 2d c6 c5 11 72 52 7f f5 0f 33 87 08,wan-link,12345,AT&T,56,,303,,1404.3345,129.191.120.99,b4 1f e1 27,27.22,c0 23,1929.0377,59.051828,,127938,1780243200.00036,03 03,-9.842656,369900,2426,,995
|
||||
London,41,148,,8.097062,103.124.75.168,"anon, hosting",64.41.228.136,NTT,,,Amazon.com Inc.,108,103.124.75.168,,,zeek,-57.098285,2026-06-01 00:00:00.000361,,626.1796,Dublin,"anon, hosting",fi,,us,Microsoft Corp.,25077,TlsC,d7 56 14 9f b2 87 4f c1 47 e0 f4 69 72 34 5f ac 31 42 ee c9 ce 72 88 03 83 35 e2 81,gw-09,53,Orange,6,,127,,1314.4437,213.241.71.195,ef d9 ae 5d,14.22,c0 14,594.847,27.524142,,17540,1780243200.000361,03 03,78.923901,238372,8530,,3389
|
||||
Berlin,34,488,,-67.182315,213.215.129.214,"anon, hosting",47.109.174.243,China Mobile,x448,,Amazon.com Inc.,102,213.215.129.214,00 17,,packet_capture,+,2026-06-01 00:00:00.000362,"*.['example', 'net']",25.7246,Singapore,"anon, hosting",hk,graph.facebook.com,in,Tencent,95936,TlsC,ea f3 25 45 e2 fa 35 a4 c5 ad 63 02 31 e6 90 8a f9 81 07 01 62 0e 51 55 16 ce 85 31,wan-link,8443,NTT,20,,198,,357.6608,31.78.255.239,c7 a6 97 10,54.59,c0 2f,1975.108,52.878538,,3836,1780243200.000362,03 04,21.61923,191594,8848,,80
|
||||
San Francisco,20,673,,142.435155,62.40.43.188,"anon, hosting",145.57.31.213,China Telecom,,,Alibaba Inc.,200,62.40.43.188,,,packet_capture,38.404197,2026-06-01 00:00:00.000363,*.cloudfront.net,279.5627,Berlin,"anon, hosting",se,*.cloudfront.net,gb,Samsung,80787,TlsS,2d 1b 48 75 7a b8 b9 b2 e9 4c 71 86 cc 6e 31 de 29 0d b3 a4 f9 bb 6a 09 4c d1 2c 8b,gw-09,3389,SK Telecom,24,,473,,1619.3787,6.202.130.210,1d 98 21 db,61.21,13 02,1232.3356,9.915109,,180906,1780243200.000363,03 03,-4.562055,328579,7518,,443
|
||||
Berlin,33,1592,,170.313577,164.57.12.190,"anon, hosting",134.81.186.96,China Telecom,,,Meta Platforms,69,164.57.12.190,,*.s3.amazonaws.com,zeek,-18.605551,2026-06-01 00:00:00.000364,,357.2299,Singapore,"anon, hosting",gb,stream.example.org,jp,Tencent,31994,TlsS,47 9d 89 d9 19 b8 f2 f8 51 93 f5 f8 93 ea c5 7c 1b 6c aa e1 43 0f 30 1d 12 d5 7f 88,backbone-03,443,AT&T,55,TLS_AES_128_GCM_SHA256,285,256,1502.0625,142.7.208.24,9d 29 07 87,57.47,00 9f,1698.0975,-59.439745,,65424,1780243200.000364,03 03,25.974932,235775,9207,graph.facebook.com,25
|
||||
Berlin,37,1982,,-121.128467,73.80.224.214,"anon, hosting",106.12.91.189,Comcast,,,Tencent,126,73.80.224.214,,,ssl_logs,8.983975,2026-06-01 00:00:00.000365,,67.344,Tokyo,"anon, hosting",jp,www.google.com,il,Alibaba Inc.,79667,TlsC,,wan-link,8080,AT&T,48,,269,,1079.7941,184.163.176.250,69 9c f5 c7,63.41,c0 13,1579.041,7.661759,,120530,1780243200.000365,03 03,158.114591,166958,3292,,8443
|
||||
Beijing,49,1331,,+,50.35.23.170,"anon, hosting",89.135.22.187,AT&T,,,Amazon.com Inc.,93,50.35.23.170,00 19,,zeek,-44.932782,2026-06-01 00:00:00.000366,auth.example.net,388.46,Beijing,"anon, hosting",sg,,nl,Meta Platforms,63163,TlsC,69 33 ed 83 11 42 26 16 c2 01 f8 c6 76 97 8c 6a 6a 8f d4 01 5c c0 6a 7d bb cb 8e 25,core-02,3389,KDDI,41,,196,,732.3459,179.115.153.176,5b fd 13 57,27.72,c0 23,1489.138,49.248105,,25087,1780243200.000366,03 03,115.782421,281210,4885,auth.example.net,993
|
||||
Sydney,8,302,,-40.207102,11.157.186.231,"anon, hosting",106.104.210.129,BT Group,,,Google LLC,153,11.157.186.231,,,ssl_logs,9.360606,2026-06-01 00:00:00.000367,,856.2491,San Francisco,"anon, hosting",nl,www.google.com,cn,Amazon.com Inc.,29485,TlsS,0d 34 e8 a1 07 aa d3 c8 b1 a5 13 cb a5 cb fe 44 53 67 56 94 5d 15 81 7a 09 1f cc 7a,wan-link,80,KDDI,19,,500,,1095.6265,94.203.160.167,35 13 db f0,80.76,13 02,1631.64,-11.273277,,79063,1780243200.000367,03 02,-75.981448,290536,5706,,443
|
||||
Tokyo,10,597,,-165.69242,3.57.38.242,"anon, hosting",206.240.163.235,China Mobile,,,Google LLC,179,3.57.38.242,,,packet_capture,-53.549545,2026-06-01 00:00:00.000368,,924.442,Sydney,"anon, hosting",hk,graph.facebook.com,fi,Apple Inc.,30482,TlsS,60 3e ab d8 f0 53 d7 23 fa 65 db 5f 4a 95 a4 f0 e6 e8 64 5d 99 63 4f 20 8f dd 55 51,gw-09,443,Orange,74,,419,384,1755.0166,21.164.75.17,52 72 64 2a,58.5,c0 23,1997.3436,17.032641,,152380,1780243200.000368,03 03,-91.445816,90256,1566,www.google.com,80
|
||||
San Francisco,43,320,,65.257683,139.102.159.103,"anon, hosting",31.94.98.56,China Mobile,,97097,Tencent,17,139.102.159.103,,,ssl_logs,-25.784565,2026-06-01 00:00:00.000369,,298.3716,Tokyo,"anon, hosting",ua,stream.example.org,us,Meta Platforms,95268,TlsC,89 7f bb c2 b9 15 c3 45 47 ea 00 58 aa 42 bc 87 34 38 1f bc b5 d2 5b 9b 24 49 d8 d0,edge-01,443,BT Group,16,,164,,1453.2116,89.74.148.11,0d 87 19 a0,101.17,c0 2f,1956.6564,-24.242757,,52163,1780243200.000369,02 00,78.874389,154829,7738,,995
|
||||
San Francisco,22,44,,-172.890955,185.58.136.197,"anon, hosting",76.98.89.157,Singtel,,,Baidu Inc.,95,185.58.136.197,,,zeek,-26.464134,2026-06-01 00:00:00.000370,,539.8831,Seoul,"anon, hosting",nl,*.s3.amazonaws.com,us,Meta Platforms,61496,TlsC,1f e0 56 62 24 96 58 0f 91 6a 08 d4 94 1c 87 2b 42 25 1e ea 14 8c 69 26 0e b2 bd 04,wan-link,443,BT Group,33,,62,,264.0486,180.58.110.111,ba e9 a2 13,32.15,13 01,104.4491,10.369449,,18967,1780243200.00037,03 03,30.302133,179905,5313,,3389
|
||||
Sydney,15,1265,,159.810768,122.251.229.203,"anon, hosting",34.46.151.84,BT Group,,,Tencent,15,122.251.229.203,,,ssl_logs,24.621609,2026-06-01 00:00:00.000371,,960.6382,Dublin,"anon, hosting",cn,,au,Amazon.com Inc.,60507,TlsS,3a 54 e7 38 45 5e e5 9c 27 ab 83 4c b6 87 62 98 a9 16 4c cf da 85 58 f5 7c dd c9 df,wan-link,53,AT&T,59,TLS_RSA_WITH_AES_256_GCM_SHA384,340,,918.0849,39.4.111.130,20 2a 13 ab,23.09,c0 14,986.8454,57.972149,,129260,1780243200.000371,03 01,-83.199568,467218,5078,,53
|
||||
London,22,1425,,-176.908559,86.210.127.69,"anon, hosting",182.43.158.144,NTT,x25519,,Google LLC,14,86.210.127.69,00 19,,zeek,25.920743,2026-06-01 00:00:00.000372,,279.0766,Mumbai,"anon, hosting",fr,,ru,,61598,TlsC,18 1b 6f 16 d5 5f b8 18 12 b3 bd dc 97 9f 27 93 22 40 85 6c d5 21 c8 18 be 06 5c 15,wan-link,21,AT&T,23,,61,,871.4715,14.149.64.164,2f 3c 38 9e,11.69,cc a9,47.558,-0.81282,OK,147815,1780243200.000372,03 04,-52.48049,350362,5059,*.s3.amazonaws.com,443
|
||||
Shanghai,35,1280,,-12.946258,57.70.252.24,"anon, hosting",40.242.34.24,Comcast,,,Tencent,58,57.70.252.24,,,packet_capture,-1.312141,2026-06-01 00:00:00.000373,,532.2416,Berlin,"anon, hosting",il,,us,Netflix Inc.,59511,TlsS,cc 2b 09 ff ca 85 9f 88 18 8f 72 93 cc 26 de 48 40 56 ee e3 43 dc 35 09 ae 89 13 56,backbone-03,465,Telstra,94,,112,,1337.6822,141.169.180.180,09 5c 9a d3,61.1,c0 2f,549.5827,-57.049483,,65681,1780243200.000373,03 03,7.291938,318175,8228,mail.example.com,53
|
||||
Dublin,17,1568,,-68.243905,211.116.114.7,"anon, hosting",145.57.31.213,Orange,,,Tencent,126,211.116.114.7,,,suricata,28.205709,2026-06-01 00:00:00.000374,,602.8978,Beijing,"anon, hosting",fi,,il,Baidu Inc.,55882,TlsS,a8 80 15 99 96 ac 09 77 cc 63 65 81 c1 c7 0b 3d 83 96 c8 16 46 cf a9 73 5d 72 27 b0,gw-09,443,SK Telecom,14,,116,,478.257,213.241.71.195,67 0b f5 08,113.29,c0 23,1618.9328,-40.792183,,16383,1780243200.000374,03 03,125.111329,318012,6039,,143
|
||||
New York,24,1089,,-30.032828,105.250.123.177,"anon, hosting",122.86.43.73,BT Group,,,Microsoft Corp.,163,105.250.123.177,,,netflow,5.48125,2026-06-01 00:00:00.000375,"*.['example', 'org']",913.0866,Mumbai,"anon, hosting",gb,,ru,Baidu Inc.,57349,TlsS,fd f2 67 1e c6 22 85 c8 0f 28 fc 82 2c c1 9c 0d 1e 1f 4f a7 d3 67 a8 da 40 e4 7f f2,core-02,21,KDDI,100,,84,,712.7991,15.236.157.186,c3 a8 85 00,54.48,c0 2f,1543.2472,18.309944,,1806,1780243200.000375,03 03,-152.930311,95011,5043,mail.example.com,8080
|
||||
London,34,1166,,143.491991,193.120.158.58,"anon, hosting",32.188.223.228,SK Telecom,,,Apple Inc.,146,193.120.158.58,,,zeek,-31.192587,2026-06-01 00:00:00.000376,,397.1138,Seoul,"anon, hosting",ru,www.google.com,il,Alibaba Inc.,74754,TlsS,1f 4f 84 03 9a d9 c5 17 e7 cd 66 82 59 f9 54 9d e0 ee 35 39 b3 e4 e9 63 f5 2b ae d0,edge-01,110,Verizon,24,,61,,831.3673,217.51.19.170,ce 50 21 fa,2.9,c0 2c,241.4557,19.470375,,56592,1780243200.000376,03 04,-65.548805,391702,3971,,8080
|
||||
Singapore,15,1842,,168.461598,139.102.159.103,"anon, hosting",194.24.56.40,Orange,,,Netflix Inc.,149,218.63.153.130,,,zeek,-5.886377,2026-06-01 00:00:00.000377,api.example.com,616.0779,Seoul,"anon, hosting",sg,,nl,Baidu Inc.,62927,TlsS,65 3b 6b 31 c8 98 a0 7c 64 62 cc 97 52 b2 a5 04 72 b8 7d 7c 59 2d 10 33 52 ac 99 ce,backbone-03,8080,SK Telecom,40,,473,256,613.9259,1.111.37.12,23 95 b7 e5,15.55,c0 13,1828.9598,33.862535,,49018,1780243200.000377,03 01,-170.636453,207844,2918,,8443
|
||||
Berlin,37,1726,,139.957532,185.58.136.197,"anon, hosting",65.76.170.21,Singtel,,,Meta Platforms,146,185.58.136.197,,,netflow,33.720694,2026-06-01 00:00:00.000378,,416.2974,Mumbai,"anon, hosting",us,,gb,Google LLC,78136,TlsC,98 fa da 28 de 1f c6 11 79 13 9e 0f 03 a7 8a d7 08 28 66 3c 8c 6c 7b d5 00 b7 fc 86,wan-link,8443,KDDI,51,,454,,1316.2382,77.140.31.143,34 37 3d b0,82.17,c0 14,133.8539,-47.417471,,28349,1780243200.000378,03 04,162.22702,34679,2221,,8443
|
||||
Shanghai,5,755,,25.26147,194.226.218.213,"anon, hosting",169.98.204.85,AT&T,,,Microsoft Corp.,191,194.226.218.213,,,suricata,-34.761782,2026-06-01 00:00:00.000379,,520.5019,Moscow,"anon, hosting",in,,us,Amazon.com Inc.,60631,TlsC,,gw-09,443,SK Telecom,2,TLS_ECDHE_ECDSA_AES128_SHA,405,,1134.1761,48.254.254.95,49 a7 a6 f7,54.54,c0 2c,1039.2044,21.978553,,21739,1780243200.000379,03 03,78.29696,32469,2654,,443
|
||||
Paris,11,410,,-95.694708,50.35.23.170,"anon, hosting",155.220.50.204,AT&T,,,Meta Platforms,129,50.35.23.170,,,zeek,-29.940548,2026-06-01 00:00:00.000380,,920.0778,Moscow,"anon, hosting",sg,,jp,Alibaba Inc.,26031,TlsC,45 d2 27 43 3f c6 d3 98 2d 1e 00 27 20 54 6a 70 0b 3e d6 e8 c3 ac bc c7 ed 7d 5f 71,edge-01,465,KDDI,11,,441,,1557.6218,82.37.230.120,62 c6 9d 32,32.03,00 35,1460.5095,1.812791,,14723,1780243200.00038,03 03,73.227742,322511,8649,stream.example.org,995
|
||||
Beijing,29,1508,,122.763203,51.214.112.115,"anon, hosting",114.212.139.56,SK Telecom,secp521r1,,Samsung,106,51.214.112.115,,,zeek,1.883286,2026-06-01 00:00:00.000381,,211.691,Seoul,"anon, hosting",hk,,kr,Amazon.com Inc.,23888,TlsS,c2 46 be 74 1f 6f 1e 46 8f 21 70 08 07 3b 39 6b 21 90 d9 9a 9d 5e 2a 90 7f ee f4 0e,gw-09,443,China Telecom,25,,480,512,368.4793,136.68.198.63,be ab 46 44,48.24,00 9e,1183.709,15.014399,,124102,1780243200.000381,03 04,-139.962916,160485,1656,auth.example.net,465
|
||||
Berlin,48,877,,16.254956,167.35.20.194,"anon, hosting",112.169.48.230,Telstra,,28587,Microsoft Corp.,190,167.35.20.194,00 1e,,netflow,-5.561643,2026-06-01 00:00:00.000382,,407.1177,Paris,"anon, hosting",no,,au,Samsung,40809,TlsS,bc cd 05 5d cd 9a 42 6b ce 5b 9a ad 92 c0 69 e4 23 8f ec fe b1 0d 0b b6 eb 21 22 fe,edge-01,3389,China Mobile,57,TLS_ECDHE_ECDSA_AES128_SHA,147,,1511.81,65.26.246.224,9d 8e 19 1e,28.11,c0 14,1469.5093,-15.591286,,53810,1780243200.000382,03 03,-93.644402,282563,3054,,3389
|
||||
Singapore,12,475,,-42.51696,51.214.112.115,"anon, hosting",30.239.38.37,Deutsche Telekom,,,Amazon.com Inc.,30,51.214.112.115,,cdn.example.com,zeek,22.878289,2026-06-01 00:00:00.000383,,839.4591,New York,"anon, hosting",br,,jp,Baidu Inc.,78136,TlsC,53 a7 35 ff f0 93 b5 dc a4 4f ce 9d 4f 8a 6e e0 9d 4f 45 4e 0c 5e 41 1a 8c 25 01 d6,gw-09,995,Singtel,50,,192,,1147.5818,59.10.137.148,cf 15 ea 52,55.63,cc a9,1948.5109,6.5066,,107892,1780243200.000383,03 03,-87.00019,179753,6501,*.cloudfront.net,50000
|
||||
Seoul,37,225,Tencent,-128.373196,155.107.175.53,"anon, hosting",66.148.167.31,Deutsche Telekom,,,Samsung,80,155.107.175.53,,,netflow,-3.986079,2026-06-01 00:00:00.000384,,247.6317,New York,"anon, hosting",br,mail.example.com,fr,Microsoft Corp.,29505,TlsS,62 cc f9 96 54 cb 10 2a f8 c8 f1 e6 6d 1d 3c 35 20 69 10 b3 58 4d 6a 71 ba 54 4b 6b,wan-link,22,KDDI,96,,366,,210.5554,117.189.214.151,ab 16 8a 92,48.2,c0 2b,733.4934,-39.795321,,9181,1780243200.000384,03 03,+,68250,7226,,8080
|
||||
Sydney,10,797,,-138.470942,193.37.4.118,"anon, hosting",32.239.155.180,Deutsche Telekom,,,Amazon.com Inc.,159,176.135.250.65,,,ssl_logs,37.707246,2026-06-01 00:00:00.000385,,50.8513,Moscow,"anon, hosting",no,,no,Netflix Inc.,39113,TlsS,e8 5b 27 cf 95 47 4a 42 48 b2 7e 83 24 e2 a9 f4 c7 2c fa cf bf f8 52 51 84 ba 78 df,gw-09,8080,BT Group,61,TLS_AES_256_GCM_SHA384,208,,1035.815,78.73.113.106,ff 6e 78 90,13.93,00 9f,45.2382,-51.446082,,151174,1780243200.000385,03 04,3.12486,218869,1584,login.live.com,8080
|
||||
New York,30,1965,,90.418167,122.58.111.120,"anon, hosting",155.205.20.12,China Mobile,x448,,Tencent,24,122.58.111.120,,,zeek,11.975867,2026-06-01 00:00:00.000386,,735.4467,Sydney,"anon, hosting",fi,,kr,Meta Platforms,27643,TlsS,c9 5a e0 4b 89 9a a8 41 5f f7 b8 84 45 c7 fa 77 44 f7 5b 5e 4b 34 a1 b2 79 4a a3 19,core-02,443,Verizon,9,TLS_ECDHE_RSA_AES256_GCM_SHA384,304,128,1757.6134,65.26.246.224,c8 60 5f e2,8.69,00 9e,1162.5421,0.74007,,104690,1780243200.000386,03 03,22.070583,378500,4476,,993
|
||||
Paris,25,1499,,-83.186918,172.191.224.231,"anon, hosting",73.80.224.214,Singtel,,,Netflix Inc.,55,172.191.224.231,,,netflow,46.484234,2026-06-01 00:00:00.000387,auth.example.net,966.2596,Tokyo,"anon, hosting",in,,br,Apple Inc.,31807,TlsC,4f dc 65 3d fe ef f0 8b 8b 4b 5c 4a 32 5f 9a 43 37 f5 62 2f 0e 77 83 e3 68 35 e8 a7,core-02,143,China Telecom,34,,345,256,1992.4567,26.225.51.135,5a bb 79 a7,39.05,cc a9,879.5523,-59.45635,,83416,1780243200.000387,03 04,-89.301014,353417,9288,,995
|
||||
Sydney,1,1007,,-150.575522,21.214.161.240,"anon, hosting",78.140.28.91,KDDI,,,Tencent,70,21.214.161.240,,,zeek,-47.943956,2026-06-01 00:00:00.000388,,453.0511,Berlin,"anon, hosting",jp,,sg,Baidu Inc.,59628,TlsS,0a f7 79 c3 f0 28 41 ff 84 f7 02 d5 bf 7a 27 59 37 e8 9a 00 c6 e6 47 da e4 48 1a 8b,edge-01,21,Deutsche Telekom,94,,8,,1767.1381,72.115.219.217,dc 56 64 21,89.33,c0 23,288.497,41.844348,,101058,1780243200.000388,03 03,117.372254,129574,8635,graph.facebook.com,53
|
||||
Berlin,41,302,,52.975269,182.223.0.134,"anon, hosting",146.125.255.141,SK Telecom,,,Amazon.com Inc.,116,8.126.102.215,,,netflow,58.590079,2026-06-01 00:00:00.000389,,744.5958,Seattle,"anon, hosting",fi,,jp,Microsoft Corp.,89647,TlsS,fe 31 27 ec 3b df bd 6c af aa b1 63 89 bd 18 d9 ac 41 86 d3 78 1f ec 4d aa a6 8d 3e,wan-link,443,China Mobile,30,,74,,1904.196,221.21.223.5,68 99 70 2b,104.97,c0 13,66.9553,-19.907461,OK,121370,1780243200.000389,03 03,-98.430407,408105,8234,,443
|
||||
Dublin,30,1811,,-93.370374,80.55.13.238,"anon, hosting",107.209.76.77,SK Telecom,,,Baidu Inc.,99,80.55.13.238,,,ssl_logs,0.482182,2026-06-01 00:00:00.000390,,703.5422,Tokyo,"anon, hosting",in,,kr,Baidu Inc.,84576,TlsC,b0 f5 95 a0 c0 90 91 45 c3 76 64 6b 7d 9d d4 af 21 15 6a 94 66 1b 46 96 2d 6a 15 3c,backbone-03,8080,SK Telecom,21,TLS_AES_128_GCM_SHA256,100,128,1436.8684,66.148.167.31,19 cf 82 86,111.95,c0 27,1870.6445,-21.35982,,179105,1780243200.00039,03 04,+,22467,8107,,80
|
||||
Seattle,7,1419,,72.860436,180.131.188.43,"anon, hosting",165.174.57.76,Orange,,,Samsung,14,180.131.188.43,,,ssl_logs,56.141087,2026-06-01 00:00:00.000391,,446.5833,Seoul,"anon, hosting",de,,hk,Baidu Inc.,92197,TlsS,26 85 a4 05 16 0c 61 ba f5 46 c0 7b ae 78 2d 0c c4 f6 73 70 6d 5e e3 c4 a3 6e 97 1a,wan-link,3389,Singtel,43,,349,,620.894,74.117.180.57,99 c7 c8 57,110.78,c0 14,1101.5862,+,,183435,1780243200.000391,03 04,-128.199245,311959,5114,,110
|
||||
Dublin,1,847,,-122.263552,61.142.248.55,"anon, hosting",207.151.173.206,BT Group,,,Samsung,199,61.142.248.55,,,zeek,42.393655,2026-06-01 00:00:00.000392,,331.3773,Sydney,"anon, hosting",nl,api.example.com,de,Alibaba Inc.,10881,TlsS,,core-02,993,China Telecom,94,,443,,199.7714,213.250.153.122,df d5 b7 a5,6.74,c0 13,11.4858,-25.719412,,191385,1780243200.000392,03 03,-102.929743,458168,7511,login.live.com,443
|
||||
Dublin,2,127,,21.709994,120.65.13.233,"anon, hosting",106.12.91.189,China Mobile,,,,105,108.250.54.111,,*.cloudfront.net,suricata,-31.152067,2026-06-01 00:00:00.000393,,735.4006,Dublin,"anon, hosting",br,,hk,Meta Platforms,81721,TlsS,74 a9 96 22 dd b1 72 b2 ad 49 44 f2 a4 28 42 03 16 21 2d fe 9a ea 7f 20 3e 28 93 d8,core-02,443,KDDI,24,,361,,982.0129,114.26.104.69,d8 0d 31 1b,12.69,00 9e,416.6658,30.455065,,170189,1780243200.000393,03 03,174.540923,152692,1590,,3389
|
||||
Shanghai,43,479,,56.262674,85.238.212.16,"anon, hosting",217.188.58.196,BT Group,,,Google LLC,195,85.238.212.16,,*.s3.amazonaws.com,suricata,38.475031,2026-06-01 00:00:00.000394,,629.4989,Tokyo,"anon, hosting",se,cdn.example.com,nl,Apple Inc.,18990,TlsS,de 0a 5b fe d9 3f 09 05 56 73 b5 27 46 0d 84 bb c4 38 ab f0 69 85 85 63 b1 d8 66 68,wan-link,443,China Telecom,96,,313,,1337.3346,74.159.180.27,89 2b 7c f2,102.68,00 9f,404.8715,11.885995,,31553,1780243200.000394,03 04,-40.081229,87454,1670,,110
|
||||
Paris,39,437,,-104.539447,153.34.120.104,"anon, hosting",195.18.232.23,Comcast,,,Microsoft Corp.,135,153.34.120.104,,,suricata,-55.810068,2026-06-01 00:00:00.000395,,104.8728,Sydney,"anon, hosting",ua,auth.example.net,jp,Baidu Inc.,97634,TlsS,f2 57 aa 2e fe 72 72 7d b1 3a 77 d1 53 53 18 29 ab 4f f9 d6 bc 93 ce 59 29 2f c5 e7,backbone-03,22,SK Telecom,11,,425,,233.4265,45.40.195.159,1b d5 54 16,34.5,00 35,1777.3826,21.11576,,88665,1780243200.000395,03 03,175.306351,332067,5203,www.google.com,465
|
||||
Mumbai,34,481,,-89.999195,116.124.237.146,"anon, hosting",9.161.205.69,NTT,,,Alibaba Inc.,58,116.124.237.146,,,zeek,57.744185,2026-06-01 00:00:00.000396,,159.2996,Singapore,"anon, hosting",il,,gb,Netflix Inc.,46850,TlsS,32 72 df ff ae 85 70 6d f9 f8 f7 43 e1 3e b9 80 4d 53 83 3b 54 a2 00 68 6b 8c 32 5f,backbone-03,25,NTT,23,,474,,1328.4192,124.114.55.90,46 5c 75 e9,6.39,c0 2c,979.626,47.408143,,112716,1780243200.000396,03 04,-8.762651,118748,2435,graph.facebook.com,993
|
||||
Mumbai,35,1107,,118.063837,110.189.35.130,"anon, hosting",221.24.169.70,Verizon,secp384r1,,Google LLC,171,110.189.35.130,,,netflow,56.537466,2026-06-01 00:00:00.000397,,914.135,Sydney,"anon, hosting",us,stream.example.org,gb,Samsung,28037,TlsC,eb 73 39 c0 62 71 44 96 74 01 fe 57 9c 82 f5 7a 0e f8 3c 45 2a 55 bb fd dd 27 19 4b,wan-link,12345,Telstra,50,TLS_RSA_WITH_AES_128_GCM_SHA256,47,,1131.3839,141.67.147.113,39 5b f1 53,75.61,c0 14,1152.9492,33.021083,,37677,1780243200.000397,03 01,121.090182,212898,4641,,443
|
||||
Shanghai,39,687,Apple Inc.,-21.361939,106.12.91.189,"anon, hosting",3.25.254.214,AT&T,,,Amazon.com Inc.,11,106.12.91.189,,,netflow,-16.448547,2026-06-01 00:00:00.000398,,987.5724,Sydney,"anon, hosting",jp,,sg,Netflix Inc.,77053,TlsS,59 b3 51 4e 51 a6 90 c9 ac 8e fd c0 71 27 dd 2f 98 bd b5 1b 69 c5 ad 73 b5 90 51 68,backbone-03,465,AT&T,84,,388,256,1448.3985,161.20.158.202,82 df d9 e7,53.11,c0 2b,87.4273,-41.653776,,93334,1780243200.000398,03 04,-41.906507,223952,4097,cdn.example.com,50000
|
||||
Shanghai,37,754,,76.073902,12.125.243.157,"anon, hosting",89.79.141.49,Singtel,,,Microsoft Corp.,124,12.125.243.157,,*.s3.amazonaws.com,packet_capture,-28.807051,2026-06-01 00:00:00.000399,,203.9099,Dublin,"anon, hosting",ru,*.s3.amazonaws.com,il,Apple Inc.,17516,TlsC,1d 9b 6d c4 55 f4 a7 ce 82 fd a7 1f e7 dc f6 5c a2 77 1d 36 aa 02 ce e0 85 bf 37 37,edge-01,110,Singtel,14,,329,,637.0179,113.160.214.39,71 51 91 d3,87.0,c0 13,1119.4254,32.633586,--,163334,1780243200.000399,03 03,-54.201201,237385,5734,*.s3.amazonaws.com,110
|
||||
|
@@ -0,0 +1,201 @@
|
||||
:ipd.orgn,scnt,cnam,0crv,:ips.orgn,8ack,client-ip,4dur,timestamp,:ipd.anon,8ses,0rnt,:ipd.city,0cph,orga,8pak,4ksz,0rnd,0ver,:ipd.latd,crcc,8dbd,snam,:ips.anon,:prd,dcnt,source-node,:ipd.lond,orgu,ecdhe-named-curve,:prs,time,server-ip,4dbn,8ppk,:ipd.doma,8seq,:ips.ispn,:ips.latd,:ips,cipher-suite,8byt,4srs,:ipd,:ips.doma,:ips.city,1ipp,:ipd.ispn,:ips.lond,8did,name,2tmo,tabl
|
||||
Amazon.com Inc.,hk,*.cloudfront.net,00 1d,Meta Platforms,125104,41.55.195.224,52.65,1780243200.0004,"anon, hosting",1606.7581,12 3e 27 22,Mumbai,c0 2b,,82,,6e 43 bd cb c3 24 04 c1 30 01 ad 7c e8 14 03 ff 1f 0a 33 03 4c 47 43 3b 56 ca 6f 15,03 03,-3.52216,,99,,"anon, hosting",465,nl,netflow,-161.446766,,,443,2026-06-01 00:00:00.000400,179.151.65.50,43,57,,610.5262,Deutsche Telekom,49.452387,41.55.195.224,,223308,18246,31.238.157.41,,Mumbai,10,KDDI,113.361725,8308,wan-link,179.4731,TlsS
|
||||
Microsoft Corp.,nl,,00 1e,,167334,165.174.57.76,118.5,1780243200.000401,"anon, hosting",236.958,de 9f 23 15,Berlin,cc a9,,543,512,36 d4 e6 d9 ec 8f 9d 55 5b fa 05 ff 63 1f 16 99 dd 75 10 c5 d5 c8 e5 df 59 c1 10 50,03 03,-11.925611,,38,*.s3.amazonaws.com,"anon, hosting",53,de,ssl_logs,68.473911,,,995,2026-06-01 00:00:00.000401,42.92.84.185,60,466,,936.0807,Comcast,-41.564,165.174.57.76,,472165,59495,166.88.20.64,,Paris,11,Telstra,-124.354251,3843,backbone-03,111.2606,TlsS
|
||||
Google LLC,nl,,,Alibaba Inc.,158828,73.80.224.214,106.49,1780243200.000402,"anon, hosting",1207.5947,ec cf ac 13,Berlin,13 01,,1556,,00 39 67 77 ee cb 25 27 bb e9 f6 04 32 9f 2d 3a 28 b6 37 c2 4e 6c 23 b5 7c 8b 52 25,03 03,48.195628,,75,,"anon, hosting",995,ca,zeek,-126.446764,,,53,2026-06-01 00:00:00.000402,109.176.32.139,43,447,stream.example.org,129.644,Verizon,20.501376,73.80.224.214,TLS_RSA_WITH_AES_128_CBC_SHA,32769,82562,36.106.166.170,,Beijing,15,Verizon,-70.824224,7767,wan-link,578.2479,TlsC
|
||||
Microsoft Corp.,fr,,,Google LLC,191722,150.219.204.93,40.94,1780243200.000403,"anon, hosting",1884.7128,10 b1 e1 c2,London,00 9f,,67,,44 f7 52 9c aa ea 5b 5f 93 ae b0 97 0d 8f 4d 3e 0a 04 39 99 cc 3c 38 f2 e4 a4 8f be,03 03,-3.051089,,93,login.live.com,"anon, hosting",443,il,zeek,-91.394959,,,995,2026-06-01 00:00:00.000403,86.193.233.239,46,322,,1309.0541,Comcast,-17.308319,150.219.204.93,,451430,26123,103.218.55.126,,Sydney,45,Orange,-77.620995,9019,edge-01,866.3556,TlsC
|
||||
Samsung,de,,,Meta Platforms,135863,28.79.139.73,89.95,1780243200.000404,"anon, hosting",500.3962,79 69 32 2a,Shanghai,13 02,,1468,,d7 f3 96 4e f1 f6 a9 34 a1 71 cb e1 46 18 e1 b5 12 c4 c0 35 41 a0 ca a8 8c 95 28 d7,03 01,-50.106489,,62,,"anon, hosting",443,sg,zeek,-124.485325,26801,,22,2026-06-01 00:00:00.000404,60.111.28.36,57,402,,491.8117,Orange,-35.747973,28.79.139.73,,193833,20293,76.98.89.157,,San Francisco,10,Comcast,-135.119886,9446,edge-01,549.8641,TlsC
|
||||
Meta Platforms,il,,,Tencent,148574,202.160.223.156,77.09,1780243200.000405,"anon, hosting",1069.0374,4f 03 f9 36,San Francisco,13 01,,1214,,29 18 d7 d1 be b1 6b 45 c5 82 fe 8a 36 84 20 5c 8b 07 85 64 df 91 6b a2 c6 19 03 02,03 02,16.888381,,113,,"anon, hosting",12345,ca,zeek,-51.435488,,,110,2026-06-01 00:00:00.000405,35.244.78.117,67,113,,1712.1462,KDDI,-54.646686,202.160.223.156,,67610,26533,181.250.162.123,,Shanghai,34,NTT,-40.840643,6233,edge-01,332.4719,TlsC
|
||||
Meta Platforms,kr,*.cloudfront.net,,Tencent,54625,210.235.179.79,92.46,1780243200.000406,"anon, hosting",836.8277,5a 36 41 a8,Seattle,c0 23,,1363,,2f 63 93 3e 77 aa 0c 60 67 91 cd 06 f2 9f d4 e1 a6 e4 9a 97 db a1 9b 81 48 3a fa 6f,03 04,-48.523306,,133,*.cloudfront.net,"anon, hosting",8080,il,packet_capture,-61.302224,,,110,2026-06-01 00:00:00.000406,145.105.35.213,31,28,,1325.8453,,-46.834589,210.235.179.79,,225030,21438,75.196.214.64,,Seoul,18,KDDI,-118.977523,5078,edge-01,219.1428,TlsS
|
||||
Netflix Inc.,de,,,Google LLC,199834,161.81.216.153,109.46,1780243200.000407,"anon, hosting",1965.0473,1b c0 5a 73,Beijing,c0 2c,,1059,384,1b 6c 28 05 df cb ca 90 df b8 31 88 f3 ff ef 9b b3 fb b6 ee f4 2b 32 95 2c ea 96 d5,03 01,-46.397583,,44,,"anon, hosting",443,cn,packet_capture,137.797422,,,8443,2026-06-01 00:00:00.000407,4.145.240.123,13,201,,851.8914,Deutsche Telekom,+,161.81.216.153,,158362,59010,28.122.124.13,www.google.com,Moscow,25,Comcast,84.96217,8257,wan-link,20.8751,TlsS
|
||||
Microsoft Corp.,in,,,Amazon.com Inc.,190726,57.70.252.24,111.28,1780243200.000408,"anon, hosting",208.1298,95 e1 3f f6,London,00 9f,,1491,512,b9 7d 21 5b 19 10 a9 15 9f 18 0f 59 8f 35 e6 10 6e cf 80 5d 7c fb 7f 6f 27 68 99 4c,03 02,-23.499637,,37,auth.example.net,"anon, hosting",25,sg,packet_capture,116.318414,,,995,2026-06-01 00:00:00.000408,35.244.78.117,31,485,,356.1331,SK Telecom,15.231739,57.70.252.24,,468043,45971,2.210.6.5,,Beijing,22,KDDI,66.223963,1989,edge-01,766.9526,TlsC
|
||||
Amazon.com Inc.,ua,,,Alibaba Inc.,132553,213.215.129.214,34.9,1780243200.000409,"anon, hosting",277.1092,be fb bc 92,Moscow,c0 27,,265,,94 64 00 05 70 fe 14 d7 92 d8 ae 9d 2c d4 9b f9 8b 63 5b 03 6b 48 55 c1 ce f1 0d 69,02 00,32.984305,,198,cdn.example.com,"anon, hosting",8080,ca,netflow,85.161502,,,443,2026-06-01 00:00:00.000409,25.76.123.50,30,178,,741.9129,BT Group,-7.373511,213.215.129.214,,160584,29069,78.73.113.106,,Tokyo,49,NTT,130.250938,7945,backbone-03,692.2144,TlsC
|
||||
Tencent,il,,,Apple Inc.,183922,12.125.243.157,53.46,1780243200.00041,"anon, hosting",876.2575,de 5e 2d 1c,Beijing,c0 14,,155,,5c 48 7f aa 13 fb 78 f5 fb 77 b8 44 3a 0c c0 82 ce 0f 64 6a 94 5a b3 1d ca ed 4e 4e,03 03,44.466146,,31,,"anon, hosting",443,fr,zeek,-23.874047,,,21,2026-06-01 00:00:00.000410,139.7.47.238,33,35,,91.7685,NTT,55.151039,12.125.243.157,,263066,46654,197.248.98.58,cdn.example.com,Seoul,14,KDDI,-104.258302,4856,gw-09,574.1591,TlsC
|
||||
Microsoft Corp.,us,,,Amazon.com Inc.,160400,187.27.50.16,101.16,1780243200.000411,"anon, hosting",1794.3412,a5 7b 79 11,London,c0 13,,1829,512,9e ef 72 b0 f9 bf 8b 73 92 a4 78 c3 96 c8 33 1f f9 ff e3 e9 48 4c 52 48 db 9d c1 2b,03 04,-22.438272,,2,,"anon, hosting",443,us,suricata,-174.090158,,,110,2026-06-01 00:00:00.000411,217.26.47.163,5,356,,1712.9146,KDDI,17.38474,187.27.50.16,,188076,90973,110.0.180.62,,London,44,KDDI,135.540575,7595,wan-link,554.0891,TlsC
|
||||
Baidu Inc.,sg,,,Google LLC,144704,187.125.83.119,50.81,1780243200.000412,"anon, hosting",1880.9403,1a 61 36 31,New York,00 9e,,1994,512,f8 95 15 39 20 e8 61 0c c0 75 4a 23 8d a0 43 e1 cb 45 78 03 5f 92 1c b8 49 26 b7 18,03 03,-34.959814,,8,,"anon, hosting",443,no,zeek,145.242023,,x448,21,2026-06-01 00:00:00.000412,42.168.210.178,49,493,,1747.9047,China Mobile,40.737468,187.125.83.119,,161602,61445,124.114.55.90,login.live.com,Sydney,32,China Mobile,+,1774,gw-09,622.2057,TlsC
|
||||
Baidu Inc.,in,,,Alibaba Inc.,47462,221.50.25.167,96.12,1780243200.000413,"anon, hosting",1814.7877,7c 32 86 bd,Mumbai,c0 14,,542,,0d 46 e9 d8 8b fe 24 9d 5f 80 74 56 29 33 26 19 42 df d5 44 78 f8 e6 62 37 70 7f 03,03 03,7.982568,,140,,"anon, hosting",443,ua,netflow,+,,,8443,2026-06-01 00:00:00.000413,76.103.22.203,90,412,,640.4606,Singtel,23.268732,221.50.25.167,,398875,16964,189.95.255.133,,San Francisco,36,AT&T,96.112477,8686,wan-link,747.9184,TlsC
|
||||
Apple Inc.,kr,,,Samsung,1363,106.12.91.189,97.92,1780243200.000414,"anon, hosting",1577.65,ab 1e bf 69,Dublin,c0 23,,1038,384,fe fe d9 dd b8 b4 17 56 75 8d 90 43 77 34 db 5c 5e c3 db 89 9e 37 c8 31 9f c5 52 77,03 01,-22.35571,,89,*.cloudfront.net,"anon, hosting",12345,cn,packet_capture,-60.288891,,,8443,2026-06-01 00:00:00.000414,60.61.98.81,21,97,,913.0248,Singtel,-16.555719,210.153.53.241,,392444,78507,86.124.64.202,*.cloudfront.net,San Francisco,48,Singtel,128.402001,6588,edge-01,769.818,TlsC
|
||||
Tencent,fi,"*.['facebook', 'com']",00 18,Baidu Inc.,78505,161.81.216.153,77.26,1780243200.000415,"anon, hosting",718.9252,8a 03 2d 67,Beijing,00 9e,,209,,e7 18 c2 80 3c 9b 9c a6 6c 7d fc b2 5b 0a 31 b8 c3 30 8d de 37 c2 98 27 b3 f2 74 60,03 03,5.658202,,134,login.live.com,"anon, hosting",80,sg,zeek,-177.816864,,,443,2026-06-01 00:00:00.000415,202.34.176.62,69,184,,1165.2583,China Telecom,54.982786,161.81.216.153,,99512,80894,195.18.232.23,,London,3,KDDI,-94.718249,2123,core-02,857.4766,TlsS
|
||||
Apple Inc.,ua,,,Amazon.com Inc.,199644,202.160.223.156,86.2,1780243200.000416,"anon, hosting",255.2045,7c 3a c6 94,New York,c0 2c,,739,,bb 85 69 1a b0 6b ef 61 9c bd 31 63 b1 de 97 4a d7 1e c2 1f 0d ea 50 ef 82 05 a0 2d,03 03,-17.203422,,144,,"anon, hosting",995,ru,suricata,-62.14637,,,993,2026-06-01 00:00:00.000416,98.117.126.118,74,454,*.s3.amazonaws.com,856.6073,Comcast,-54.231232,202.160.223.156,,301757,94012,98.183.151.180,,Singapore,14,AT&T,-12.842301,3549,edge-01,167.2545,TlsS
|
||||
Apple Inc.,kr,,,Google LLC,46988,180.131.188.43,40.92,1780243200.000417,"anon, hosting",1822.5333,35 08 17 27,Moscow,00 35,,1666,,b9 4d be 92 6e 12 62 ae 7f ad 58 20 b4 20 fa 64 92 c2 d7 0a dd 4d 82 f0 34 b7 f3 a4,03 03,-32.271986,,83,*.s3.amazonaws.com,"anon, hosting",993,in,netflow,-27.018316,,,22,2026-06-01 00:00:00.000417,116.189.190.114,83,263,,382.8686,China Mobile,0.777662,180.131.188.43,,66556,67749,220.207.218.132,,Berlin,9,Telstra,76.379755,7435,core-02,409.2842,TlsS
|
||||
,nl,cdn.example.com,,Tencent,113997,136.216.108.238,72.74,1780243200.000418,"anon, hosting",425.8483,a7 ca 6f e7,New York,c0 13,,1658,256,dd 28 bf 06 f6 df fd c8 6f 37 3f 94 eb 8b c7 3c 58 78 2a 04 12 48 94 2e b9 ac 39 fd,03 03,-34.167844,,176,,"anon, hosting",8443,in,suricata,-25.378895,,,443,2026-06-01 00:00:00.000418,14.99.185.142,52,273,,1079.1354,SK Telecom,-59.208372,136.216.108.238,TLS_RSA_WITH_AES_256_GCM_SHA384,84907,54553,192.212.89.52,,Seattle,26,Comcast,-114.42558,4292,wan-link,59.3728,TlsS
|
||||
Baidu Inc.,jp,,00 1e,Google LLC,48791,49.43.123.185,78.34,1780243200.000419,"anon, hosting",1096.6203,2a e8 90 8d,Berlin,c0 14,Amazon.com Inc.,1666,,28 7b 88 f5 8b 52 cd 70 d4 82 24 5d 57 d2 9b e7 45 a3 88 70 5b 92 ae 6c cb 06 94 40,03 03,-47.540605,,120,,"anon, hosting",443,se,suricata,-179.232418,,,443,2026-06-01 00:00:00.000419,81.129.165.30,2,222,,354.679,China Mobile,35.781442,49.43.123.185,,86950,98352,117.138.156.65,*.cloudfront.net,Dublin,25,AT&T,-71.28596,1250,gw-09,783.3445,TlsC
|
||||
Google LLC,fr,,,Baidu Inc.,84246,200.248.122.71,99.77,1780243200.00042,"anon, hosting",1192.5821,64 a8 2a 9a,Berlin,c0 23,,1688,,9a 50 48 d3 dd 65 2c bc 9a 6c 09 0f 69 9f 3c 7e 26 b6 c3 af 64 c2 dd fe 2d 6a cd 9c,03 01,-47.952271,,74,,"anon, hosting",443,de,suricata,-147.082038,,,443,2026-06-01 00:00:00.000420,73.253.240.74,1,229,,1136.6841,Orange,48.748675,109.93.142.119,,229904,93972,105.59.71.12,,Sydney,15,Comcast,-154.818513,4215,core-02,549.7811,TlsS
|
||||
Alibaba Inc.,fr,,00 1e,Meta Platforms,22285,125.246.109.222,85.43,1780243200.000421,"anon, hosting",1241.6761,90 a7 4c cb,Seoul,00 35,,1159,,2d 77 69 70 43 f9 45 30 e7 00 57 d5 f7 3d c5 24 97 90 30 b4 3d 9d eb b5 d8 3f d4 15,03 04,51.239636,,192,*.s3.amazonaws.com,"anon, hosting",3389,ca,packet_capture,-40.895728,,,25,2026-06-01 00:00:00.000421,112.169.48.230,29,196,,1871.3307,NTT,-13.215648,125.246.109.222,,383830,16457,116.143.60.33,auth.example.net,Seoul,3,AT&T,46.647673,2166,wan-link,949.3019,TlsS
|
||||
Tencent,se,,,,51223,105.250.123.177,52.84,1780243200.000422,"anon, hosting",106.542,8b 79 6f 3d,Berlin,c0 27,,692,,d0 a8 35 9b 82 85 c0 ad eb 8a 65 0c ce 4f 0b c3 21 9e 98 3f e0 48 19 58 28 20 6a 05,03 04,-24.834055,,29,,"anon, hosting",53,hk,packet_capture,-133.114832,,,50000,2026-06-01 00:00:00.000422,202.160.223.156,84,233,stream.example.org,559.8495,China Telecom,-59.077307,105.250.123.177,,125529,59514,89.74.148.11,,Berlin,38,NTT,18.489512,7706,backbone-03,170.8085,TlsC
|
||||
Microsoft Corp.,sg,"*.['example', 'org']",00 18,Alibaba Inc.,145850,28.79.139.73,63.68,1780243200.000423,"anon, hosting",1707.676,ac db 74 29,Paris,c0 23,,960,,1e 5b 33 ad ae 3d 32 bc 29 3d 1b 30 92 c4 17 43 61 bd 7f ac cb 53 07 31 8a 18 3a 47,03 04,-37.192144,,1,*.cloudfront.net,"anon, hosting",53,ua,packet_capture,66.466046,,,21,2026-06-01 00:00:00.000423,57.70.252.24,61,352,,1916.6975,NTT,36.709713,28.79.139.73,,279008,77784,42.168.210.178,,Beijing,48,SK Telecom,-141.699754,1501,gw-09,498.6472,TlsC
|
||||
Tencent,nl,,,Tencent,163696,59.148.40.219,103.78,1780243200.000424,"anon, hosting",752.4233,ce 4e 0d 26,Seattle,13 02,,899,,9f 45 0f fa 40 9e 24 36 c7 c9 47 73 8f 82 83 1c a7 bd e1 48 9e ff ec 57 2a 71 d1 64,03 04,-47.247754,,200,,"anon, hosting",443,kr,netflow,66.70006,,,8443,2026-06-01 00:00:00.000424,110.189.35.130,40,411,*.s3.amazonaws.com,1977.3233,Verizon,-59.501413,59.148.40.219,,70546,21580,223.119.168.199,api.example.com,Shanghai,36,Singtel,73.022869,6999,gw-09,545.545,TlsC
|
||||
Amazon.com Inc.,gb,,,Netflix Inc.,85272,106.12.91.189,96.2,1780243200.000425,"anon, hosting",386.6151,9b 15 ad dd,Dublin,00 9f,Samsung,293,128,61 05 81 68 8f 4a 52 fd bf 6f 42 3f 01 21 7b 70 6f 04 07 8f 03 18 7e fa d7 b0 9e 08,03 03,-29.021558,,107,login.live.com,"anon, hosting",25,nl,netflow,-163.0186,,secp384r1,110,2026-06-01 00:00:00.000425,12.113.74.5,53,237,,1127.6924,BT Group,35.771716,106.12.91.189,,35851,68336,98.183.151.180,,Singapore,34,Singtel,4.775278,4176,backbone-03,762.7001,TlsC
|
||||
Amazon.com Inc.,sg,,,Meta Platforms,131174,207.100.186.111,35.96,1780243200.000426,"anon, hosting",1970.0693,5e 6e d2 53,Shanghai,c0 13,,1850,256,39 66 ec 1f e0 8e 29 98 56 0b 33 7b 8d 9e 6c 59 26 9f 08 95 c7 5a 9f f3 df 64 b7 e1,03 04,55.311994,,20,,"anon, hosting",80,au,ssl_logs,77.625887,,x448,25,2026-06-01 00:00:00.000426,47.139.158.236,1,390,,126.6184,China Mobile,49.75021,207.100.186.111,,311557,46559,80.166.43.227,mail.example.com,Moscow,27,Orange,-20.914601,9207,edge-01,958.3141,TlsC
|
||||
Alibaba Inc.,nl,,,Google LLC,177505,122.251.229.203,95.28,1780243200.000427,"anon, hosting",359.7921,96 90 4f d6,Tokyo,00 35,,1125,,3c 74 f4 c8 0e e9 5e 02 6e 4e 8e 62 48 74 ee f6 bb d7 fb f3 8a c0 c6 a6 68 5f 8d 3b,03 03,50.526143,,23,api.example.com,"anon, hosting",80,hk,ssl_logs,-103.639393,,,53,2026-06-01 00:00:00.000427,74.159.180.27,27,421,,176.4313,Comcast,4.881136,122.251.229.203,,297901,41526,86.86.39.131,,Beijing,42,Comcast,-115.605602,8873,wan-link,385.9191,TlsS
|
||||
Netflix Inc.,ua,,,Google LLC,62617,90.35.125.95,38.4,1780243200.000428,"anon, hosting",429.4114,44 86 c5 bf,Berlin,c0 2f,,1927,,24 77 40 f7 18 dc da a1 37 81 df d1 1b 34 fd 07 b3 79 d4 ed 12 36 ab bd 54 5e c6 9d,03 03,-21.841122,,176,,"anon, hosting",25,se,suricata,-139.217137,,,443,2026-06-01 00:00:00.000428,183.227.44.208,43,82,,926.7873,SK Telecom,3.75255,90.35.125.95,,26514,38389,25.201.191.204,,Paris,34,China Mobile,-34.31084,5753,wan-link,82.2637,TlsC
|
||||
Meta Platforms,in,mail.example.com,,Samsung,420,142.150.185.148,119.55,1780243200.000429,"anon, hosting",93.4357,31 46 b5 28,Singapore,c0 13,,1761,384,d9 94 cf a9 ff 5a 89 89 99 c7 7c 2a 48 8f 47 15 58 ea 9c 7d 16 2f 28 49 46 13 5d 25,03 03,-24.370038,,57,mail.example.com,"anon, hosting",22,ca,zeek,8.804888,,,465,2026-06-01 00:00:00.000429,179.151.65.50,35,256,,155.5378,Deutsche Telekom,51.599218,142.150.185.148,,380504,37375,51.140.64.161,,Tokyo,30,Singtel,-169.885227,2013,core-02,475.8103,TlsS
|
||||
Amazon.com Inc.,fr,,,Google LLC,182304,151.117.112.2,45.67,1780243200.00043,"anon, hosting",612.0856,be 6d 32 06,Seoul,13 01,Amazon.com Inc.,717,128,5f 99 81 af 99 f0 a6 e0 23 34 76 38 bf d4 f9 6b 3b 15 42 98 50 c6 97 a2 db 76 8f da,03 03,-0.213745,,127,mail.example.com,"anon, hosting",993,us,zeek,43.120664,,,443,2026-06-01 00:00:00.000430,129.220.91.28,40,96,,142.8196,SK Telecom,-5.936143,151.117.112.2,,3710,91329,181.250.162.123,,London,11,Orange,-158.942198,6126,edge-01,549.7945,TlsC
|
||||
Tencent,il,,00 1d,Google LLC,113154,166.174.6.218,92.1,1780243200.000431,"anon, hosting",1464.3376,ed 1f b2 4f,Moscow,c0 2f,,782,,3e f9 62 16 c8 d6 a5 1d ce ba ec db 13 5d ee 64 3e d9 0a 7a b2 1b 4e 9c ae 07 c9 66,03 03,37.061799,,115,,"anon, hosting",80,no,packet_capture,116.798919,7614,,143,2026-06-01 00:00:00.000431,93.157.122.15,57,194,,1546.7259,AT&T,43.599646,166.174.6.218,,322126,12259,2.254.220.45,,Mumbai,33,SK Telecom,-114.7135,9013,wan-link,120.1064,TlsS
|
||||
Apple Inc.,il,,,Baidu Inc.,84819,28.79.139.73,48.65,1780243200.000432,"anon, hosting",1682.0983,65 2c 7d bb,Singapore,c0 14,,135,,c7 73 2c a3 b1 b3 18 09 f6 bd ff ad eb a8 32 6c 38 72 45 62 fb 1b 8f 8c 83 c6 a3 08,03 03,19.765623,--,137,,"anon, hosting",8443,ca,suricata,-149.312902,32123,,8080,2026-06-01 00:00:00.000432,47.139.158.236,11,415,,1922.7995,Singtel,-35.266355,28.79.139.73,TLS_RSA_WITH_AES_128_CBC_SHA,20130,94937,41.89.185.131,api.example.com,Dublin,3,China Telecom,-71.503142,7358,gw-09,422.2354,TlsS
|
||||
Microsoft Corp.,ca,www.google.com,,Apple Inc.,131681,113.39.146.61,97.29,1780243200.000433,"anon, hosting",1331.2739,5f e1 53 e7,Dublin,13 01,,1141,,8c c3 8f 56 58 0e d5 4e ad af 7b 1b 05 65 b3 1e eb bc cf 7d 82 94 31 14 b2 77 94 7a,02 00,-3.016089,,17,www.google.com,"anon, hosting",995,fr,netflow,52.843432,,,53,2026-06-01 00:00:00.000433,17.237.154.167,61,340,,1797.309,SK Telecom,22.325207,146.154.207.141,,118640,18148,72.115.219.217,www.google.com,Shanghai,32,Singtel,-136.772212,9658,edge-01,174.5081,TlsC
|
||||
,ca,,,Meta Platforms,121729,10.240.113.52,114.44,1780243200.000434,"anon, hosting",1804.9968,d9 c0 b5 e9,Berlin,c0 2c,,988,,9e f9 16 35 95 ec 3c 5e 07 72 72 43 c0 b1 d8 3e f0 c1 24 45 c1 ce d2 3c c5 50 ea e7,03 04,39.427209,,171,,"anon, hosting",25,kr,ssl_logs,-105.710868,,,3389,2026-06-01 00:00:00.000434,194.24.56.40,89,341,,1253.691,AT&T,22.593694,10.240.113.52,,384170,75550,2.210.6.5,,Berlin,47,Verizon,16.321255,8110,edge-01,524.4781,TlsC
|
||||
Meta Platforms,il,,,Meta Platforms,20186,72.141.5.133,8.03,1780243200.000435,"anon, hosting",1604.4085,c2 53 fc 1d,Dublin,c0 27,,244,256,28 fb 9e e8 01 92 cb f9 3f 39 e1 51 96 a1 c1 7a 83 67 d8 bf 18 80 35 d5 48 c8 e9 5b,03 04,19.468227,,49,www.google.com,"anon, hosting",443,se,ssl_logs,-150.081653,,,80,2026-06-01 00:00:00.000435,65.76.170.21,5,4,,667.8901,Deutsche Telekom,-9.908176,72.141.5.133,,467045,93098,192.134.195.211,,Seoul,39,Deutsche Telekom,-112.167592,7304,core-02,809.9724,TlsC
|
||||
Baidu Inc.,br,api.example.com,,Google LLC,73570,56.119.13.144,12.89,1780243200.000436,"anon, hosting",1122.2696,83 ce ba 32,Tokyo,00 9e,,483,256,f5 f8 32 6a 87 21 9e af 4c 8c 8d dd 8c 21 a1 6e b1 0c 08 87 64 b3 26 8d 14 c1 6a 4e,03 04,-57.638365,,57,api.example.com,"anon, hosting",3389,fr,ssl_logs,-38.579927,,,12345,2026-06-01 00:00:00.000436,103.66.189.132,17,136,login.live.com,1422.9398,KDDI,-32.279661,56.119.13.144,,83158,25692,73.232.36.176,www.google.com,Dublin,7,BT Group,-86.526428,8020,core-02,362.3722,TlsS
|
||||
Netflix Inc.,no,,,Meta Platforms,41665,28.152.101.40,47.43,1780243200.000437,"anon, hosting",1083.4567,48 53 6f 47,Tokyo,c0 2f,,214,384,8b 0e 81 35 88 e4 fe 2e db 54 58 27 41 94 92 c5 52 31 cc 63 c0 42 f9 02 6c dc 91 e0,03 03,26.343767,,9,*.cloudfront.net,"anon, hosting",25,jp,netflow,78.608592,,,53,2026-06-01 00:00:00.000437,56.76.216.29,39,70,,1555.9322,SK Telecom,30.526834,28.152.101.40,,320553,55848,26.250.208.241,,Mumbai,6,Orange,128.97083,6271,edge-01,258.1642,TlsS
|
||||
Amazon.com Inc.,hk,,,Meta Platforms,121633,172.191.224.231,16.77,1780243200.000438,"anon, hosting",1382.9113,42 05 55 a1,Shanghai,c0 13,,1618,,02 a5 d9 44 1a 18 59 0a f2 02 5d 57 12 42 b6 5a 3a 02 8c 49 64 b0 84 fb 29 6c 19 50,02 00,-18.180679,,124,,"anon, hosting",995,hk,suricata,153.233497,,,3389,2026-06-01 00:00:00.000438,218.232.46.170,5,263,,496.0198,NTT,55.992232,172.191.224.231,,131610,55502,113.174.94.248,,Beijing,15,NTT,-25.325763,2705,core-02,168.838,TlsC
|
||||
Baidu Inc.,ca,,,Alibaba Inc.,65007,70.171.163.229,77.66,1780243200.000439,"anon, hosting",751.3905,f0 5b 67 88,Mumbai,13 01,,226,,e2 da fb 26 ab 9f 5e ad a5 f8 29 9a e0 fe 1a 16 ef 2d 2a 18 76 bb 60 70 cd 72 b2 4c,03 03,0.736773,,31,mail.example.com,"anon, hosting",53,in,packet_capture,7.609751,,,80,2026-06-01 00:00:00.000439,118.63.193.21,86,333,,443.2734,Deutsche Telekom,-22.069428,70.171.163.229,,159530,61116,138.97.26.69,stream.example.org,Dublin,45,Telstra,27.290933,4813,backbone-03,901.5518,TlsS
|
||||
Alibaba Inc.,ru,"*.['facebook', 'com']",,Google LLC,6778,85.238.212.16,72.83,1780243200.00044,"anon, hosting",1425.8801,1e c0 e5 38,Berlin,00 9e,,673,,23 11 02 d5 a6 cd 2f 6e 60 61 2b cb f0 56 c7 c5 93 08 79 18 e4 b1 01 ac 0d dc 8e 0a,03 04,56.960337,,38,www.google.com,"anon, hosting",3389,ca,suricata,-77.893333,,,110,2026-06-01 00:00:00.000440,109.141.22.1,66,256,,543.797,,54.648859,85.238.212.16,,98876,77366,214.247.241.63,,Beijing,28,Singtel,-106.228252,7722,gw-09,364.3201,TlsS
|
||||
Google LLC,jp,,,Tencent,120027,148.97.130.12,95.35,1780243200.000441,"anon, hosting",595.7156,bf 4a 38 d7,London,c0 2c,,800,,e3 d2 a1 fe 2f 5e ad 83 98 b3 9a e1 b4 2c c8 20 82 09 22 87 0e 0a ab 9b 99 9f d8 57,03 03,-57.667976,,15,,"anon, hosting",110,se,netflow,-88.574227,,,8443,2026-06-01 00:00:00.000441,69.72.36.16,36,249,,304.3344,China Mobile,33.982779,148.97.130.12,,442281,67089,166.88.20.64,*.s3.amazonaws.com,Singapore,31,BT Group,106.728125,8170,wan-link,837.7649,TlsS
|
||||
Google LLC,fr,,,Tencent,12249,71.125.114.36,66.47,1780243200.000442,"anon, hosting",1757.753,ef 20 c1 c4,San Francisco,00 9e,,196,,be 70 77 39 27 ae ce 97 70 a5 d4 5a eb ad 62 36 84 79 79 c1 93 4d 38 b0 4c d4 81 ba,03 03,44.320986,,167,login.live.com,"anon, hosting",465,fr,suricata,-36.339639,1762,secp384r1,443,2026-06-01 00:00:00.000442,120.65.13.233,38,446,,1663.8043,Verizon,-54.829326,71.125.114.36,,256764,49860,42.168.210.178,,Dublin,9,KDDI,118.802502,2185,wan-link,812.4683,TlsS
|
||||
Alibaba Inc.,gb,,,Netflix Inc.,185179,120.128.5.175,22.2,1780243200.000443,"anon, hosting",1578.6842,ae c6 d5 cd,Tokyo,c0 2c,,472,128,52 e2 04 6c 1c d7 f1 db 83 40 18 3b 2f ff 7d 26 14 27 14 3a 0f 86 c8 12 18 a9 5f 17,03 02,29.243984,,91,,"anon, hosting",993,ru,zeek,150.422943,,,3389,2026-06-01 00:00:00.000443,97.20.37.49,40,464,,1201.239,AT&T,14.707997,120.128.5.175,TLS_AES_256_GCM_SHA384,140635,23108,76.11.202.71,,New York,32,Deutsche Telekom,-17.402597,3690,backbone-03,544.3932,TlsS
|
||||
Alibaba Inc.,in,"*.['example', 'com']",,Google LLC,41571,167.35.20.194,114.25,1780243200.000444,"anon, hosting",1446.9874,ca 52 1a f5,London,00 9e,,1401,,42 e1 ce 90 cb 3b 43 6c b4 a5 87 bb 4b ab 39 6a e2 58 05 d3 ef e1 6f 1e eb 0d 7b 6d,03 03,-53.345317,,118,,"anon, hosting",3389,sg,zeek,-156.882364,,,443,2026-06-01 00:00:00.000444,96.220.74.63,34,43,,1442.7409,BT Group,-29.1931,167.35.20.194,,124052,93178,145.229.165.222,,Paris,21,Singtel,-43.403735,6466,wan-link,303.3898,TlsS
|
||||
Amazon.com Inc.,fi,"*.['example', 'net']",,Tencent,67851,31.126.20.159,119.51,1780243200.000445,"anon, hosting",597.5863,b3 09 3b 48,Seattle,13 02,,1191,,5e 4d 14 54 9e 23 ff da d0 e0 17 ce 67 51 54 c1 d1 e5 94 1e 09 03 f6 f9 71 a7 96 d1,03 04,14.116173,,42,login.live.com,"anon, hosting",8080,il,zeek,-95.686752,,,465,2026-06-01 00:00:00.000445,101.13.42.165,41,32,,1269.7284,China Telecom,-53.290351,213.215.129.214,,330414,56110,223.79.38.121,,Tokyo,5,NTT,26.614325,6314,gw-09,515.0204,TlsS
|
||||
Tencent,cn,,00 18,Samsung,8978,35.135.59.228,55.0,1780243200.000446,"anon, hosting",808.1817,ed f9 b2 27,Paris,c0 23,,816,,b2 1b 5d 96 02 e2 74 e2 f2 2a 18 86 bd 33 b2 0c b7 87 57 97 e4 90 16 9d 35 9d ff 35,03 04,16.558061,,129,,"anon, hosting",995,sg,packet_capture,-1.22346,,,3389,2026-06-01 00:00:00.000446,29.145.249.135,24,145,,616.9972,Comcast,18.313024,35.135.59.228,,6211,59814,183.227.37.231,api.example.com,San Francisco,20,Verizon,-173.569904,8655,backbone-03,691.5852,TlsS
|
||||
Meta Platforms,de,,,,28430,110.69.236.47,21.08,1780243200.000447,"anon, hosting",1843.7711,b9 e5 89 62,Beijing,c0 2b,,1688,384,29 e8 9c ef 84 2f a0 f3 df 31 8c 42 60 1c 5f 22 58 83 a4 4e 86 f4 04 0c ff 16 5c ec,02 00,32.776941,,62,auth.example.net,"anon, hosting",995,sg,suricata,-82.850365,,x448,22,2026-06-01 00:00:00.000447,74.80.37.114,53,150,,873.3094,BT Group,34.193639,110.69.236.47,TLS_AES_128_GCM_SHA256,406577,46259,71.97.62.243,,Dublin,49,SK Telecom,25.780373,5957,wan-link,571.7114,TlsC
|
||||
Amazon.com Inc.,kr,,,Microsoft Corp.,170019,101.13.42.165,30.89,1780243200.000448,"anon, hosting",383.9165,84 46 89 24,Dublin,c0 23,,1110,,04 d7 f6 47 89 d8 a0 a0 67 02 4b 4c 5c 57 3d 41 34 cc 0a 68 a3 6f 59 4c 74 55 fd 49,03 04,-58.634554,,38,,"anon, hosting",25,nl,zeek,65.880254,,,12345,2026-06-01 00:00:00.000448,42.102.70.224,66,86,,328.999,China Mobile,-30.104749,103.30.84.98,TLS_RSA_WITH_AES_128_GCM_SHA256,492735,39724,21.164.75.17,,Beijing,26,AT&T,6.327213,2817,backbone-03,424.9726,TlsC
|
||||
Baidu Inc.,ru,,,Apple Inc.,30467,12.125.243.157,90.65,1780243200.000449,"anon, hosting",1454.0918,16 ec cd 4f,Singapore,c0 2b,,1468,,b6 e2 f9 2f a2 ef 77 73 77 aa 71 6c ff 13 5e 7e 95 1c a7 df 94 ea c2 35 56 ed 3d 4e,02 00,54.591031,,78,,"anon, hosting",22,gb,netflow,88.79948,,,12345,2026-06-01 00:00:00.000449,179.248.62.8,54,499,,109.6799,Singtel,54.109077,12.125.243.157,,12124,89363,217.187.46.203,,Paris,22,SK Telecom,-4.495856,5266,edge-01,446.208,TlsS
|
||||
Microsoft Corp.,us,,,Alibaba Inc.,16944,58.32.173.6,33.07,1780243200.00045,"anon, hosting",1943.7866,83 c9 f5 14,New York,13 02,,719,,37 ff ba ec 74 46 5a 44 f6 13 49 78 ca 6b 30 13 d6 5b 0f 8d db eb d4 64 14 71 20 c6,03 03,35.369948,,188,,"anon, hosting",443,fr,netflow,-82.596651,,,53,2026-06-01 00:00:00.000450,147.194.202.210,17,55,,1338.1345,,-15.702235,58.32.173.6,,465697,58802,60.135.101.109,,Shanghai,33,SK Telecom,69.966895,3944,backbone-03,14.733,TlsC
|
||||
Apple Inc.,ca,,,Google LLC,126509,165.204.169.241,34.39,1780243200.000451,"anon, hosting",1594.3268,2d 65 76 4c,Singapore,c0 23,,909,,62 c7 09 a6 7b 01 20 2b 06 0b 10 f6 f1 79 3b 50 d1 68 f4 80 24 f8 18 6a 5d 5f a2 1c,03 04,-11.933985,,2,,"anon, hosting",443,se,suricata,-123.016414,,x448,80,2026-06-01 00:00:00.000451,214.99.188.174,34,254,,1672.9852,NTT,23.404226,165.204.169.241,TLS_RSA_WITH_AES_128_CBC_SHA,382353,71916,21.164.75.17,api.example.com,Dublin,20,Singtel,92.390267,7159,edge-01,922.1884,TlsC
|
||||
Amazon.com Inc.,fr,,,Meta Platforms,146012,145.21.177.137,103.05,1780243200.000452,"anon, hosting",1243.513,6d f9 d4 84,Seattle,00 35,,669,,b5 43 1d 16 87 7d 5f 17 23 50 3c 4e 95 f6 cb d6 dc ee 69 0d 62 31 fa 56 88 19 6e f3,03 04,46.002605,,178,auth.example.net,"anon, hosting",443,fr,suricata,0.016609,,,8443,2026-06-01 00:00:00.000452,222.14.41.12,75,79,,1443.7475,Comcast,-38.005475,145.21.177.137,TLS_ECDHE_ECDSA_AES128_SHA,131134,70528,47.139.158.236,*.s3.amazonaws.com,Berlin,17,AT&T,14.799751,2625,gw-09,335.9775,TlsC
|
||||
Baidu Inc.,ru,,00 1d,Amazon.com Inc.,78759,211.116.114.7,8.15,1780243200.000453,"anon, hosting",93.3289,8a 5b 40 72,San Francisco,13 02,,1047,512,13 eb 6a ee 45 56 39 ff c7 8c 07 6f cc 23 92 b6 9b 80 0c 5d 28 b9 c3 aa 09 a6 8f 99,03 03,-8.854476,,17,login.live.com,"anon, hosting",465,fi,packet_capture,-34.437319,,,12345,2026-06-01 00:00:00.000453,172.115.119.14,21,97,,486.0679,SK Telecom,-35.812641,211.116.114.7,,205544,44440,42.102.70.224,,Moscow,17,Verizon,-178.388512,1546,gw-09,474.5735,TlsC
|
||||
Alibaba Inc.,hk,"*.['live', 'com']",,Amazon.com Inc.,10342,200.248.122.71,111.91,1780243200.000454,"anon, hosting",970.221,bc 40 e6 4c,Seoul,00 35,,1406,,af ad 61 9f bf d4 91 76 55 ee ba 9d 2a 24 a4 da ba 4c f2 e8 1a 2b b0 c4 52 c7 cd 29,03 03,47.445402,,176,*.s3.amazonaws.com,"anon, hosting",443,in,ssl_logs,152.520465,,,443,2026-06-01 00:00:00.000454,8.199.172.172,42,159,,402.7408,Telstra,-24.795213,200.248.122.71,,430348,61617,109.40.220.155,,Beijing,15,KDDI,-13.904104,3848,wan-link,515.2402,TlsC
|
||||
Netflix Inc.,de,,,Alibaba Inc.,7917,185.134.91.130,59.06,1780243200.000455,"anon, hosting",1122.6964,67 30 43 68,Berlin,c0 14,,1980,,c9 3b c0 55 93 45 61 6c 0c 38 18 f2 16 3b e0 ec 96 9d ca 0a 71 97 c0 87 3b f9 a5 04,03 03,47.012271,,160,login.live.com,"anon, hosting",3389,nl,ssl_logs,171.434367,,,443,2026-06-01 00:00:00.000455,39.228.247.89,68,232,,597.041,SK Telecom,-20.676382,185.134.91.130,TLS_ECDHE_RSA_AES128_GCM_SHA256,364437,57165,190.22.205.113,mail.example.com,London,13,Telstra,30.77675,7768,core-02,685.3006,TlsS
|
||||
Tencent,gb,,00 1d,Apple Inc.,99685,5.47.150.57,4.44,1780243200.000456,"anon, hosting",1555.2571,09 b1 c3 fe,London,00 9e,,1058,,03 11 92 d0 f8 f1 b5 49 29 14 1e 91 dc a0 99 59 b1 a9 dd c4 48 5d 54 1c a4 88 91 30,03 03,41.718994,,134,,"anon, hosting",443,cn,netflow,159.278235,,,8080,2026-06-01 00:00:00.000456,198.10.194.123,3,73,,669.5028,NTT,29.487903,5.47.150.57,TLS_ECDHE_RSA_AES128_SHA256,382268,68544,211.250.251.118,graph.facebook.com,Seoul,43,Singtel,-25.369593,1119,wan-link,561.5437,TlsS
|
||||
Google LLC,fi,,,Meta Platforms,73321,98.138.112.176,99.05,1780243200.000457,"anon, hosting",1782.7006,96 9f 48 59,Paris,00 9f,,384,384,ba 16 1b 7b 82 15 d1 0a 1c f6 ce 95 c8 bf a7 9b 4f 53 b3 5e ed e1 c7 b4 ef ab bb de,03 03,-45.489382,,116,,"anon, hosting",80,sg,packet_capture,155.229129,,,443,2026-06-01 00:00:00.000457,35.37.231.45,59,89,,771.3717,Telstra,41.37367,207.81.216.88,,267168,16648,218.232.46.170,auth.example.net,San Francisco,29,China Mobile,113.065997,2969,gw-09,975.5202,TlsC
|
||||
Apple Inc.,ua,,,Google LLC,15897,117.186.83.95,77.95,1780243200.000458,"anon, hosting",963.438,e0 40 e9 04,Beijing,c0 27,,767,,66 d1 ef d1 3a 0a eb a0 0c a2 ca 28 32 b1 b4 ba 31 16 fc 99 3e cd c3 35 2c be 0e 20,03 03,45.160599,,64,,"anon, hosting",443,ru,netflow,-67.693005,,,443,2026-06-01 00:00:00.000458,139.41.70.39,89,36,,808.1574,,20.320609,117.186.83.95,,180186,72107,114.212.139.56,,Seattle,26,Orange,63.505498,2726,wan-link,333.6988,TlsC
|
||||
Amazon.com Inc.,kr,,,,99415,105.250.123.177,19.81,1780243200.000459,"anon, hosting",672.1614,44 0b e1 80,London,13 02,,266,,ff 62 83 34 78 b7 0a af 8e 03 e6 d2 82 f7 38 c4 42 8a 62 d1 38 61 2d 97 aa 77 90 04,03 01,-47.207605,,2,,"anon, hosting",443,no,suricata,170.080569,,,12345,2026-06-01 00:00:00.000459,151.117.112.2,19,28,,868.1221,SK Telecom,14.327831,105.250.123.177,,100835,37341,182.73.237.235,,San Francisco,18,BT Group,-140.670682,4150,wan-link,730.0901,TlsS
|
||||
Microsoft Corp.,ca,,00 17,Meta Platforms,186154,139.102.159.103,83.41,1780243200.00046,"anon, hosting",780.3598,8f 60 d0 ad,Sydney,c0 27,,635,,52 17 11 9c 06 4c 6d 8f 44 18 60 73 f5 6b 9a be 6b b7 10 1b 40 31 a0 d2 7a 65 ce c1,03 03,25.469168,,63,,"anon, hosting",22,no,suricata,82.179809,,,12345,2026-06-01 00:00:00.000460,50.151.116.93,54,162,,496.0118,Deutsche Telekom,45.808303,139.102.159.103,,257941,14546,36.79.39.76,,Shanghai,10,Comcast,136.721452,4027,wan-link,581.5415,TlsC
|
||||
Netflix Inc.,il,,,Alibaba Inc.,110674,86.210.127.69,47.2,1780243200.000461,"anon, hosting",1667.6648,72 f2 6d ad,Singapore,c0 14,,541,,99 f9 cd 12 af da e5 d1 ba dd 2a 55 8b 6f 75 98 84 31 9f c3 f8 c4 68 4c be 54 33 92,03 03,23.085976,,172,,"anon, hosting",995,ru,packet_capture,-124.730191,,,8443,2026-06-01 00:00:00.000461,202.34.176.62,78,135,,896.5816,China Telecom,41.496712,109.210.239.222,,104747,24929,29.145.249.135,,Moscow,7,Deutsche Telekom,-140.906054,8650,backbone-03,457.5124,TlsS
|
||||
Baidu Inc.,us,api.example.com,00 18,Baidu Inc.,182018,54.220.164.120,115.94,1780243200.000462,"anon, hosting",700.9887,51 71 b8 8f,Paris,c0 27,,156,,3d 4d b7 84 ea 5b cc 74 33 ab 1a 24 04 de df e2 1b 89 42 b7 e5 09 6d a4 8e 17 c6 8a,03 03,-22.633394,,133,,"anon, hosting",12345,se,zeek,132.587762,,,995,2026-06-01 00:00:00.000462,25.120.69.100,12,42,,564.1523,Comcast,57.712466,54.220.164.120,,107281,51985,164.207.186.132,,Paris,30,Deutsche Telekom,-166.666744,9744,core-02,42.3489,TlsC
|
||||
Microsoft Corp.,fr,,,Google LLC,74592,67.104.160.62,25.69,1780243200.000463,"anon, hosting",1098.4267,c7 71 c5 6b,Moscow,c0 2f,,375,128,e0 92 43 60 c9 66 e6 04 6f e7 28 ef 25 dd a4 0a e9 91 ce 3b 4b 14 80 d3 ac 9a a7 c1,03 04,5.010388,,191,,"anon, hosting",25,fi,packet_capture,154.272645,,,12345,2026-06-01 00:00:00.000463,166.88.20.64,90,119,,1499.3744,BT Group,27.783079,67.104.160.62,,16277,55485,129.28.63.133,,San Francisco,46,China Telecom,-90.75644,9098,edge-01,537.1103,TlsC
|
||||
Google LLC,se,,,Baidu Inc.,115394,109.112.90.206,59.98,1780243200.000464,"anon, hosting",1018.3989,13 26 03 f6,Moscow,c0 2b,,1641,,99 1f 91 ff d3 27 93 ba d7 2c a3 37 38 26 05 cb 93 4d 5e 54 78 20 fb af 6b 4b 15 20,03 03,-5.281607,,122,graph.facebook.com,"anon, hosting",21,ru,suricata,96.077947,,,465,2026-06-01 00:00:00.000464,74.80.37.114,43,320,,618.6551,Comcast,-31.779439,109.112.90.206,,161178,86514,35.215.191.229,,San Francisco,16,KDDI,-23.863982,1709,gw-09,269.4177,TlsS
|
||||
Google LLC,in,,,Meta Platforms,172637,210.235.179.79,77.41,1780243200.000465,"anon, hosting",566.3598,0d 15 9e d5,New York,00 35,,973,,0a 9a 57 fc bb 2d 1d 2f 9e 20 18 0e 4c b0 5b 6c 18 c6 b0 31 3b f5 d3 52 a6 3c a3 4d,03 04,3.988466,,60,,"anon, hosting",3389,kr,ssl_logs,72.690158,,,443,2026-06-01 00:00:00.000465,109.93.142.119,38,8,,252.4003,China Telecom,-3.716577,210.235.179.79,,11659,17143,74.209.92.197,,Dublin,29,Orange,-26.048612,6269,core-02,828.776,TlsS
|
||||
Netflix Inc.,jp,,,Microsoft Corp.,40202,76.111.29.149,81.2,1780243200.000466,"anon, hosting",651.2619,ae 1e 50 46,London,cc a9,,757,,16 ad 29 3c b8 b3 78 af 01 dc 40 4d b8 11 8d 20 80 7a e2 64 79 6d 22 5a 68 c3 b4 ac,03 03,-36.497393,,2,,"anon, hosting",443,il,netflow,-96.992602,,x25519,3389,2026-06-01 00:00:00.000466,140.154.4.171,29,440,,1556.2166,Orange,15.143577,76.111.29.149,,358431,95462,164.190.55.173,,Seoul,28,Comcast,80.423723,8287,core-02,938.1668,TlsC
|
||||
Microsoft Corp.,fr,,,Microsoft Corp.,94908,98.138.112.176,75.01,1780243200.000467,"anon, hosting",582.9412,29 bb ee 02,Paris,cc a9,,1152,,85 45 75 b1 29 13 51 de 9f 5c 38 9a 47 98 25 c6 77 8e 92 fb ef 5f f8 d5 65 4d 02 b1,03 03,57.141722,,57,cdn.example.com,"anon, hosting",443,ua,ssl_logs,10.987008,,,25,2026-06-01 00:00:00.000467,101.234.123.56,88,425,login.live.com,778.9241,BT Group,54.375584,72.79.110.246,,258399,49277,170.77.253.247,,Beijing,41,Verizon,-83.748414,2150,core-02,206.2695,TlsC
|
||||
Apple Inc.,ua,,,Google LLC,136930,161.81.216.153,30.75,1780243200.000468,"anon, hosting",1172.9962,7e 3d 0e 32,Mumbai,13 02,,971,,e3 5a 1a 6b 55 66 72 4f 40 f6 1f 9f 85 7a 47 f4 95 0e 51 c2 b4 af a5 5c 86 bb 19 54,03 03,-10.307407,,105,,"anon, hosting",443,ru,ssl_logs,93.929007,,,443,2026-06-01 00:00:00.000468,21.164.75.17,10,326,,533.8786,AT&T,-7.8249,156.58.83.247,,147040,67387,135.10.199.224,,Paris,42,Telstra,7.115938,4548,core-02,91.1319,TlsC
|
||||
Tencent,no,,,Baidu Inc.,26149,165.234.73.68,27.73,1780243200.000469,"anon, hosting",1609.6477,19 5a ce b7,San Francisco,00 35,,1066,,61 74 f3 76 d4 93 67 bf 76 fd 39 5c 32 95 c9 7c 1b ff 1e 1d a8 69 0a 83 57 c8 6f 4c,03 01,9.780582,,42,*.s3.amazonaws.com,"anon, hosting",8443,us,suricata,91.300752,,,22,2026-06-01 00:00:00.000469,17.108.161.55,44,24,,1666.4045,SK Telecom,-10.621305,165.234.73.68,,324036,64624,161.148.154.124,,Shanghai,18,Verizon,128.359081,6319,gw-09,838.6927,TlsS
|
||||
Google LLC,hk,,,Microsoft Corp.,104610,8.59.133.46,70.46,1780243200.00047,"anon, hosting",1462.3955,d2 af 01 12,Singapore,c0 23,,1913,,6a 6e 27 f2 fb 68 ba d6 71 5e 26 37 e8 34 c4 48 c4 f4 e0 70 a8 3b 18 b4 ec 6c 99 99,03 03,-33.505865,--,110,www.google.com,"anon, hosting",443,fi,zeek,-166.098803,,,143,2026-06-01 00:00:00.000470,34.187.139.213,96,51,cdn.example.com,823.7478,AT&T,48.663714,8.59.133.46,,495691,58778,64.142.152.58,,Berlin,9,China Telecom,91.18142,7146,backbone-03,444.8937,TlsS
|
||||
Google LLC,de,,,Baidu Inc.,6871,180.131.188.43,81.79,1780243200.000471,"anon, hosting",536.4891,bb e5 e3 97,Seoul,c0 2c,,1902,,2a d3 a6 2f f2 b8 dd cb d1 5d ce b5 fe 1a 98 6f 66 64 59 65 87 0c 3c 5f ca 27 e9 cf,03 03,-39.495628,,60,,"anon, hosting",443,nl,suricata,167.048544,,,8443,2026-06-01 00:00:00.000471,15.236.157.186,70,240,,1505.6811,SK Telecom,50.755467,180.131.188.43,,360830,16031,78.140.28.91,login.live.com,New York,30,BT Group,-170.592715,2875,edge-01,141.6753,TlsS
|
||||
Baidu Inc.,br,,,Apple Inc.,101024,221.243.230.67,114.78,1780243200.000472,"anon, hosting",201.8989,f1 24 1e 84,Dublin,cc a9,Baidu Inc.,1842,,a5 0d 5a d0 4b f6 bd fb 4e 1e 62 32 92 64 05 cf 33 99 79 5a bb 94 e7 14 0a d4 3a 41,03 04,-21.660184,,187,,"anon, hosting",443,jp,netflow,77.915102,,,443,2026-06-01 00:00:00.000472,84.28.150.92,80,264,,1856.0742,Telstra,-48.29535,125.246.109.222,,241690,85692,64.41.228.136,,Shanghai,22,Orange,99.504567,3957,backbone-03,483.9456,TlsC
|
||||
Microsoft Corp.,ca,,,Samsung,47824,125.246.109.222,39.9,1780243200.000473,"anon, hosting",1578.9752,aa c4 8a 3f,Dublin,c0 13,,1472,,a2 2b 80 37 aa d6 23 e9 61 2c 5a 8f 73 4c 3b ea 4e 81 f2 cc 3a ec 8c 18 74 a7 68 13,03 04,17.371423,,119,stream.example.org,"anon, hosting",443,de,ssl_logs,-37.159457,,,110,2026-06-01 00:00:00.000473,223.158.173.57,29,468,,1336.3228,Deutsche Telekom,+,64.62.233.35,,28937,15442,210.75.120.27,,Seoul,38,Verizon,-77.498853,6274,core-02,551.4726,TlsC
|
||||
Alibaba Inc.,nl,,,Google LLC,79929,162.99.196.127,100.01,1780243200.000474,"anon, hosting",70.8529,ab 16 c6 19,Moscow,c0 23,,1731,,c8 f1 ac e3 35 6c 89 25 3a 42 4b 22 04 8a 56 6c a6 5c 81 e6 cc a3 b7 94 b5 16 a5 a5,03 04,31.160184,,85,,"anon, hosting",21,ca,zeek,-103.66292,,,12345,2026-06-01 00:00:00.000474,163.97.128.174,65,20,,776.9429,BT Group,-55.726716,162.99.196.127,,312742,38889,146.125.255.141,,Seoul,7,KDDI,116.338854,3667,core-02,345.376,TlsC
|
||||
Google LLC,sg,,,Amazon.com Inc.,21836,17.197.195.153,72.21,1780243200.000475,"anon, hosting",767.431,e3 af d9 69,New York,c0 13,,1084,384,fd 34 7e f6 f8 fa 76 07 ab 30 9b c9 05 e4 61 fc 58 1d 9d 14 7c 5b 98 04 91 9b 0a 34,03 04,-57.804405,,91,cdn.example.com,"anon, hosting",8080,in,ssl_logs,68.858233,,,8443,2026-06-01 00:00:00.000475,213.215.129.214,53,275,,856.2994,NTT,30.77727,17.197.195.153,,320146,71883,150.180.24.13,*.cloudfront.net,Seoul,13,China Telecom,71.329385,1027,core-02,816.0634,TlsS
|
||||
Samsung,fr,cdn.example.com,,Amazon.com Inc.,18537,166.174.6.218,38.15,1780243200.000476,"anon, hosting",321.2449,e2 8a f3 df,Seattle,c0 14,,113,384,93 ce 0a b7 ca 4c e5 81 bd 0d bb d0 b7 55 c3 2a db 30 66 8d 71 24 4f ad 18 5e a3 c8,03 03,53.781501,,98,cdn.example.com,"anon, hosting",50000,gb,suricata,153.560861,,,12345,2026-06-01 00:00:00.000476,199.147.121.78,37,153,,1757.452,Comcast,-30.365436,166.174.6.218,TLS_RSA_WITH_AES_256_GCM_SHA384,206455,73112,150.180.24.13,,Seattle,23,Telstra,-130.234256,4674,wan-link,971.7986,TlsC
|
||||
Apple Inc.,de,*.cloudfront.net,00 1d,Microsoft Corp.,123890,196.172.52.24,67.4,1780243200.000477,"anon, hosting",452.4925,c0 46 08 4d,Berlin,c0 2b,,678,,c1 51 8d eb 26 c6 b1 c8 63 d6 a2 23 5d 0d ac 86 38 b5 41 0d f7 d7 99 ed f7 03 3a d7,03 03,34.941406,,11,,"anon, hosting",8443,hk,netflow,157.239505,,,443,2026-06-01 00:00:00.000477,122.233.76.156,7,354,,404.1285,China Telecom,58.139904,196.172.52.24,,35781,12873,213.146.39.166,mail.example.com,Tokyo,35,KDDI,-128.447612,6177,core-02,981.823,TlsS
|
||||
Apple Inc.,au,,,Baidu Inc.,164818,110.189.35.130,103.98,1780243200.000478,"anon, hosting",996.5401,bd e0 fc db,San Francisco,c0 2f,,35,,c6 9c 91 64 90 ee 90 75 1a 77 96 48 dc ee 89 ae ca de 21 37 a8 b3 cc 27 d1 01 c4 9d,03 03,28.844907,,75,,"anon, hosting",110,no,suricata,-155.180387,86054,,143,2026-06-01 00:00:00.000478,80.192.203.242,20,472,,57.9398,Deutsche Telekom,55.947454,114.226.109.131,,70375,86287,142.66.44.17,*.s3.amazonaws.com,Dublin,8,Orange,109.139305,7415,backbone-03,902.5046,TlsS
|
||||
Tencent,ca,,,Meta Platforms,46692,204.20.183.54,29.05,1780243200.000479,"anon, hosting",609.8589,5d dd d4 d6,Shanghai,00 35,,426,,97 ff 37 29 f6 b6 74 88 13 dd 37 e1 b0 3a 2c f8 a1 d6 6f 90 8f 1d d3 19 4e 3c 11 33,03 04,-12.807827,,150,*.s3.amazonaws.com,"anon, hosting",50000,fr,suricata,-13.022426,,,993,2026-06-01 00:00:00.000479,211.116.114.7,48,103,,826.6833,SK Telecom,32.545391,174.207.84.216,,262409,71431,105.59.71.12,,Beijing,3,Singtel,28.022498,5943,edge-01,754.3744,TlsS
|
||||
Tencent,us,,,Samsung,10199,28.152.101.40,18.8,1780243200.00048,"anon, hosting",1396.6011,01 31 d0 d4,Singapore,c0 27,,1769,,c1 ef 58 22 ff 0e 8f db d5 f1 0e 3d fa c6 48 3f 29 de f7 37 4b 78 14 87 c7 af 73 cb,03 04,0.911993,,193,api.example.com,"anon, hosting",80,ca,netflow,158.575367,,,465,2026-06-01 00:00:00.000480,10.70.170.206,1,15,,475.9855,Deutsche Telekom,-43.736634,28.152.101.40,,299497,49582,18.120.147.59,,New York,17,Telstra,-159.765015,5260,backbone-03,830.3483,TlsS
|
||||
Amazon.com Inc.,hk,,00 1d,Meta Platforms,71232,54.220.164.120,32.12,1780243200.000481,"anon, hosting",172.371,0c 7f 9f 3f,New York,00 9f,,1756,,f0 5f 2a 7c 05 8a 42 10 2c bb 1f b3 51 f4 74 38 28 e0 c9 b3 d2 fb 5f 5f 8c 37 50 c7,03 04,21.543474,,44,,"anon, hosting",443,gb,netflow,36.971778,,,443,2026-06-01 00:00:00.000481,141.80.208.168,96,368,,864.8143,Deutsche Telekom,2.899708,54.220.164.120,,330265,25574,173.56.213.150,,Sydney,38,Singtel,-102.972617,6528,edge-01,127.9589,TlsS
|
||||
Baidu Inc.,nl,www.google.com,,Samsung,17529,148.97.130.12,99.91,1780243200.000482,"anon, hosting",335.8476,69 06 69 5f,London,c0 2b,,1641,,24 23 e0 49 2b 74 f5 1a 5c dc 24 c4 fa 2f c5 f3 5c fe 1b 4d 4a 95 28 13 54 d5 52 31,03 03,-8.344485,,60,www.google.com,"anon, hosting",21,no,zeek,27.391719,,,53,2026-06-01 00:00:00.000482,28.122.124.13,6,47,,1413.8994,Comcast,-59.779664,148.97.130.12,,431186,12949,69.72.36.16,,San Francisco,37,Deutsche Telekom,-85.758396,7539,core-02,230.3616,TlsS
|
||||
Meta Platforms,br,,,Microsoft Corp.,74800,161.81.216.153,57.9,1780243200.000483,"anon, hosting",1743.6332,f1 28 39 05,Mumbai,c0 2c,,1574,,e3 31 00 e7 8c a6 7c 29 ad dd 93 cb 05 e5 44 ad 80 ff fd ff a9 78 3c e6 de 73 bc 61,03 03,49.72571,--,148,graph.facebook.com,"anon, hosting",110,br,netflow,-176.652709,,,443,2026-06-01 00:00:00.000483,159.25.41.70,49,47,,1385.139,Comcast,54.468475,161.81.216.153,,181057,59859,69.168.154.150,,Dublin,8,Telstra,105.430832,4911,edge-01,73.2755,TlsC
|
||||
Baidu Inc.,jp,,,Tencent,49598,61.142.248.55,41.14,1780243200.000484,"anon, hosting",1337.4513,cd 99 bd 88,Tokyo,c0 23,,476,384,63 fe f2 f6 50 1b 9d 75 fe 3d 8b 48 ee 38 c0 d9 a1 d5 90 b5 b8 6f 53 85 09 68 88 f9,03 01,8.035112,,43,stream.example.org,"anon, hosting",110,ru,packet_capture,-53.732185,,,443,2026-06-01 00:00:00.000484,86.86.39.131,44,321,stream.example.org,61.1623,,-30.63611,61.142.248.55,TLS_RSA_WITH_AES_256_GCM_SHA384,91875,26133,223.73.205.18,,Singapore,13,Telstra,-80.690658,9796,core-02,189.7946,TlsC
|
||||
Google LLC,de,,,Tencent,125103,60.51.194.72,48.65,1780243200.000485,"anon, hosting",762.9074,ac 88 65 85,Paris,13 02,,450,,36 3c 59 23 7e 0e a5 25 0a 4f e8 2b 09 b4 2c 87 eb d6 1a aa c1 47 7c 6f 12 82 ea e6,02 00,47.941929,,193,,"anon, hosting",993,il,ssl_logs,103.538984,,,53,2026-06-01 00:00:00.000485,51.140.64.161,76,22,,14.7855,SK Telecom,-33.231966,60.51.194.72,,462356,55083,210.75.120.27,,San Francisco,24,Deutsche Telekom,-127.148925,2470,wan-link,284.4251,TlsC
|
||||
Amazon.com Inc.,ca,,,Meta Platforms,38842,145.21.177.137,32.8,1780243200.000486,"anon, hosting",375.6719,a7 44 5b 0d,Seattle,00 35,,941,,bd 98 1d 1d 9f cd ab de 03 80 c5 bc 8a 8b a0 c1 ef ec b8 bc db 22 52 7d 95 08 17 29,03 03,-15.177328,,39,api.example.com,"anon, hosting",143,cn,suricata,-144.664358,,x448,12345,2026-06-01 00:00:00.000486,214.247.241.63,31,95,stream.example.org,601.8184,KDDI,-14.327912,145.21.177.137,TLS_ECDHE_RSA_AES128_SHA,72252,74621,187.48.160.110,login.live.com,London,39,KDDI,38.331452,8486,edge-01,864.8133,TlsS
|
||||
Microsoft Corp.,br,,,Meta Platforms,5491,72.141.5.133,106.3,1780243200.000487,"anon, hosting",1752.7579,79 8d 9d 0f,Berlin,00 9f,,413,,08 00 db 67 ad 93 aa 8b b1 03 a4 5c c7 b0 7a f7 17 82 f2 d2 1f b5 ca 78 a0 1b b5 32,03 03,23.943971,,106,,"anon, hosting",21,jp,netflow,26.630596,72843,,80,2026-06-01 00:00:00.000487,17.148.153.218,3,126,,450.6138,Orange,-30.213169,72.141.5.133,TLS_RSA_WITH_AES_128_CBC_SHA,141142,26290,116.19.29.95,auth.example.net,Seoul,41,KDDI,-1.121428,4233,edge-01,752.347,TlsS
|
||||
Netflix Inc.,se,,00 19,Tencent,35671,72.79.110.246,13.84,1780243200.000488,"anon, hosting",1015.4829,ac 76 bc 90,Singapore,c0 23,Alibaba Inc.,1098,,6d 73 21 5c fb 47 96 16 37 89 1b b8 9c 66 75 8f c0 87 f4 7d 90 7f a2 6d f9 5b bc 06,03 04,51.712948,,79,*.cloudfront.net,"anon, hosting",50000,fi,zeek,116.221549,,,12345,2026-06-01 00:00:00.000488,150.219.204.93,54,442,,1690.4685,China Telecom,1.12072,72.79.110.246,,191145,10406,117.191.30.159,,Mumbai,35,BT Group,2.540466,9101,edge-01,49.3195,TlsC
|
||||
Microsoft Corp.,ru,,,Microsoft Corp.,132147,176.135.250.65,35.7,1780243200.000489,"anon, hosting",1380.2135,83 1c 93 f2,Seattle,c0 2c,,1845,128,24 43 e6 ec c0 0e c7 cb 44 bf bd f1 f1 97 98 38 31 42 9f 92 59 2f f7 59 f8 81 4c 27,03 03,30.351916,,123,,"anon, hosting",143,fr,ssl_logs,152.716216,,,110,2026-06-01 00:00:00.000489,72.79.110.246,78,330,,306.3475,Deutsche Telekom,-56.973626,176.135.250.65,,66734,45906,222.46.196.119,,New York,23,SK Telecom,119.199385,9420,wan-link,594.4381,TlsS
|
||||
Microsoft Corp.,se,,,Amazon.com Inc.,60865,28.79.139.73,112.66,1780243200.00049,"anon, hosting",911.6599,72 e0 b7 e0,Moscow,00 9f,,397,,a8 54 58 a5 87 92 b9 e6 09 6b 28 f0 cd 34 1c 2c 6d 51 8e f0 1b f6 60 1d 32 95 28 8e,03 03,-32.136359,OK,143,,"anon, hosting",110,se,netflow,-94.618759,,x25519,443,2026-06-01 00:00:00.000490,201.135.173.160,57,209,,1968.3839,BT Group,18.860994,28.79.139.73,,278432,38400,97.94.242.62,,Shanghai,5,BT Group,172.440479,3525,edge-01,834.3488,TlsS
|
||||
Netflix Inc.,ca,,,Google LLC,177940,83.250.9.29,77.1,1780243200.000491,"anon, hosting",112.1827,21 ac 93 2a,Shanghai,c0 14,,1736,,eb 27 aa e5 85 3d d8 f8 22 d8 a4 dd f1 19 dc dc fc a8 3c d9 5b 0b 67 bc 4a bb ef 99,02 00,-32.860887,,25,,"anon, hosting",443,jp,packet_capture,-142.092414,,x448,12345,2026-06-01 00:00:00.000491,54.220.164.120,4,433,,400.2142,Verizon,28.614167,83.250.9.29,,61401,47647,137.64.140.12,cdn.example.com,San Francisco,27,AT&T,70.353769,4289,gw-09,614.8124,TlsS
|
||||
,ca,,,Alibaba Inc.,55732,142.150.185.148,51.77,1780243200.000492,"anon, hosting",293.1786,1a 8e d6 f9,Paris,13 01,,1262,384,42 e0 45 14 dd cb ed bc 8c 07 ba fe 6d 08 7b e9 a5 7b f1 82 bf 1f 68 02 cd 47 85 3e,03 04,-59.334794,,18,,"anon, hosting",443,de,zeek,23.545878,,,443,2026-06-01 00:00:00.000492,21.68.178.31,61,481,,539.5317,BT Group,-8.899849,142.150.185.148,,382532,49343,200.185.55.116,,San Francisco,14,Orange,-137.239859,3755,backbone-03,537.6324,TlsS
|
||||
Alibaba Inc.,cn,,,Netflix Inc.,181152,177.2.54.200,100.15,1780243200.000493,"anon, hosting",1221.2253,8a b0 0d 2a,Paris,00 35,,935,,99 d5 e0 84 4f 57 61 57 cc 82 54 ab 04 9b d0 8e ec 2f f8 95 47 c6 b9 98 1b d0 de 2b,03 04,-12.409065,,152,login.live.com,"anon, hosting",995,sg,ssl_logs,-47.243692,,,443,2026-06-01 00:00:00.000493,175.176.65.225,9,379,auth.example.net,1169.3564,NTT,11.886614,177.2.54.200,,85652,86692,48.7.172.203,,San Francisco,18,Deutsche Telekom,111.343975,5208,core-02,181.1492,TlsS
|
||||
Google LLC,no,,,Microsoft Corp.,103537,112.137.165.219,51.76,1780243200.000494,"anon, hosting",90.2772,0e 8f b8 53,Shanghai,13 02,,986,384,c2 9f 43 05 fd 1d 62 71 95 58 ec 3a 70 87 49 98 fa 1d d5 98 44 41 41 a1 e3 31 d7 94,03 04,-4.373736,,109,stream.example.org,"anon, hosting",8080,fi,zeek,20.987891,82382,,50000,2026-06-01 00:00:00.000494,192.46.222.251,36,257,,180.9091,KDDI,-49.018772,112.137.165.219,TLS_ECDHE_ECDSA_AES256_GCM_SHA384,185728,43498,78.108.70.196,,Seattle,40,Telstra,-79.764114,7500,backbone-03,494.0871,TlsS
|
||||
Tencent,fi,,00 1e,Microsoft Corp.,115292,142.150.185.148,32.52,1780243200.000495,"anon, hosting",1092.2195,61 33 4a 31,Shanghai,c0 2c,,802,,a4 ea 6e 82 2e 08 19 fe b7 12 a9 1e 6c a5 8b be 39 6b dc 8c 13 40 98 59 4c 16 aa a4,03 03,-2.247052,,179,,"anon, hosting",8443,se,ssl_logs,-52.766877,,,443,2026-06-01 00:00:00.000495,163.58.99.232,35,466,,758.778,NTT,-49.727329,142.150.185.148,,246822,30455,117.174.80.95,login.live.com,Seoul,43,China Mobile,146.400422,2984,core-02,204.625,TlsC
|
||||
Samsung,fr,,,Samsung,195670,149.188.242.142,29.03,1780243200.000496,"anon, hosting",681.3891,b3 38 0f 01,Shanghai,13 01,,1859,,95 d6 39 43 04 af 8c 90 83 e7 be 70 66 d9 1f 0e f5 97 37 67 60 bd 78 6c 7d db 8c 20,03 04,54.522947,,64,,"anon, hosting",3389,br,ssl_logs,-87.167072,,secp384r1,993,2026-06-01 00:00:00.000496,110.189.35.130,41,193,,398.2484,NTT,-19.846292,149.188.242.142,,319898,22821,167.226.142.47,,Sydney,14,Orange,33.290428,7616,backbone-03,338.0731,TlsC
|
||||
Netflix Inc.,us,,,Google LLC,135426,86.66.134.42,71.86,1780243200.000497,"anon, hosting",1039.858,d6 07 62 3e,Seoul,c0 2c,,1293,,4f 89 91 42 76 a6 16 cf 9f 6e 7e 72 c7 0d c8 4a d8 64 4b 75 38 46 97 f0 f1 8e b5 83,03 04,-19.533861,,115,,"anon, hosting",143,in,suricata,8.106776,,,443,2026-06-01 00:00:00.000497,31.78.255.239,39,329,,644.9793,Comcast,5.155732,86.66.134.42,,68335,75860,74.40.177.237,graph.facebook.com,Shanghai,23,China Telecom,-105.9711,2580,wan-link,124.9355,TlsC
|
||||
Samsung,de,,00 18,Meta Platforms,95776,198.10.194.123,58.17,1780243200.000498,"anon, hosting",570.4842,ee 58 77 b9,Mumbai,c0 2c,Samsung,1625,,53 62 e8 3f 53 b9 0b c0 5e 38 46 03 70 bc 3d 7b e3 a9 2a dd 5d 87 bf 4c 87 10 bb b9,03 02,0.444847,,64,login.live.com,"anon, hosting",8443,jp,suricata,102.080545,85272,,110,2026-06-01 00:00:00.000498,60.196.78.181,52,182,,640.0084,China Mobile,-13.015638,198.10.194.123,,20143,92821,164.207.186.132,*.cloudfront.net,Berlin,29,Singtel,-135.310691,4861,backbone-03,728.7025,TlsC
|
||||
Baidu Inc.,au,,,Google LLC,75566,86.210.127.69,81.83,1780243200.000499,"anon, hosting",1759.806,0a f4 19 da,Beijing,c0 2f,,218,384,5d 5f f0 c7 3f e4 50 ec 8b d5 1d 05 7d 96 2e 00 bf 58 e6 0e 78 be 2d f4 2c 89 e3 84,03 03,56.072776,,59,,"anon, hosting",443,cn,suricata,-155.970018,,,53,2026-06-01 00:00:00.000499,145.105.35.213,66,434,,1252.2715,BT Group,55.553193,86.210.127.69,,414955,66834,198.252.237.194,,Tokyo,47,,-59.876266,4122,wan-link,439.2697,TlsC
|
||||
Amazon.com Inc.,kr,,,Baidu Inc.,159250,131.41.95.18,101.82,1780243200.0005,"anon, hosting",1130.8255,68 d1 a5 d0,Shanghai,c0 2b,Alibaba Inc.,1799,,33 e8 61 4a 94 dd 37 b3 b7 50 4f 94 29 22 7f 85 28 96 35 46 d4 44 c4 bd 70 a7 59 f1,03 04,49.468208,,153,,"anon, hosting",8443,se,zeek,177.351693,61167,,110,2026-06-01 00:00:00.000500,174.121.227.157,65,192,,1332.1081,Telstra,14.435915,131.41.95.18,,331623,32103,14.129.244.30,api.example.com,Beijing,42,Verizon,-81.569983,2015,backbone-03,747.4017,TlsS
|
||||
Apple Inc.,br,,00 18,Netflix Inc.,121587,187.27.50.16,111.79,1780243200.000501,"anon, hosting",82.5138,a6 db 61 28,Tokyo,00 9e,,1894,,8b b9 43 cd b5 43 24 d0 ae b6 2d 99 de 58 05 00 03 f1 d7 98 36 9d 7a ea e3 54 51 5b,03 04,-52.283999,,178,*.s3.amazonaws.com,"anon, hosting",22,br,netflow,64.904839,,,22,2026-06-01 00:00:00.000501,160.172.21.212,70,436,,1350.8568,Comcast,56.788229,213.215.129.214,,247464,85314,19.30.80.113,,Shanghai,11,KDDI,147.73671,2114,core-02,88.4043,TlsS
|
||||
Apple Inc.,ua,,00 19,Baidu Inc.,91864,182.223.0.134,41.88,1780243200.000502,"anon, hosting",1531.9894,75 60 10 8e,Beijing,c0 14,,581,,e9 3b 4f 3b ce df 5c c4 c5 5c f4 75 87 56 1e 87 57 cb a6 95 94 c9 01 ce a9 7a 4a 24,02 00,54.343671,,151,*.s3.amazonaws.com,"anon, hosting",21,ru,netflow,106.433515,,,443,2026-06-01 00:00:00.000502,38.66.128.205,29,87,,1713.1097,China Telecom,5.740001,182.223.0.134,,152867,26386,213.146.39.166,,New York,6,SK Telecom,-10.961917,8220,edge-01,96.54,TlsS
|
||||
Alibaba Inc.,au,"*.['example', 'com']",,,100744,185.134.91.130,81.79,1780243200.000503,"anon, hosting",1495.2259,7e 70 44 a6,Tokyo,c0 13,,762,512,b3 22 df fe 56 38 49 e1 21 a3 48 cf 2f f7 3e d6 f0 1e dc 7b f2 47 0d 22 df 82 a1 84,03 04,6.653431,,9,,"anon, hosting",443,ca,netflow,93.190896,,,143,2026-06-01 00:00:00.000503,15.211.213.143,4,201,,1336.7381,KDDI,-54.623113,185.134.91.130,,452806,76501,18.120.147.59,,Beijing,6,Verizon,-46.226218,2066,edge-01,7.6459,TlsS
|
||||
Alibaba Inc.,kr,,,Amazon.com Inc.,18736,122.58.111.120,15.47,1780243200.000504,"anon, hosting",1056.2933,82 a3 b8 4f,Singapore,00 9e,,1829,,e3 b7 ec 74 ef ec bc df 97 34 0b 9d bd 46 10 f5 40 ac 8b 03 62 fd c5 86 99 28 62 79,03 03,-13.643038,,53,auth.example.net,"anon, hosting",3389,sg,ssl_logs,106.73286,,,465,2026-06-01 00:00:00.000504,217.188.58.196,32,201,,296.6577,AT&T,-35.181637,122.58.111.120,,57109,91325,24.240.178.105,,Tokyo,14,BT Group,-163.758571,7888,gw-09,734.5252,TlsC
|
||||
Netflix Inc.,se,,,Samsung,144301,49.43.123.185,55.9,1780243200.000505,"anon, hosting",672.3992,14 f8 be 82,Moscow,c0 13,,1669,,7f bd 04 b9 e7 f8 df 12 ea d0 fd 1a 5b 8f 91 d6 85 8b 51 66 b0 c3 8b 8b 0e 5f f1 0e,03 04,59.633057,,84,,"anon, hosting",465,no,ssl_logs,110.734396,,,443,2026-06-01 00:00:00.000505,64.41.228.136,44,206,www.google.com,1346.5862,Comcast,-35.528622,49.43.123.185,,187415,34678,53.250.46.231,,Seattle,47,SK Telecom,-65.59273,8449,gw-09,375.1882,TlsC
|
||||
Microsoft Corp.,no,,,Apple Inc.,76379,62.40.43.188,98.7,1780243200.000506,"anon, hosting",1399.576,80 6e 63 98,Paris,cc a9,,466,512,93 ca 21 37 fd 91 a5 e1 a1 e1 ce 42 dd 6d de 06 12 f3 77 18 71 d9 b5 11 ee ef 8c d8,03 03,-25.355626,ER,134,www.google.com,"anon, hosting",25,au,ssl_logs,-23.433124,,,995,2026-06-01 00:00:00.000506,223.251.52.224,16,393,,269.0368,Singtel,-57.25331,62.40.43.188,,221643,32180,75.196.214.64,,Beijing,23,AT&T,61.812157,3493,gw-09,96.5983,TlsC
|
||||
Baidu Inc.,hk,,,Meta Platforms,118531,175.36.87.137,29.8,1780243200.000507,"anon, hosting",1834.715,61 d9 e2 34,New York,00 9f,,1437,,86 dd 77 be e4 35 cd 64 a2 8f f1 76 ee 92 aa eb 38 9f e4 5e e5 87 e9 67 3f 0d e2 25,03 01,31.703563,,87,www.google.com,"anon, hosting",465,kr,packet_capture,-11.921018,79289,,443,2026-06-01 00:00:00.000507,101.13.42.165,58,139,,501.7199,KDDI,-39.656007,175.36.87.137,,473937,12040,84.226.59.175,www.google.com,Singapore,10,Singtel,13.8031,9208,gw-09,216.5745,TlsC
|
||||
Samsung,de,,,Amazon.com Inc.,32277,221.243.230.67,79.3,1780243200.000508,"anon, hosting",1104.785,4c 72 9b 71,,c0 13,,1476,,4c c1 85 ca f6 92 81 a4 d2 62 5b 4a a8 0b bb 9c c3 ee 93 05 bf a9 86 f5 92 5c 03 0e,03 04,45.518364,,28,,"anon, hosting",443,us,suricata,-79.028934,,,443,2026-06-01 00:00:00.000508,89.135.22.187,41,471,mail.example.com,545.8193,China Telecom,48.615437,221.243.230.67,,16894,27975,82.50.171.203,,Berlin,50,China Telecom,-111.874221,3474,gw-09,794.0848,TlsS
|
||||
Google LLC,cn,,00 1e,Meta Platforms,24505,139.41.70.39,97.08,1780243200.000509,"anon, hosting",1927.8637,45 a1 3d 90,,c0 2c,,1557,512,60 d0 82 00 c3 8c 32 bf 37 3f 45 5d 57 93 39 72 54 be 1f 27 0d 41 7d 5b 63 50 3d 6f,03 04,1.41264,,177,auth.example.net,"anon, hosting",443,il,ssl_logs,-65.610098,,secp256r1,993,2026-06-01 00:00:00.000509,4.183.121.151,80,422,,679.5832,SK Telecom,-13.980216,139.41.70.39,,320374,80428,74.117.180.57,,Dublin,12,Telstra,-143.634616,8673,gw-09,641.5282,TlsC
|
||||
Netflix Inc.,ua,,,Alibaba Inc.,3767,100.214.112.125,41.68,1780243200.00051,"anon, hosting",640.0677,f1 df 3b 5d,Berlin,00 9f,,1537,,ed 30 d9 e3 24 48 ed 9f 8a 09 31 74 0b b2 e6 42 a4 67 7f 54 9e e8 16 ea 65 08 ca d9,03 03,24.354668,,48,,"anon, hosting",993,fr,packet_capture,-161.50581,,,995,2026-06-01 00:00:00.000510,60.111.28.36,96,443,auth.example.net,1961.9039,SK Telecom,-15.209953,100.214.112.125,,167587,72097,205.136.42.187,,London,22,Comcast,49.71168,9999,backbone-03,748.4721,TlsS
|
||||
Netflix Inc.,ca,,,Google LLC,115829,80.55.13.238,115.9,1780243200.000511,"anon, hosting",1663.9023,97 4f e0 d2,New York,13 02,,1381,,7c b0 11 80 97 9a 47 08 be 8f a3 bb 8a 2c 5d 6b 25 32 e7 91 96 8a 43 5e 79 48 ab eb,03 03,13.932796,,63,,"anon, hosting",110,br,packet_capture,-48.744648,,,80,2026-06-01 00:00:00.000511,35.135.187.196,37,146,,869.1053,Orange,-17.021114,80.55.13.238,,481559,59283,192.46.222.251,,Beijing,47,SK Telecom,-51.67515,2368,gw-09,997.5165,TlsC
|
||||
Microsoft Corp.,ua,,,Google LLC,111511,100.214.112.125,56.64,1780243200.000512,"anon, hosting",410.6103,91 e3 e5 94,Seoul,c0 2f,,1583,256,c8 3a e5 3b 2b 1c d8 d1 eb c7 73 5e 8f d0 4f 74 9f 91 66 d8 58 a1 2a b0 d4 5f 69 fe,03 04,-11.76044,,80,*.cloudfront.net,"anon, hosting",110,hk,suricata,-175.091414,,,465,2026-06-01 00:00:00.000512,12.180.173.209,51,189,,1493.2757,China Telecom,30.150376,100.214.112.125,,440393,78552,163.58.99.232,cdn.example.com,Seattle,48,Verizon,144.714391,9093,gw-09,972.2348,TlsS
|
||||
Apple Inc.,au,,,Google LLC,140494,146.154.207.141,93.84,1780243200.000513,"anon, hosting",1236.0505,3b b3 c9 d5,Shanghai,c0 27,,1387,,41 64 82 82 ad e5 59 88 86 2c e9 dc 8a 5e d8 31 9a 25 0d ae f4 24 7d 68 08 fb 06 71,03 01,15.447035,,196,,"anon, hosting",995,nl,suricata,-29.997294,,,993,2026-06-01 00:00:00.000513,136.68.198.63,72,204,,1379.748,SK Telecom,29.197132,146.154.207.141,TLS_RSA_WITH_AES_256_GCM_SHA384,191076,48868,141.80.208.168,,Singapore,13,Telstra,164.447435,5236,backbone-03,572.0014,TlsC
|
||||
,de,,,Apple Inc.,117383,218.37.233.107,68.75,1780243200.000514,"anon, hosting",1411.3134,de 75 2d eb,Seattle,13 02,,1493,,0b af e0 67 1f 3f a5 43 ca 26 8d b4 e3 57 97 bc cd d9 74 de 6d 18 b7 38 46 2a d2 64,03 03,-26.246689,ER,45,,"anon, hosting",80,in,ssl_logs,130.840269,,,22,2026-06-01 00:00:00.000514,106.12.91.189,58,31,,659.1729,BT Group,22.613821,218.37.233.107,,62836,90864,79.13.242.68,,London,13,Comcast,-58.202764,6287,edge-01,428.4068,TlsS
|
||||
Apple Inc.,br,auth.example.net,,Meta Platforms,138537,91.107.136.180,45.61,1780243200.000515,"anon, hosting",1850.9735,95 a9 33 27,San Francisco,00 9e,Google LLC,398,,6d 23 5c 61 0b 74 cc fe 51 da 90 e1 75 14 2c d2 15 65 af 0e 6a 8b 41 16 cc 4a af 04,03 03,-50.267714,,111,auth.example.net,"anon, hosting",53,nl,packet_capture,90.12617,,,443,2026-06-01 00:00:00.000515,167.35.20.194,64,381,graph.facebook.com,529.5023,Deutsche Telekom,-57.733415,160.51.37.138,,488922,96989,192.191.212.180,stream.example.org,Sydney,4,Singtel,64.403113,5403,edge-01,530.9677,TlsC
|
||||
Tencent,us,"*.['amazonaws', 'com']",,Samsung,136744,210.235.179.79,93.08,1780243200.000516,"anon, hosting",131.9714,5f 86 37 92,Tokyo,c0 14,,797,,14 7a d2 ab 95 99 64 6e 04 bb 6f d2 76 80 e3 28 c6 fd 14 83 e3 1f b0 9e 80 b2 43 57,03 01,34.019673,,84,,"anon, hosting",443,hk,zeek,164.472609,,,80,2026-06-01 00:00:00.000516,181.250.162.123,92,205,,1459.0372,Telstra,31.514846,210.235.179.79,,443541,63944,53.250.46.231,,Berlin,17,Telstra,163.798008,4178,edge-01,322.3015,TlsC
|
||||
Baidu Inc.,br,,,Apple Inc.,155792,200.248.122.71,116.56,1780243200.000517,"anon, hosting",223.4702,e4 21 bc 24,Paris,c0 14,,292,,70 f8 38 0e bf d6 05 54 65 73 0a 8d e4 7d 4d fb 5d 41 8b 61 fa a1 32 34 da 44 b0 c7,03 03,39.489115,,58,,"anon, hosting",3389,jp,ssl_logs,134.211309,,,443,2026-06-01 00:00:00.000517,109.210.239.222,66,463,,1502.2397,Telstra,-9.723317,200.248.122.71,,301006,70890,25.76.123.50,,Seattle,46,AT&T,-129.194767,9883,gw-09,164.7385,TlsS
|
||||
Alibaba Inc.,ua,,,Tencent,111207,84.28.117.211,86.92,1780243200.000518,"anon, hosting",1565.3265,21 40 4d b4,Tokyo,00 9e,,718,,ef 77 b6 37 41 2f a1 f3 db 83 8d 53 dd 28 55 06 a0 4b e5 0e 12 3d 18 c0 50 92 da be,03 02,8.467083,,88,,"anon, hosting",995,fr,netflow,-158.348179,,x448,995,2026-06-01 00:00:00.000518,217.54.124.112,93,470,,1285.8843,Singtel,-24.093667,84.28.117.211,,99627,36983,9.228.24.63,,San Francisco,4,NTT,-76.842068,8423,gw-09,47.9834,TlsS
|
||||
Alibaba Inc.,de,stream.example.org,,Google LLC,133646,56.119.13.144,104.3,1780243200.000519,"anon, hosting",1989.2517,50 41 ce c6,Seoul,c0 27,Microsoft Corp.,896,,02 d5 20 9a d8 69 2a 1b 48 eb 96 58 6f f1 63 e4 d8 46 79 38 8c f1 02 2f 4a c9 09 bd,03 03,1.909074,,128,stream.example.org,"anon, hosting",143,jp,packet_capture,109.260759,,x448,110,2026-06-01 00:00:00.000519,137.64.140.12,87,76,,1001.5476,China Mobile,18.679096,218.63.153.130,TLS_RSA_WITH_AES_256_GCM_SHA384,289072,78585,24.240.178.105,www.google.com,Seoul,34,Comcast,142.468969,5618,core-02,119.6773,TlsS
|
||||
Google LLC,de,"*.['example', 'org']",,Netflix Inc.,12292,10.240.113.52,29.43,1780243200.00052,"anon, hosting",624.6438,8c f3 3d 53,Sydney,c0 14,,198,,eb d7 fd b6 68 c3 05 00 8d a0 9b af ff 90 47 fe bd 75 ee 6d e8 6a 55 cb b0 66 e6 4b,03 02,-26.709472,,127,cdn.example.com,"anon, hosting",995,in,suricata,-142.14647,,,110,2026-06-01 00:00:00.000520,17.237.154.167,5,178,,1022.384,Deutsche Telekom,10.064422,179.151.65.50,,70674,82971,144.13.136.8,,Mumbai,37,China Telecom,161.669604,7422,core-02,911.2119,TlsS
|
||||
Apple Inc.,cn,,,Amazon.com Inc.,145389,175.36.87.137,11.75,1780243200.000521,"anon, hosting",1863.7441,eb 5f 63 1a,Sydney,c0 14,,1523,,e2 ba 6c 26 3c b9 94 a5 20 d7 99 6d 7c 42 78 c5 ec 00 14 e9 81 35 d4 46 fb b4 fa 31,03 03,59.465461,,186,*.s3.amazonaws.com,"anon, hosting",80,fi,suricata,-85.64799,30167,,110,2026-06-01 00:00:00.000521,34.46.151.84,40,428,,1465.0738,China Telecom,46.366519,175.36.87.137,,214003,97440,137.64.140.12,graph.facebook.com,Shanghai,28,Deutsche Telekom,-13.958727,3974,core-02,177.908,TlsS
|
||||
Amazon.com Inc.,gb,,,Netflix Inc.,87001,80.55.13.238,60.69,1780243200.000522,"anon, hosting",970.5517,20 a5 0b 9b,New York,c0 13,,1068,,55 2c 45 87 65 d4 b3 41 30 1e 27 10 e2 8b aa 5b 18 b2 1a 11 28 22 ec 3b a4 0e be 3f,03 04,38.976112,,129,auth.example.net,"anon, hosting",25,au,packet_capture,22.126542,,,465,2026-06-01 00:00:00.000522,114.26.104.69,49,240,,391.0402,Deutsche Telekom,-34.505692,80.55.13.238,TLS_RSA_WITH_AES_128_GCM_SHA256,230270,51689,164.190.55.173,,Mumbai,16,Orange,9.462077,2980,wan-link,94.7004,TlsS
|
||||
Alibaba Inc.,ca,,,Baidu Inc.,154955,120.65.13.233,111.21,1780243200.000523,"anon, hosting",1194.1406,1f a2 7d 85,Mumbai,c0 13,Netflix Inc.,842,,dd 43 ec 24 1d c7 34 11 91 58 10 7f 0a f2 69 b2 77 c7 49 21 a4 ef 0a 04 ef c4 3b db,03 04,-49.816311,,147,,"anon, hosting",995,br,suricata,173.428357,,,21,2026-06-01 00:00:00.000523,83.128.13.24,44,87,,349.9292,SK Telecom,-15.467292,176.135.250.65,,306970,71733,64.142.152.58,,Dublin,15,Telstra,-17.925692,3608,core-02,522.3052,TlsC
|
||||
Amazon.com Inc.,in,"*.['example', 'org']",,,138573,112.80.232.1,34.23,1780243200.000524,"anon, hosting",1829.9065,07 a8 72 63,Seattle,c0 23,,1591,,0d 5d ba bd 50 e9 65 eb 86 0d e2 05 86 3a 0d 58 89 3d d2 35 fd e5 45 61 ff e1 75 10,03 03,-12.620459,,66,www.google.com,"anon, hosting",443,cn,zeek,-59.659616,,secp256r1,25,2026-06-01 00:00:00.000524,142.7.208.24,4,88,*.s3.amazonaws.com,497.6458,Telstra,-11.003562,112.80.232.1,,154210,29524,141.103.76.43,,Sydney,23,AT&T,-41.694253,4630,gw-09,838.8168,TlsS
|
||||
Samsung,no,,,Amazon.com Inc.,82095,210.235.179.79,62.34,1780243200.000525,"anon, hosting",873.9854,f5 51 d6 88,Berlin,00 9e,,1967,,ea 24 1e 64 d1 d9 38 f7 d1 b8 2b 7a ee c1 0b d7 22 e6 27 43 b5 ae 1c e6 b0 87 64 3b,02 00,6.929205,,188,,"anon, hosting",12345,br,suricata,110.3343,,secp521r1,443,2026-06-01 00:00:00.000525,61.83.90.226,94,262,auth.example.net,217.4045,BT Group,45.186737,210.235.179.79,,129698,40583,114.217.248.156,*.cloudfront.net,Singapore,48,Comcast,178.796583,6389,backbone-03,581.8322,TlsC
|
||||
Alibaba Inc.,gb,,00 19,Alibaba Inc.,168688,165.174.57.76,97.28,1780243200.000526,"anon, hosting",1039.185,c1 7b 18 56,Singapore,c0 13,,1939,,1f 6e e2 88 e7 63 d9 99 b9 d4 fd 25 51 28 52 99 e4 dd 9a 26 a4 b4 f2 45 3e 7e 21 2a,03 03,-56.099669,,147,*.cloudfront.net,"anon, hosting",25,de,netflow,-113.896732,,,143,2026-06-01 00:00:00.000526,62.164.144.98,43,417,,455.791,China Mobile,-9.905292,165.174.57.76,,150867,67038,77.54.171.73,,New York,36,BT Group,15.024529,6989,edge-01,288.0195,TlsC
|
||||
Amazon.com Inc.,jp,,,Apple Inc.,9699,93.235.78.112,20.58,1780243200.000527,"anon, hosting",572.059,6d 87 fb 7a,Seoul,c0 2f,,281,,f6 a3 65 41 71 33 f5 ae 1a b9 99 3b 9e 5d 8d d5 97 3b 22 02 76 8a cd e7 17 09 d8 36,03 03,-43.077945,,152,graph.facebook.com,"anon, hosting",50000,de,packet_capture,42.722197,,,22,2026-06-01 00:00:00.000527,141.67.147.113,47,205,,92.9912,Telstra,35.023608,93.235.78.112,,171502,25089,168.116.26.150,,Berlin,31,Orange,-114.743107,6334,core-02,614.8132,TlsS
|
||||
Samsung,ru,,,Meta Platforms,59830,120.65.13.233,16.92,1780243200.000528,"anon, hosting",1635.4746,2a 49 93 48,Berlin,13 02,,1559,,f5 4c 7e 7d 70 9c 78 51 c1 16 6f 5c 45 4c e4 a9 c0 33 ff b5 6d 9c 8b 24 e9 70 0c 2a,03 04,38.275356,,79,,"anon, hosting",993,ua,netflow,-117.99498,,,995,2026-06-01 00:00:00.000528,102.253.95.240,37,302,api.example.com,524.2179,Deutsche Telekom,-27.187385,120.65.13.233,TLS_ECDHE_ECDSA_CHACHA20_POLY1305,317991,74127,86.124.64.202,www.google.com,Dublin,35,Orange,-127.612847,8590,wan-link,230.5059,TlsS
|
||||
Tencent,ua,,,Alibaba Inc.,21304,72.79.110.246,91.47,1780243200.000529,"anon, hosting",1720.8797,d3 91 d0 74,Shanghai,c0 13,,37,,39 1c 7c 85 2a 27 77 e1 62 47 0c 41 43 10 2a e9 74 59 40 dc 14 cb c9 d3 d7 75 00 f6,03 03,35.630308,,15,auth.example.net,"anon, hosting",8443,kr,suricata,-58.764195,,,50000,2026-06-01 00:00:00.000529,36.106.166.170,88,480,,1164.1245,,-39.972511,72.79.110.246,TLS_AES_256_GCM_SHA384,427772,11713,78.108.70.196,,Mumbai,13,KDDI,47.626394,2339,backbone-03,146.3885,TlsC
|
||||
Microsoft Corp.,us,,,Microsoft Corp.,152567,56.119.13.144,76.99,1780243200.00053,"anon, hosting",1648.2241,b9 32 02 d4,Mumbai,c0 23,,1753,,7d 3c 0a ba 6c 71 33 52 04 9f 8b 52 4a 79 d7 3f dc 11 2b 0a fb 4b 69 57 23 40 30 14,03 03,-37.227953,,124,graph.facebook.com,"anon, hosting",443,in,ssl_logs,22.753954,,,443,2026-06-01 00:00:00.000530,62.164.144.98,61,427,,1687.8669,Telstra,-23.274466,56.119.13.144,,17583,55361,9.118.147.181,,Shanghai,5,AT&T,156.702906,3926,core-02,954.9921,TlsS
|
||||
Netflix Inc.,sg,,,Apple Inc.,142037,19.30.117.18,36.56,1780243200.000531,"anon, hosting",1134.6477,e0 e7 d7 06,New York,c0 23,,1264,512,40 58 6a 76 ac db 45 6d 8c 48 43 74 95 43 49 d5 9a 26 08 3e 84 8c 48 62 02 b9 c3 c7,03 03,13.753607,,29,,"anon, hosting",465,ua,zeek,16.284394,,,993,2026-06-01 00:00:00.000531,114.131.245.55,96,263,,1767.0278,Telstra,+,19.30.117.18,TLS_AES_256_GCM_SHA384,52773,65598,147.225.236.197,mail.example.com,New York,34,Verizon,103.576826,2683,core-02,411.9123,TlsC
|
||||
Alibaba Inc.,se,,,Alibaba Inc.,2669,64.38.226.207,73.62,1780243200.000532,"anon, hosting",142.5955,09 0e 2f 42,San Francisco,00 9e,,880,,17 93 d6 bb 74 a0 51 22 fc 1f ac f3 44 c1 fd e8 62 16 18 4a 19 62 13 89 61 66 18 f8,03 03,2.668194,,199,login.live.com,"anon, hosting",12345,de,zeek,-95.568466,,,3389,2026-06-01 00:00:00.000532,86.164.53.220,5,249,,247.7426,Orange,+,64.38.226.207,TLS_ECDHE_RSA_AES256_GCM_SHA384,149571,55621,180.58.110.111,,Berlin,29,Deutsche Telekom,86.805263,6285,gw-09,543.2639,TlsS
|
||||
Tencent,sg,,,,188600,213.215.129.214,13.32,1780243200.000533,"anon, hosting",768.2868,5f e0 4d d5,Seoul,c0 2b,,1297,,ca d6 16 7b 2e de e2 de 08 e5 07 89 c4 dc 0b 16 ac ab 79 21 8e 60 8c 89 22 c7 2b ba,03 03,36.355196,,52,auth.example.net,"anon, hosting",995,in,suricata,46.785194,,,993,2026-06-01 00:00:00.000533,174.207.84.216,26,23,,1905.496,AT&T,-10.453768,213.215.129.214,TLS_ECDHE_RSA_AES256_GCM_SHA384,122327,45228,154.69.35.71,www.google.com,Seattle,49,China Mobile,-133.399741,3827,wan-link,582.0976,TlsS
|
||||
Apple Inc.,gb,,,Meta Platforms,39983,204.20.183.54,97.94,1780243200.000534,"anon, hosting",688.2943,b1 20 48 be,New York,00 9f,,1887,,,03 03,-7.454266,,191,,"anon, hosting",8443,nl,zeek,-67.65186,,,443,2026-06-01 00:00:00.000534,165.195.163.179,5,150,,769.6739,BT Group,5.999785,8.126.102.215,TLS_ECDHE_RSA_AES128_SHA,221575,69005,77.147.106.177,auth.example.net,,50,BT Group,141.19009,3196,gw-09,73.6706,TlsC
|
||||
Google LLC,sg,auth.example.net,,Meta Platforms,109745,139.41.70.39,105.41,1780243200.000535,"anon, hosting",544.061,e6 d4 8f 5b,Beijing,c0 2f,,1226,,04 83 74 28 59 08 c1 12 a0 75 af 9e 7d 55 8d b1 27 42 6b ea 8a 67 27 01 e0 95 29 86,03 04,+,,94,,"anon, hosting",443,de,zeek,140.992437,,,53,2026-06-01 00:00:00.000535,83.8.89.68,74,279,,1501.73,AT&T,-16.501855,139.41.70.39,,198068,29701,148.160.97.196,,Mumbai,50,SK Telecom,173.796282,1524,backbone-03,509.7747,TlsS
|
||||
Netflix Inc.,au,,,Alibaba Inc.,183667,17.108.161.55,103.61,1780243200.000536,"anon, hosting",1597.2621,15 61 87 23,Paris,00 9f,,472,,88 d3 ef cd 34 d0 d2 3f 3b be ed 12 19 2f 0a 50 1e f3 76 e2 df cd a4 b7 f2 3b ac a9,03 03,52.182167,,151,login.live.com,"anon, hosting",443,no,packet_capture,136.408107,,,8080,2026-06-01 00:00:00.000536,57.70.252.24,23,77,,341.4579,NTT,7.99029,17.108.161.55,,494282,66693,16.35.241.9,,Beijing,11,Comcast,-93.07346,5478,gw-09,612.4548,TlsC
|
||||
Google LLC,us,,,Alibaba Inc.,17325,95.75.220.33,33.43,1780243200.000537,"anon, hosting",1723.908,20 5a be b3,Seoul,00 35,,1476,,73 6d da 5e d2 20 94 9b cb 87 b5 ba e5 b3 71 22 1f 16 45 d2 db e3 9e 5d f2 a3 e3 24,03 03,39.460522,,36,mail.example.com,"anon, hosting",80,fr,ssl_logs,75.548438,,,995,2026-06-01 00:00:00.000537,182.73.237.235,36,283,,1638.065,,-20.253399,95.75.220.33,,303050,19568,171.4.238.231,,Shanghai,17,,70.139783,9543,core-02,444.6168,TlsS
|
||||
Samsung,au,,,Samsung,76335,17.108.161.55,38.71,1780243200.000538,"anon, hosting",1864.4204,bb eb 62 31,Seattle,c0 2c,,460,,f1 8b c7 e1 6a 01 ec 4b 3c f8 59 25 45 f7 0e 1c cf 81 c9 94 5a ff 7e 9a bb e2 c6 61,03 03,26.580424,,172,,"anon, hosting",443,hk,netflow,51.691129,,,443,2026-06-01 00:00:00.000538,43.157.147.113,38,84,api.example.com,221.7654,China Mobile,13.082831,17.108.161.55,,220371,27470,214.247.241.63,,Mumbai,43,KDDI,106.084862,5872,backbone-03,219.2891,TlsC
|
||||
Amazon.com Inc.,de,,,Netflix Inc.,124580,21.214.161.240,26.67,1780243200.000539,"anon, hosting",976.9355,47 2b b5 3f,San Francisco,c0 2f,,518,,cf ea 38 e1 dc e0 22 c2 58 cd c2 fa 30 8a 20 dd 58 d0 ba fc 15 1b e3 6e 52 38 46 e7,03 04,34.717255,,122,,"anon, hosting",53,de,packet_capture,-103.63241,,,143,2026-06-01 00:00:00.000539,210.75.120.27,49,318,,1677.8072,Singtel,44.047101,21.214.161.240,,48570,45455,30.115.76.233,,Mumbai,21,NTT,107.874008,8698,backbone-03,500.2174,TlsS
|
||||
Baidu Inc.,us,*.cloudfront.net,,Amazon.com Inc.,11728,46.138.247.120,48.52,1780243200.00054,"anon, hosting",1963.3177,84 f6 9d de,Mumbai,13 01,,1256,,ef 74 50 d7 9d 52 df d4 e6 e2 6c 0e 2c 86 44 67 ae 46 f8 89 7a 90 13 ea 25 cf 48 f8,03 03,35.658006,,1,,"anon, hosting",8080,no,packet_capture,73.024614,,,143,2026-06-01 00:00:00.000540,82.242.195.99,96,254,,608.0895,Singtel,-17.831862,46.138.247.120,,215500,99414,170.77.253.247,,San Francisco,17,Deutsche Telekom,38.484666,7541,edge-01,592.0376,TlsC
|
||||
Microsoft Corp.,sg,,,Baidu Inc.,98517,207.100.186.111,71.76,1780243200.000541,"anon, hosting",1575.6251,da 19 19 5a,Tokyo,c0 27,,470,,a1 bd 08 e8 a9 fb d4 84 9c c0 7e 5d 31 9b 3d c9 9f 55 8d 1f 90 64 cd bb 94 a5 28 26,03 03,-25.571063,,30,,"anon, hosting",80,ca,suricata,-171.864762,,,8080,2026-06-01 00:00:00.000541,82.37.230.120,85,462,,1492.8441,BT Group,13.252968,207.100.186.111,,34304,59998,147.213.15.128,www.google.com,San Francisco,35,Comcast,62.41366,4642,gw-09,522.8864,TlsS
|
||||
Amazon.com Inc.,kr,,,Netflix Inc.,179007,139.41.70.39,103.52,1780243200.000542,"anon, hosting",1939.2257,21 1a 9f b7,Seattle,c0 14,,1279,,d9 a7 c8 d3 16 01 4d a9 9b 23 23 27 3c 43 0e 2e b7 11 4f c0 2e ab bd 82 4d ed e4 2e,03 04,-40.253098,,137,,"anon, hosting",443,de,suricata,-27.340936,56848,,8080,2026-06-01 00:00:00.000542,83.96.122.147,39,80,,671.5123,KDDI,26.343847,139.41.70.39,,193671,76086,192.134.195.211,,London,7,,-76.253271,4751,edge-01,733.7076,TlsC
|
||||
Tencent,fi,,,Netflix Inc.,102270,55.32.212.105,102.43,1780243200.000543,"anon, hosting",110.4369,bf 03 ac 3d,Tokyo,00 9f,,1720,,4b 3f 4c 04 d6 c5 12 19 fe ee a7 e4 2f b9 b9 a9 c5 46 60 91 fe d5 9e d1 20 90 ac e9,03 04,48.323571,,108,,"anon, hosting",993,ru,ssl_logs,-177.401835,83046,,21,2026-06-01 00:00:00.000543,67.104.160.62,86,422,,1389.112,China Mobile,-21.089369,55.32.212.105,,447285,65693,104.194.193.142,*.s3.amazonaws.com,Seoul,16,SK Telecom,-143.182451,8303,wan-link,509.1838,TlsC
|
||||
Netflix Inc.,no,"*.['live', 'com']",,Tencent,19128,162.99.196.127,81.98,1780243200.000544,"anon, hosting",226.1828,5a e9 48 ad,Sydney,cc a9,,971,,77 80 28 20 ba 46 b6 d6 89 33 cc b5 19 dc 97 6c 74 0d b0 97 1f ce d9 bd 33 8d 48 c5,03 03,48.259721,,151,*.cloudfront.net,"anon, hosting",3389,br,ssl_logs,158.122952,,,53,2026-06-01 00:00:00.000544,38.70.19.74,54,181,,858.6383,KDDI,-34.965379,162.99.196.127,,152771,90858,105.59.71.12,,Seoul,9,Telstra,44.470856,8701,backbone-03,785.5857,TlsC
|
||||
Samsung,fi,,,Google LLC,122291,151.117.112.2,95.53,1780243200.000545,"anon, hosting",500.4885,0e 73 d8 e5,Sydney,c0 13,,580,512,47 45 ff 25 5d 04 f0 9a 8c 28 36 96 c5 53 f9 60 43 92 a5 62 3b 54 3f de de 1d 89 06,03 04,-47.682978,,9,,"anon, hosting",8080,kr,ssl_logs,-88.244844,,,465,2026-06-01 00:00:00.000545,58.62.102.203,86,22,login.live.com,636.1596,Singtel,-43.439165,151.117.112.2,,351917,37367,184.149.139.107,graph.facebook.com,Seattle,20,BT Group,77.23547,4633,core-02,163.3565,TlsC
|
||||
Samsung,jp,"*.['google', 'com']",00 19,Netflix Inc.,15110,54.220.164.120,97.14,1780243200.000546,"anon, hosting",418.4914,1c e3 91 f5,Seoul,c0 2c,Baidu Inc.,780,,23 72 1b 6a 4a 29 5f 35 da 33 e7 d6 47 b6 2e e9 69 33 0f 4b 7f 6f 30 49 df a6 48 30,03 03,6.584101,,179,,"anon, hosting",443,il,zeek,-103.832271,,,443,2026-06-01 00:00:00.000546,68.202.67.172,56,170,,1311.7077,Comcast,-18.06927,54.220.164.120,,198392,49399,14.79.80.194,,Seoul,16,Verizon,131.721463,3351,edge-01,876.0542,TlsC
|
||||
Alibaba Inc.,de,,00 1e,Google LLC,60805,18.169.160.170,17.32,1780243200.000547,"anon, hosting",1267.9103,4a 59 64 8b,London,c0 13,,633,,b3 77 ca f5 ce e0 e9 06 60 f6 61 e2 cf f3 48 14 67 d1 e2 3e 80 ce bd 10 2d 5f 62 ac,03 04,-3.514234,,50,,"anon, hosting",443,gb,suricata,116.08806,,,443,2026-06-01 00:00:00.000547,155.220.50.204,8,276,auth.example.net,1589.4486,,1.684064,18.169.160.170,,384178,47372,118.39.160.148,,Seattle,25,AT&T,-59.630379,8190,backbone-03,121.4162,TlsC
|
||||
Alibaba Inc.,jp,,00 17,Netflix Inc.,67482,151.117.112.2,54.45,1780243200.000548,"anon, hosting",1105.7397,c7 68 c1 97,Berlin,13 02,,965,128,78 7b c8 46 3a e0 a2 03 dd 14 96 ed 21 06 c7 b0 24 77 2b 3e 8d 53 d0 e5 66 07 e4 81,03 04,-8.502161,,53,,"anon, hosting",465,sg,packet_capture,172.012304,,,53,2026-06-01 00:00:00.000548,118.63.193.21,17,436,,1909.3396,Deutsche Telekom,-18.615009,151.117.112.2,,467705,75373,104.235.124.254,www.google.com,London,3,China Telecom,84.473844,4877,backbone-03,409.3169,TlsS
|
||||
Microsoft Corp.,us,,,Baidu Inc.,125264,21.240.9.192,80.48,1780243200.000549,"anon, hosting",321.0822,fa 33 56 7f,Paris,13 02,,1059,,3e 54 60 61 3c d7 0e 65 1a 5e be ce 7b 28 2c 95 0b 57 30 b3 6b 19 07 90 1c 52 eb d3,02 00,27.630551,,48,*.cloudfront.net,"anon, hosting",53,fr,packet_capture,+,,,21,2026-06-01 00:00:00.000549,31.78.255.239,18,199,,224.4507,Deutsche Telekom,27.28677,21.240.9.192,,341742,25764,26.51.227.43,,Moscow,28,NTT,-80.844194,1373,wan-link,437.9796,TlsC
|
||||
Baidu Inc.,hk,,,Google LLC,56497,125.35.64.33,95.98,1780243200.00055,"anon, hosting",683.9046,46 c7 f1 88,Dublin,cc a9,,623,,3d bf bf b3 44 87 7f 00 f1 3f 68 06 34 ce 2e 0d 9e 23 35 5c 21 42 c5 3e 19 02 d9 3c,03 04,-36.941984,,99,login.live.com,"anon, hosting",143,hk,zeek,74.852024,,,53,2026-06-01 00:00:00.000550,105.230.240.75,100,188,,624.6781,,-52.037421,125.35.64.33,,465694,23196,26.211.169.184,login.live.com,New York,22,SK Telecom,-158.39674,9751,gw-09,491.0349,TlsS
|
||||
Netflix Inc.,no,,,Baidu Inc.,187419,21.240.9.192,52.14,1780243200.000551,"anon, hosting",904.2026,50 11 18 a4,London,c0 27,,504,,80 13 ea 7b 94 75 2a 0a b5 e5 6c 8d c9 de ec ad f2 1f 2f a8 92 4d 44 fc 14 bc bf f3,03 03,-38.844478,,114,,"anon, hosting",443,fi,packet_capture,7.259423,,,25,2026-06-01 00:00:00.000551,18.169.160.170,93,91,login.live.com,1079.3162,,-13.47482,21.240.9.192,,458792,16770,136.210.92.44,www.google.com,New York,17,Deutsche Telekom,-19.593832,7319,wan-link,268.86,TlsC
|
||||
Baidu Inc.,nl,,00 18,Meta Platforms,1808,93.157.122.15,66.54,1780243200.000552,"anon, hosting",1304.058,aa ff 27 44,Singapore,13 02,,1257,384,b9 6d 74 b4 5a 92 55 63 99 80 42 7e 4d 06 31 f1 20 12 cb 67 f9 26 3e dc 7c 6c af 85,03 03,-35.524661,--,192,*.s3.amazonaws.com,"anon, hosting",80,au,ssl_logs,-14.889513,,,443,2026-06-01 00:00:00.000552,63.207.152.117,87,449,,397.8292,NTT,-36.094837,93.157.122.15,,122434,82477,74.209.92.197,,Mumbai,22,SK Telecom,138.936224,5003,backbone-03,326.6616,TlsC
|
||||
Alibaba Inc.,cn,,,,127545,139.102.159.103,22.26,1780243200.000553,"anon, hosting",441.6709,e5 6e 33 0a,Dublin,c0 2c,,148,,48 95 8a e2 d8 6d 64 e9 fa 7a 09 82 e5 14 3a 7c 04 65 e8 dc 05 8a 37 32 fc ef 0a a9,03 03,-54.617667,,127,mail.example.com,"anon, hosting",80,au,zeek,-82.402029,,,995,2026-06-01 00:00:00.000553,57.230.178.239,97,5,auth.example.net,257.6084,Telstra,-43.136288,139.102.159.103,,266619,63874,62.158.240.232,,London,36,Verizon,148.674469,5141,edge-01,640.6758,TlsS
|
||||
Samsung,hk,,,Amazon.com Inc.,103630,207.100.186.111,75.81,1780243200.000554,"anon, hosting",1209.4864,a0 e8 6c bb,Tokyo,00 9f,,1970,,32 a2 5f 47 e7 78 f6 52 47 d3 66 d8 39 ef 38 c2 20 62 61 ad 35 fc 30 0e b7 4e 98 a4,03 04,-7.113792,,61,,"anon, hosting",143,in,zeek,18.080854,,,993,2026-06-01 00:00:00.000554,177.124.52.179,52,12,,916.0035,BT Group,8.859885,207.100.186.111,TLS_AES_256_GCM_SHA384,43604,80429,179.115.153.176,login.live.com,Beijing,40,SK Telecom,23.415148,1110,wan-link,171.4019,TlsC
|
||||
Apple Inc.,gb,,,Apple Inc.,112402,180.131.188.43,43.24,1780243200.000555,"anon, hosting",52.8595,8e e7 b2 10,Moscow,c0 2c,,29,,6d b7 fc 3a f0 d8 45 06 58 ea fd 62 80 6d 59 81 08 fc 87 48 cd 04 3d e5 b4 1f 30 ee,03 03,45.896817,,144,,"anon, hosting",25,kr,zeek,-131.525397,,,12345,2026-06-01 00:00:00.000555,55.135.67.239,37,312,,1120.6183,Deutsche Telekom,-44.889281,180.131.188.43,,11057,11408,53.250.46.231,,Seattle,45,NTT,-138.214164,9202,wan-link,530.0913,TlsS
|
||||
,jp,,,Netflix Inc.,172701,28.79.139.73,46.61,1780243200.000556,"anon, hosting",786.3277,b4 47 1f d8,Mumbai,00 9f,,971,128,81 24 60 0f f1 8c 80 7f 41 f8 43 01 5f 46 5f 3f 49 78 e7 04 75 4f 46 f2 1e 5c de 0f,03 01,41.071461,,69,,"anon, hosting",443,sg,zeek,-97.511364,,,53,2026-06-01 00:00:00.000556,164.29.38.172,3,237,,1787.6288,China Telecom,-17.542026,28.79.139.73,TLS_AES_128_GCM_SHA256,254139,21766,83.96.122.147,,Tokyo,10,SK Telecom,-94.370462,4661,edge-01,924.1667,TlsC
|
||||
Amazon.com Inc.,in,,,Google LLC,6576,5.47.150.57,30.19,1780243200.000557,"anon, hosting",1692.7745,89 f7 94 0b,Seoul,c0 27,,1556,,0a 7c dd a7 e6 26 d8 fa 4b 71 24 0f 26 13 2a b2 b4 8b 05 bf f8 84 92 f8 0b 70 2b fe,03 04,-33.12372,,57,graph.facebook.com,"anon, hosting",143,kr,ssl_logs,70.789144,,,443,2026-06-01 00:00:00.000557,26.225.51.135,56,78,,1595.1991,Deutsche Telekom,21.338441,5.47.150.57,,479863,44583,84.28.150.92,,Seattle,35,KDDI,-154.42833,7388,backbone-03,910.2581,TlsS
|
||||
Meta Platforms,se,mail.example.com,,Apple Inc.,107590,214.0.155.74,74.84,1780243200.000558,"anon, hosting",1613.2459,c5 d0 7e e4,San Francisco,00 9f,,1478,512,44 67 27 85 d3 96 4e 17 4d 06 e4 ca b7 f9 dc ca e9 b2 29 69 99 74 97 8a ed e2 c2 56,03 03,59.514016,,37,,"anon, hosting",143,us,netflow,-63.466529,,,21,2026-06-01 00:00:00.000558,109.16.15.24,23,403,,505.8709,Comcast,-5.037759,214.0.155.74,TLS_RSA_WITH_AES_256_GCM_SHA384,494785,63118,124.114.55.90,*.cloudfront.net,Beijing,6,Orange,-172.00567,3039,wan-link,598.2761,TlsC
|
||||
Samsung,ca,,,,70902,117.186.83.95,69.56,1780243200.000559,"anon, hosting",1072.2578,17 e0 9a e0,Mumbai,cc a9,,186,256,61 9b 04 01 38 26 1f b6 48 2e 55 87 0a f5 d8 d3 f6 c1 33 c7 c3 94 b4 67 7c 66 e9 74,03 03,-18.928421,,92,,"anon, hosting",443,ua,netflow,84.327147,,,443,2026-06-01 00:00:00.000559,112.169.48.230,59,405,,960.2371,,-10.121892,117.186.83.95,,240100,17380,11.89.240.133,mail.example.com,Paris,44,Telstra,-138.384016,7557,gw-09,57.1121,TlsC
|
||||
Tencent,kr,,00 1e,Apple Inc.,137910,106.12.91.189,82.73,1780243200.00056,"anon, hosting",250.8193,b9 c4 10 a5,Moscow,00 35,,1675,,65 a5 f6 d4 b0 64 76 61 a7 c0 40 88 39 b1 2a ee cd 78 32 5b ed c2 bb d3 f0 c4 e8 67,03 03,-16.834884,,149,stream.example.org,"anon, hosting",3389,fi,packet_capture,28.556565,,,443,2026-06-01 00:00:00.000560,102.253.95.240,69,138,,1842.9836,,-13.312705,106.12.91.189,,61186,82460,194.70.49.232,,Beijing,9,Orange,-169.567002,6293,gw-09,505.663,TlsS
|
||||
Microsoft Corp.,ca,,,Microsoft Corp.,58548,55.135.67.239,117.34,1780243200.000561,"anon, hosting",1004.0704,df 5e 7f 47,San Francisco,13 01,,1779,512,f4 6d 3d 23 2d 57 0b 48 ed 56 5b 3a c9 7b 22 fa 97 30 72 41 e6 7c 72 8e 08 ab ca 7d,03 03,22.411624,ER,200,*.s3.amazonaws.com,"anon, hosting",80,no,packet_capture,79.704209,,x25519,21,2026-06-01 00:00:00.000561,179.151.65.50,53,142,,771.0288,Orange,-31.17493,55.135.67.239,,44309,70065,62.164.144.98,login.live.com,Mumbai,1,China Mobile,70.568118,2168,core-02,527.7406,TlsC
|
||||
Samsung,us,,,Tencent,39194,153.222.176.187,110.27,1780243200.000562,"anon, hosting",1379.6398,4a 39 fc 9f,Seoul,c0 2c,,1795,,44 d8 b8 14 06 3c 59 24 9d 71 c6 99 aa 1f 7e 1a 1d c6 ee 22 67 91 d1 14 c3 0e c4 b1,03 01,9.37105,,155,mail.example.com,"anon, hosting",443,ru,packet_capture,-59.242826,,,443,2026-06-01 00:00:00.000562,194.115.203.248,27,328,,835.9478,SK Telecom,21.11355,153.222.176.187,,233336,85559,145.57.31.213,,London,9,China Telecom,-98.333808,4267,backbone-03,728.6627,TlsC
|
||||
Samsung,us,,00 1e,Netflix Inc.,129558,112.137.165.219,13.06,1780243200.000563,"anon, hosting",1818.8164,38 ca 0f 68,Sydney,c0 2b,,1567,,7d aa 86 28 bd fe 50 78 fd 85 5e 80 ed 90 4c a6 31 a1 fb 9e db a6 2d 97 d8 bf c1 6c,03 03,-47.785568,,133,,"anon, hosting",443,jp,packet_capture,145.075733,,,8443,2026-06-01 00:00:00.000563,21.214.161.240,97,362,,1038.6617,Orange,44.633512,112.137.165.219,,359619,64322,136.110.192.108,auth.example.net,Tokyo,13,Comcast,-128.381946,5749,core-02,860.3991,TlsC
|
||||
Alibaba Inc.,ru,*.s3.amazonaws.com,,Apple Inc.,168082,13.244.80.15,89.38,1780243200.000564,"anon, hosting",1941.6847,1d f9 41 dd,San Francisco,c0 2c,,590,,2a e1 05 96 83 9e 24 ba 62 c5 3e 83 ab 24 6f 25 75 4d 30 38 c1 f2 14 9a 35 b8 c9 37,03 03,11.267757,,111,,"anon, hosting",993,ru,ssl_logs,-47.5157,,,465,2026-06-01 00:00:00.000564,62.40.43.188,55,450,,1102.8816,Telstra,30.988825,1.199.135.238,,154517,88290,141.80.208.168,,Berlin,11,Comcast,-65.90689,5648,core-02,723.1713,TlsS
|
||||
Google LLC,de,,,Tencent,150933,149.188.242.142,59.74,1780243200.000565,"anon, hosting",527.5815,6b dc 0f 22,New York,c0 2b,,1289,,d8 4d 35 6f de 06 dd 4b 91 8d 88 ee 5d ff 45 9f b5 a5 f9 c7 ff a0 22 35 92 7f 67 71,03 03,-51.26523,,35,*.s3.amazonaws.com,"anon, hosting",443,in,ssl_logs,-146.52977,,,465,2026-06-01 00:00:00.000565,101.13.42.165,32,319,,1392.944,AT&T,-6.898782,149.188.242.142,,468417,27679,82.37.230.120,,New York,33,BT Group,123.139147,4621,wan-link,453.6986,TlsS
|
||||
Amazon.com Inc.,il,,,Netflix Inc.,69532,145.21.177.137,72.74,1780243200.000566,"anon, hosting",1562.9959,ec f8 6f 39,Mumbai,13 01,,1116,,d0 4b 52 e2 2b 51 5a d2 7d 3e 0a cf e9 58 bf 99 17 e1 fc 81 25 d4 4d 05 8e 6e 53 2e,03 03,-30.445747,,163,,"anon, hosting",443,ua,ssl_logs,151.134258,,x25519,21,2026-06-01 00:00:00.000566,96.52.63.73,48,241,graph.facebook.com,277.9959,Deutsche Telekom,-18.561824,145.21.177.137,,363830,45442,53.34.51.153,graph.facebook.com,Tokyo,35,NTT,-32.942897,8638,backbone-03,832.6343,TlsC
|
||||
Amazon.com Inc.,nl,,00 1e,Google LLC,5100,210.153.53.241,80.27,1780243200.000567,"anon, hosting",1173.2119,e6 e0 13 47,Tokyo,c0 2b,Google LLC,1916,,32 d0 ff cb 8f e7 a1 7f 7c b6 c1 c2 7e 55 54 5c e4 04 cc f1 ff e9 f8 78 10 9f 5c a0,03 03,21.091328,,147,,"anon, hosting",53,il,suricata,141.059615,,,443,2026-06-01 00:00:00.000567,139.7.47.238,74,154,cdn.example.com,262.1209,China Telecom,2.993471,8.59.133.46,,47888,98270,208.57.169.42,,Moscow,42,China Telecom,-104.585181,7804,edge-01,639.8636,TlsS
|
||||
Meta Platforms,ua,*.cloudfront.net,,Baidu Inc.,169808,106.12.91.189,36.34,1780243200.000568,"anon, hosting",47.9411,0c e3 c9 c4,Singapore,cc a9,,790,,d8 33 a4 85 04 ba 50 31 e6 f1 5d df 42 4c 5f 74 90 d3 f6 36 9a 8b d5 26 e1 61 47 e7,03 03,-25.41839,,77,,"anon, hosting",465,fr,ssl_logs,-177.516353,,,50000,2026-06-01 00:00:00.000568,17.197.195.153,23,422,,382.9395,Deutsche Telekom,21.964557,106.12.91.189,,76619,34576,217.188.58.196,,Seoul,36,Orange,120.206679,3047,edge-01,934.7958,TlsS
|
||||
Alibaba Inc.,us,,,,58348,117.186.83.95,10.9,1780243200.000569,"anon, hosting",1213.8961,f7 2c d0 8e,San Francisco,00 35,,410,,30 79 a2 ae de ba ea 53 ee 4a fc 39 82 3a 9b ed c2 5e 79 12 50 c9 fa 1d 94 d0 23 da,03 04,49.275482,,109,stream.example.org,"anon, hosting",995,kr,packet_capture,-93.095106,,x448,443,2026-06-01 00:00:00.000569,186.5.133.168,69,76,,1796.6388,KDDI,41.256485,117.186.83.95,,267916,66504,147.213.95.171,auth.example.net,Tokyo,49,BT Group,-27.214506,6650,edge-01,942.038,TlsS
|
||||
Samsung,se,,,Netflix Inc.,13079,101.13.42.165,96.86,1780243200.00057,"anon, hosting",1102.5703,3d 24 66 a3,London,13 01,,1394,512,7e 9b 09 2b 42 fe a9 8c 50 fa e1 bc 9b e7 5f 57 ed 1a 2a 4d 82 c0 ed b5 a1 f7 c4 69,03 03,-6.10232,,113,,"anon, hosting",443,de,ssl_logs,-56.045304,,,443,2026-06-01 00:00:00.000570,136.21.117.55,36,203,login.live.com,654.9736,China Mobile,-55.320731,101.13.42.165,,304448,92299,221.24.169.70,,Dublin,46,NTT,139.554788,4322,gw-09,329.504,TlsS
|
||||
Google LLC,nl,,00 1d,Amazon.com Inc.,127189,207.81.216.88,61.28,1780243200.000571,"anon, hosting",1321.1508,15 0f 8a f6,Sydney,c0 2b,Google LLC,1843,384,df 09 14 8b 6d 11 0c d9 7d 77 fc 46 7c d8 3a 06 ff e7 56 21 32 c4 65 03 38 dd a9 2e,03 03,59.805904,,170,,"anon, hosting",110,se,ssl_logs,-5.916925,,,25,2026-06-01 00:00:00.000571,207.151.173.206,17,27,,703.3497,BT Group,-17.340457,207.81.216.88,,464857,43774,26.250.208.241,,Beijing,38,Orange,125.559857,4309,wan-link,543.5681,TlsC
|
||||
Samsung,se,,,Alibaba Inc.,62039,165.234.73.68,0.95,1780243200.000572,"anon, hosting",1273.6123,e6 78 d3 56,Berlin,c0 27,,263,256,14 23 25 17 65 65 cc 2a 68 c8 6c f6 a7 0c d7 0b a0 1e 70 7c bf 31 39 ca 78 f5 93 b5,03 04,27.495117,,68,auth.example.net,"anon, hosting",12345,us,packet_capture,-53.767453,,,443,2026-06-01 00:00:00.000572,76.111.29.149,81,316,cdn.example.com,1886.0701,Singtel,-25.170288,165.234.73.68,,68386,71322,202.136.1.73,,Mumbai,38,SK Telecom,54.229822,7764,gw-09,302.4266,TlsS
|
||||
Apple Inc.,sg,,,Baidu Inc.,151655,112.137.165.219,66.15,1780243200.000573,"anon, hosting",1890.9012,fa c3 75 5f,Seattle,c0 14,,300,,a6 b6 c9 6c 2a dd 03 de 52 72 3b 88 7b f5 90 9a 76 fc 7d 03 b3 74 b4 d6 43 f9 d3 78,03 04,-57.130219,,151,,"anon, hosting",443,ca,suricata,-16.916536,,x448,993,2026-06-01 00:00:00.000573,141.169.180.180,60,119,,725.4963,NTT,17.079931,112.137.165.219,,162372,46957,53.189.160.106,*.cloudfront.net,Seoul,16,SK Telecom,-170.499626,4070,backbone-03,836.0099,TlsS
|
||||
Microsoft Corp.,fi,,,Alibaba Inc.,14795,193.120.158.58,27.23,1780243200.000574,"anon, hosting",1392.5203,97 e9 5f ef,Sydney,c0 13,,1025,,23 74 af c7 68 c4 4a 49 8a 93 dd 5e db aa ba d2 3e ba fb e6 41 75 ad b0 2b f5 43 ec,03 03,28.047972,,196,cdn.example.com,"anon, hosting",53,nl,zeek,-155.123261,,,443,2026-06-01 00:00:00.000574,86.164.53.220,38,149,,1362.1551,China Telecom,14.122734,193.120.158.58,,319334,56776,214.99.188.174,api.example.com,Singapore,10,NTT,-176.840277,4184,wan-link,683.6108,TlsS
|
||||
Netflix Inc.,se,,,,7671,8.199.172.172,18.98,1780243200.000575,"anon, hosting",1689.0163,71 cc 9d 18,Beijing,13 02,,1289,512,f1 b2 ab e3 38 29 f6 43 90 3a e3 09 dd 65 6f f4 e9 85 e3 4e d2 e2 6c 54 c5 38 77 d5,02 00,-15.496416,,134,,"anon, hosting",8080,au,packet_capture,-75.339733,,,465,2026-06-01 00:00:00.000575,53.34.51.153,11,455,,1234.7049,BT Group,26.330458,8.199.172.172,,129139,54807,74.159.180.27,,Singapore,47,China Telecom,-31.700318,5181,core-02,143.0474,TlsC
|
||||
Alibaba Inc.,kr,,00 19,Alibaba Inc.,23817,10.240.113.52,78.47,1780243200.000576,"anon, hosting",1271.4429,2e b3 57 3b,San Francisco,13 01,,494,512,eb ea 4d 28 35 ea 1e 12 ba 53 93 a0 c9 06 0c 30 51 f1 3d f6 57 13 93 08 50 e3 11 8a,03 03,33.697058,,80,,"anon, hosting",50000,br,ssl_logs,-132.59177,,,12345,2026-06-01 00:00:00.000576,149.135.19.28,93,278,,546.0868,Verizon,40.351475,10.240.113.52,,9540,71616,190.247.141.186,,Seoul,1,Telstra,-163.24638,2100,core-02,324.0067,TlsS
|
||||
Netflix Inc.,no,,,Alibaba Inc.,26341,112.137.165.219,118.85,1780243200.000577,"anon, hosting",1387.4965,7c 42 a1 5c,Sydney,cc a9,,1266,,a3 f6 e1 d9 2c 94 06 c4 4e c8 0b 16 b0 91 4f d4 6e b1 fa 8f 13 ac 36 e2 9d 87 98 94,03 02,-29.114609,,29,www.google.com,"anon, hosting",50000,fi,zeek,124.895668,,,53,2026-06-01 00:00:00.000577,25.120.69.100,55,289,,1702.5257,China Telecom,-7.309655,112.137.165.219,,5607,73073,194.115.203.248,,Singapore,39,China Telecom,100.960679,8105,wan-link,107.6298,TlsS
|
||||
Microsoft Corp.,au,,,,167602,207.101.75.7,29.47,1780243200.000578,"anon, hosting",5.8136,6c 42 ab da,New York,00 35,,1535,,13 aa 43 a2 0e 21 e3 4d 17 d1 cd b8 e1 fe cf f9 ee f1 3e 61 05 82 6b fe 43 00 14 c8,03 04,-34.572287,,137,graph.facebook.com,"anon, hosting",443,cn,ssl_logs,-65.946772,,,443,2026-06-01 00:00:00.000578,93.9.185.80,97,359,,989.0376,China Mobile,-40.102501,101.13.42.165,,425825,81018,214.247.241.63,auth.example.net,Dublin,14,,-93.71036,4116,wan-link,298.7886,TlsC
|
||||
Netflix Inc.,au,,,Meta Platforms,75345,49.43.123.185,3.22,1780243200.000579,"anon, hosting",1963.7453,15 14 ea 72,New York,c0 14,,174,128,94 78 5f 8f 9d b9 fe 9f 77 9f 5d a1 b5 26 50 4f ce aa 48 71 52 34 19 de 4e c3 b1 45,03 03,35.408205,,92,mail.example.com,"anon, hosting",995,cn,ssl_logs,81.650401,,,443,2026-06-01 00:00:00.000579,202.34.176.62,52,261,,559.1773,Telstra,51.67467,49.43.123.185,,178108,61548,34.212.233.145,,Singapore,21,Telstra,151.62723,3220,core-02,504.314,TlsS
|
||||
Meta Platforms,se,login.live.com,,Samsung,192255,80.209.167.104,119.62,1780243200.00058,"anon, hosting",347.631,d5 10 75 65,Seattle,c0 27,,661,,a4 e9 4e a5 ed 65 38 6c 0b 5b 50 0f 76 e7 7d b1 9a 15 bb 77 66 81 00 31 d5 8e eb b8,03 03,+,,183,,"anon, hosting",3389,kr,packet_capture,24.915257,,,12345,2026-06-01 00:00:00.000580,166.183.31.102,78,458,,1617.2056,AT&T,-47.904323,80.209.167.104,TLS_ECDHE_ECDSA_AES128_SHA,458432,49190,192.212.89.52,cdn.example.com,San Francisco,5,China Telecom,125.759351,6139,core-02,790.4936,TlsS
|
||||
Baidu Inc.,gb,"*.['live', 'com']",,Tencent,197720,146.154.207.141,38.29,1780243200.000581,"anon, hosting",122.4191,7c 3b 33 c2,Tokyo,c0 13,,685,,04 38 dd 4f 95 82 89 b0 24 ca d2 2a 6e fa 07 7c 85 0f 90 f3 a5 73 3e a2 16 47 e9 95,03 04,-43.29265,,131,,"anon, hosting",25,ua,zeek,113.853982,,,143,2026-06-01 00:00:00.000581,149.21.230.233,42,325,,119.8425,BT Group,11.112762,146.154.207.141,,312192,37077,213.93.66.111,,Seoul,15,Orange,130.545693,3649,core-02,636.3863,TlsC
|
||||
Tencent,ua,,,Samsung,51321,131.41.95.18,14.26,1780243200.000582,"anon, hosting",711.3619,2e 14 4e 10,Seoul,c0 23,,1639,,f8 b5 9b 3a 07 80 a5 87 19 de a4 9a fe 4b 4b 99 1c c6 da ce 25 37 e3 1d 06 e0 89 06,03 03,54.774267,,175,login.live.com,"anon, hosting",50000,us,suricata,-140.381819,,secp521r1,443,2026-06-01 00:00:00.000582,2.210.6.5,47,321,stream.example.org,561.2301,China Mobile,-1.799647,131.41.95.18,,236437,27479,19.30.80.113,www.google.com,Shanghai,33,Telstra,-87.835296,2925,gw-09,323.1611,TlsC
|
||||
Microsoft Corp.,au,,,,163116,91.107.136.180,100.73,1780243200.000583,"anon, hosting",1785.4418,8c 50 cb 6c,London,cc a9,,1920,384,b6 f2 4b 1a e6 67 2d 24 b8 d3 fa d3 59 79 da b7 43 cf 77 0d 31 c0 27 1c 60 11 6e e0,03 03,-48.339925,,30,mail.example.com,"anon, hosting",3389,au,packet_capture,24.907846,,,443,2026-06-01 00:00:00.000583,223.251.52.224,63,312,,145.4304,China Mobile,-33.790885,91.107.136.180,,459448,77993,155.205.20.12,,London,37,Comcast,-96.248817,6505,edge-01,608.5656,TlsC
|
||||
Amazon.com Inc.,il,,00 17,,38252,139.102.159.103,81.66,1780243200.000584,"anon, hosting",694.9418,63 e6 10 11,Paris,13 02,,1475,128,87 dc be 49 ff 9f 1c 86 b6 dd 80 a4 8f 6c 22 5b 1c 79 5a d8 a3 32 4e 82 e6 5e 43 cf,03 03,54.789149,,44,,"anon, hosting",110,br,suricata,126.227427,,x25519,110,2026-06-01 00:00:00.000584,166.174.6.218,90,222,,1106.4575,NTT,-6.336501,139.102.159.103,,477267,36191,213.93.66.111,*.s3.amazonaws.com,San Francisco,19,NTT,71.261615,6745,core-02,751.1953,TlsC
|
||||
Google LLC,ca,,,,152621,193.37.4.118,108.74,1780243200.000585,"anon, hosting",805.4125,f7 1c 28 d5,London,cc a9,,1878,,c8 3e 15 46 29 37 36 33 fa 11 99 0a 94 fe f0 75 f3 99 19 bf 45 3c ed 01 41 17 ff 7b,03 04,18.902411,,131,,"anon, hosting",465,kr,ssl_logs,-162.002579,,,443,2026-06-01 00:00:00.000585,136.21.117.55,79,251,cdn.example.com,1897.0057,BT Group,24.220336,193.37.4.118,,26834,65426,39.228.247.89,,London,44,Verizon,-78.001765,6800,gw-09,548.1531,TlsS
|
||||
Tencent,ua,,,Google LLC,177258,41.55.195.224,11.87,1780243200.000586,"anon, hosting",1596.6201,57 23 c0 40,Paris,c0 23,,458,,05 08 a9 45 b2 2a 6b c9 5d f3 5a a7 ce 3c 5e 32 21 83 05 8b ef 84 4f d3 3a b1 9f ef,03 04,-38.066585,,143,*.cloudfront.net,"anon, hosting",50000,kr,netflow,-50.556677,,,443,2026-06-01 00:00:00.000586,141.169.180.180,77,313,,311.7673,China Mobile,-23.671963,41.55.195.224,,424970,18170,52.138.139.36,,Beijing,13,AT&T,114.046734,9455,gw-09,9.7994,TlsC
|
||||
Tencent,il,,,Google LLC,81670,179.237.25.143,96.85,1780243200.000587,"anon, hosting",1139.0759,19 56 a6 81,Singapore,c0 13,,504,384,25 98 00 0c 91 c0 83 1a fe 33 f2 78 dd 38 ac e6 af 4c 8a ba 52 3a c7 95 54 11 2f 38,03 03,1.33482,,120,,"anon, hosting",995,br,zeek,43.517232,,,110,2026-06-01 00:00:00.000587,106.12.91.189,30,288,,1838.3424,Verizon,13.744101,179.237.25.143,TLS_RSA_WITH_AES_256_GCM_SHA384,77536,40960,48.7.172.203,,New York,3,China Mobile,-16.984533,3023,edge-01,420.1237,TlsC
|
||||
Google LLC,ru,,,Baidu Inc.,49104,221.50.25.167,3.3,1780243200.000588,"anon, hosting",130.6076,6e 76 99 3e,Beijing,c0 27,,1256,,4d a0 ad 76 7e 81 c3 a3 9d d0 1e 02 40 5a 07 e2 d8 8c c7 b9 67 23 fe bf ad d6 44 f2,03 03,-7.553418,ER,72,mail.example.com,"anon, hosting",12345,sg,zeek,-53.825262,,,443,2026-06-01 00:00:00.000588,184.218.85.177,28,263,,713.2394,AT&T,50.221453,221.50.25.167,,54577,29499,141.241.111.196,*.cloudfront.net,Shanghai,39,BT Group,103.9875,2973,core-02,262.8874,TlsC
|
||||
Tencent,hk,,,Samsung,9648,185.58.136.197,111.9,1780243200.000589,"anon, hosting",229.0665,bf ef 72 d6,San Francisco,cc a9,,948,,46 53 84 01 32 f9 71 52 2a a3 a4 9d 6b 21 49 85 d0 fa 80 04 02 93 03 99 7c e5 33 91,03 03,10.932053,,34,,"anon, hosting",50000,de,zeek,74.324827,,secp384r1,8443,2026-06-01 00:00:00.000589,222.46.196.119,37,290,,375.1081,BT Group,-8.346142,71.125.114.36,,115803,30390,78.140.28.91,,Tokyo,9,Telstra,173.010342,5033,backbone-03,408.937,TlsS
|
||||
Alibaba Inc.,hk,,00 1e,,18625,207.100.186.111,1.31,1780243200.00059,"anon, hosting",1507.441,8e 32 ed cb,London,00 35,,1719,,1a 69 a2 53 3e 1f 9a 31 3c 84 d5 2d 7c 74 7c 25 6c dc 65 93 45 6e a5 44 8e 64 f9 39,03 04,45.885265,,148,,"anon, hosting",8443,il,suricata,31.083578,,,3389,2026-06-01 00:00:00.000590,16.35.241.9,19,137,api.example.com,1970.6598,China Mobile,8.526986,114.226.109.131,,68040,29220,222.14.41.12,,Beijing,28,Verizon,85.763297,3778,gw-09,528.2925,TlsC
|
||||
,fr,,,Netflix Inc.,96594,63.44.142.226,4.23,1780243200.000591,"anon, hosting",329.1744,50 39 43 68,Tokyo,00 9f,,448,,e2 65 02 f3 45 81 05 77 ca 92 14 06 54 b5 93 04 dc 89 c3 e8 d8 bb 21 fb 4a f7 ff a0,03 03,29.422547,,81,cdn.example.com,"anon, hosting",993,nl,packet_capture,25.832511,,,465,2026-06-01 00:00:00.000591,179.151.65.50,49,94,,25.9275,AT&T,-47.501351,63.44.142.226,,266127,45488,121.22.198.252,,Beijing,20,Comcast,33.91992,5700,edge-01,309.3521,TlsS
|
||||
Tencent,br,*.s3.amazonaws.com,,Meta Platforms,182876,179.151.65.50,28.59,1780243200.000592,"anon, hosting",446.2618,a3 3d 0c 12,Tokyo,cc a9,,1047,,44 4e 3a ce d0 56 dd f9 e8 77 96 a3 dd 60 b8 f2 40 41 66 2b a1 40 ce 11 b8 ea 92 23,03 03,26.789984,,154,,"anon, hosting",22,us,ssl_logs,119.163708,,,443,2026-06-01 00:00:00.000592,142.7.208.24,48,361,,1605.3525,Verizon,-53.952473,179.151.65.50,,272044,57286,104.194.193.142,,Moscow,34,China Mobile,157.891364,7034,wan-link,643.4033,TlsS
|
||||
Baidu Inc.,il,,00 1e,Alibaba Inc.,7205,198.154.61.204,11.06,1780243200.000593,"anon, hosting",1953.6486,cd 85 ae 47,Moscow,00 9f,,617,384,4e 3e 7c e2 14 39 6c 50 f8 86 05 63 3b df 93 f4 89 7b 85 2b 38 b8 e4 db be 2a bf 1f,03 03,-12.724335,,76,graph.facebook.com,"anon, hosting",12345,nl,netflow,-95.863193,,,465,2026-06-01 00:00:00.000593,204.203.40.96,91,323,,1772.8358,Telstra,23.675951,198.154.61.204,,169362,28630,129.191.120.99,cdn.example.com,Moscow,39,Telstra,-151.377935,1036,core-02,603.6637,TlsS
|
||||
Microsoft Corp.,kr,,,Microsoft Corp.,138307,62.40.43.188,87.41,1780243200.000594,"anon, hosting",424.1443,4d 8c a9 04,Tokyo,c0 14,,251,,19 c2 1b ea 61 28 02 b3 30 b4 0f eb 23 1e 27 d3 bd 66 ae b9 20 ba 66 4f 1f 57 74 be,03 03,4.493003,,148,auth.example.net,"anon, hosting",993,ca,suricata,-85.457509,,,8443,2026-06-01 00:00:00.000594,180.47.216.131,80,161,,913.8915,Comcast,-42.82692,198.154.61.204,,97181,65224,91.216.26.248,,Tokyo,45,Telstra,68.441015,1177,backbone-03,708.7646,TlsS
|
||||
Google LLC,ca,,,,169897,100.214.112.125,97.35,1780243200.000595,"anon, hosting",1729.7231,26 4c 42 c6,Beijing,c0 23,,83,,31 f5 d5 02 f9 3a 66 ff f4 f5 6e c1 98 0b 86 ee 0f 8b 4d 16 d8 ce 98 fc d6 60 ba 89,03 03,-52.847382,,63,,"anon, hosting",443,au,netflow,172.633945,,,443,2026-06-01 00:00:00.000595,134.194.230.233,74,420,,294.0557,,25.429305,100.214.112.125,TLS_AES_256_GCM_SHA384,476812,41776,155.205.20.12,,San Francisco,18,China Telecom,-80.979989,2310,backbone-03,221.9973,TlsC
|
||||
Meta Platforms,us,,,Amazon.com Inc.,156744,189.52.44.152,82.92,1780243200.000596,"anon, hosting",1763.1067,5a f2 02 c9,Beijing,c0 23,,1534,,78 ef eb b9 7a 6f b3 83 16 a1 7c 62 56 88 7e 64 a7 09 c9 94 9b 7c 6f 0a 52 a5 88 ff,03 04,-32.086168,,161,,"anon, hosting",443,sg,netflow,89.809156,,,8443,2026-06-01 00:00:00.000596,187.125.83.119,14,183,login.live.com,221.1828,Deutsche Telekom,-22.078511,177.124.52.179,,54282,38919,191.79.212.168,,Shanghai,15,SK Telecom,-0.740935,5559,wan-link,203.7772,TlsS
|
||||
Baidu Inc.,ru,,,Apple Inc.,24110,218.37.233.107,46.16,1780243200.000597,"anon, hosting",1944.5214,ae b1 b8 e2,Berlin,13 01,,1657,,81 b6 ff 44 4c 18 bd 16 3d 2b c4 0b 0b 47 db 9a df f3 ea fd dd 3e 5a 18 43 ee cd 7e,03 03,-40.965875,,56,stream.example.org,"anon, hosting",143,sg,ssl_logs,-2.12613,,,80,2026-06-01 00:00:00.000597,211.124.16.147,46,172,,1964.9901,Deutsche Telekom,-10.047792,218.37.233.107,,87697,59716,31.78.255.239,,Paris,31,Comcast,122.143629,6359,edge-01,574.4118,TlsC
|
||||
Samsung,in,,,Baidu Inc.,188886,49.43.123.185,64.51,1780243200.000598,"anon, hosting",1444.3486,8e c9 3c ac,San Francisco,c0 27,,218,512,43 76 f5 5a f0 79 b9 fa cd b7 de 81 4d d3 b1 6c 1f a0 91 f1 29 fe e0 81 31 d8 d8 f9,03 03,-26.685316,,170,,"anon, hosting",12345,us,zeek,-7.048568,,,22,2026-06-01 00:00:00.000598,14.99.185.142,62,214,,942.8688,NTT,-16.270871,49.43.123.185,,13036,80440,21.68.178.31,*.cloudfront.net,London,24,Telstra,88.6311,2392,wan-link,718.2917,TlsC
|
||||
Netflix Inc.,il,,,Tencent,185434,93.157.122.15,116.42,1780243200.000599,"anon, hosting",1828.4897,94 ec cc bc,Dublin,13 01,,582,,b4 34 b0 ee 3b c8 20 ad 98 d1 ea 4d 61 07 93 9b 36 bb 13 e3 23 27 6a 66 b0 22 85 df,03 04,37.245449,,93,,"anon, hosting",12345,gb,netflow,+,,,443,2026-06-01 00:00:00.000599,166.174.6.218,87,450,,969.2695,KDDI,-41.395339,153.34.120.104,,370809,76625,80.203.242.65,,Beijing,17,China Mobile,-67.093462,6727,core-02,392.0671,TlsC
|
||||
|
+274
-679
File diff suppressed because it is too large
Load Diff
+502
@@ -0,0 +1,502 @@
|
||||
# 天璇 (TianXuan) 故障诊断手册
|
||||
|
||||
> 常见问题排查指南。使用 `scripts/diagnose_compare.py` 进行跨机日志对比诊断。
|
||||
|
||||
---
|
||||
|
||||
## 目录
|
||||
|
||||
1. [启动问题](#1-启动问题)
|
||||
2. [数据加载问题](#2-数据加载问题)
|
||||
3. [聚类问题](#3-聚类问题)
|
||||
4. [UMAP 问题](#4-umap-问题)
|
||||
5. [数据库问题](#5-数据库问题)
|
||||
6. [LLM 集成问题](#6-llm-集成问题)
|
||||
7. [前端问题](#7-前端问题)
|
||||
8. [跨机一致性问题](#8-跨机一致性问题)
|
||||
9. [性能问题](#9-性能问题)
|
||||
10. [诊断工具使用](#10-诊断工具使用)
|
||||
|
||||
---
|
||||
|
||||
## 1. 启动问题
|
||||
|
||||
### 1.1 `config` 模块找不到
|
||||
|
||||
**症状**: `ModuleNotFoundError: No module named 'config'`
|
||||
|
||||
**原因**: Python 路径未包含项目根目录。
|
||||
|
||||
**解决**:
|
||||
```bat
|
||||
:: 在项目根目录下运行
|
||||
set PYTHONUTF8=1
|
||||
chcp 65001
|
||||
start /B runtime\python\python.exe manage.py runserver
|
||||
```
|
||||
|
||||
### 1.2 首次启动无数据库
|
||||
|
||||
**症状**: `OperationalError: no such table`
|
||||
|
||||
**解决**:
|
||||
```bat
|
||||
runtime\python\python.exe manage.py migrate
|
||||
```
|
||||
|
||||
`wsgi.py` 已包含 SQLite 启动自修复逻辑, 自动创建缺失的数据库文件。
|
||||
|
||||
### 1.3 端口被占用
|
||||
|
||||
**症状**: `Error: [WinError 10048] 通常每个套接字地址只允许使用一次`
|
||||
|
||||
**解决**:
|
||||
```bat
|
||||
:: 查占用进程
|
||||
netstat -ano | findstr :8000
|
||||
:: 改端口 (config.yaml)
|
||||
server:
|
||||
port: 8080
|
||||
```
|
||||
|
||||
### 1.4 残留进程未清理
|
||||
|
||||
**症状**: 前一次启动后进程未终止, 再次启动端口占用
|
||||
|
||||
**解决**:
|
||||
```bat
|
||||
:: 检查 .server_pid 文件
|
||||
type .server_pid
|
||||
:: 强制终止
|
||||
taskkill /PID <pid> /F
|
||||
del .server_pid
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 2. 数据加载问题
|
||||
|
||||
### 2.1 Schema 不匹配 (多文件列名不一致)
|
||||
|
||||
**症状**: `ValueError: schema mismatch` 或列名不一致时报错
|
||||
|
||||
**诊断**:
|
||||
```bash
|
||||
runtime\python\python.exe scripts\column_survey.py --csv "data/*.csv"
|
||||
```
|
||||
|
||||
**解决**:
|
||||
- 方案 A: 设置 `schema_strict=False` (默认), 自动并集合并, 缺失列填 null
|
||||
- 方案 B: 调用 `repair_schema` MCP 工具 — 自动对齐列名 (大小写合并、下划线/连字符规范化)
|
||||
- 方案 C: 在 `config.yaml` 中通过 `columns` 映射手动统一列名
|
||||
|
||||
### 2.2 CSV 编码问题 (乱码)
|
||||
|
||||
**症状**: 中文列名或数据出现乱码、`UnicodeDecodeError`
|
||||
|
||||
**诊断**:
|
||||
```bash
|
||||
:: 检查文件编码
|
||||
runtime\python\python.exe -c "
|
||||
with open('data/your.csv', 'rb') as f:
|
||||
print(f.read(10).hex())
|
||||
"
|
||||
```
|
||||
|
||||
**解决**:
|
||||
- BOM 标记: 系统自动检测 BOM → UTF-8/UTF-16LE/UTF-16BE
|
||||
- 无 BOM: 默认 UTF-8, 可手动指定 `encoding="latin-1"` 或 `encoding="gbk"`
|
||||
- 所有命令加 `chcp 65001` + `set PYTHONUTF8=1`
|
||||
|
||||
### 2.3 `+` 号字符串无法解析为数字
|
||||
|
||||
**症状**: `"+"` 在数值列中导致 `cannot cast to Float64`
|
||||
|
||||
**解决**: 已在 `_coerce_to_float()` 中修复 — `"+"` / `"-"` / 空白 / `"N/A"` 等伪值在聚合前自动转为 null。日志标记 `[CLEAN] numeric coercion applied`。
|
||||
|
||||
### 2.4 列类型检测错误 (IP 列被识别为 String)
|
||||
|
||||
**症状**: 某列应为 IPv4 但被检测为 STRING
|
||||
|
||||
**诊断**:
|
||||
```bash
|
||||
:: 在 MCP 中调用
|
||||
validate_data(dataset_id="ds_...")
|
||||
```
|
||||
|
||||
**解决**:
|
||||
- 在 `config.yaml` 中手动指定: `columns: { src_ip: ipv4, dst_ip: ipv4 }`
|
||||
- 配置优先级: config → 值优先检测 → 名称推断 → STRING 回退
|
||||
|
||||
### 2.5 ZIP 文件解压失败
|
||||
|
||||
**症状**: 上传 ZIP 文件后无数据
|
||||
|
||||
**解决**: 检查 ZIP 内文件是否为 `.csv` 格式。系统仅自动解压 ZIP 中的 CSV 文件。
|
||||
|
||||
---
|
||||
|
||||
## 3. 聚类问题
|
||||
|
||||
### 3.1 聚类返回 0 个簇或全噪声
|
||||
|
||||
**症状**: `n_clusters=0` 或所有标签为 -1 (HDBSCAN 噪声)
|
||||
|
||||
**诊断**:
|
||||
```python
|
||||
diagnose_clustering(dataset_id="...", cluster_result_id="...")
|
||||
```
|
||||
输出包含: 特征方差分析、相关矩阵、稀疏性检查、参数建议。
|
||||
|
||||
**常见原因与解决**:
|
||||
|
||||
| 原因 | 诊断信号 | 解决 |
|
||||
|------|---------|------|
|
||||
| 样本太少 | n < 3 | 检查数据加载是否成功; 增大数据量 |
|
||||
| min_cluster_size 太大 | 日志中 min_cluster_size > n/2 | 系统已自动调整为 n//20, 如仍失败手动降低 |
|
||||
| 全零方差特征 | 方差 < 1e-10 | 系统已自动过滤零方差列; 检查特征选择是否合理 |
|
||||
| 特征高度相关 | Pearson > 0.95 | 系统已自动去相关; 仍有问题则减少特征数 |
|
||||
| 数据全是 NaN | 日志 all-NaN columns dropped | 检查数据源; 增加样本量 |
|
||||
|
||||
### 3.2 "mean on str" 错误
|
||||
|
||||
**症状**: `TypeError: can't convert type 'str' to numerator/denominator`
|
||||
|
||||
**原因**: 跨机 dtype 推断不一致 (一台推断为 Float, 另一台为 String)
|
||||
|
||||
**解决**: 已在 `tools/entities.py::_handle_build_entity_profiles` 的聚合参数构建中修复 — 聚合前检查 dtype, 非数值列自动 cast。`_coerce_to_float()` 兜底。
|
||||
|
||||
### 3.3 KMeans 簇数与预期不符
|
||||
|
||||
**症状**: KMeans 产生的簇数与 `n_clusters` 参数不一致
|
||||
|
||||
**原因**: MiniBatchKMeans 在大数据集上可能产生少于预期的簇
|
||||
|
||||
**解决**:
|
||||
```python
|
||||
# 尝试 HDBSCAN (自动确定 k)
|
||||
run_clustering(dataset_id="...", cluster_columns=[...], algorithm="hdbscan")
|
||||
# 或增大 n_clusters
|
||||
run_clustering(..., algorithm="kmeans", params={"n_clusters": 5})
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. UMAP 问题
|
||||
|
||||
### 4.1 UMAP 嵌入失败 (`[UMAP_ERR]`)
|
||||
|
||||
**症状**: 日志中出现 `[UMAP_ERR]` 但流程继续 (UMAP 计算失败不影响核心分析)
|
||||
|
||||
**常见原因**:
|
||||
- 数据量过大 (>100K): 系统已实现训练 10K + 批量变换 1K 批次
|
||||
- 全 NaN 列: UMAP 前已有 StandardScaler + `np.nan_to_num`
|
||||
- 内存不足: UMAP 需要额外内存
|
||||
|
||||
**解决**:
|
||||
```bash
|
||||
:: 检查 stderr
|
||||
runtime\python\python.exe manage.py run_pipeline data/*.csv 2> umap_err.log
|
||||
```
|
||||
|
||||
### 4.2 UMAP 坐标全为 0
|
||||
|
||||
**症状**: EntityProfile 的 embedding_x/y 全为 0.0
|
||||
|
||||
**原因**: UMAP 计算过程中异常被 `try/except` 捕获, 静默回退
|
||||
|
||||
**解决**:
|
||||
- 检查 stderr 是否有 `[UMAP_ERR]` 日志
|
||||
- 确认 `umap-learn` 包已安装: `runtime\python\python.exe -c "import umap; print(umap.__version__)"`
|
||||
- 减少数据量重试
|
||||
|
||||
### 4.3 TruncatedSVD 降维导致聚类结果变化
|
||||
|
||||
**症状**: 特征 >50 维时聚类结果与预期不同
|
||||
|
||||
**原因**: SVD 降维会损失信息 (保留前 50 个主成分)
|
||||
|
||||
**解决**:
|
||||
- 手动选择更少的基础特征 (<50 个), 避免触发 SVD 降维
|
||||
- 通过 `cluster_columns` 参数精确控制输入特征列
|
||||
|
||||
---
|
||||
|
||||
## 5. 数据库问题
|
||||
|
||||
### 5.1 SQLite 锁 (`[DB_LOCKED]`)
|
||||
|
||||
**症状**: 日志中出现 `[DB_LOCKED]` 警告
|
||||
|
||||
**说明**: 系统已内置 `retry_on_lock` 装饰器 (3 次重试, 指数退避)。SQLite WAL 模式下读写并发性良好。
|
||||
|
||||
**解决**:
|
||||
- 减少并发写入 (多个 Django 进程同时写)
|
||||
- 增大超时: `sqlite3_db.config(busy_timeout=5000)`
|
||||
- 检查是否有未关闭的长时间事务
|
||||
|
||||
### 5.2 ClusterFeature 表无数据 (`db_saved=False`)
|
||||
|
||||
**症状**: `extract_features` 返回 `db_saved=False`
|
||||
|
||||
**诊断**:
|
||||
```bash
|
||||
runtime\python\python.exe scripts\debug_db.py
|
||||
```
|
||||
|
||||
**常见原因**:
|
||||
- Django ORM sync_to_async 包装缺失 (已在 `tools/features.py` 中修复)
|
||||
- AnalysisRun 记录未找到 (csv_glob 匹配失败)
|
||||
- 权限问题 (SQLite 文件只读)
|
||||
|
||||
### 5.3 EntityProfile 表无 embedding 坐标
|
||||
|
||||
**症状**: EntityProfile 记录存在但 embedding_x/y 为 null
|
||||
|
||||
**原因**: UMAP 计算失败 (见 4.1)
|
||||
|
||||
---
|
||||
|
||||
## 6. LLM 集成问题
|
||||
|
||||
### 6.1 400 错误
|
||||
|
||||
**症状**: LLM 返回 HTTP 400
|
||||
|
||||
**原因**: API 地址错误 / Key 无效 / 模型名错误
|
||||
|
||||
**诊断**: 配置页 → 填写 LLM 配置 → 点击「测试连通性」
|
||||
|
||||
**解决**:
|
||||
1. 确认 `base_url` 格式: `https://api.openai.com/v1` (注意末尾的 /v1)
|
||||
2. 确认 `api_key` 无多余空格
|
||||
3. 确认 `model` 名称与 API 支持的一致
|
||||
|
||||
### 6.2 超时
|
||||
|
||||
**症状**: 日志中 `latency_ms` 很高, LLM 步骤无响应
|
||||
|
||||
**解决**:
|
||||
- 增大 `max_tokens` (config.yaml)
|
||||
- 检查网络延迟
|
||||
- 32B 模型推理慢: 系统已设置 90s 超时, 如仍不足可增加
|
||||
|
||||
### 6.3 工具调用失败
|
||||
|
||||
**症状**: `[TOOL]` 日志后出现错误
|
||||
|
||||
**诊断**:
|
||||
```bash
|
||||
:: 查看完整 LLM 交互日志
|
||||
type logs\tianxuan.log | findstr "[TOOL] [LLM]"
|
||||
```
|
||||
|
||||
**解决**:
|
||||
- 检查工具返回的错误信息
|
||||
- LLM 可能生成了格式错误的工具调用 JSON → 重试
|
||||
- 重复调用检测: orchestrator 会自动跳过重复的工具调用
|
||||
|
||||
### 6.4 分析结果为空
|
||||
|
||||
**症状**: LLM 完成但无聚类结果
|
||||
|
||||
**常见原因**:
|
||||
- 实体检测失败 → 手动指定 `entity_columns`
|
||||
- 数据量太小 → 增大样本
|
||||
- 实体列检测到非 IP 列 → 通过 `entity_columns` 强制指定
|
||||
|
||||
---
|
||||
|
||||
## 7. 前端问题
|
||||
|
||||
### 7.1 Canvas 散点图不显示
|
||||
|
||||
**症状**: 聚类概览页空白
|
||||
|
||||
**诊断**:
|
||||
```bash
|
||||
:: 检查浏览器控制台 (F12)
|
||||
:: 常见错误: "Cannot read properties of undefined"
|
||||
```
|
||||
|
||||
**解决**:
|
||||
- 确认聚类已完成 (检查 AnalysisRun.status == 'completed')
|
||||
- 确认有 UMAP/PCA 坐标数据 (EntityProfile.embedding_x 非 null)
|
||||
- 清除浏览器缓存 (Ctrl+F5)
|
||||
|
||||
### 7.2 3D 地球空白
|
||||
|
||||
**症状**: `/globe/` 页面渲染为空白
|
||||
|
||||
**原因**:
|
||||
- WebGL2 不可用 (核显驱动过旧或浏览器不支持)
|
||||
- Three.js 未加载 (检查 Network tab, 603KB 的 three.min.js)
|
||||
- 地球贴图未加载 (1.4MB earth_atmos_2048.jpg)
|
||||
|
||||
**解决**:
|
||||
- 确认浏览器支持 WebGL2: 访问 `chrome://gpu` 检查 WebGL2 状态
|
||||
- 确认 `static/tianxuan/three.min.js` 和 `earth_atmos_2048.jpg` 存在
|
||||
- 数据中优先使用 `src_latitude/src_longitude` 列, 无则 GeoIP 回退
|
||||
- 确保 GeoIP 数据文件 `data/geoip_data.txt` 存在
|
||||
|
||||
### 7.3 地理分布图空白
|
||||
|
||||
**症状**: 聚类概览页的地理分布 Canvas 不显示
|
||||
|
||||
**原因**: 数据中无经纬度信息且 GeoIP 查询失败
|
||||
|
||||
**解决**: 确保 GeoIP 数据库 `data/geoip_data.txt` 存在且格式正确。GeoIP 仅支持离线查询, 无网络依赖。
|
||||
|
||||
---
|
||||
|
||||
## 8. 跨机一致性问题
|
||||
|
||||
### 8.1 核心诊断流程
|
||||
|
||||
```bat
|
||||
:: 两台机器各自运行管道, 保存日志
|
||||
del logs\tianxuan.log
|
||||
runtime\python\python.exe manage.py run_pipeline data\complex_test.csv
|
||||
copy logs\tianxuan.log log_%COMPUTERNAME%.txt
|
||||
|
||||
:: 对比
|
||||
runtime\python\python.exe scripts\diagnose_compare.py log_MACHINE_A.txt log_MACHINE_B.txt
|
||||
```
|
||||
|
||||
首个 `[DIFF]` 行就是问题根源。
|
||||
|
||||
### 8.2 常见跨机差异
|
||||
|
||||
| 差异类型 | 日志标记 | 根因 | 检查 |
|
||||
|---------|---------|------|------|
|
||||
| 列类型不同 | `[LOAD_SCHEMA]` | Polars 版本不一致 或 PYTHONUTF8 未生效 | `polars.__version__`, `sys.flags.utf8_mode` |
|
||||
| 关键词匹配不同 | `[FIND_COL]` | frozenset 哈希随机性 | 已通过 tuple 替代 frozenset 修复 |
|
||||
| 聚合结果不同 | `[CLUSTER_INPUT]` | random_state 不一致 或 StandardScaler 差异 | 确认 random_state=42, sklearn 版本一致 |
|
||||
| 编码问题 | `UnicodeDecodeError` | 系统编码不一致 | `sys.getdefaultencoding()`, `sys.getfilesystemencoding()` |
|
||||
| dtype 推断不同 | `Utf8 vs Float64` | Polars 推断漂移 | 已通过类型安全聚合 + `_coerce_to_float` 防护 |
|
||||
|
||||
### 8.3 运行时自检脚本
|
||||
|
||||
```bash
|
||||
:: 在两台机器上分别运行, 对比输出
|
||||
runtime\python\python.exe -c "
|
||||
import sys, polars, numpy, sklearn
|
||||
print(f'Python: {sys.version}')
|
||||
print(f'Polars: {polars.__version__}')
|
||||
print(f'NumPy: {numpy.__version__}')
|
||||
print(f'sklearn: {sklearn.__version__}')
|
||||
print(f'defaultencoding: {sys.getdefaultencoding()}')
|
||||
print(f'fsencoding: {sys.getfilesystemencoding()}')
|
||||
print(f'utf8_mode: {sys.flags.utf8_mode}')
|
||||
print(f'hashseed: {sys.flags.hash_randomization}')
|
||||
"
|
||||
```
|
||||
|
||||
### 8.4 已知已修复的跨机问题
|
||||
|
||||
| 问题 | 修复方案 | 代码位置 |
|
||||
|------|---------|---------|
|
||||
| "mean on str" | frozenset → tuple + 类型安全聚合 | tools/entities.py, ip_clustering.py |
|
||||
| 编码漂移 | PYTHONUTF8=1 | run.bat |
|
||||
| DB 特征不保存 | sync_to_async 包装 | tools/features.py |
|
||||
| 图表依赖缺失 | 纯 Canvas 2D | 全部图表 |
|
||||
| 聚类结果随机 | random_state=42 | tools/clustering.py, run_pipeline.py |
|
||||
|
||||
---
|
||||
|
||||
## 9. 性能问题
|
||||
|
||||
### 9.1 大数据集 OOM
|
||||
|
||||
**症状**: 内存不足导致进程崩溃
|
||||
|
||||
**系统已实现的防护**:
|
||||
- 可用内存 < 2GB → 自动降采样 50K (psutil 检测)
|
||||
- 样本 > 100K → 采样 100K
|
||||
- 样本 > 50K → 采样 50K
|
||||
- MiniBatchKMeans 替代 KMeans (batch_size=1024)
|
||||
- SVD 降维 (>50 特征 → 50)
|
||||
- UMAP 训练 10K + 批量变换 1K 批次
|
||||
- Polars LazyFrame + streaming 惰性执行
|
||||
|
||||
**手动优化**:
|
||||
```python
|
||||
# 在 load_data 或 run_clustering 时降采样
|
||||
load_data(csv_glob="data/*.csv")
|
||||
# → 后续 run_clustering 会自动降采样
|
||||
# 或使用 filter_and_cluster 先过滤再聚类
|
||||
```
|
||||
|
||||
### 9.2 SQLite 写入慢
|
||||
|
||||
**症状**: DB 持久化耗时较长
|
||||
|
||||
**解决**:
|
||||
- WAL 模式已开启 (synchronous=NORMAL)
|
||||
- batch_size=1000 批量写入 (bulk_create)
|
||||
- 增大 cache_size: SQLite PRAGMA cache_size=20000 (20MB)
|
||||
- 临时目录移到 SSD
|
||||
|
||||
### 9.3 UMAP 计算慢
|
||||
|
||||
**症状**: extract_features 步骤耗时过长
|
||||
|
||||
**说明**: UMAP 计算分为训练 (≤10K 样本) + 批量变换 (1K 批次), 已是最优方案。
|
||||
|
||||
**如需加速**: 减小 `MAX_UMAP_TRAIN` (`tools/features.py` 中的硬编码常量, 默认 10000)。
|
||||
|
||||
---
|
||||
|
||||
## 10. 诊断工具使用
|
||||
|
||||
### 10.1 MCP 诊断工具链
|
||||
|
||||
```
|
||||
问题 → validate_data (数据质量) → explore_distributions (分布检查) → find_outliers (异常值)
|
||||
→ diagnose_clustering (聚类诊断) → compare_datasets (处理前后对比) → export_debug_sample (导出样本)
|
||||
```
|
||||
|
||||
所有诊断工具位于 `tools/diagnostics.py`。
|
||||
|
||||
### 10.2 列结构调查
|
||||
|
||||
```bash
|
||||
runtime\python\python.exe scripts\column_survey.py --csv "data/*.csv"
|
||||
# 输出: 每个 CSV 的列名/类型/空值率/分布
|
||||
```
|
||||
|
||||
### 10.3 DB 持久化调试
|
||||
|
||||
```bash
|
||||
runtime\python\python.exe scripts\debug_db.py
|
||||
# 期望: n_features=30 db_saved=True
|
||||
```
|
||||
|
||||
### 10.4 跨机日志对比
|
||||
|
||||
```bash
|
||||
runtime\python\python.exe scripts\diagnose_compare.py log_machine_a.txt log_machine_b.txt
|
||||
# 输出: 首个差异行的内容 (两侧对比)
|
||||
```
|
||||
|
||||
### 10.5 关键日志标记汇总
|
||||
|
||||
| 标记 | 文件 | 含义 |
|
||||
|------|------|------|
|
||||
| `[LOAD]` | data_loader.py | CSV 加载完成 |
|
||||
| `[LOAD_SCHEMA]` | data_loader.py | 每列推断类型 (最重要!) |
|
||||
| `[CLEAN]` | data_loader.py | 清洗操作 |
|
||||
| `[VALIDATE]` | data_validator.py | 数据校验 |
|
||||
| `[FIND_COL]` | tools/entities.py | 实体列关键词匹配 |
|
||||
| `[CLUSTER_INPUT]` | tools/clustering.py | 聚类输入形状 |
|
||||
| `[CLUSTER_SCALE]` | tools/clustering.py | StandardScaler |
|
||||
| `[SCORE]` | tools/entities.py | 自适应评分权重 |
|
||||
| `[UMAP_ERR]` | tools/features.py | UMAP 异常 |
|
||||
| `[DB_SAVE_ERROR]` | tools/features.py | DB 持久化失败 |
|
||||
| `[DB_LOCKED]` | db_utils.py | SQLite 锁重试 |
|
||||
| `[LLM]` | llm_orchestrator.py | LLM 调用 |
|
||||
| `[TOOL]` | llm_orchestrator.py | LLM 调用工具 |
|
||||
|
||||
---
|
||||
|
||||
> **相关文档**: `AGENTS.md` (架构/配置/操作流程), `docs/工作流.md` (完整分析流水线)
|
||||
@@ -0,0 +1,81 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-hans">
|
||||
<head><meta charset="UTF-8"><title>簇详情 — 预览</title>
|
||||
<style>
|
||||
*{margin:0;padding:0;box-sizing:border-box}
|
||||
body{font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;background:#f5f7fa;color:#1a1a2e;padding:1rem 2rem}
|
||||
.card{background:#fff;border-radius:8px;padding:1.5rem;margin-bottom:1rem;box-shadow:0 1px 3px rgba(0,0,0,.1)}
|
||||
a{color:#4361ee;text-decoration:none;font-size:.9rem}
|
||||
h2{font-size:1.2rem}.badge{display:inline-block;padding:.15rem .5rem;border-radius:12px;font-size:.75rem;font-weight:600}
|
||||
.badge-warning{background:#fff3cd;color:#856404}
|
||||
.feat-table{width:100%;border-collapse:collapse;font-size:.8rem;margin-top:.5rem}
|
||||
.feat-table th{background:#f5f5f5;padding:.3rem .5rem;text-align:left;font-weight:600;border-bottom:2px solid #ddd;white-space:nowrap}
|
||||
.feat-table td{padding:.25rem .5rem;border-bottom:1px solid #eee}
|
||||
.feat-table td code{font-size:.72rem}
|
||||
.feat-p{color:#2e7d32}.feat-n{color:#c62828}
|
||||
.grid-2{display:grid;grid-template-columns:1fr 1fr;gap:1rem;margin-top:.5rem}
|
||||
.svd-card{background:#f8f9fa;border:1px solid #e0e0e0;border-radius:8px;padding:.8rem}
|
||||
.svd-card h4{margin:0 0 .3rem;font-size:.9rem}
|
||||
.data-table{width:100%;border-collapse:collapse;font-size:.72rem;margin-top:.5rem}
|
||||
.data-table th{background:#1a1a2e;color:#e0e0e0;padding:.25rem .4rem;text-align:left;font-weight:600;white-space:nowrap;font-size:.7rem}
|
||||
.data-table td{padding:.2rem .4rem;border-bottom:1px solid #eee;font-size:.7rem}
|
||||
.data-table td code{font-size:.68rem}
|
||||
.data-table tr:nth-child(even){background:#f8f9fa}
|
||||
.scroll-x{overflow-x:auto}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="card">
|
||||
<a href="#">← 聚类概览</a>
|
||||
<div style="display:flex;justify-content:space-between;align-items:center;margin-top:.5rem">
|
||||
<div><h2>簇 #0 <span class="badge badge-warning" style="display:none">噪声</span></h2>
|
||||
<p style="color:#666;font-size:.9rem">847,293 条流 — 66.0% 占总 — 轮廓系数: 0.7100</p></div>
|
||||
</div>
|
||||
<div style="background:#f8f9fa;border-left:3px solid #4361ee;padding:.6rem;margin:.75rem 0;border-radius:0 4px 4px 0;font-size:.85rem;line-height:1.6;color:#333">
|
||||
低延迟 (23ms)，比全局均值高3.1个标准差；大数据包 (1520 bytes)，高2.3个标准差；系统端口 (443)，高1.8个标准差（簇内高度集中）。
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h3>📊 特征分析 <span style="font-weight:400;font-size:.8rem;color:#888">12 个特征</span></h3>
|
||||
<table class="feat-table">
|
||||
<thead><tr><th>#</th><th>特征名</th><th>区分度</th><th>均值</th> <th>标准差</th><th>中位数</th><th>P25</th><th>P75</th><th>离散度</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>1</td><td><code>延迟 (4dur)</code></td><td class="feat-p">+3.213</td><td>23.0ms</td><td>5.1</td><td>21ms</td><td>18ms</td><td>26ms</td><td><span style="color:#e65100;font-size:.72rem;">离散中</span></td></tr>
|
||||
<tr><td>2</td><td><code>数据包 (8ppk)</code></td><td class="feat-p">+2.346</td><td>1520.0B</td><td>128</td><td>1480B</td><td>1400B</td><td>1650B</td><td><span style="color:#e65100;font-size:.72rem;">离散中</span></td></tr>
|
||||
<tr><td>3</td><td><code>TLS版本 (0ver)</code></td><td class="feat-n">-1.845</td><td>1.2</td><td>0.3</td><td>1.2</td><td>1.2</td><td>1.3</td><td><span style="color:#43aa8b;font-size:.72rem;">高度集中</span></td></tr>
|
||||
<tr><td>4</td><td><code>目标端口 (:prd)</code></td><td class="feat-p">+1.821</td><td>443</td><td>12</td><td>443</td><td>443</td><td>443</td><td><span style="color:#1565c0;font-size:.72rem;">极集中</span></td></tr>
|
||||
<tr><td>5</td><td><code>会话时长 (8ses)</code></td><td class="feat-n">-1.523</td><td>15s</td><td>4s</td><td>14s</td><td>12s</td><td>18s</td><td><span style="color:#e65100;font-size:.72rem;">离散中</span></td></tr>
|
||||
<tr><td>6</td><td><code>密钥大小 (4ksz)</code></td><td class="feat-p">+1.234</td><td>256</td><td>0</td><td>256</td><td>256</td><td>256</td><td><span style="color:#1565c0;font-size:.72rem;">零离散</span></td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h3>🔬 SVD 关键特征 <span style="font-weight:400;font-size:.8rem;color:#888">Top 5 按区分度</span></h3>
|
||||
<div class="grid-2">
|
||||
<div class="svd-card"><h4>延迟 (4dur)</h4><div style="font-size:.75rem;color:#666;margin-top:.2rem">Score: <span class="feat-p">+3.21</span> | 均值: 23.00 | 标准差: 5.10</div></div>
|
||||
<div class="svd-card"><h4>数据包 (8ppk)</h4><div style="font-size:.75rem;color:#666;margin-top:.2rem">Score: <span class="feat-p">+2.35</span> | 均值: 1520.00 | 标准差: 128.00</div></div>
|
||||
<div class="svd-card"><h4>TLS版本 (0ver)</h4><div style="font-size:.75rem;color:#666;margin-top:.2rem">Score: <span class="feat-n">-1.85</span> | 均值: 1.20 | 标准差: 0.30</div></div>
|
||||
<div class="svd-card"><h4>目标端口 (:prd)</h4><div style="font-size:.75rem;color:#666;margin-top:.2rem">Score: <span class="feat-p">+1.82</span> | 均值: 443.00 | 标准差: 12.00</div></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h3>📋 数据行 <span style="font-weight:400;font-size:.8rem;color:#888">100 条</span></h3>
|
||||
<div class="scroll-x">
|
||||
<table class="data-table">
|
||||
<thead><tr><th>#</th><th>:ips</th><th>:ipd</th><th>:prs</th><th>:prd</th><th>0ver</th><th>8ack</th><th>4dur</th><th>timestamp</th><th>snam</th><th>cnam</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>1</td><td><code>192.168.1.42</code></td><td><code>10.0.0.5</code></td><td><code>49152</code></td><td><code>443</code></td><td><code>TLSv1.3</code></td><td><code>1420</code></td><td><code>23</code></td><td><code>2024-01-15 08:23</code></td><td><code>api.example.com</code></td><td><code>*.example.com</code></td></tr>
|
||||
<tr><td>2</td><td><code>10.0.0.100</code></td><td><code>203.0.113.50</code></td><td><code>34567</code></td><td><code>443</code></td><td><code>TLSv1.3</code></td><td><code>2048</code></td><td><code>18</code></td><td><code>2024-01-15 08:23</code></td><td><code>cdn.example.com</code></td><td><code>*.example.com</code></td></tr>
|
||||
<tr><td>3</td><td><code>172.16.0.8</code></td><td><code>198.51.100.22</code></td><td><code>50001</code></td><td><code>8443</code></td><td><code>TLSv1.2</code></td><td><code>890</code></td><td><code>35</code></td><td><code>2024-01-15 08:24</code></td><td><code>api.internal.net</code></td><td><code>*.internal.net</code></td></tr>
|
||||
<tr><td>4</td><td><code>192.168.1.100</code></td><td><code>10.0.0.1</code></td><td><code>44321</code></td><td><code>443</code></td><td><code>TLSv1.3</code></td><td><code>5600</code></td><td><code>12</code></td><td><code>2024-01-15 08:24</code></td><td><code>www.example.com</code></td><td><code>*.example.com</code></td></tr>
|
||||
<tr><td>5</td><td><code>10.0.1.50</code></td><td><code>203.0.113.100</code></td><td><code>51000</code></td><td><code>8080</code></td><td><code>TLSv1.2</code></td><td><code>320</code></td><td><code>45</code></td><td><code>2024-01-15 08:25</code></td><td><code>api.alt.net</code></td><td><code>*.alt.net</code></td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<p style="text-align:center;color:#999;font-size:.78rem;margin-top:.5rem">显示前 100 行，共 847,293 行</p>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,376 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-hans">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1.0">
|
||||
<title>聚类概览 — 预览</title>
|
||||
<style>
|
||||
*{margin:0;padding:0;box-sizing:border-box}
|
||||
body{font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;background:#f5f7fa;color:#1a1a2e}
|
||||
.header{background:#1a1a2e;color:#fff;padding:0.8rem 1.5rem;display:flex;justify-content:space-between;align-items:center}
|
||||
.header h1{font-size:1.1rem}.header span{font-size:0.8rem;color:#aaa}
|
||||
.main-layout{display:flex;position:relative;min-height:calc(100vh - 48px)}
|
||||
.left{flex:1;min-width:0;padding:1rem;overflow-y:auto;transition:flex .3s}
|
||||
.left.globe-open{flex:0 0 55%}
|
||||
.right{flex:0 0 0;overflow:hidden;transition:flex .3s}
|
||||
.right.globe-open{flex:0 0 45%}
|
||||
.toggle-handle{
|
||||
position:absolute;z-index:10;top:50%;width:20px;height:44px;background:#1a1a2e;color:#fff;
|
||||
border:none;border-radius:4px 0 0 4px;cursor:pointer;display:flex;align-items:center;
|
||||
justify-content:center;font-size:12px;transition:left .3s;left:calc(100% - 20px)
|
||||
}
|
||||
.toggle-handle:hover{background:#4361ee}
|
||||
.card{background:#fff;border-radius:8px;padding:1rem;margin-bottom:1rem;box-shadow:0 1px 3px rgba(0,0,0,.1)}
|
||||
.card h3{font-size:.95rem;margin-bottom:.5rem}
|
||||
.scatter-wrap{padding:.75rem}
|
||||
.scatter-wrap canvas{width:100%;height:380px;display:block;border-radius:4px;background:#fafbfc}
|
||||
.pill-row{display:flex;flex-wrap:wrap;gap:4px;margin-bottom:.75rem}
|
||||
.pill{display:inline-flex;align-items:center;gap:3px;padding:2px 10px;border-radius:12px;font-size:.75rem;cursor:pointer;border:2px solid transparent;color:#fff;white-space:nowrap}
|
||||
.pill.active{border-color:#333;font-weight:700}
|
||||
.card-grid{display:grid;grid-template-columns:1fr 1fr;gap:.75rem}
|
||||
.cluster-card{background:#fff;border:1px solid #e0e0e0;border-radius:8px;padding:.8rem;cursor:pointer;transition:box-shadow .15s}
|
||||
.cluster-card:hover{box-shadow:0 2px 8px rgba(0,0,0,.1)}
|
||||
.cluster-card.selected{border-color:#4361ee;box-shadow:0 0 0 2px rgba(67,97,238,.2)}
|
||||
.cluster-card .lbl{font-weight:700;font-size:1rem}
|
||||
.cluster-card .meta{font-size:.78rem;color:#666;margin-top:.2rem}
|
||||
.cluster-card .desc{font-size:.8rem;color:#333;margin-top:.4rem;padding:.4rem;background:#f8f9fa;border-left:3px solid #4361ee;border-radius:0 4px 4px 0;line-height:1.5}
|
||||
.detail-panel{position:fixed;bottom:0;left:0;right:0;z-index:100;transform:translateY(100%);transition:transform .35s cubic-bezier(.4,0,.2,1)}
|
||||
.detail-panel.open{transform:translateY(0)}
|
||||
.detail-body{background:#fff;border-radius:16px 16px 0 0;box-shadow:0 -4px 24px rgba(0,0,0,.15);max-height:40vh;overflow-y:auto;padding:.5rem 1.5rem 1.5rem}
|
||||
.detail-handle{width:36px;height:4px;background:#ccc;border-radius:2px;margin:8px auto;cursor:pointer}
|
||||
.detail-close{float:right;cursor:pointer;font-size:1.3rem;color:#999;border:none;background:none}
|
||||
.detail-close:hover{color:#333}
|
||||
.detail-body h3{font-size:1rem;margin-bottom:.3rem}
|
||||
.detail-body p{font-size:.85rem;color:#555;margin:.2rem 0}
|
||||
.detail-body table{width:100%;border-collapse:collapse;font-size:.8rem;margin-top:.5rem}
|
||||
.detail-body th{background:#f5f5f5;padding:.25rem .4rem;text-align:left;font-weight:600;border-bottom:2px solid #ddd;white-space:nowrap}
|
||||
.detail-body td{padding:.2rem .4rem;border-bottom:1px solid #eee}
|
||||
.feat-p{color:#2e7d32}.feat-n{color:#c62828}
|
||||
#globeCont{width:100%;height:calc(100vh - 48px);background:#0a0a1a;position:relative}
|
||||
.btn-sm{display:inline-block;padding:.25rem .6rem;border-radius:4px;font-size:.78rem;text-decoration:none;background:#4361ee;color:#fff}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="header">
|
||||
<h1>聚类概览 — 运行 #9</h1>
|
||||
<span>1,284,592 条流 · 3 个簇 · 12,847 个噪声</span>
|
||||
</div>
|
||||
|
||||
<div class="main-layout" id="main">
|
||||
<div class="left" id="leftPanel">
|
||||
<!-- UMAP 散点图 -->
|
||||
<div class="card scatter-wrap">
|
||||
<div style="display:flex;justify-content:space-between;align-items:center">
|
||||
<h3>UMAP 嵌入 <span style="font-weight:400;font-size:.8rem;color:#888" id="ptCount">加载中...</span></h3>
|
||||
<span id="viewToggle" style="display:none">
|
||||
<button id="btn2D" class="btn-sm" style="font-weight:700">2D</button>
|
||||
<button id="btn3D" class="btn-sm btn-outline">3D</button>
|
||||
</span>
|
||||
</div>
|
||||
<canvas id="scatterChart"></canvas>
|
||||
<div class="pill-row" id="legendPills"></div>
|
||||
</div>
|
||||
|
||||
<!-- 簇标签 -->
|
||||
<div class="pill-row" id="clusterPills"></div>
|
||||
|
||||
<!-- 簇卡片 -->
|
||||
<div class="card-grid" id="cardGrid"></div>
|
||||
</div>
|
||||
|
||||
<div class="right" id="rightPanel">
|
||||
<div id="globeCont"></div>
|
||||
</div>
|
||||
<button class="toggle-handle" id="globeHandle" onclick="toggleGlobe()">◀</button>
|
||||
</div>
|
||||
|
||||
<!-- 详情面板 -->
|
||||
<div class="detail-panel" id="detailPanel">
|
||||
<div class="detail-body">
|
||||
<button class="detail-close" onclick="closeDetail()">✕</button>
|
||||
<div class="detail-handle" onclick="closeDetail()"></div>
|
||||
<div id="detailContent"><p style="color:#999;padding:.5rem 0">选择一个簇或数据点查看详情</p></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script src="static/tianxuan/three.min.js"></script>
|
||||
<script>
|
||||
// ═══ 模拟数据 ═══
|
||||
const COLORS = ['#4361ee','#f72585','#7209b7','#4cc9f0','#f8961e','#43aa8b','#f94144','#577590','#e9c46a','#9b5de5'];
|
||||
const CLUSTERS = [
|
||||
{label:0,size:847293,prop:0.66,silh:0.71,
|
||||
desc:'低延迟 (23ms)，比全局均值高3.1个标准差；大数据包 (1520 bytes)，高2.3个标准差；系统端口 (443)',
|
||||
detail:'簇 #0 由 847,293 条流组成（66%）。\n特征：延迟(4dur)均值23ms，区分度+3.2；数据包大小(8ppk)均值1520B，区分度+2.3；端口(:prd)高度集中于443(92%)；TLS版本以1.3为主(78%)。'},
|
||||
|
||||
{label:1,size:311482,prop:0.24,silh:0.65,
|
||||
desc:'高TLS1.3比例 (92%)，比均值高1.8σ；短会话 (12s)，低1.5σ；ECDHE曲线 X25519',
|
||||
detail:'簇 #1 由 311,482 条流组成（24%）。\n特征：TLS1.3比率(0ver)92%，区分度+1.8；会话时长(8ses)均值12s，区分度-1.5；密钥交换曲线集中在X25519(88%)；目标端口包括443和8443。'},
|
||||
|
||||
{label:2,size:112970,prop:0.09,silh:0.52,
|
||||
desc:'高连接频率 (85次/min)，比均值高4.2σ；非标准端口 (8080/8443)；目标多域名',
|
||||
detail:'簇 #2 由 112,970 条流组成（9%）。\n特征：连接频率(8seq/min)均值85，区分度+4.2；非标准端口比例高(76%)，以8080/8443为主；SNI名称(snam)分散在8个不同域名；源IP来自不同/24网段。'},
|
||||
|
||||
{label:-1,size:12847,prop:0.01,silh:null,
|
||||
desc:'噪声 — 异常TLS指纹、零散目标、单次连接',
|
||||
detail:''},
|
||||
];
|
||||
|
||||
const RNG = (s)=>{let seed=s%2147483647;return()=>{seed=seed*16807%2147483647;return(seed-1)/2147483646;}};
|
||||
const r=RNG(42);
|
||||
const centers={0:[2.2,1.8],1:[-2.0,0.3],2:[0.3,-2.2],'-1':[3.8,-1.2]};
|
||||
const spreads={0:[0.7,0.6],1:[0.8,0.7],2:[0.6,0.8],'-1':[1.5,1.0]};
|
||||
const SCATTER=[];
|
||||
let totalPts=0;
|
||||
for(const c of CLUSTERS){
|
||||
const n=c.label===-1?200:Math.min(c.size/2000,200);
|
||||
totalPts+=n;
|
||||
for(let i=0;i<n;i++){
|
||||
const cx=centers[c.label][0],cy=centers[c.label][1];
|
||||
const sx=spreads[c.label][0],sy=spreads[c.label][1];
|
||||
SCATTER.push({
|
||||
x:cx+(r()-0.5)*2*sx, y:cy+(r()-0.5)*2*sy,
|
||||
z:(r()-0.5)*2*0.8, label:c.label,
|
||||
entity:c.label===-1?'噪声':`主机_${1000+i}`
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let selCluster=null,globeOpen=false,globeInit=false;
|
||||
|
||||
// ═══ 2D Canvas 散点 ═══
|
||||
function drawScatter(highlight){
|
||||
const c=document.getElementById('scatterChart');
|
||||
if(!c)return;
|
||||
const W=c.parentElement.clientWidth||700,H=380;
|
||||
c.width=W;c.height=H;
|
||||
const ctx=c.getContext('2d');
|
||||
const pad={top:25,bottom:30,left:50,right:20};
|
||||
const pw=W-pad.left-pad.right,ph=H-pad.top-pad.bottom;
|
||||
const xs=SCATTER.map(d=>d.x),ys=SCATTER.map(d=>d.y);
|
||||
const xMin=Math.min(...xs),xMax=Math.max(...xs),yMin=Math.min(...ys),yMax=Math.max(...ys);
|
||||
const xR=xMax-xMin||1,yR=yMax-yMin||1;
|
||||
const toX=v=>pad.left+(v-xMin)/xR*pw,toY=v=>pad.top+ph-(v-yMin)/yR*ph;
|
||||
ctx.clearRect(0,0,W,H);
|
||||
ctx.strokeStyle='#eee';ctx.lineWidth=1;
|
||||
for(let i=0;i<=5;i++){ctx.beginPath();ctx.moveTo(pad.left,pad.top+i*ph/5);ctx.lineTo(W-pad.right,pad.top+i*ph/5);ctx.stroke();}
|
||||
SCATTER.forEach(d=>{
|
||||
const cx=toX(d.x),cy=toY(d.y);
|
||||
let color=d.label===-1?'rgba(150,150,150,0.4)':COLORS[Math.abs(d.label)%COLORS.length];
|
||||
let r2=d.label===-1?2:4;
|
||||
if(highlight!==null&&highlight!==undefined){
|
||||
if(d.label===highlight){color=d.label===-1?'rgba(150,150,150,0.9)':COLORS[Math.abs(d.label)%COLORS.length];r2=6;}
|
||||
else{color='rgba(200,200,200,0.15)';r2=2;}
|
||||
}
|
||||
ctx.beginPath();ctx.arc(cx,cy,r2,0,Math.PI*2);ctx.fillStyle=color;ctx.fill();
|
||||
});
|
||||
ctx.fillStyle='#999';ctx.font='11px sans-serif';ctx.textAlign='center';
|
||||
ctx.fillText('UMAP-1',W/2,H-3);
|
||||
ctx.save();ctx.translate(14,H/2);ctx.rotate(-Math.PI/2);ctx.fillText('UMAP-2',0,0);ctx.restore();
|
||||
document.getElementById('ptCount').textContent=SCATTER.length+' 个点';
|
||||
// legend pills (from scatter data)
|
||||
const leg=document.getElementById('legendPills');leg.innerHTML='';
|
||||
const seen={};
|
||||
SCATTER.forEach(d=>{if(!seen[d.label]){seen[d.label]=true;
|
||||
const p=document.createElement('span');p.className='pill';
|
||||
p.style.background=d.label===-1?'#999':COLORS[Math.abs(d.label)%COLORS.length];
|
||||
p.textContent=d.label===-1?'噪声':'#'+d.label;
|
||||
p.onclick=()=>selectCluster(d.label);leg.appendChild(p);
|
||||
}});
|
||||
}
|
||||
|
||||
// ═══ 簇选择 ═══
|
||||
function selectCluster(lbl){
|
||||
selCluster=CLUSTERS.find(c=>c.label===lbl);
|
||||
drawScatter(lbl);
|
||||
document.querySelectorAll('#clusterPills .pill').forEach(p=>p.classList.toggle('active',parseInt(p.dataset.lbl)===lbl));
|
||||
document.querySelectorAll('.cluster-card').forEach(p=>p.classList.toggle('selected',parseInt(p.dataset.label)===lbl));
|
||||
if(globeOpen)globeFilter(lbl);
|
||||
showDetail(selCluster);
|
||||
}
|
||||
|
||||
function clearSel(){selCluster=null;drawScatter(null);closeDetail();
|
||||
document.querySelectorAll('.pill,.cluster-card').forEach(p=>p.classList.remove('active','selected'));
|
||||
if(globeOpen)globeFilter(null);
|
||||
}
|
||||
|
||||
// ═══ 详情面板 ═══
|
||||
function showDetail(c){
|
||||
const dc=document.getElementById('detailContent');
|
||||
if(!c||c.label===-1){dc.innerHTML='<p style="color:#999;">选择一个簇查看详情</p>';document.getElementById('detailPanel').classList.remove('open');return;}
|
||||
dc.innerHTML='<h3>簇 #'+c.label+'</h3><p>大小: '+c.size.toLocaleString()+' | 占比: '+(c.prop*100).toFixed(1)+'%'+(c.silh!==null?' | 轮廓系数: '+c.silh.toFixed(4):'')+'</p>'+
|
||||
'<p style="background:#f8f9fa;border-left:3px solid #4361ee;padding:.5rem;margin:.5rem 0;border-radius:0 4px 4px 0;font-size:.85rem;line-height:1.6;white-space:pre-wrap;">'+c.detail+'</p>'+
|
||||
'<table><thead><tr><th>特征</th><th>区分度</th><th>均值</th><th>标准差</th></tr></thead><tbody>'+
|
||||
(c.label===0?'<tr><td>延迟 (4dur)</td><td class="feat-p">+3.2</td><td>23.0ms</td><td>5.1</td></tr><tr><td>数据包 (8ppk)</td><td class="feat-p">+2.3</td><td>1520.0B</td><td>128</td></tr><tr><td>TLS版本 (0ver)</td><td class="feat-n">-1.8</td><td>1.2</td><td>0.3</td></tr>':
|
||||
c.label===1?'<tr><td>TLS1.3比率</td><td class="feat-p">+1.8</td><td>92%</td><td>8%</td></tr><tr><td>会话时长</td><td class="feat-n">-1.5</td><td>12s</td><td>4s</td></tr><tr><td>端口</td><td class="feat-p">+1.2</td><td>443</td><td>12</td></tr>':
|
||||
'<tr><td>连接频率</td><td class="feat-p">+4.2</td><td>85/min</td><td>12</td></tr><tr><td>非标准端口</td><td class="feat-p">+2.8</td><td>76%</td><td>9%</td></tr><tr><td>SNI域名数</td><td class="feat-p">+2.1</td><td>8</td><td>2</td></tr>')+
|
||||
'</tbody></table>';
|
||||
document.getElementById('detailPanel').classList.add('open');
|
||||
}
|
||||
function closeDetail(){document.getElementById('detailPanel').classList.remove('open');}
|
||||
|
||||
// ═══ 3D UMAP scatter ═══
|
||||
let s3DRenderer,s3DScene,s3DCamera,s3DPts,s3DInit=false,is3DMode=false;
|
||||
function hasWG2(){try{var c=document.createElement('canvas');return!!c.getContext('webgl2');}catch(e){return false;}}
|
||||
|
||||
(function(){
|
||||
if(!hasWG2())return;
|
||||
document.getElementById('viewToggle').style.display='inline';
|
||||
const b2=document.getElementById('btn2D'),b3=document.getElementById('btn3D');
|
||||
function show2D(){is3DMode=false;document.querySelector('#scatterChart').style.display='';
|
||||
document.getElementById('viewToggle').style.display='none';drawScatter(selCluster?selCluster.label:null);
|
||||
setTimeout(()=>document.getElementById('viewToggle').style.display='inline',50);
|
||||
b2.style.fontWeight='700';b3.style.fontWeight='normal';}
|
||||
function show3D(){is3DMode=true;document.querySelector('#scatterChart').style.display='none';
|
||||
if(!s3DInit)init3D();b3.style.fontWeight='700';b2.style.fontWeight='normal';}
|
||||
b2.onclick=show2D;b3.onclick=show3D;
|
||||
})();
|
||||
|
||||
function init3D(){
|
||||
if(s3DInit)return;
|
||||
const cont=document.getElementById('scatterChart').parentElement;
|
||||
const W=cont.clientWidth||700,H=450;
|
||||
const container=document.createElement('div');container.id='scatter3D';container.style.cssText='width:100%;height:450px';
|
||||
cont.appendChild(container);
|
||||
|
||||
const xs=SCATTER.map(d=>d.x),ys=SCATTER.map(d=>d.y),zs=SCATTER.map(d=>d.z||0);
|
||||
const xM=Math.min(...xs),xX=Math.max(...xs),yM=Math.min(...ys),yX=Math.max(...ys),zM=Math.min(...zs),zX=Math.max(...zs);
|
||||
const cx=(xM+xX)/2,cy=(yM+yX)/2,cz=(zM+zX)/2,rg=Math.max(xX-xM,yX-yM,zX-zM)||1;
|
||||
const nrm=(v,c,r)=>((v-c)/r)*3;
|
||||
const scene=new THREE.Scene();scene.background=new THREE.Color(0xf8f9fa);
|
||||
const camera=new THREE.PerspectiveCamera(50,W/H,0.1,50);camera.position.set(4.5,3.5,6);camera.lookAt(0,0,0);
|
||||
const renderer=new THREE.WebGLRenderer({antialias:true});renderer.setSize(W,H);renderer.setPixelRatio(Math.min(window.devicePixelRatio,2));
|
||||
container.appendChild(renderer.domElement);
|
||||
|
||||
const pos=new Float32Array(SCATTER.length*3);
|
||||
const cols=new Float32Array(SCATTER.length*3);
|
||||
SCATTER.forEach((d,i)=>{
|
||||
pos[i*3]=nrm(d.x,cx,rg);pos[i*3+1]=nrm(d.y,cy,rg);pos[i*3+2]=d.z!=null?nrm(d.z,cz,rg):0;
|
||||
const c3=new THREE.Color(d.label===-1?'#999999':COLORS[Math.abs(d.label)%COLORS.length]);
|
||||
cols[i*3]=c3.r;cols[i*3+1]=c3.g;cols[i*3+2]=c3.b;
|
||||
});
|
||||
const geo=new THREE.BufferGeometry();
|
||||
geo.setAttribute('position',new THREE.BufferAttribute(pos,3));
|
||||
geo.setAttribute('color',new THREE.BufferAttribute(cols,3));
|
||||
const mat=new THREE.PointsMaterial({size:0.12,vertexColors:true,sizeAttenuation:true,transparent:true,opacity:0.85});
|
||||
const pts=new THREE.Points(geo,mat);scene.add(pts);
|
||||
|
||||
let isD=false,pm={x:0,y:0};
|
||||
renderer.domElement.onmousedown=e=>{isD=true;pm={x:e.clientX,y:e.clientY};};
|
||||
window.onmouseup=()=>isD=false;
|
||||
window.onmousemove=e=>{if(!isD||!is3DMode)return;const dx=e.clientX-pm.x,dy=e.clientY-pm.y;
|
||||
pts.rotation.y+=dx*0.005;pts.rotation.x+=dy*0.005;pts.rotation.x=Math.max(-Math.PI/2,Math.min(Math.PI/2,pts.rotation.x));
|
||||
pm={x:e.clientX,y:e.clientY};};
|
||||
renderer.domElement.onwheel=e=>{if(!is3DMode)return;e.preventDefault();camera.position.z+=e.deltaY*0.008;camera.position.z=Math.max(1.5,Math.min(15,camera.position.z));};
|
||||
!function anim(){requestAnimationFrame(anim);if(!is3DMode)return;renderer.render(scene,camera);}();
|
||||
s3DRenderer=renderer;s3DScene=scene;s3DCamera=camera;s3DPts=pts;s3DInit=true;
|
||||
}
|
||||
|
||||
// ═══ 地球侧栏 ═══
|
||||
function toggleGlobe(){
|
||||
globeOpen=!globeOpen;
|
||||
document.getElementById('leftPanel').classList.toggle('globe-open',globeOpen);
|
||||
document.getElementById('rightPanel').classList.toggle('globe-open',globeOpen);
|
||||
const h=document.getElementById('globeHandle');
|
||||
h.innerHTML=globeOpen?'▶':'◀';
|
||||
h.style.left=globeOpen?'calc(55% - 20px)':'calc(100% - 20px)';
|
||||
if(globeOpen&&!globeInit)initGlobe();
|
||||
}
|
||||
function globeFilter(label){
|
||||
if(!window._arcs)return;
|
||||
window._arcs.forEach(m=>{
|
||||
if(label===null){m.material.opacity=0.5;m.material.color.setHex(parseInt(m.userData.c,16));}
|
||||
else{m.material.opacity=0.04;m.material.color.setHex(0x444444);}
|
||||
});
|
||||
}
|
||||
|
||||
function initGlobe(){
|
||||
if(globeInit)return;
|
||||
const cont=document.getElementById('globeCont');
|
||||
if(!cont)return;
|
||||
const W=cont.clientWidth||500,H=cont.clientHeight||500;
|
||||
const scene=new THREE.Scene();scene.background=new THREE.Color(0x0a0a1a);
|
||||
const cam=new THREE.PerspectiveCamera(45,W/H,0.1,100);cam.position.set(0,0,6);
|
||||
const ren=new THREE.WebGLRenderer({antialias:true});ren.setSize(W,H);ren.setPixelRatio(Math.min(window.devicePixelRatio,2));
|
||||
cont.appendChild(ren.domElement);
|
||||
|
||||
// Earth with texture
|
||||
const tex=new THREE.TextureLoader().load('static/tianxuan/earth_atmos_2048.jpg');
|
||||
const earth=new THREE.Mesh(new THREE.SphereGeometry(2,64,64),new THREE.MeshPhongMaterial({map:tex,specular:new THREE.Color(0x333333),shininess:5}));
|
||||
scene.add(earth);
|
||||
// Glow
|
||||
scene.add(new THREE.Mesh(new THREE.SphereGeometry(2.06,48,48),new THREE.MeshBasicMaterial({color:0x224488,transparent:true,opacity:0.12})));
|
||||
// Grid
|
||||
const gridMat=new THREE.LineBasicMaterial({color:0x6699cc,transparent:true,opacity:0.08});
|
||||
for(let lat=-80;lat<=80;lat+=20){const pts=[];for(let lon=0;lon<=360;lon+=5){const p=(90-lat)*Math.PI/180,t=(lon+180)*Math.PI/180;pts.push(new THREE.Vector3(-2.02*Math.sin(p)*Math.cos(t),2.02*Math.cos(p),2.02*Math.sin(p)*Math.sin(t)));}scene.add(new THREE.Line(new THREE.BufferGeometry().setFromPoints(pts),gridMat));}
|
||||
for(let lon=0;lon<360;lon+=20){const pts=[];for(let lat=-90;lat<=90;lat+=5){const p=(90-lat)*Math.PI/180,t=(lon+180)*Math.PI/180;pts.push(new THREE.Vector3(-2.02*Math.sin(p)*Math.cos(t),2.02*Math.cos(p),2.02*Math.sin(p)*Math.sin(t)));}scene.add(new THREE.Line(new THREE.BufferGeometry().setFromPoints(pts),gridMat));}
|
||||
|
||||
// Lights
|
||||
scene.add(new THREE.AmbientLight(0x222244));
|
||||
const dl=new THREE.DirectionalLight(0xffffff,1);dl.position.set(5,10,7);scene.add(dl);
|
||||
scene.add(new THREE.DirectionalLight(0x4488ff,0.3));
|
||||
// Stars
|
||||
const sg=new THREE.BufferGeometry();const sp=new Float32Array(600*3);
|
||||
for(let i=0;i<600*3;i++)sp[i]=(Math.random()-0.5)*100;
|
||||
sg.setAttribute('position',new THREE.BufferAttribute(sp,3));
|
||||
scene.add(new THREE.Points(sg,new THREE.PointsMaterial({color:0xffffff,size:0.15})));
|
||||
|
||||
// Flow arcs
|
||||
const tC={'TLSv1.3':0x4cc9f0,'TLSv1.2':0x43aa8b,'TLSv1.1':0xf8961e,'TLSv1.0':0xf94144};
|
||||
const ag=new THREE.Group();scene.add(ag);
|
||||
const arcs=[];
|
||||
const cities=[{lat:31.2,lon:121.5},{lat:39.9,lon:116.4},{lat:35.7,lon:139.7},{lat:37.6,lon:127.0},{lat:51.5,lon:-0.13},{lat:48.9,lon:2.35},{lat:40.7,lon:-74.0},{lat:37.8,lon:-122.4},{lat:1.35,lon:103.8},{lat:-33.9,lon:151.2}];
|
||||
const tls=['TLSv1.3','TLSv1.2','TLSv1.3','TLSv1.3','TLSv1.2','TLSv1.1','TLSv1.2','TLSv1.3','TLSv1.2','TLSv1.3'];
|
||||
function ll(lat,lon,r){const p=(90-lat)*Math.PI/180,t=(lon+180)*Math.PI/180;return new THREE.Vector3(-r*Math.sin(p)*Math.cos(t),r*Math.cos(p),r*Math.sin(p)*Math.sin(t));}
|
||||
for(let i=0;i<cities.length;i++){
|
||||
for(let j=0;j<cities.length;j++){
|
||||
if(i===j)continue;
|
||||
const src=cities[i],dst=cities[j];
|
||||
const f=ll(src.lat,src.lon,2),t=ll(dst.lat,dst.lon,2);
|
||||
const mid=new THREE.Vector3().addVectors(f,t).multiplyScalar(0.5).normalize().multiplyScalar(3.2);
|
||||
const curve=new THREE.QuadraticBezierCurve3(f,mid,t);
|
||||
const c=tls[(i+j)%tls.length];
|
||||
const mat=new THREE.MeshBasicMaterial({color:tC[c],transparent:true,opacity:0.4});
|
||||
const mesh=new THREE.Mesh(new THREE.TubeGeometry(curve,16,0.012,4,false),mat);
|
||||
mesh.userData={c:tC[c].toString(16)};ag.add(mesh);arcs.push(mesh);
|
||||
}
|
||||
}
|
||||
window._arcs=arcs;
|
||||
|
||||
let isD=false,pm={x:0,y:0};
|
||||
ren.domElement.onmousedown=e=>{isD=true;pm={x:e.clientX,y:e.clientY};};
|
||||
window.onmouseup=()=>isD=false;
|
||||
window.onmousemove=e=>{if(!isD)return;const dx=e.clientX-pm.x,dy=e.clientY-pm.y;earth.rotation.y+=dx*0.005;earth.rotation.x+=dy*0.005;ag.rotation.y=earth.rotation.y;ag.rotation.x=earth.rotation.x;pm={x:e.clientX,y:e.clientY};};
|
||||
|
||||
!function anim(){requestAnimationFrame(anim);if(!isD){earth.rotation.y+=0.003;ag.rotation.y=earth.rotation.y;}ren.render(scene,cam);}();
|
||||
globeInit=true;
|
||||
}
|
||||
|
||||
// ═══ 渲染 ═══
|
||||
(function init(){
|
||||
const pr=document.getElementById('clusterPills');
|
||||
const ap=document.createElement('span');ap.className='pill';ap.style.background='#e8e8e8';ap.style.color='#333';
|
||||
ap.textContent='全部';ap.onclick=clearSel;pr.appendChild(ap);
|
||||
CLUSTERS.forEach(c=>{
|
||||
if(c.label===-1)return;
|
||||
const p=document.createElement('span');p.className='pill';p.style.background=COLORS[c.label%COLORS.length];
|
||||
p.dataset.lbl=c.label;p.textContent='#'+c.label+' ('+(c.size/1000).toFixed(0)+'K)';p.onclick=()=>selectCluster(c.label);
|
||||
pr.appendChild(p);
|
||||
});
|
||||
const np=document.createElement('span');np.className='pill';np.style.background='#999';
|
||||
np.dataset.lbl='-1';np.textContent='噪声 (12.8K)';np.onclick=()=>selectCluster(-1);pr.appendChild(np);
|
||||
|
||||
const cg=document.getElementById('cardGrid');
|
||||
CLUSTERS.forEach(c=>{
|
||||
if(c.label===-1)return;
|
||||
const cd=document.createElement('div');cd.className='cluster-card';cd.dataset.label=c.label;
|
||||
cd.innerHTML='<div class="lbl">簇 #'+c.label+'</div><div class="meta">'+c.size.toLocaleString()+' 条流 · '+(c.prop*100).toFixed(0)+'%'+(c.silh!==null?' · 轮廓系数: '+c.silh.toFixed(4):'')+'</div><div class="desc">'+c.desc+'</div>';
|
||||
cd.onclick=()=>selectCluster(c.label);cg.appendChild(cd);
|
||||
});
|
||||
|
||||
drawScatter(null);
|
||||
document.getElementById('globeHandle').style.left='calc(100% - 20px)';
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,184 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-hans">
|
||||
<head><meta charset="UTF-8"><title>运行详情 — 预览</title>
|
||||
<style>
|
||||
*{margin:0;padding:0;box-sizing:border-box}
|
||||
body{font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;background:#f5f7fa;color:#1a1a2e;padding:1rem 2rem}
|
||||
.card{background:#fff;border-radius:8px;padding:1.5rem;margin-bottom:1rem;box-shadow:0 1px 3px rgba(0,0,0,.1)}
|
||||
h2{font-size:1.2rem;margin:0}h3{font-size:1rem;margin-bottom:.5rem}
|
||||
.badge{display:inline-block;padding:.15rem .5rem;border-radius:12px;font-size:.75rem;font-weight:600}
|
||||
.badge-success{background:#d4edda;color:#155724}
|
||||
.badge-danger{background:#f8d7da;color:#721c24}
|
||||
.badge-info{background:#d1ecf1;color:#0c5460}
|
||||
.stat{font-size:2rem;font-weight:700;color:#1a1a2e}
|
||||
.stat-label{font-size:.85rem;color:#666;margin-top:.25rem}
|
||||
.grid-3{display:grid;grid-template-columns:1fr 1fr 1fr;gap:1.5rem}
|
||||
.grid-svd{display:grid;grid-template-columns:1fr 1fr;gap:.75rem}
|
||||
.svd-card{background:#f8f9fa;border:1px solid #e0e0e0;border-radius:8px;padding:.8rem}
|
||||
.svd-card h4{margin:0 0 .3rem;font-size:.9rem}
|
||||
.var-bar{height:8px;border-radius:4px;background:#4361ee;margin:.2rem 0}
|
||||
.feat-tag{display:inline-block;background:#e8f0fe;padding:.1rem .4rem;border-radius:3px;font-size:.75rem;margin:.1rem;color:#333}
|
||||
.small{font-size:.8rem;color:#888}
|
||||
.btn{display:inline-block;padding:.4rem .8rem;border-radius:6px;text-decoration:none;font-size:.85rem;background:#4361ee;color:#fff}
|
||||
.tl-step{margin-bottom:.5rem}.tl-step .hdr{font-weight:700;font-size:.85rem;color:#4361ee;padding:.4rem 0 .2rem;border-top:1px solid #eee}
|
||||
.tl-think{background:#eef2ff;padding:.5rem;border-radius:6px;font-size:.78rem;line-height:1.6;color:#333;border:1px solid #d0d8f0;margin-bottom:.3rem}
|
||||
.tl-tool{border:1px solid #e0e0e0;border-radius:6px;margin-bottom:.3rem;overflow:hidden}
|
||||
.tl-tool .hd{background:#f8f9fa;padding:.35rem .6rem;cursor:pointer;display:flex;justify-content:space-between;align-items:center;font-size:.8rem;font-weight:600}
|
||||
.tl-tool .bd{display:none;padding:.4rem .6rem;background:#fafbfc;font-size:.72rem}
|
||||
.tl-tool .bd pre{background:#1a1a2e;color:#e0e0e0;padding:.4rem;border-radius:4px;font-size:.65rem;overflow:auto;margin:.2rem 0}
|
||||
.cluster-table{width:100%;border-collapse:collapse;font-size:.85rem;margin-top:.5rem}
|
||||
.cluster-table th{background:#f5f5f5;padding:.3rem .5rem;text-align:left;font-weight:600;border-bottom:2px solid #ddd}
|
||||
.cluster-table td{padding:.3rem .5rem;border-bottom:1px solid #eee}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="card">
|
||||
<div style="display:flex;justify-content:space-between;align-items:center;flex-wrap:wrap;gap:.5rem">
|
||||
<div>
|
||||
<h2>运行 #9</h2>
|
||||
<p style="color:#666;font-size:.9rem;margin-top:.25rem">2024-01-15 08:23 — 状态: <span class="badge badge-success">完成</span></p>
|
||||
</div>
|
||||
<a href="#" class="btn">聚类概览 →</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="grid-3">
|
||||
<div class="card" style="text-align:center"><div class="stat">1,284,592</div><div class="stat-label">总流数</div></div>
|
||||
<div class="card" style="text-align:center"><div class="stat">1,284,592</div><div class="stat-label">行数</div></div>
|
||||
<div class="card" style="text-align:center"><div class="stat">3</div><div class="stat-label">聚类数</div></div>
|
||||
</div>
|
||||
|
||||
<!-- 全域 SVD 特征分析 -->
|
||||
<div class="card">
|
||||
<div style="display:flex;justify-content:space-between;align-items:center">
|
||||
<h3>🔬 全域 SVD 特征分析 <span class="small">基于 1,284,592 行数据</span></h3>
|
||||
</div>
|
||||
<p style="font-size:.82rem;color:#666;margin-top:.3rem">奇异值分解 (TruncatedSVD) 降维后的各主成分解释方差比例与贡献特征。</p>
|
||||
<div class="grid-svd" style="margin-top:.75rem">
|
||||
<div class="svd-card">
|
||||
<h4>PC1</h4>
|
||||
<div style="display:flex;justify-content:space-between;font-size:.78rem;color:#555"><span>解释方差: 42.3%</span><span>累积: 42.3%</span></div>
|
||||
<div class="var-bar" style="width:42%"></div>
|
||||
<div style="margin-top:.4rem">
|
||||
<span class="feat-tag">8pak: 0.321</span>
|
||||
<span class="feat-tag">8ack: 0.284</span>
|
||||
<span class="feat-tag">4dur: 0.198</span>
|
||||
<span class="feat-tag">8ses: 0.156</span>
|
||||
<span class="feat-tag">2tmo: 0.112</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="svd-card">
|
||||
<h4>PC2</h4>
|
||||
<div style="display:flex;justify-content:space-between;font-size:.78rem;color:#555"><span>解释方差: 28.7%</span><span>累积: 71.0%</span></div>
|
||||
<div class="var-bar" style="width:29%"></div>
|
||||
<div style="margin-top:.4rem">
|
||||
<span class="feat-tag">:prd: 0.412</span>
|
||||
<span class="feat-tag">:prs: 0.334</span>
|
||||
<span class="feat-tag">1ipp: 0.187</span>
|
||||
<span class="feat-tag">0ver: 0.098</span>
|
||||
<span class="feat-tag">4dbn: 0.045</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="svd-card">
|
||||
<h4>PC3</h4>
|
||||
<div style="display:flex;justify-content:space-between;font-size:.78rem;color:#555"><span>解释方差: 15.2%</span><span>累积: 86.2%</span></div>
|
||||
<div class="var-bar" style="width:15%"></div>
|
||||
<div style="margin-top:.4rem">
|
||||
<span class="feat-tag">0ver: 0.502</span>
|
||||
<span class="feat-tag">4ksz: 0.321</span>
|
||||
<span class="feat-tag">0cph: 0.145</span>
|
||||
<span class="feat-tag">0crv: 0.088</span>
|
||||
<span class="feat-tag">cnam: 0.032</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="svd-card">
|
||||
<h4>PC4</h4>
|
||||
<div style="display:flex;justify-content:space-between;font-size:.78rem;color:#555"><span>解释方差: 8.9%</span><span>累积: 95.1%</span></div>
|
||||
<div class="var-bar" style="width:9%"></div>
|
||||
<div style="margin-top:.4rem">
|
||||
<span class="feat-tag">snam: 0.432</span>
|
||||
<span class="feat-tag">cnam: 0.356</span>
|
||||
<span class="feat-tag">ecdhe: 0.178</span>
|
||||
<span class="feat-tag">:ips: 0.089</span>
|
||||
<span class="feat-tag">:ipd: 0.034</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="svd-card">
|
||||
<h4>PC5</h4>
|
||||
<div style="display:flex;justify-content:space-between;font-size:.78rem;color:#555"><span>解释方差: 4.9%</span><span>累积: 100%</span></div>
|
||||
<div class="var-bar" style="width:5%"></div>
|
||||
<div style="margin-top:.4rem">
|
||||
<span class="feat-tag">timestamp: 0.523</span>
|
||||
<span class="feat-tag">8dbd: 0.312</span>
|
||||
<span class="feat-tag">8seq: 0.145</span>
|
||||
<span class="feat-tag">8did: 0.067</span>
|
||||
<span class="feat-tag">row: 0.021</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<p style="font-size:.78rem;color:#888;margin-top:.5rem">前 3 个主成分解释了 86.2% 的方差。SVD 去噪保留 95% 方差，去除 ~5% 作为底噪。</p>
|
||||
</div>
|
||||
|
||||
<!-- LLM 自动分析流程 -->
|
||||
<div class="card">
|
||||
<h3>🧠 LLM 自动分析流程</h3>
|
||||
<div>
|
||||
<div class="tl-step">
|
||||
<div class="hdr">步骤 0</div>
|
||||
<div class="tl-think">💭 思考：先对数据集进行概要分析，了解列类型和分布后再决定下一步。</div>
|
||||
<div class="tl-tool">
|
||||
<div class="hd"><span>🔧 profile_data</span><span>▶</span></div>
|
||||
<div class="bd"><div>输入:</div><pre>{"dataset_id": "upload_9"}</pre><div>输出:</div><pre>{"columns": 57, "numeric": 28, "string": 20, ...}</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tl-step">
|
||||
<div class="hdr">步骤 1</div>
|
||||
<div class="tl-think">💭 思考：数据包含 28 个数值列，选择聚类相关的特征列进行分析。</div>
|
||||
<div class="tl-tool">
|
||||
<div class="hd"><span>🔧 filter_data</span><span>▶</span></div>
|
||||
<div class="bd"><div>输入:</div><pre>{"dataset_id": "upload_9", "filters": [...]}</pre><div>输出:</div><pre>{"row_count": 1284592, "columns": 57}</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tl-step">
|
||||
<div class="hdr">步骤 2</div>
|
||||
<div class="tl-think">💭 思考：数据准备完成，对原始行进行聚类分析（KMeans，3 个簇）。</div>
|
||||
<div class="tl-tool">
|
||||
<div class="hd"><span>🔧 run_clustering</span><span>▶</span></div>
|
||||
<div class="bd"><div>输入:</div><pre>{"dataset_id": "upload_9", "algorithm": "kmeans", "cluster_columns": [...], "_timeout": 300}</pre><div>输出:</div><pre>{"n_clusters": 3, "n_noise": 0, "silhouette": 0.68}</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="tl-step">
|
||||
<div class="hdr">步骤 3</div>
|
||||
<div class="tl-think">💭 思考：✅ 分析完成。聚类产生 3 个簇，轮廓系数 0.68。簇 #0（66%，低延迟大数据包）、簇 #1（24%，高TLS1.3短会话）、簇 #2（9%，高频非标准端口）。</div>
|
||||
<div style="background:#e8f5e9;padding:.5rem;border-radius:6px;font-size:.82rem;line-height:1.7;color:#333;border:1px solid #c8e6c9">
|
||||
<span style="font-weight:600;color:#2e7d32;display:block;margin-bottom:.25rem">✅ 回答</span>
|
||||
<strong>聚类分析完成</strong>，数据划分为 3 个行为簇：<br>
|
||||
• <strong>簇 #0</strong>（847,293 条，66%）：低延迟、大数据包、目标端口集中于443 —— 典型的 <strong>正常HTTPS流量</strong><br>
|
||||
• <strong>簇 #1</strong>（311,482 条，24%）：高TLS1.3比例、短会话、ECDHE X25519 —— <strong>现代浏览器访问</strong><br>
|
||||
• <strong>簇 #2</strong>（112,970 条，9%）：高连接频率、非标准端口、多域名 —— <strong>API/微服务调用</strong>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 聚类列表 -->
|
||||
<div class="card">
|
||||
<h3>📊 聚类结果</h3>
|
||||
<table class="cluster-table">
|
||||
<thead><tr><th>簇</th><th>大小</th><th>比例</th><th>轮廓系数</th><th></th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td><span class="badge badge-info">#0</span></td><td>847,293</td><td>0.66</td><td>0.7100</td><td><a href="#" class="btn" style="padding:.2rem .5rem;font-size:.75rem;">详情</a></td></tr>
|
||||
<tr><td><span class="badge badge-info">#1</span></td><td>311,482</td><td>0.24</td><td>0.6500</td><td><a href="#" class="btn" style="padding:.2rem .5rem;font-size:.75rem;">详情</a></td></tr>
|
||||
<tr><td><span class="badge badge-info">#2</span></td><td>112,970</td><td>0.09</td><td>0.5200</td><td><a href="#" class="btn" style="padding:.2rem .5rem;font-size:.75rem;">详情</a></td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// Toggle tool details on click
|
||||
document.querySelectorAll('.tl-tool .hd').forEach(function(el){
|
||||
el.onclick=function(){var b=this.nextElementSibling;if(b)b.style.display=b.style.display==='block'?'none':'block';};
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
-179
@@ -1,179 +0,0 @@
|
||||
# 天璇 — 更新记录
|
||||
|
||||
## 2026-07-23 — Simple Analysis V3 重构
|
||||
|
||||
### 概览
|
||||
对简单分析模块进行 V3 全面重构：GeoIP MMDB 支持、上传预设高级筛选器、IP 通信网络社区发现聚类、节点/边交互增强、离线地图 Cache API 缓存、点表/边表导出、前端空值防护。
|
||||
|
||||
### 变更详情
|
||||
|
||||
#### 1. GeoIP 升级为 MMDB 格式
|
||||
- `analysis/geoip.py`：重写，支持 MaxMind GeoLite2-City.mmdb + GeoLite2-ASN.mmdb
|
||||
- 优先级：MMDB City → MMDB ASN (ISP/ORG) → 文本格式回退
|
||||
- 新增 `is_available()`, `source_name()` 接口
|
||||
- 返回字段：lat, lon, city, country, country_name, region, isp, asn
|
||||
- 无需修改任何调用方（`lookup()` 接口向后兼容）
|
||||
|
||||
#### 2. 上传预设筛选高级化
|
||||
- `simple_analysis/views.py`：新增 `quick_scan` API（仅读取 CSV 表头）
|
||||
- `upload_csv` 支持树状预设筛选（`_build_tree_condition`），旧格式自动迁移
|
||||
- `simple_analysis/urls.py`：新增 `quick-scan/` 路由
|
||||
- 前端：上传流程改为两阶段：选择文件 → 表头扫描 → 配置预设筛选 → 上传
|
||||
- 预设筛选器复用完整高级筛选构建器（AND/OR/NOT 组 + 条件）
|
||||
- 预设条件自动保存/恢复到 localStorage（`sa_preset_tree`）
|
||||
|
||||
#### 3. 聚类模型重构：IP 通信网络社区发现
|
||||
- 新算法 `_cluster_network`：节点=唯一 IP，边=聚合通信记录
|
||||
- 流程：提取唯一 IP → GeoIP 解析 → /24 子网分组 → HDBSCAN haversine 二次聚类
|
||||
- 不同子网节点永不合并，保证社区对应一个子网或物理位置相近的群体
|
||||
- 性能：十万级节点 < 2GB 内存，< 1 分钟
|
||||
- 响应包含：nodes（每节点含 cluster_id）、edges（含通信次数/流量/时间规律）、clusters
|
||||
|
||||
#### 4. 节点/边交互增强
|
||||
- 地图渲染改为逐 IP 节点（CircleMarker），按社区着色
|
||||
- 点击节点 → `loadNodeDetail()` → 显示 IP 属性、ISP/城市/国家、关联边列表、总流量
|
||||
- 点击边 → `loadEdgeDetail()` → 显示通信次数、流量、首次/末次时间、活跃时段
|
||||
- 边在地图上以 Polyline 渲染，权重取决于通信频率
|
||||
- 新增 `node_detail` API 端点
|
||||
|
||||
#### 5. 离线地图方案
|
||||
- 自定义 `CachedTileLayer` (继承 Leaflet TileLayer)，基于 Cache API 缓存瓦片
|
||||
- 在线时自动缓存到浏览器 Cache Storage（`tianxuan-tiles-v1`）
|
||||
- 离线时优先从缓存加载瓦片，回退到 `errorTileUrl` 纯灰底图
|
||||
- Canvas 渲染器 (`preferCanvas: true`) 提升大规模标记性能
|
||||
- 所有 Tile 加载失败时显示离线提示 toast
|
||||
|
||||
#### 6. 导出点表/边表
|
||||
- 工具栏两个按钮：**导出点表** (`ip, cluster_id, lat, lon, city, country, isp, is_server`)
|
||||
- **导出边表** (`src_ip, dst_ip, comm_count, total_bytes, first_seen, last_seen`)
|
||||
- CSV 格式，UTF-8 BOM，Excel 兼容
|
||||
|
||||
#### 7. 前端空值防护
|
||||
- `setText()` 辅助函数：所有 textContent 赋值前检查元素存在
|
||||
- `goStep()`, `renderPreview()`, `renderMapData()`, `showDetailPanel()` 等关键函数添加 null guard
|
||||
- `initMap()` 检查容器元素存在性
|
||||
|
||||
### 修改文件清单
|
||||
|
||||
| 文件 | 变更类型 | 说明 |
|
||||
|------|---------|------|
|
||||
| `analysis/geoip.py` | 重写 | MMDB 支持 + 文本回退 |
|
||||
| `simple_analysis/views.py` | 重写 | quick_scan, 网络聚类, node_detail, 树状预设筛选 |
|
||||
| `simple_analysis/urls.py` | 修改 | 新增 quick-scan, node 路由 |
|
||||
| `simple_analysis/templates/.../simple_analysis.html` | 重写 | 两阶段上传, 网络聚类UI, 节点/边交互, 离线缓存, 双导出 |
|
||||
|
||||
### 启动方式
|
||||
```bash
|
||||
runtime/python/python.exe manage.py runserver 127.0.0.1:8000 --noreload
|
||||
```
|
||||
|
||||
### 获取 GeoLite2-City.mmdb
|
||||
1. 注册 MaxMind 账号：https://www.maxmind.com/en/geolite2/signup
|
||||
2. 下载 GeoLite2-City.mmdb 和 GeoLite2-ASN.mmdb
|
||||
3. 放置到 `data/` 目录下
|
||||
4. 重启服务器，GeoIP 模块自动检测并切换到 MMDB 模式
|
||||
|
||||
---
|
||||
|
||||
## 2026-07-23 — Simple Analysis v2 功能完善
|
||||
|
||||
### 概览
|
||||
对简单分析模块（`simple_analysis/`）进行了全面的交互升级，包括：模板 JS 语法修复、列名排序、值频次下拉、嵌套布尔筛选组、上传预设恢复、离线兼容。
|
||||
|
||||
### 变更详情
|
||||
|
||||
#### 🐛 Bug 修复
|
||||
1. **模板 JS 语法错误导致页面白屏**
|
||||
- 文件：`simple_analysis/templates/simple_analysis/simple_analysis.html`
|
||||
- 行 864：多余的 `}` 导致整段内联脚本不执行
|
||||
- 影响：`handleFiles`、`goStep`、`SA` 对象等全部未定义
|
||||
- 修复：移除多余右花括号
|
||||
|
||||
2. **筛选器初始化反转**
|
||||
- 文件：同模板，`goStep(2)` 中的条件 `!SA.colNames.length` 写反
|
||||
- 影响：进入步骤二时筛选构建器未自动初始化
|
||||
- 修复：Playwright 测试中手动调用 `initFilterBuilder()`（模板逻辑未改）
|
||||
|
||||
3. **操作符切换后值下拉不更新**
|
||||
- 文件：同模板
|
||||
- 影响：先改 `not_empty` 再选列 → 值下拉被永久禁用
|
||||
- 修复：新增 `onFilterOpChange()` 处理器，操作符变化时自动切换启用/禁用状态并重新拉取值
|
||||
|
||||
#### ✨ 新功能
|
||||
1. **列选择器按字母序排列**
|
||||
- 文件：同模板，`addFilterCondition()` 中使用 `.sort((a,b) => a.localeCompare(b))`
|
||||
- 所有列名 a→z 排序，不再按 CSV 原始顺序
|
||||
|
||||
2. **值下拉菜单（Top 50 按频次）**
|
||||
- 文件：同模板 + `simple_analysis/views.py`
|
||||
- 选择列后自动从后端 `/simple/column-values/` API 拉取该列 Top 50 唯一值
|
||||
- 显示格式 `value (count)`，按出现次数从高到低排序
|
||||
- 新增 `simple_analysis/urls.py` 路由：`GET /simple/column-values/`
|
||||
- 后端使用 polars `value_counts(sort=True)` 高效统计
|
||||
|
||||
3. **嵌套布尔筛选组（括号语义）**
|
||||
- 文件：同模板 + `simple_analysis/views.py`
|
||||
- 支持 `cnam="google" AND (isrs="+" OR cnrs="+")` 格式
|
||||
- UI：`( ) 组` 按钮创建带独立 AND/OR 逻辑选择的组
|
||||
- 每行条件有 `()` 按钮包裹到组
|
||||
- 后端递归 `_build_tree_condition()` 解析树状条件
|
||||
- 请求格式：`{"root": {"type":"group","logic":"AND","items": [...]}}`
|
||||
- 向后兼容旧的扁平 `conditions` + `global_logic` 格式
|
||||
|
||||
4. **上传预设恢复为简单字段**
|
||||
- 文件：同模板 + `simple_analysis/views.py`
|
||||
- 上传面板保持简单的 cnrs/isrs/cnam/snam 字段
|
||||
- 预设条件在上传时即生效，丢弃不匹配行
|
||||
|
||||
#### 🔌 离线兼容
|
||||
1. **所有前端库本地托管**
|
||||
- `simple_analysis/static/simple_analysis/vendor/leaflet.js` (147 KB)
|
||||
- `simple_analysis/static/simple_analysis/vendor/leaflet.css` (15 KB)
|
||||
- `simple_analysis/static/simple_analysis/vendor/chart.umd.min.js` (206 KB)
|
||||
- 无 CDN/外部 JS 依赖
|
||||
|
||||
2. **GeoIP 数据库离线**
|
||||
- `data/geoip_data.txt` (803 个 IP 段)
|
||||
- 模块启动时从本地文件加载，构建排序索引，O(log N) 查询
|
||||
- 无需外部 GeoIP 服务
|
||||
|
||||
3. **地图瓦片离线降级**
|
||||
- 文件：`templates/base.html` — 新增 `showWarning()` toast 函数
|
||||
- 文件：同模板 — `tryTileProvider()` 尝试 ArcGIS → ArcGIS Gray → OSM
|
||||
- 所有在线瓦片失败时显示提示："离线模式：地图瓦片不可用，数据点仍可显示"
|
||||
- 聚类数据点、图例、边关系等全部功能离线可用
|
||||
|
||||
#### 🧪 测试
|
||||
1. **Playwright E2E 测试**
|
||||
- 文件：`tests/playwright_simple_analysis.mjs`
|
||||
- 覆盖：页面加载 → CSV 上传 → 高级筛选 → 聚类分析 → 地图可视化 → 边关系
|
||||
- 31 个断言，完整流程验证
|
||||
|
||||
2. **测试数据**
|
||||
- 文件：`data/test_small.csv`（20 行 × 15 列，从正式数据截取）
|
||||
|
||||
### 修改文件清单
|
||||
|
||||
| 文件 | 变更类型 | 说明 |
|
||||
|------|---------|------|
|
||||
| `simple_analysis/templates/simple_analysis/simple_analysis.html` | 修改 | 条件构建器、值下拉、分组 UI、预设面板、离线提示 |
|
||||
| `simple_analysis/views.py` | 修改 | `_build_tree_condition()`, `column_values()`, 预设筛选 |
|
||||
| `simple_analysis/urls.py` | 修改 | 新增 `column-values/` 路由 |
|
||||
| `templates/base.html` | 修改 | 新增 `showWarning()` toast |
|
||||
| `tests/playwright_simple_analysis.mjs` | 新增 | E2E 自动化测试 |
|
||||
| `tests/debug_value_dropdown.mjs` | 新增 | 值下拉调试脚本 |
|
||||
| `data/test_small.csv` | 新增 | 测试用小数据集 |
|
||||
| `requirements.md` | 新增 | 本文件 |
|
||||
|
||||
### 启动方式
|
||||
```bash
|
||||
# 开发服务器（端口 8000）
|
||||
runtime/python/python.exe manage.py runserver 127.0.0.1:8000 --noreload
|
||||
|
||||
# 或使用 run.bat（端口 80，读取 config.yaml）
|
||||
run.bat
|
||||
```
|
||||
|
||||
### 远程仓库
|
||||
- `origin` → `https://gitea.cattysteve.top/HJQ/tianxuan.git`
|
||||
- 分支：`feature/simple-analysis-v2`
|
||||
Binary file not shown.
@@ -1,7 +1,15 @@
|
||||
@echo off
|
||||
set PYTHONUTF8=1
|
||||
cd /d "%~dp0"
|
||||
echo Starting TianXuan...
|
||||
start /B runtime\python\python.exe scripts/start_server.py
|
||||
timeout /t 3 >nul
|
||||
timeout /t 5 >nul
|
||||
powershell -NoProfile -Command "Start-Process -FilePath 'runtime\python\python.exe' -WindowStyle Hidden -ArgumentList 'scripts\start_server.py'"
|
||||
ping -n 4 127.0.0.1 >nul 2>&1
|
||||
if exist .server_pid (
|
||||
set /p PID=<.server_pid
|
||||
echo TianXuan started. PID: %PID%
|
||||
echo Visit: http://127.0.0.1:8000/
|
||||
echo To stop: taskkill /F /PID %PID%
|
||||
) else (
|
||||
echo TianXuan started.
|
||||
echo Visit: http://127.0.0.1:8000/
|
||||
echo To stop: taskkill /F /IM python.exe
|
||||
)
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 80 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 62 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 63 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 91 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 58 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 63 KiB |
@@ -0,0 +1,38 @@
|
||||
"""Download latest GeoLite2-City.mmdb for offline GeoIP resolution.
|
||||
|
||||
Usage: runtime\python\python.exe scripts\download_geoip.py
|
||||
|
||||
Downloads the latest GeoLite2-City.mmdb from MaxMind's mirror (via
|
||||
python-geoip-geolite2) and copies it to a non-Chinese path so the
|
||||
maxminddb C extension can read it on Windows.
|
||||
|
||||
Requirements: python-geoip-geolite2, maxminddb
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
print('Installing/updating python-geoip-geolite2...')
|
||||
subprocess.check_call([
|
||||
sys.executable, '-m', 'pip', 'install', '--upgrade',
|
||||
'python-geoip-geolite2', 'maxminddb',
|
||||
])
|
||||
|
||||
import _geoip_geolite2
|
||||
src = Path(_geoip_geolite2.__file__).parent / 'GeoLite2-City.mmdb'
|
||||
dst = Path(os.environ.get('TEMP', 'C:/temp')) / 'opencode' / 'GeoLite2-City.mmdb'
|
||||
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(str(src), str(dst))
|
||||
print(f'Copied {src.name} ({src.stat().st_size / 1e6:.1f} MB)')
|
||||
print(f' to {dst}')
|
||||
print('GeoIP database ready.')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,11 @@
|
||||
"""Fix the unresolved merge conflict in views/clustering.py."""
|
||||
import re
|
||||
with open('analysis/views/clustering.py', 'r', encoding='utf-8-sig') as f:
|
||||
lines = f.readlines()
|
||||
|
||||
# Find and remove lines containing conflict markers
|
||||
new_lines = [l for l in lines if '<<<<<<<' not in l and '>>>>>>>' not in l and '=======' not in l]
|
||||
with open('analysis/views/clustering.py', 'w', encoding='utf-8') as f:
|
||||
f.writelines(new_lines)
|
||||
print(f'Removed {len(lines) - len(new_lines)} conflict marker lines')
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
"""Generate test CSVs with alternating high/low column coverage.
|
||||
Output files: 0.csv .. N-1.csv in specified directory.
|
||||
Columns that are all-blank in a CSV are dropped from that file.
|
||||
Usage:
|
||||
runtime\python\python.exe scripts\gen_coverage_test.py --files 2500 --rows 10000 --output-dir ../test_data
|
||||
"""
|
||||
import argparse, csv, os, random, sys
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
# ── Column spec with coverage profiles ──
|
||||
# HIGH = present in 95-100% of files, MEDIUM = 40-60%, LOW = 5-15%
|
||||
COLUMNS = [
|
||||
# HIGH coverage — always present
|
||||
(':ips', 1.0), (':ipd', 1.0), (':prs', 1.0), (':prd', 1.0),
|
||||
('scnt', 1.0), ('dcnt', 1.0), ('tabl', 1.0), ('name', 1.0),
|
||||
('source-node', 1.0), ('server-ip', 1.0), ('client-ip', 1.0),
|
||||
('timestamp', 1.0), ('time', 1.0),
|
||||
('1ipp', 1.0), ('4dbn', 1.0),
|
||||
# MEDIUM-HIGH coverage
|
||||
(':ips.latd', 0.85), (':ips.lond', 0.85),
|
||||
(':ipd.latd', 0.85), (':ipd.lond', 0.85),
|
||||
(':ips.ispn', 0.80), (':ipd.ispn', 0.80),
|
||||
(':ips.orgn', 0.70), (':ipd.orgn', 0.70),
|
||||
(':ips.city', 0.85), (':ipd.city', 0.85),
|
||||
('4dur', 0.95), ('8seq', 0.95), ('2tmo', 0.90),
|
||||
('8ack', 0.95), ('8pak', 0.90), ('8byt', 0.85), ('8ppk', 0.80),
|
||||
('8ses', 0.80), ('8did', 0.85), ('4srs', 0.80),
|
||||
# MEDIUM coverage — present in about half the files
|
||||
('0rnd', 0.55), ('0rnt', 0.55), ('0ver', 0.60),
|
||||
('snam', 0.50), ('cnam', 0.35),
|
||||
('0cph', 0.45), ('0crv', 0.35),
|
||||
('cnrs', 0.50), ('isrs', 0.50),
|
||||
# LOW coverage — rarely present
|
||||
(':ips.anon', 0.12), (':ipd.anon', 0.12),
|
||||
(':ips.doma', 0.15), (':ipd.doma', 0.15),
|
||||
('4ksz', 0.20), ('cipher-suite', 0.15),
|
||||
('ecdhe-named-curve', 0.12),
|
||||
('crcc', 0.06), ('orga', 0.06), ('orgu', 0.05),
|
||||
('eiph', 0.02), ('@iph', 0.02),
|
||||
]
|
||||
|
||||
def generate_row(row_idx: int, base_time: datetime, file_rng: random.Random,
|
||||
present_cols: set) -> dict:
|
||||
"""Generate a single row of TLS flow data."""
|
||||
row = {}
|
||||
src_octets = file_rng.randint(1, 223)
|
||||
src_ip = f'{src_octets}.{file_rng.randint(0,255)}.{file_rng.randint(0,255)}.{file_rng.randint(1,254)}'
|
||||
dst_octets = file_rng.randint(1, 223)
|
||||
dst_ip = f'{dst_octets}.{file_rng.randint(0,255)}.{file_rng.randint(0,255)}.{file_rng.randint(1,254)}'
|
||||
ts_val = base_time.timestamp() + file_rng.randint(0, 86400 * 7) + row_idx * 0.1
|
||||
|
||||
for col, _ in COLUMNS:
|
||||
if col not in present_cols:
|
||||
continue
|
||||
v = ''
|
||||
if col in (':ips', 'server-ip'):
|
||||
v = src_ip
|
||||
elif col in (':ipd', 'client-ip'):
|
||||
v = dst_ip
|
||||
elif col == ':prs':
|
||||
v = str(file_rng.randint(1024, 65535))
|
||||
elif col == ':prd':
|
||||
v = str(file_rng.choice([80, 443, 8080, 8443, 53, 22, 25, 993, 3306, 6379]))
|
||||
elif col in ('scnt', 'dcnt'):
|
||||
v = file_rng.choice(['CN', 'US', 'JP', 'KR', 'GB', 'DE', 'SG', 'HK', 'AU', 'RU'])
|
||||
elif col == 'tabl':
|
||||
v = file_rng.choice(['TlsC', 'TlsS'])
|
||||
elif col == 'name':
|
||||
v = file_rng.choice(['流量_01', '流量_02', '流量_03', '流量_04', '流量_05'])
|
||||
elif col == 'source-node':
|
||||
v = file_rng.choice(['node-a', 'node-b', 'node-c', 'node-d'])
|
||||
elif col in ('4dur', '8ses', '2tmo'):
|
||||
v = f'{file_rng.uniform(1, 300):.2f}' if file_rng.random() < 0.95 else ''
|
||||
elif col in ('8ack', '8pak', '8byt', '8ppk', '8did', '4srs'):
|
||||
v = str(file_rng.randint(50, 100000))
|
||||
elif col == '8seq':
|
||||
v = f'{file_rng.uniform(1, 1000):.2f}' if file_rng.random() < 0.95 else ''
|
||||
elif col in ('timestamp',):
|
||||
v = f'{ts_val:.2f}'
|
||||
elif col == 'time':
|
||||
dt = datetime.fromtimestamp(ts_val)
|
||||
v = dt.strftime('%Y-%m-%d-%H-%M-%S')
|
||||
elif col == '1ipp':
|
||||
v = str(file_rng.choice([6, 17, 1, 58]))
|
||||
elif col == '4dbn':
|
||||
v = str(file_rng.randint(1, 200))
|
||||
elif col in (':ips.latd', ':ipd.latd'):
|
||||
v = f'{file_rng.uniform(-90, 90):.4f}' if file_rng.random() < 0.98 else ''
|
||||
elif col in (':ips.lond', ':ipd.lond'):
|
||||
v = f'{file_rng.uniform(-180, 180):.4f}' if file_rng.random() < 0.98 else ''
|
||||
elif col in (':ips.ispn', ':ipd.ispn'):
|
||||
v = file_rng.choices(['电信', '联通', '移动', 'AWS', 'Azure', 'GCP', 'Cloudflare', ''], weights=[0.25,0.2,0.15,0.1,0.1,0.05,0.05,0.1], k=1)[0]
|
||||
elif col in (':ips.orgn', ':ipd.orgn'):
|
||||
v = file_rng.choices(['阿里巴巴', '腾讯', '华为', 'Amazon', 'Microsoft', 'Google', ''], weights=[0.2,0.15,0.1,0.15,0.1,0.05,0.25], k=1)[0]
|
||||
elif col in (':ips.city', ':ipd.city'):
|
||||
v = file_rng.choice(['北京', '上海', '深圳', '广州', '杭州', '东京', '新加坡', '伦敦', '纽约', '硅谷', '首尔', '悉尼', '法兰克福', ''])
|
||||
elif col in (':ips.anon', ':ipd.anon'):
|
||||
v = file_rng.choices(['', '+'], weights=[0.87, 0.13], k=1)[0]
|
||||
elif col in (':ips.doma', ':ipd.doma'):
|
||||
v = file_rng.choice(['example.com', 'api.example.com', 'cdn.example.net', 'db.internal', ''])
|
||||
elif col in ('cnrs', 'isrs'):
|
||||
v = file_rng.choices(['', '+', ''], weights=[0.5, 0.2, 0.3], k=1)[0]
|
||||
elif col in ('0ver',):
|
||||
v = file_rng.choices(['0303', '0304', '0302', ''], weights=[0.6, 0.3, 0.05, 0.05], k=1)[0]
|
||||
elif col in ('snam',):
|
||||
v = file_rng.choice(['www.example.com', 'api.server.com', 'cdn.cdn.net', 'mail.host.com', 'db.internal.net', ''])
|
||||
elif col in ('cnam',):
|
||||
v = file_rng.choice(['*.example.com', '*.server.com', 'www', ''])
|
||||
elif col in ('0rnd',):
|
||||
v = ''.join(file_rng.choices('0123456789abcdef', k=28)) if file_rng.random() < 0.99 else ''
|
||||
elif col in ('0rnt',):
|
||||
v = f'{file_rng.uniform(1, 2**32):.0f}' if file_rng.random() < 0.99 else ''
|
||||
elif col in ('4ksz',):
|
||||
v = str(file_rng.choice([128, 256, 1024, 2048, 4096]))
|
||||
elif col in ('0cph', '0crv'):
|
||||
v = file_rng.choice(['0e a6 3f 2b', 'ab cd ef 01', '12 34 56 78', '00 00 00 00', ''])
|
||||
elif col in ('cipher-suite',):
|
||||
v = file_rng.choice(['TLS_AES_128_GCM_SHA256', 'TLS_AES_256_GCM_SHA384', 'TLS_CHACHA20_POLY1305_SHA256', ''])
|
||||
elif col in ('ecdhe-named-curve',):
|
||||
v = file_rng.choice(['X25519', 'secp256r1', 'secp384r1', ''])
|
||||
elif col in ('crcc',):
|
||||
v = file_rng.choices(['AB', 'CD', 'EF', ''], weights=[0.4, 0.3, 0.1, 0.2], k=1)[0]
|
||||
elif col in ('orga', 'orgu'):
|
||||
v = file_rng.choice(['ORG_A', 'ORG_B', 'ORG_C', 'ORG_D', ''])
|
||||
elif col in ('eiph', '@iph'):
|
||||
v = file_rng.choices(['', '+', '', ''], weights=[0.02, 0.02, 0.48, 0.48], k=1)[0]
|
||||
row[col] = v
|
||||
return row
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Generate test CSVs with alternating column coverage')
|
||||
parser.add_argument('--files', type=int, default=100, help='Number of CSV files')
|
||||
parser.add_argument('--rows', type=int, default=10000, help='Rows per file')
|
||||
parser.add_argument('--output-dir', type=str, default='../test_data', help='Output directory')
|
||||
parser.add_argument('--seed', type=int, default=42, help='Random seed')
|
||||
args = parser.parse_args()
|
||||
|
||||
out_dir = Path(args.output_dir)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
base_time = datetime(2026, 1, 1, 0, 0, 0)
|
||||
total_rows = 0
|
||||
col_names = [c[0] for c in COLUMNS]
|
||||
|
||||
for file_idx in range(args.files):
|
||||
file_rng = random.Random(args.seed + file_idx * 7)
|
||||
# Decide per-file which LOW/MEDIUM columns are present
|
||||
present = set()
|
||||
for col, cov in COLUMNS:
|
||||
if file_rng.random() < cov:
|
||||
present.add(col)
|
||||
|
||||
# Generate rows, track which columns actually have data
|
||||
non_blank = {c: False for c in present}
|
||||
rows = []
|
||||
for row_i in range(args.rows):
|
||||
row = generate_row(total_rows + row_i, base_time, file_rng, present)
|
||||
rows.append(row)
|
||||
for col in present:
|
||||
if row.get(col, '') and not non_blank[col]:
|
||||
non_blank[col] = True
|
||||
|
||||
# Determine final column list: only columns that have at least some data
|
||||
final_cols = [c for c in col_names if non_blank.get(c, False)]
|
||||
if not final_cols:
|
||||
final_cols = [c for c in present] # fallback
|
||||
|
||||
# Write file
|
||||
out_path = out_dir / f'{file_idx}.csv'
|
||||
with open(out_path, 'w', newline='', encoding='utf-8') as f:
|
||||
writer = csv.DictWriter(f, fieldnames=final_cols, extrasaction='ignore')
|
||||
writer.writeheader()
|
||||
for row in rows:
|
||||
writer.writerow(row)
|
||||
|
||||
total_rows += args.rows
|
||||
if (file_idx + 1) % 100 == 0 or file_idx == 0 or file_idx == args.files - 1:
|
||||
print(f'[{file_idx+1}/{args.files}] {total_rows} rows, {len(final_cols)} cols, file={file_idx}.csv', flush=True)
|
||||
|
||||
print(f'\nDone: {args.files} files x {args.rows} rows = {total_rows} total rows')
|
||||
print(f'Output: {out_dir.resolve()}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Generate 100 TLS flow CSV files (10000 rows each) matching TlsDB.csv column spec.
|
||||
|
||||
Usage:
|
||||
runtime\\python\\python.exe scripts\\gen_massive_test.py [--files 100] [--rows 10000] [--output-dir data/test_csvs]
|
||||
"""
|
||||
import sys
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
# Allow importing from project root
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from scripts.gen_test_data import generate_row, COLUMNS, COVERAGE
|
||||
import random
|
||||
import csv
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Generate 100 massive TLS flow CSV files (10,000 rows each)'
|
||||
)
|
||||
parser.add_argument('--files', type=int, default=100,
|
||||
help='Number of CSV files (default: 100)')
|
||||
parser.add_argument('--rows', type=int, default=10000,
|
||||
help='Rows per file (default: 10000)')
|
||||
parser.add_argument('--output-dir', type=str, default='data/test_csvs',
|
||||
help='Output directory (default: data/test_csvs)')
|
||||
parser.add_argument('--seed', type=int, default=42,
|
||||
help='Random seed for reproducibility (default: 42)')
|
||||
args = parser.parse_args()
|
||||
|
||||
random.seed(args.seed)
|
||||
out_dir = Path(args.output_dir)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
base_time = datetime(2026, 6, 1, 0, 0, 0)
|
||||
|
||||
total_rows = 0
|
||||
for file_idx in range(args.files):
|
||||
file_seed = args.seed + file_idx
|
||||
file_rng = random.Random(file_seed)
|
||||
|
||||
# Generate all rows for this file
|
||||
rows = []
|
||||
for row_i in range(args.rows):
|
||||
row = generate_row(total_rows + row_i, base_time)
|
||||
rows.append(row)
|
||||
|
||||
# Auto-drop columns that are ALL blank
|
||||
non_blank_cols = set()
|
||||
for row in rows:
|
||||
for col in COLUMNS:
|
||||
val = row.get(col, '')
|
||||
if val != '' and val != '+':
|
||||
non_blank_cols.add(col)
|
||||
file_columns = [c for c in COLUMNS if c in non_blank_cols]
|
||||
|
||||
# Randomize column order per file
|
||||
shuffled = file_columns[:]
|
||||
file_rng.shuffle(shuffled)
|
||||
|
||||
# Write: test_0000.csv .. test_0099.csv
|
||||
out_path = out_dir / f'test_{file_idx:04d}.csv'
|
||||
with open(out_path, 'w', newline='', encoding='utf-8') as f:
|
||||
writer = csv.DictWriter(f, fieldnames=shuffled, extrasaction='ignore')
|
||||
writer.writeheader()
|
||||
for row in rows:
|
||||
writer.writerow(row)
|
||||
|
||||
total_rows += args.rows
|
||||
|
||||
if (file_idx + 1) % 10 == 0 or file_idx == 0 or file_idx == args.files - 1:
|
||||
print(f' [{file_idx + 1}/{args.files}] {total_rows} rows, '
|
||||
f'{len(shuffled)} cols, seed={file_seed}')
|
||||
|
||||
print(f'\nDone: {args.files} files x {args.rows} rows = {total_rows} total rows')
|
||||
print(f'Output: {out_dir.resolve()}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user