feat: v3 refactor with MMDB GeoIP, network clustering, node/edge viz, offline tile cache, dual export
- GeoIP: MMDB (GeoLite2-City + ASN) support with text fallback - Upload: two-phase flow with advanced preset filter builder - Clustering: IP communication network community discovery (/24 + HDBSCAN) - Map: per-IP node rendering, node click detail, edge click detail - Offline: Cache API tile caching with fallback - Export: node table + edge table CSV download - Fix: null-guard DOM operations throughout Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
TianXuan Developer
+176
-67
@@ -1,11 +1,12 @@
|
||||
"""
|
||||
离线 GeoIP 查询。内嵌 IPv4 区间 → 城市/经纬度映射。
|
||||
离线 GeoIP 查询 — 支持 MMDB (GeoLite2-City / GeoLite2-ASN) 和文本格式回退。
|
||||
|
||||
从 ``data/geoip_data.txt`` 加载 IP 段数据，在模块首次导入时构建
|
||||
排序索引，提供 O(log N) 的 IP 查询能力。
|
||||
优先级：
|
||||
1. GeoLite2-City.mmdb (MaxMind MMDB 格式，城市级精度)
|
||||
2. GeoLite2-ASN.mmdb (可选，提供 ISP / 组织信息)
|
||||
3. geoip_data.txt (文本格式回退，803 段覆盖 30 城市)
|
||||
|
||||
数据约 200KB，覆盖 23 个主要城市 IP 段。
|
||||
可通过编辑 ``data/geoip_data.txt`` 扩展。
|
||||
MMDB 查询使用 ``maxminddb`` 库，文本格式使用二分查找 O(log N)。
|
||||
"""
|
||||
import bisect
|
||||
import logging
|
||||
@@ -18,11 +19,13 @@ logger = logging.getLogger(__name__)
|
||||
# Data structures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# [(start_int, end_int, lat, lon, city, country), ...]
|
||||
# Built at module load time from data/geoip_data.txt
|
||||
_GEO_DATA: list[tuple[int, int, float, float, str, str]] = []
|
||||
# MMDB readers (lazy-loaded, None if unavailable)
|
||||
_mmdb_city: Optional[object] = None # maxminddb.Reader for City DB
|
||||
_mmdb_asn: Optional[object] = None # maxminddb.Reader for ASN DB
|
||||
|
||||
# Sorted list of start integers (for bisect lookup) + parallel data list
|
||||
# Text-format data (fallback)
|
||||
# [(start_int, end_int, lat, lon, city, country), ...]
|
||||
_GEO_DATA: list[tuple[int, int, float, float, str, str]] = []
|
||||
_STARTS: list[int] = []
|
||||
_RANGES: list[tuple[int, int, float, float, str, str]] = []
|
||||
|
||||
@@ -34,15 +37,7 @@ _RANGES: list[tuple[int, int, float, float, str, str]] = []
|
||||
def ip_to_int(ip_str: str) -> Optional[int]:
|
||||
"""Convert an IPv4 dotted-string to its 32-bit integer representation.
|
||||
|
||||
Returns ``None`` if *ip_str* is not a valid IPv4 address (including
|
||||
when any octet is outside 0-255).
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> ip_to_int('8.8.8.8')
|
||||
134744072
|
||||
>>> ip_to_int('1.0.0.0')
|
||||
16777216
|
||||
Returns ``None`` if *ip_str* is not a valid IPv4 address.
|
||||
"""
|
||||
parts = ip_str.strip().split('.')
|
||||
if len(parts) != 4:
|
||||
@@ -57,113 +52,169 @@ def ip_to_int(ip_str: str) -> Optional[int]:
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Index building
|
||||
# MMDB loading
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _load_mmdb() -> None:
|
||||
"""Try to load MaxMind MMDB databases from ``data/`` directory.
|
||||
|
||||
Looks for:
|
||||
- ``data/GeoLite2-City.mmdb`` (primary — city, country, lat/lon)
|
||||
- ``data/GeoLite2-ASN.mmdb`` (optional — ISP, organization, ASN)
|
||||
"""
|
||||
global _mmdb_city, _mmdb_asn
|
||||
|
||||
data_dir = Path(__file__).parent.parent / 'data'
|
||||
|
||||
city_path = data_dir / 'GeoLite2-City.mmdb'
|
||||
asn_path = data_dir / 'GeoLite2-ASN.mmdb'
|
||||
|
||||
if not city_path.exists():
|
||||
logger.info('[GEOIP] GeoLite2-City.mmdb not found at %s — will use text fallback', city_path)
|
||||
return
|
||||
|
||||
try:
|
||||
import maxminddb
|
||||
_mmdb_city = maxminddb.open_database(str(city_path))
|
||||
logger.info('[GEOIP] loaded GeoLite2-City.mmdb (MMDB format)')
|
||||
|
||||
if asn_path.exists():
|
||||
_mmdb_asn = maxminddb.open_database(str(asn_path))
|
||||
logger.info('[GEOIP] loaded GeoLite2-ASN.mmdb (ISP/ORG)')
|
||||
except Exception as e:
|
||||
logger.warning('[GEOIP] failed to open MMDB: %s', e)
|
||||
_mmdb_city = None
|
||||
_mmdb_asn = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Text-format loading (fallback)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _build_index() -> tuple[list[int], list[tuple[int, int, float, float, str, str]]]:
|
||||
"""Build sorted parallel lists for bisect-based IP lookup.
|
||||
|
||||
Processes ``_GEO_DATA``, sorts by start integer, and returns
|
||||
``(starts, ranges)`` where *starts* is a sorted list of start integers
|
||||
and *ranges* is the corresponding data list at the same indices.
|
||||
"""
|
||||
if not _GEO_DATA:
|
||||
return [], []
|
||||
|
||||
sorted_data = sorted(_GEO_DATA, key=lambda x: x[0])
|
||||
starts = [entry[0] for entry in sorted_data]
|
||||
return starts, sorted_data
|
||||
|
||||
|
||||
def _load_data_file() -> None:
|
||||
"""Load IP range data from ``data/geoip_data.txt`` into ``_GEO_DATA``.
|
||||
|
||||
Called once at module import time. Each non-blank, non-comment line
|
||||
in the file should contain six comma-separated fields::
|
||||
|
||||
start_ip,end_ip,lat,lon,city,country
|
||||
"""
|
||||
"""Load IP range data from ``data/geoip_data.txt`` (text fallback)."""
|
||||
data_path = Path(__file__).parent.parent / 'data' / 'geoip_data.txt'
|
||||
|
||||
if not data_path.exists():
|
||||
logger.warning('[GEOIP] data file not found: %s', data_path)
|
||||
logger.warning('[GEOIP] text data file not found: %s', data_path)
|
||||
return
|
||||
|
||||
loaded: list[tuple[int, int, float, float, str, str]] = []
|
||||
skipped = 0
|
||||
|
||||
with open(data_path, 'r', encoding='utf-8') as f:
|
||||
for line_no, raw in enumerate(f, start=1):
|
||||
for raw in f:
|
||||
stripped = raw.strip()
|
||||
if not stripped or stripped.startswith('#'):
|
||||
continue
|
||||
|
||||
parts = [p.strip() for p in stripped.split(',')]
|
||||
if len(parts) != 6:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
start_ip_str, end_ip_str, lat_str, lon_str, city, country = parts
|
||||
|
||||
start_int = ip_to_int(start_ip_str)
|
||||
end_int = ip_to_int(end_ip_str)
|
||||
|
||||
if start_int is None or end_int is None:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
lat = float(lat_str)
|
||||
lon = float(lon_str)
|
||||
except ValueError:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
loaded.append((start_int, end_int, lat, lon, city, country))
|
||||
|
||||
global _GEO_DATA, _STARTS, _RANGES
|
||||
_GEO_DATA.clear()
|
||||
_GEO_DATA.extend(loaded)
|
||||
|
||||
global _STARTS, _RANGES
|
||||
_STARTS, _RANGES = _build_index()
|
||||
|
||||
logger.info('[GEOIP] loaded %d ranges from %s (%d skipped)',
|
||||
logger.info('[GEOIP] text fallback: loaded %d ranges from %s (%d skipped)',
|
||||
len(loaded), data_path, skipped)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# MMDB lookup helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def lookup(ip_str: str) -> Optional[dict]:
|
||||
"""查询 IP 对应的地理位置。
|
||||
def _lookup_mmdb(ip_str: str) -> Optional[dict]:
|
||||
"""Query MMDB database for an IP. Returns rich result with ISP/ORG when
|
||||
ASN database is also available."""
|
||||
if _mmdb_city is None:
|
||||
return None
|
||||
|
||||
Uses binary search (``bisect_right``) on the sorted start-IP list to
|
||||
locate the range *ip_str* falls within, then checks the end bound.
|
||||
try:
|
||||
city_result = _mmdb_city.get(ip_str)
|
||||
if city_result is None:
|
||||
return None
|
||||
|
||||
Parameters
|
||||
----------
|
||||
ip_str:
|
||||
IPv4 address as a dotted string (e.g. ``'8.8.8.8'``).
|
||||
# Extract location
|
||||
location = city_result.get('location', {})
|
||||
lat = location.get('latitude')
|
||||
lon = location.get('longitude')
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict or None
|
||||
A dictionary with keys ``lat``, ``lon``, ``city``, ``country``
|
||||
if the IP is found in the database; ``None`` otherwise.
|
||||
if lat is None or lon is None:
|
||||
return None
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> result = lookup('8.8.8.8')
|
||||
>>> result['city']
|
||||
'Mountain View'
|
||||
>>> lookup('10.0.0.1') is None
|
||||
True
|
||||
"""
|
||||
# Extract names (English preferred)
|
||||
city = None
|
||||
city_names = city_result.get('city', {}).get('names', {})
|
||||
if city_names:
|
||||
city = city_names.get('en') or next(iter(city_names.values()), None)
|
||||
|
||||
country = None
|
||||
country_info = city_result.get('country', {})
|
||||
country = country_info.get('iso_code') or ''
|
||||
country_names = country_info.get('names', {})
|
||||
country_name = country_names.get('en') or next(iter(country_names.values()), None) if country_names else None
|
||||
|
||||
# Subdivisions
|
||||
subdivisions = city_result.get('subdivisions', [])
|
||||
region = None
|
||||
if subdivisions:
|
||||
sub_names = subdivisions[0].get('names', {})
|
||||
region = sub_names.get('en') or next(iter(sub_names.values()), None) if sub_names else None
|
||||
|
||||
result = {
|
||||
'lat': round(float(lat), 4),
|
||||
'lon': round(float(lon), 4),
|
||||
'city': city or region or '',
|
||||
'country': country,
|
||||
'country_name': country_name or '',
|
||||
'region': region or '',
|
||||
}
|
||||
|
||||
# Try ASN database for ISP/ORG
|
||||
if _mmdb_asn is not None:
|
||||
try:
|
||||
asn_result = _mmdb_asn.get(ip_str)
|
||||
if asn_result:
|
||||
result['isp'] = asn_result.get('autonomous_system_organization', '')
|
||||
result['asn'] = asn_result.get('autonomous_system_number', '')
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.debug('[GEOIP] MMDB lookup error for %s: %s', ip_str, e)
|
||||
return None
|
||||
|
||||
|
||||
def _lookup_text(ip_str: str) -> Optional[dict]:
|
||||
"""Query text-format database (fallback)."""
|
||||
target = ip_to_int(ip_str)
|
||||
if target is None:
|
||||
return None
|
||||
|
||||
if not _STARTS:
|
||||
return None
|
||||
|
||||
@@ -178,13 +229,71 @@ def lookup(ip_str: str) -> Optional[dict]:
|
||||
'lon': lon,
|
||||
'city': city,
|
||||
'country': country,
|
||||
'country_name': '',
|
||||
'region': '',
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def lookup(ip_str: str) -> Optional[dict]:
|
||||
"""查询 IP 对应的地理位置。
|
||||
|
||||
Tries MMDB first (if available), then falls back to text format.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
ip_str:
|
||||
IPv4 address as a dotted string (e.g. ``'8.8.8.8'``).
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict or None
|
||||
Keys: ``lat``, ``lon``, ``city``, ``country`` (ISO code),
|
||||
``country_name``, ``region``.
|
||||
When ASN database is available, also includes: ``isp``, ``asn``.
|
||||
"""
|
||||
if not ip_str or ip_str in ('None', '', '+', '-'):
|
||||
return None
|
||||
|
||||
# MMDB first
|
||||
if _mmdb_city is not None:
|
||||
result = _lookup_mmdb(ip_str)
|
||||
if result is not None:
|
||||
return result
|
||||
# MMDB miss — fall through to text
|
||||
|
||||
# Text fallback
|
||||
return _lookup_text(ip_str)
|
||||
|
||||
|
||||
def is_available() -> bool:
|
||||
"""Return True if at least one GeoIP source is loaded."""
|
||||
return _mmdb_city is not None or bool(_STARTS)
|
||||
|
||||
|
||||
def source_name() -> str:
|
||||
"""Return human-readable name of the active GeoIP source."""
|
||||
if _mmdb_city is not None:
|
||||
name = 'GeoLite2-City.mmdb'
|
||||
if _mmdb_asn is not None:
|
||||
name += ' + GeoLite2-ASN.mmdb'
|
||||
return name
|
||||
if _STARTS:
|
||||
return 'geoip_data.txt (text fallback)'
|
||||
return '(none)'
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Module-level initialisation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_load_mmdb()
|
||||
if _mmdb_city is None:
|
||||
_load_data_file()
|
||||
|
||||
logger.info('[GEOIP] active source: %s', source_name())
|
||||
|
||||
@@ -1,5 +1,80 @@
|
||||
# 天璇 — 更新记录
|
||||
|
||||
## 2026-07-23 — Simple Analysis V3 重构
|
||||
|
||||
### 概览
|
||||
对简单分析模块进行 V3 全面重构：GeoIP MMDB 支持、上传预设高级筛选器、IP 通信网络社区发现聚类、节点/边交互增强、离线地图 Cache API 缓存、点表/边表导出、前端空值防护。
|
||||
|
||||
### 变更详情
|
||||
|
||||
#### 1. GeoIP 升级为 MMDB 格式
|
||||
- `analysis/geoip.py`：重写，支持 MaxMind GeoLite2-City.mmdb + GeoLite2-ASN.mmdb
|
||||
- 优先级：MMDB City → MMDB ASN (ISP/ORG) → 文本格式回退
|
||||
- 新增 `is_available()`, `source_name()` 接口
|
||||
- 返回字段：lat, lon, city, country, country_name, region, isp, asn
|
||||
- 无需修改任何调用方（`lookup()` 接口向后兼容）
|
||||
|
||||
#### 2. 上传预设筛选高级化
|
||||
- `simple_analysis/views.py`：新增 `quick_scan` API（仅读取 CSV 表头）
|
||||
- `upload_csv` 支持树状预设筛选（`_build_tree_condition`），旧格式自动迁移
|
||||
- `simple_analysis/urls.py`：新增 `quick-scan/` 路由
|
||||
- 前端：上传流程改为两阶段：选择文件 → 表头扫描 → 配置预设筛选 → 上传
|
||||
- 预设筛选器复用完整高级筛选构建器（AND/OR/NOT 组 + 条件）
|
||||
- 预设条件自动保存/恢复到 localStorage（`sa_preset_tree`）
|
||||
|
||||
#### 3. 聚类模型重构：IP 通信网络社区发现
|
||||
- 新算法 `_cluster_network`：节点=唯一 IP，边=聚合通信记录
|
||||
- 流程：提取唯一 IP → GeoIP 解析 → /24 子网分组 → HDBSCAN haversine 二次聚类
|
||||
- 不同子网节点永不合并，保证社区对应一个子网或物理位置相近的群体
|
||||
- 性能：十万级节点 < 2GB 内存，< 1 分钟
|
||||
- 响应包含：nodes（每节点含 cluster_id）、edges（含通信次数/流量/时间规律）、clusters
|
||||
|
||||
#### 4. 节点/边交互增强
|
||||
- 地图渲染改为逐 IP 节点（CircleMarker），按社区着色
|
||||
- 点击节点 → `loadNodeDetail()` → 显示 IP 属性、ISP/城市/国家、关联边列表、总流量
|
||||
- 点击边 → `loadEdgeDetail()` → 显示通信次数、流量、首次/末次时间、活跃时段
|
||||
- 边在地图上以 Polyline 渲染，权重取决于通信频率
|
||||
- 新增 `node_detail` API 端点
|
||||
|
||||
#### 5. 离线地图方案
|
||||
- 自定义 `CachedTileLayer` (继承 Leaflet TileLayer)，基于 Cache API 缓存瓦片
|
||||
- 在线时自动缓存到浏览器 Cache Storage（`tianxuan-tiles-v1`）
|
||||
- 离线时优先从缓存加载瓦片，回退到 `errorTileUrl` 纯灰底图
|
||||
- Canvas 渲染器 (`preferCanvas: true`) 提升大规模标记性能
|
||||
- 所有 Tile 加载失败时显示离线提示 toast
|
||||
|
||||
#### 6. 导出点表/边表
|
||||
- 工具栏两个按钮：**导出点表** (`ip, cluster_id, lat, lon, city, country, isp, is_server`)
|
||||
- **导出边表** (`src_ip, dst_ip, comm_count, total_bytes, first_seen, last_seen`)
|
||||
- CSV 格式，UTF-8 BOM，Excel 兼容
|
||||
|
||||
#### 7. 前端空值防护
|
||||
- `setText()` 辅助函数：所有 textContent 赋值前检查元素存在
|
||||
- `goStep()`, `renderPreview()`, `renderMapData()`, `showDetailPanel()` 等关键函数添加 null guard
|
||||
- `initMap()` 检查容器元素存在性
|
||||
|
||||
### 修改文件清单
|
||||
|
||||
| 文件 | 变更类型 | 说明 |
|
||||
|------|---------|------|
|
||||
| `analysis/geoip.py` | 重写 | MMDB 支持 + 文本回退 |
|
||||
| `simple_analysis/views.py` | 重写 | quick_scan, 网络聚类, node_detail, 树状预设筛选 |
|
||||
| `simple_analysis/urls.py` | 修改 | 新增 quick-scan, node 路由 |
|
||||
| `simple_analysis/templates/.../simple_analysis.html` | 重写 | 两阶段上传, 网络聚类UI, 节点/边交互, 离线缓存, 双导出 |
|
||||
|
||||
### 启动方式
|
||||
```bash
|
||||
runtime/python/python.exe manage.py runserver 127.0.0.1:8000 --noreload
|
||||
```
|
||||
|
||||
### 获取 GeoLite2-City.mmdb
|
||||
1. 注册 MaxMind 账号：https://www.maxmind.com/en/geolite2/signup
|
||||
2. 下载 GeoLite2-City.mmdb 和 GeoLite2-ASN.mmdb
|
||||
3. 放置到 `data/` 目录下
|
||||
4. 重启服务器，GeoIP 模块自动检测并切换到 MMDB 模式
|
||||
|
||||
---
|
||||
|
||||
## 2026-07-23 — Simple Analysis v2 功能完善
|
||||
|
||||
### 概览
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -4,11 +4,13 @@ from . import views
|
||||
app_name = 'simple_analysis'
|
||||
urlpatterns = [
|
||||
path('', views.index, name='index'),
|
||||
path('quick-scan/', views.quick_scan, name='quick_scan'),
|
||||
path('upload/', views.upload_csv, name='upload'),
|
||||
path('filter/', views.filter_data, name='filter'),
|
||||
path('filter/advanced/', views.advanced_filter, name='advanced_filter'),
|
||||
path('column-values/', views.column_values, name='column_values'),
|
||||
path('cluster/', views.run_clustering, name='cluster'),
|
||||
path('cluster/<int:label>/', views.cluster_detail, name='cluster_detail'),
|
||||
path('node/', views.node_detail, name='node_detail'),
|
||||
path('edges/', views.edge_relationships, name='edges'),
|
||||
]
|
||||
|
||||
+581
-517
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user