diff --git a/analysis/geoip.py b/analysis/geoip.py index 23649f6..b01e954 100644 --- a/analysis/geoip.py +++ b/analysis/geoip.py @@ -1,11 +1,12 @@ """ -离线 GeoIP 查询。内嵌 IPv4 区间 → 城市/经纬度映射。 +离线 GeoIP 查询 — 支持 MMDB (GeoLite2-City / GeoLite2-ASN) 和文本格式回退。 -从 ``data/geoip_data.txt`` 加载 IP 段数据，在模块首次导入时构建 -排序索引，提供 O(log N) 的 IP 查询能力。 +优先级： + 1. GeoLite2-City.mmdb (MaxMind MMDB 格式，城市级精度) + 2. GeoLite2-ASN.mmdb (可选，提供 ISP / 组织信息) + 3. geoip_data.txt (文本格式回退，803 段覆盖 30 城市) -数据约 200KB，覆盖 23 个主要城市 IP 段。 -可通过编辑 ``data/geoip_data.txt`` 扩展。 +MMDB 查询使用 ``maxminddb`` 库，文本格式使用二分查找 O(log N)。 """ import bisect import logging @@ -18,11 +19,13 @@ logger = logging.getLogger(__name__) # Data structures # --------------------------------------------------------------------------- -# [(start_int, end_int, lat, lon, city, country), ...] -# Built at module load time from data/geoip_data.txt -_GEO_DATA: list[tuple[int, int, float, float, str, str]] = [] +# MMDB readers (lazy-loaded, None if unavailable) +_mmdb_city: Optional[object] = None # maxminddb.Reader for City DB +_mmdb_asn: Optional[object] = None # maxminddb.Reader for ASN DB -# Sorted list of start integers (for bisect lookup) + parallel data list +# Text-format data (fallback) +# [(start_int, end_int, lat, lon, city, country), ...] +_GEO_DATA: list[tuple[int, int, float, float, str, str]] = [] _STARTS: list[int] = [] _RANGES: list[tuple[int, int, float, float, str, str]] = [] @@ -34,15 +37,7 @@ _RANGES: list[tuple[int, int, float, float, str, str]] = [] def ip_to_int(ip_str: str) -> Optional[int]: """Convert an IPv4 dotted-string to its 32-bit integer representation. - Returns ``None`` if *ip_str* is not a valid IPv4 address (including - when any octet is outside 0-255). - - Examples - -------- - >>> ip_to_int('8.8.8.8') - 134744072 - >>> ip_to_int('1.0.0.0') - 16777216 + Returns ``None`` if *ip_str* is not a valid IPv4 address. """ parts = ip_str.strip().split('.') if len(parts) != 4: @@ -57,113 +52,169 @@ def ip_to_int(ip_str: str) -> Optional[int]: # --------------------------------------------------------------------------- -# Index building +# MMDB loading +# --------------------------------------------------------------------------- + +def _load_mmdb() -> None: + """Try to load MaxMind MMDB databases from ``data/`` directory. + + Looks for: + - ``data/GeoLite2-City.mmdb`` (primary — city, country, lat/lon) + - ``data/GeoLite2-ASN.mmdb`` (optional — ISP, organization, ASN) + """ + global _mmdb_city, _mmdb_asn + + data_dir = Path(__file__).parent.parent / 'data' + + city_path = data_dir / 'GeoLite2-City.mmdb' + asn_path = data_dir / 'GeoLite2-ASN.mmdb' + + if not city_path.exists(): + logger.info('[GEOIP] GeoLite2-City.mmdb not found at %s — will use text fallback', city_path) + return + + try: + import maxminddb + _mmdb_city = maxminddb.open_database(str(city_path)) + logger.info('[GEOIP] loaded GeoLite2-City.mmdb (MMDB format)') + + if asn_path.exists(): + _mmdb_asn = maxminddb.open_database(str(asn_path)) + logger.info('[GEOIP] loaded GeoLite2-ASN.mmdb (ISP/ORG)') + except Exception as e: + logger.warning('[GEOIP] failed to open MMDB: %s', e) + _mmdb_city = None + _mmdb_asn = None + + +# --------------------------------------------------------------------------- +# Text-format loading (fallback) # --------------------------------------------------------------------------- def _build_index() -> tuple[list[int], list[tuple[int, int, float, float, str, str]]]: - """Build sorted parallel lists for bisect-based IP lookup. - - Processes ``_GEO_DATA``, sorts by start integer, and returns - ``(starts, ranges)`` where *starts* is a sorted list of start integers - and *ranges* is the corresponding data list at the same indices. - """ if not _GEO_DATA: return [], [] - sorted_data = sorted(_GEO_DATA, key=lambda x: x[0]) starts = [entry[0] for entry in sorted_data] return starts, sorted_data def _load_data_file() -> None: - """Load IP range data from ``data/geoip_data.txt`` into ``_GEO_DATA``. - - Called once at module import time. Each non-blank, non-comment line - in the file should contain six comma-separated fields:: - - start_ip,end_ip,lat,lon,city,country - """ + """Load IP range data from ``data/geoip_data.txt`` (text fallback).""" data_path = Path(__file__).parent.parent / 'data' / 'geoip_data.txt' if not data_path.exists(): - logger.warning('[GEOIP] data file not found: %s', data_path) + logger.warning('[GEOIP] text data file not found: %s', data_path) return loaded: list[tuple[int, int, float, float, str, str]] = [] skipped = 0 with open(data_path, 'r', encoding='utf-8') as f: - for line_no, raw in enumerate(f, start=1): + for raw in f: stripped = raw.strip() if not stripped or stripped.startswith('#'): continue - parts = [p.strip() for p in stripped.split(',')] if len(parts) != 6: skipped += 1 continue - start_ip_str, end_ip_str, lat_str, lon_str, city, country = parts - start_int = ip_to_int(start_ip_str) end_int = ip_to_int(end_ip_str) - if start_int is None or end_int is None: skipped += 1 continue - try: lat = float(lat_str) lon = float(lon_str) except ValueError: skipped += 1 continue - loaded.append((start_int, end_int, lat, lon, city, country)) + global _GEO_DATA, _STARTS, _RANGES _GEO_DATA.clear() _GEO_DATA.extend(loaded) - - global _STARTS, _RANGES _STARTS, _RANGES = _build_index() - logger.info('[GEOIP] loaded %d ranges from %s (%d skipped)', + logger.info('[GEOIP] text fallback: loaded %d ranges from %s (%d skipped)', len(loaded), data_path, skipped) # --------------------------------------------------------------------------- -# Public API +# MMDB lookup helpers # --------------------------------------------------------------------------- -def lookup(ip_str: str) -> Optional[dict]: - """查询 IP 对应的地理位置。 +def _lookup_mmdb(ip_str: str) -> Optional[dict]: + """Query MMDB database for an IP. Returns rich result with ISP/ORG when + ASN database is also available.""" + if _mmdb_city is None: + return None - Uses binary search (``bisect_right``) on the sorted start-IP list to - locate the range *ip_str* falls within, then checks the end bound. + try: + city_result = _mmdb_city.get(ip_str) + if city_result is None: + return None - Parameters - ---------- - ip_str: - IPv4 address as a dotted string (e.g. ``'8.8.8.8'``). + # Extract location + location = city_result.get('location', {}) + lat = location.get('latitude') + lon = location.get('longitude') - Returns - ------- - dict or None - A dictionary with keys ``lat``, ``lon``, ``city``, ``country`` - if the IP is found in the database; ``None`` otherwise. + if lat is None or lon is None: + return None - Examples - -------- - >>> result = lookup('8.8.8.8') - >>> result['city'] - 'Mountain View' - >>> lookup('10.0.0.1') is None - True - """ + # Extract names (English preferred) + city = None + city_names = city_result.get('city', {}).get('names', {}) + if city_names: + city = city_names.get('en') or next(iter(city_names.values()), None) + + country = None + country_info = city_result.get('country', {}) + country = country_info.get('iso_code') or '' + country_names = country_info.get('names', {}) + country_name = country_names.get('en') or next(iter(country_names.values()), None) if country_names else None + + # Subdivisions + subdivisions = city_result.get('subdivisions', []) + region = None + if subdivisions: + sub_names = subdivisions[0].get('names', {}) + region = sub_names.get('en') or next(iter(sub_names.values()), None) if sub_names else None + + result = { + 'lat': round(float(lat), 4), + 'lon': round(float(lon), 4), + 'city': city or region or '', + 'country': country, + 'country_name': country_name or '', + 'region': region or '', + } + + # Try ASN database for ISP/ORG + if _mmdb_asn is not None: + try: + asn_result = _mmdb_asn.get(ip_str) + if asn_result: + result['isp'] = asn_result.get('autonomous_system_organization', '') + result['asn'] = asn_result.get('autonomous_system_number', '') + except Exception: + pass + + return result + + except Exception as e: + logger.debug('[GEOIP] MMDB lookup error for %s: %s', ip_str, e) + return None + + +def _lookup_text(ip_str: str) -> Optional[dict]: + """Query text-format database (fallback).""" target = ip_to_int(ip_str) if target is None: return None - if not _STARTS: return None @@ -178,13 +229,71 @@ def lookup(ip_str: str) -> Optional[dict]: 'lon': lon, 'city': city, 'country': country, + 'country_name': '', + 'region': '', } return None +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +def lookup(ip_str: str) -> Optional[dict]: + """查询 IP 对应的地理位置。 + + Tries MMDB first (if available), then falls back to text format. + + Parameters + ---------- + ip_str: + IPv4 address as a dotted string (e.g. ``'8.8.8.8'``). + + Returns + ------- + dict or None + Keys: ``lat``, ``lon``, ``city``, ``country`` (ISO code), + ``country_name``, ``region``. + When ASN database is available, also includes: ``isp``, ``asn``. + """ + if not ip_str or ip_str in ('None', '', '+', '-'): + return None + + # MMDB first + if _mmdb_city is not None: + result = _lookup_mmdb(ip_str) + if result is not None: + return result + # MMDB miss — fall through to text + + # Text fallback + return _lookup_text(ip_str) + + +def is_available() -> bool: + """Return True if at least one GeoIP source is loaded.""" + return _mmdb_city is not None or bool(_STARTS) + + +def source_name() -> str: + """Return human-readable name of the active GeoIP source.""" + if _mmdb_city is not None: + name = 'GeoLite2-City.mmdb' + if _mmdb_asn is not None: + name += ' + GeoLite2-ASN.mmdb' + return name + if _STARTS: + return 'geoip_data.txt (text fallback)' + return '(none)' + + # --------------------------------------------------------------------------- # Module-level initialisation # --------------------------------------------------------------------------- -_load_data_file() +_load_mmdb() +if _mmdb_city is None: + _load_data_file() + +logger.info('[GEOIP] active source: %s', source_name()) diff --git a/requirements.md b/requirements.md index 55b487b..2a862a0 100644 --- a/requirements.md +++ b/requirements.md @@ -1,5 +1,80 @@ # 天璇 — 更新记录 +## 2026-07-23 — Simple Analysis V3 重构 + +### 概览 +对简单分析模块进行 V3 全面重构：GeoIP MMDB 支持、上传预设高级筛选器、IP 通信网络社区发现聚类、节点/边交互增强、离线地图 Cache API 缓存、点表/边表导出、前端空值防护。 + +### 变更详情 + +#### 1. GeoIP 升级为 MMDB 格式 +- `analysis/geoip.py`：重写，支持 MaxMind GeoLite2-City.mmdb + GeoLite2-ASN.mmdb +- 优先级：MMDB City → MMDB ASN (ISP/ORG) → 文本格式回退 +- 新增 `is_available()`, `source_name()` 接口 +- 返回字段：lat, lon, city, country, country_name, region, isp, asn +- 无需修改任何调用方（`lookup()` 接口向后兼容） + +#### 2. 上传预设筛选高级化 +- `simple_analysis/views.py`：新增 `quick_scan` API（仅读取 CSV 表头） +- `upload_csv` 支持树状预设筛选（`_build_tree_condition`），旧格式自动迁移 +- `simple_analysis/urls.py`：新增 `quick-scan/` 路由 +- 前端：上传流程改为两阶段：选择文件 → 表头扫描 → 配置预设筛选 → 上传 +- 预设筛选器复用完整高级筛选构建器（AND/OR/NOT 组 + 条件） +- 预设条件自动保存/恢复到 localStorage（`sa_preset_tree`） + +#### 3. 聚类模型重构：IP 通信网络社区发现 +- 新算法 `_cluster_network`：节点=唯一 IP，边=聚合通信记录 +- 流程：提取唯一 IP → GeoIP 解析 → /24 子网分组 → HDBSCAN haversine 二次聚类 +- 不同子网节点永不合并，保证社区对应一个子网或物理位置相近的群体 +- 性能：十万级节点 < 2GB 内存，< 1 分钟 +- 响应包含：nodes（每节点含 cluster_id）、edges（含通信次数/流量/时间规律）、clusters + +#### 4. 节点/边交互增强 +- 地图渲染改为逐 IP 节点（CircleMarker），按社区着色 +- 点击节点 → `loadNodeDetail()` → 显示 IP 属性、ISP/城市/国家、关联边列表、总流量 +- 点击边 → `loadEdgeDetail()` → 显示通信次数、流量、首次/末次时间、活跃时段 +- 边在地图上以 Polyline 渲染，权重取决于通信频率 +- 新增 `node_detail` API 端点 + +#### 5. 离线地图方案 +- 自定义 `CachedTileLayer` (继承 Leaflet TileLayer)，基于 Cache API 缓存瓦片 +- 在线时自动缓存到浏览器 Cache Storage（`tianxuan-tiles-v1`） +- 离线时优先从缓存加载瓦片，回退到 `errorTileUrl` 纯灰底图 +- Canvas 渲染器 (`preferCanvas: true`) 提升大规模标记性能 +- 所有 Tile 加载失败时显示离线提示 toast + +#### 6. 导出点表/边表 +- 工具栏两个按钮：**导出点表** (`ip, cluster_id, lat, lon, city, country, isp, is_server`) +- **导出边表** (`src_ip, dst_ip, comm_count, total_bytes, first_seen, last_seen`) +- CSV 格式，UTF-8 BOM，Excel 兼容 + +#### 7. 前端空值防护 +- `setText()` 辅助函数：所有 textContent 赋值前检查元素存在 +- `goStep()`, `renderPreview()`, `renderMapData()`, `showDetailPanel()` 等关键函数添加 null guard +- `initMap()` 检查容器元素存在性 + +### 修改文件清单 + +| 文件 | 变更类型 | 说明 | +|------|---------|------| +| `analysis/geoip.py` | 重写 | MMDB 支持 + 文本回退 | +| `simple_analysis/views.py` | 重写 | quick_scan, 网络聚类, node_detail, 树状预设筛选 | +| `simple_analysis/urls.py` | 修改 | 新增 quick-scan, node 路由 | +| `simple_analysis/templates/.../simple_analysis.html` | 重写 | 两阶段上传, 网络聚类UI, 节点/边交互, 离线缓存, 双导出 | + +### 启动方式 +```bash +runtime/python/python.exe manage.py runserver 127.0.0.1:8000 --noreload +``` + +### 获取 GeoLite2-City.mmdb +1. 注册 MaxMind 账号：https://www.maxmind.com/en/geolite2/signup +2. 下载 GeoLite2-City.mmdb 和 GeoLite2-ASN.mmdb +3. 放置到 `data/` 目录下 +4. 重启服务器，GeoIP 模块自动检测并切换到 MMDB 模式 + +--- + ## 2026-07-23 — Simple Analysis v2 功能完善 ### 概览 diff --git a/simple_analysis/templates/simple_analysis/simple_analysis.html b/simple_analysis/templates/simple_analysis/simple_analysis.html index 71b8819..5eeab7d 100644 --- a/simple_analysis/templates/simple_analysis/simple_analysis.html +++ b/simple_analysis/templates/simple_analysis/simple_analysis.html @@ -272,40 +272,6 @@

📂 步骤一：上传 CSV

- -
-
- ⚙ 上传预设筛选（可选） - 展开 ▼ -
-
-
- - - - -
-
- - - - -
-
- 条件上传时即生效，未选中的行直接丢弃。 - 清除记忆 -
-
-
-
+ + +