fix: 重写gen_multi2, 继承gen_test_data精确格式
- 复用 generate_row() 保证列格式/覆盖率/空值率完全一致 - 覆盖IP和服务字段为拓扑感知值 - cnam→1-3固定服务器IP, snam同理 - 代理出度870-1005, 服务器入度1615-2100 - 200文件×~100行, 列顺序/删减同gen_multi Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
TianXuan Developer
+246
-322
@@ -1,109 +1,98 @@
|
||||
"""Realistic network topology test data generator.
|
||||
"""Realistic network topology data generator — extends gen_test_data format.
|
||||
|
||||
Models realistic client→server communication patterns:
|
||||
- Server nodes: high-degree, many clients connect to them
|
||||
- Proxy/hub nodes: medium-high degree in both directions
|
||||
- Client nodes: moderate degree, connect to multiple services
|
||||
- Service→IP mapping: each snam/cnam maps to specific server IP ranges
|
||||
Uses gen_test_data for exact column format/coverage, overrides IPs with topology:
|
||||
- 13 server nodes (high in-degree): cnam/snam → specific server IPs
|
||||
- 5 proxy nodes (high out-degree): internal gateways
|
||||
- 144 client nodes: connect to multiple servers
|
||||
|
||||
Output format matches gen_test_data EXACTLY (columns, coverages, blanks, types).
|
||||
|
||||
Usage:
|
||||
runtime\\python\\python.exe scripts\\gen_multi2.py [--files N] [--rows N] [--seed SEED]
|
||||
Output: data/gen_multi2/
|
||||
runtime\\python\\python.exe scripts\\gen_multi2.py [--files N] [--rows N]
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import random
|
||||
import csv
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timedelta
|
||||
from collections import defaultdict
|
||||
|
||||
# ── Column spec ──
|
||||
COLUMNS = [
|
||||
':ips', ':ipd', ':prs', ':prd', 'scnt', 'dcnt',
|
||||
':ips.latd', ':ips.lond', ':ipd.latd', ':ipd.lond',
|
||||
':ips.ispn', ':ipd.ispn', ':ips.orgn', ':ipd.orgn',
|
||||
':ips.city', ':ipd.city', ':ips.anon', ':ipd.anon',
|
||||
':ips.doma', ':ipd.doma',
|
||||
'server-ip', 'client-ip', 'time', 'timestamp',
|
||||
'1ipp', '4dbn', 'tabl', '4ksz', 'cnrs', 'isrs',
|
||||
'cnam', '0ver', 'snam', '4dur', '8seq', '2tmo', 'name',
|
||||
'source-node', 'cipher-suite', 'ecdhe-named-curve',
|
||||
'0cph', '0crv', '0rnd', '0rnt',
|
||||
'8ack', '8pak', '8did', '4srs', '8ppk', '8ses', '8byt', '8dbd',
|
||||
'crcc', 'orga', 'orgu', 'eiph', '@iph',
|
||||
]
|
||||
# Add scripts/ to path to import gen_test_data
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
TLS_VERSIONS = ['03 03', '03 04', '02 00', '03 01', '03 02']
|
||||
TLS_WEIGHTS = [0.55, 0.30, 0.05, 0.05, 0.05]
|
||||
CIPHER_HEX = ['c0 2b', 'c0 2f', '13 01', '13 02', 'c0 2c', 'cc a9', 'c0 23',
|
||||
'c0 27', 'c0 13', 'c0 14', '00 9e', '00 9f', '00 35']
|
||||
CIPHER_NAMES = [
|
||||
'TLS_ECDHE_ECDSA_AES128_GCM_SHA256', 'TLS_ECDHE_RSA_AES128_GCM_SHA256',
|
||||
'TLS_AES_128_GCM_SHA256', 'TLS_AES_256_GCM_SHA384',
|
||||
'TLS_ECDHE_ECDSA_AES256_GCM_SHA384', 'TLS_ECDHE_ECDSA_CHACHA20_POLY1305',
|
||||
]
|
||||
NAMED_CURVES_HEX = ['00 1d', '00 17', '00 18', '00 19', '00 1e']
|
||||
NAMED_CURVES_TEXT = ['secp256r1', 'secp384r1', 'secp521r1', 'x25519', 'x448']
|
||||
SRC_NODES = ['packet_capture', 'ssl_logs', 'netflow', 'zeek', 'suricata']
|
||||
NODE_NAMES = ['wan-link', 'core-02', 'gw-09', 'edge-01', 'backbone-03']
|
||||
from gen_test_data import (
|
||||
COLUMNS, COVERAGE, generate_row as _base_generate_row,
|
||||
_rand_ip, _rand_hex, _maybe, _blank_or, _blank_or_plus,
|
||||
TLS_VERSIONS, TLS_WEIGHTS, CIPHER_HEX, CIPHER_NAMES,
|
||||
NAMED_CURVES_HEX, NAMED_CURVES_TEXT,
|
||||
SRC_IPS, DST_IPS, PORTS, COUNTRIES, ISPS, ORGS, CITIES,
|
||||
SERVICES, SRC_NODES, NODE_NAMES,
|
||||
)
|
||||
|
||||
# ── Realistic topology definition ──
|
||||
# ══════════════════════════════════
|
||||
# Network topology
|
||||
# ══════════════════════════════════
|
||||
|
||||
# Geo info per region
|
||||
REGIONS = {
|
||||
'CN-Beijing': {'city': 'Beijing', 'country': 'CN', 'lat': 39.90, 'lon': 116.40, 'isp': 'China Telecom', 'org': 'Alibaba Inc.'},
|
||||
'CN-Shanghai': {'city': 'Shanghai', 'country': 'CN', 'lat': 31.23, 'lon': 121.47, 'isp': 'China Mobile', 'org': 'Tencent'},
|
||||
'US-West': {'city': 'San Francisco','country': 'US', 'lat': 37.77, 'lon': -122.42, 'isp': 'AT&T', 'org': 'Google LLC'},
|
||||
'US-East': {'city': 'New York', 'country': 'US', 'lat': 40.71, 'lon': -74.00, 'isp': 'Verizon', 'org': 'Amazon.com Inc.'},
|
||||
'EU-West': {'city': 'London', 'country': 'GB', 'lat': 51.51, 'lon': -0.13, 'isp': 'BT Group', 'org': 'Microsoft Corp.'},
|
||||
'EU-Central': {'city': 'Frankfurt', 'country': 'DE', 'lat': 50.11, 'lon': 8.68, 'isp': 'Deutsche Telekom', 'org': 'Meta Platforms'},
|
||||
'KR-Seoul': {'city': 'Seoul', 'country': 'KR', 'lat': 37.57, 'lon': 126.98, 'isp': 'SK Telecom', 'org': 'Samsung'},
|
||||
'JP-Tokyo': {'city': 'Tokyo', 'country': 'JP', 'lat': 35.68, 'lon': 139.76, 'isp': 'NTT', 'org': 'Apple Inc.'},
|
||||
'SG': {'city': 'Singapore', 'country': 'SG', 'lat': 1.35, 'lon': 103.82, 'isp': 'Singtel', 'org': 'Netflix Inc.'},
|
||||
'IN-Mumbai': {'city': 'Mumbai', 'country': 'IN', 'lat': 19.08, 'lon': 72.88, 'isp': 'Tata Comms', 'org': 'Microsoft Corp.'},
|
||||
'AU-Sydney': {'city': 'Sydney', 'country': 'AU', 'lat': -33.87, 'lon': 151.21, 'isp': 'Telstra', 'org': 'Amazon.com Inc.'},
|
||||
'BR-SaoPaulo': {'city': 'Sao Paulo', 'country': 'BR', 'lat': -23.55, 'lon': -46.63, 'isp': 'Claro', 'org': 'Meta Platforms'},
|
||||
'CN-Beijing': {'city': 'Beijing', 'country': 'CN', 'lat': 39.90, 'lon': 116.40,
|
||||
'isp': 'China Telecom', 'org': 'Alibaba Inc.'},
|
||||
'CN-Shanghai': {'city': 'Shanghai', 'country': 'CN', 'lat': 31.23, 'lon': 121.47,
|
||||
'isp': 'China Mobile', 'org': 'Tencent'},
|
||||
'US-West': {'city': 'San Francisco','country': 'US', 'lat': 37.77, 'lon': -122.42,
|
||||
'isp': 'AT&T', 'org': 'Google LLC'},
|
||||
'US-East': {'city': 'New York', 'country': 'US', 'lat': 40.71, 'lon': -74.00,
|
||||
'isp': 'Verizon', 'org': 'Amazon.com Inc.'},
|
||||
'EU-West': {'city': 'London', 'country': 'GB', 'lat': 51.51, 'lon': -0.13,
|
||||
'isp': 'BT Group', 'org': 'Microsoft Corp.'},
|
||||
'EU-Central': {'city': 'Frankfurt', 'country': 'DE', 'lat': 50.11, 'lon': 8.68,
|
||||
'isp': 'Deutsche Telekom', 'org': 'Meta Platforms'},
|
||||
'KR-Seoul': {'city': 'Seoul', 'country': 'KR', 'lat': 37.57, 'lon': 126.98,
|
||||
'isp': 'SK Telecom', 'org': 'Samsung'},
|
||||
'JP-Tokyo': {'city': 'Tokyo', 'country': 'JP', 'lat': 35.68, 'lon': 139.76,
|
||||
'isp': 'NTT', 'org': 'Apple Inc.'},
|
||||
'SG': {'city': 'Singapore', 'country': 'SG', 'lat': 1.35, 'lon': 103.82,
|
||||
'isp': 'Singtel', 'org': 'Netflix Inc.'},
|
||||
'IN-Mumbai': {'city': 'Mumbai', 'country': 'IN', 'lat': 19.08, 'lon': 72.88,
|
||||
'isp': 'Tata Comms', 'org': 'Microsoft Corp.'},
|
||||
'AU-Sydney': {'city': 'Sydney', 'country': 'AU', 'lat': -33.87, 'lon': 151.21,
|
||||
'isp': 'Telstra', 'org': 'Amazon.com Inc.'},
|
||||
'BR-SaoPaulo': {'city': 'Sao Paulo', 'country': 'BR', 'lat': -23.55, 'lon': -46.63,
|
||||
'isp': 'Claro', 'org': 'Meta Platforms'},
|
||||
}
|
||||
|
||||
# Server nodes: each has a fixed IP, region, and services
|
||||
# Format: {ip: {region, services: [snam_list], categories: [cnam_list], port, is_server}}
|
||||
# Server nodes: fixed public IPs, region, services/categories
|
||||
SERVER_NODES = {
|
||||
# CDN edges — high traffic, many connections
|
||||
'203.0.113.10': {'region': 'US-West', 'services': ['cdn.example.com', '*.cloudfront.net'],
|
||||
'categories': ['TlsS', 'CDN'], 'port': 443, 'is_cdn': True},
|
||||
'cats': ['TlsS', 'CDN'], 'port': 443},
|
||||
'203.0.113.11': {'region': 'US-East', 'services': ['cdn.example.com', '*.cloudfront.net'],
|
||||
'categories': ['TlsS', 'CDN'], 'port': 443, 'is_cdn': True},
|
||||
'cats': ['TlsS', 'CDN'], 'port': 443},
|
||||
'203.0.113.12': {'region': 'EU-West', 'services': ['cdn.example.com', '*.cloudfront.net'],
|
||||
'categories': ['TlsS', 'CDN'], 'port': 443, 'is_cdn': True},
|
||||
'cats': ['TlsS', 'CDN'], 'port': 443},
|
||||
'198.51.100.20': {'region': 'CN-Shanghai','services': ['api.example.com'],
|
||||
'categories': ['TlsS', 'API'], 'port': 8443, 'is_cdn': False},
|
||||
'cats': ['TlsS', 'API'], 'port': 8443},
|
||||
'198.51.100.21': {'region': 'CN-Beijing', 'services': ['api.example.com'],
|
||||
'categories': ['TlsS', 'API'], 'port': 8443, 'is_cdn': False},
|
||||
# Mail servers
|
||||
'cats': ['TlsS', 'API'], 'port': 8443},
|
||||
'192.0.2.50': {'region': 'US-West', 'services': ['mail.example.com'],
|
||||
'categories': ['TlsS', 'MAIL'], 'port': 465, 'is_cdn': False},
|
||||
'cats': ['TlsS', 'MAIL'], 'port': 465},
|
||||
'192.0.2.51': {'region': 'EU-Central', 'services': ['mail.example.com'],
|
||||
'categories': ['TlsS', 'MAIL'], 'port': 465, 'is_cdn': False},
|
||||
# Auth servers
|
||||
'cats': ['TlsS', 'MAIL'], 'port': 465},
|
||||
'10.20.30.40': {'region': 'US-East', 'services': ['auth.example.net', 'login.live.com'],
|
||||
'categories': ['TlsS', 'AUTH'], 'port': 443, 'is_cdn': False},
|
||||
# Internal / graph API
|
||||
'cats': ['TlsS', 'AUTH'], 'port': 443},
|
||||
'172.16.100.10': {'region': 'US-West', 'services': ['graph.facebook.com'],
|
||||
'categories': ['TlsS', 'SOCIAL'], 'port': 443, 'is_cdn': False},
|
||||
# Storage
|
||||
'cats': ['TlsS', 'SOCIAL'], 'port': 443},
|
||||
'198.51.100.50': {'region': 'US-East', 'services': ['*.s3.amazonaws.com'],
|
||||
'categories': ['TlsS', 'STORAGE'], 'port': 443, 'is_cdn': False},
|
||||
# Streaming
|
||||
'cats': ['TlsS', 'STORAGE'], 'port': 443},
|
||||
'203.0.113.50': {'region': 'SG', 'services': ['stream.example.org'],
|
||||
'categories': ['TlsS', 'STREAM'], 'port': 443, 'is_cdn': False},
|
||||
'cats': ['TlsS', 'STREAM'], 'port': 443},
|
||||
'203.0.113.51': {'region': 'JP-Tokyo', 'services': ['stream.example.org'],
|
||||
'categories': ['TlsS', 'STREAM'], 'port': 443, 'is_cdn': False},
|
||||
# Search / web
|
||||
'cats': ['TlsS', 'STREAM'], 'port': 443},
|
||||
'198.51.100.60': {'region': 'US-West', 'services': ['www.google.com'],
|
||||
'categories': ['TlsS', 'SEARCH'], 'port': 443, 'is_cdn': False},
|
||||
'cats': ['TlsS', 'SEARCH'], 'port': 443},
|
||||
}
|
||||
|
||||
# Proxy/hub nodes: internal gateways that forward traffic
|
||||
# Proxy nodes: internal gateways (appear as :ips connecting to many servers)
|
||||
PROXY_NODES = {
|
||||
'10.0.1.1': {'region': 'CN-Beijing', 'org': 'Baidu Inc.'},
|
||||
'10.0.2.1': {'region': 'CN-Shanghai', 'org': 'Alibaba Inc.'},
|
||||
@@ -112,304 +101,239 @@ PROXY_NODES = {
|
||||
'10.0.5.1': {'region': 'SG', 'org': 'Netflix Inc.'},
|
||||
}
|
||||
|
||||
# Client IP pools per region (these are the ":ips" — source of traffic)
|
||||
CLIENTS_BY_REGION = defaultdict(list)
|
||||
ALL_SERVERS = list(SERVER_NODES.keys())
|
||||
ALL_PROXIES = list(PROXY_NODES.keys())
|
||||
SERVICES_LIST = list({s for sv in SERVER_NODES.values() for s in sv['services']})
|
||||
CATEGORIES_LIST = list({c for sv in SERVER_NODES.values() for c in sv['cats']})
|
||||
|
||||
|
||||
def init_clients():
|
||||
"""Generate client IPs per region."""
|
||||
clients = []
|
||||
ip_idx = 1
|
||||
# ── Client IP pools per region ──
|
||||
CLIENTS_BY_REGION = {}
|
||||
|
||||
|
||||
def init_topology(seed):
|
||||
"""Generate client IPs per region and client→server affinity."""
|
||||
random.seed(seed)
|
||||
for region_key in REGIONS:
|
||||
CLIENTS_BY_REGION[region_key] = []
|
||||
all_clients = []
|
||||
# Generate clients as public-looking IPs
|
||||
ip_counter = 1
|
||||
for region_key in REGIONS:
|
||||
region_clients = []
|
||||
# Each region gets 8-15 client IPs
|
||||
n = random.randint(8, 15)
|
||||
for _ in range(n):
|
||||
client_ip = f'192.168.{random.randint(1, 250)}.{random.randint(1, 254)}'
|
||||
region_clients.append(client_ip)
|
||||
ip_idx += 1
|
||||
CLIENTS_BY_REGION[region_key] = region_clients
|
||||
clients.extend(region_clients)
|
||||
return clients
|
||||
client_ip = f'{random.randint(1,223)}.{random.randint(0,255)}.{random.randint(0,255)}.{random.randint(1,254)}'
|
||||
CLIENTS_BY_REGION[region_key].append(client_ip)
|
||||
all_clients.append(client_ip)
|
||||
ip_counter += 1
|
||||
|
||||
# Build client→server affinity
|
||||
client_servers = {}
|
||||
for client_ip in all_clients:
|
||||
client_region = None
|
||||
for rk, cl in CLIENTS_BY_REGION.items():
|
||||
if client_ip in cl:
|
||||
client_region = rk
|
||||
break
|
||||
region_srvs = [s for s in ALL_SERVERS if SERVER_NODES[s]['region'] == client_region]
|
||||
other_srvs = [s for s in ALL_SERVERS if s not in region_srvs]
|
||||
n_srvs = random.randint(3, 8)
|
||||
chosen = random.sample(region_srvs, min(len(region_srvs), max(1, n_srvs // 2)))
|
||||
rem = n_srvs - len(chosen)
|
||||
if rem > 0 and other_srvs:
|
||||
chosen += random.sample(other_srvs, min(len(other_srvs), rem))
|
||||
client_servers[client_ip] = chosen
|
||||
|
||||
return all_clients, client_servers
|
||||
|
||||
|
||||
def rand_hex(n):
|
||||
return ' '.join(f'{random.randint(0,255):02x}' for _ in range(n))
|
||||
def make_row(row_id, base_time, client_ip, client_region, dst_ip, pattern):
|
||||
"""Generate a row using gen_test_data's exact format, then override IPs+services.
|
||||
|
||||
|
||||
def gen_rows(clients, total_rows_target, base_time):
|
||||
"""Generate rows with realistic network topology.
|
||||
|
||||
Key design:
|
||||
- Each server gets connections from many different clients (high degree).
|
||||
- Proxy nodes appear as both src (:ips) and dst (:ipd) — medium-high degree.
|
||||
- Each client connects to 3-8 different servers (realistic browsing).
|
||||
- cnam/snam determine which server IP is used as :ipd.
|
||||
Returns a dict matching all COLUMNS with proper types, blanks and coverage.
|
||||
"""
|
||||
rows = []
|
||||
servers = list(SERVER_NODES.keys())
|
||||
proxies = list(PROXY_NODES.keys())
|
||||
row_time = base_time
|
||||
# Get a base row with all the right formatting/coverages/blanks
|
||||
row = _base_generate_row(row_id, base_time)
|
||||
|
||||
# Build a client→server affinity: each client prefers certain servers based on region
|
||||
client_affinity = {}
|
||||
for client_ip in clients:
|
||||
# Find which region this client belongs to
|
||||
client_region = None
|
||||
for region_key, client_list in CLIENTS_BY_REGION.items():
|
||||
if client_ip in client_list:
|
||||
client_region = region_key
|
||||
break
|
||||
srv = SERVER_NODES.get(dst_ip)
|
||||
if not srv:
|
||||
srv = random.choice(list(SERVER_NODES.values()))
|
||||
srv_region = srv['region']
|
||||
src_geo = REGIONS.get(client_region, REGIONS['US-West'])
|
||||
srv_geo = REGIONS.get(srv_region, REGIONS['US-West'])
|
||||
|
||||
# Client connects to 3-8 servers, with preference for nearby regions
|
||||
region_servers = [
|
||||
s for s in servers
|
||||
if SERVER_NODES[s]['region'] == client_region
|
||||
] if client_region else []
|
||||
other_servers = [s for s in servers if s not in region_servers]
|
||||
snam_val = random.choice(srv['services'])
|
||||
cnam_val = random.choice(srv['cats'])
|
||||
|
||||
n_servers = random.randint(3, 8)
|
||||
chosen = random.sample(region_servers, min(len(region_servers), max(1, n_servers // 2)))
|
||||
remaining = n_servers - len(chosen)
|
||||
if remaining > 0 and other_servers:
|
||||
chosen += random.sample(other_servers, min(len(other_servers), remaining))
|
||||
client_affinity[client_ip] = chosen
|
||||
# ── Override IPs and services with topology-aware values ──
|
||||
row[':ips'] = client_ip
|
||||
row[':ipd'] = dst_ip
|
||||
row[':prs'] = random.choice(PORTS)
|
||||
row[':prd'] = srv['port']
|
||||
row['server-ip'] = dst_ip
|
||||
row['client-ip'] = client_ip
|
||||
|
||||
# ── Generate rows ──
|
||||
server_connection_counts = defaultdict(int)
|
||||
proxy_src_counts = defaultdict(int)
|
||||
# Service/category — respect coverage from TlsDB.csv
|
||||
row['snam'] = _blank_or(COVERAGE.get('snam', 1.0), snam_val)
|
||||
row['cnam'] = _blank_or(COVERAGE.get('cnam', 1.0), cnam_val)
|
||||
|
||||
for row_id in range(total_rows_target):
|
||||
# Pick a client
|
||||
client_ip = random.choice(clients)
|
||||
client_region = None
|
||||
for region_key, client_list in CLIENTS_BY_REGION.items():
|
||||
if client_ip in client_list:
|
||||
client_region = region_key
|
||||
break
|
||||
# GeoIP — use real coordinates
|
||||
slat = round(srv_geo['lat'] + random.uniform(-0.3, 0.3), 6)
|
||||
slon = round(srv_geo['lon'] + random.uniform(-0.3, 0.3), 6)
|
||||
dlat = round(srv_geo['lat'] + random.uniform(-0.1, 0.1), 6)
|
||||
dlon = round(srv_geo['lon'] + random.uniform(-0.1, 0.1), 6)
|
||||
row[':ips.latd'] = _blank_or_plus(COVERAGE.get(':ips.latd', 1.0), str(slat))
|
||||
row[':ips.lond'] = _blank_or_plus(COVERAGE.get(':ips.lond', 1.0), str(slon))
|
||||
row[':ipd.latd'] = _blank_or_plus(COVERAGE.get(':ipd.latd', 1.0), str(dlat))
|
||||
row[':ipd.lond'] = _blank_or_plus(COVERAGE.get(':ipd.lond', 1.0), str(dlon))
|
||||
row[':ips.ispn'] = _blank_or(COVERAGE.get(':ips.ispn', 1.0), src_geo['isp'])
|
||||
row[':ipd.ispn'] = _blank_or(COVERAGE.get(':ipd.ispn', 1.0), srv_geo['isp'])
|
||||
row[':ips.orgn'] = _blank_or(COVERAGE.get(':ips.orgn', 1.0), src_geo['org'])
|
||||
row[':ipd.orgn'] = _blank_or(COVERAGE.get(':ipd.orgn', 1.0), srv_geo['org'])
|
||||
row[':ips.city'] = _blank_or(COVERAGE.get(':ips.city', 1.0), src_geo['city'])
|
||||
row[':ipd.city'] = _blank_or(COVERAGE.get(':ipd.city', 1.0), srv_geo['city'])
|
||||
row['scnt'] = _blank_or(COVERAGE.get('scnt', 1.0), src_geo['country'].lower())
|
||||
row['dcnt'] = _blank_or(COVERAGE.get('dcnt', 1.0), srv_geo['country'].lower())
|
||||
|
||||
# 70%: client→server, 20%: client→proxy→server, 10%: proxy→server
|
||||
pattern = random.choices(['direct', 'via_proxy', 'proxy_only'], weights=[0.70, 0.20, 0.10])[0]
|
||||
# Domain — matches service
|
||||
row[':ips.doma'] = _blank_or(COVERAGE.get(':ips.doma', 1.0), snam_val)
|
||||
row[':ipd.doma'] = _blank_or(COVERAGE.get(':ipd.doma', 1.0), snam_val)
|
||||
|
||||
if pattern == 'direct':
|
||||
# Client connects directly to one of its servers
|
||||
dst_ip = random.choice(client_affinity.get(client_ip, servers))
|
||||
src_ip = client_ip
|
||||
conn_type = 'direct'
|
||||
# org fields
|
||||
row['orga'] = _blank_or(COVERAGE.get('orga', 1.0), src_geo['org'])
|
||||
|
||||
elif pattern == 'via_proxy':
|
||||
# Client → proxy → server (two rows)
|
||||
proxy_ip = random.choice(proxies)
|
||||
proxy_region = PROXY_NODES[proxy_ip]['region']
|
||||
dst_ip = random.choice(client_affinity.get(client_ip, servers))
|
||||
src_ip = client_ip
|
||||
conn_type = 'via_proxy'
|
||||
# We'll generate the proxy row below
|
||||
proxy_src_counts[proxy_ip] += 1
|
||||
|
||||
else: # proxy_only
|
||||
proxy_ip = random.choice(proxies)
|
||||
dst_ip = random.choice(servers)
|
||||
src_ip = proxy_ip
|
||||
conn_type = 'proxy_only'
|
||||
|
||||
server_connection_counts[dst_ip] += 1
|
||||
|
||||
# Server info
|
||||
srv_info = SERVER_NODES.get(dst_ip)
|
||||
if not srv_info:
|
||||
# This shouldn't happen but fallback
|
||||
srv_info = random.choice(list(SERVER_NODES.values()))
|
||||
|
||||
srv_region = srv_info['region']
|
||||
srv_geo = REGIONS.get(srv_region, REGIONS['US-West'])
|
||||
src_geo = REGIONS.get(client_region, REGIONS['US-West'])
|
||||
|
||||
# Service name from server's list
|
||||
snam = random.choice(srv_info['services'])
|
||||
cnam = random.choice(srv_info['categories'])
|
||||
|
||||
# TLS details
|
||||
tls_ver = random.choices(TLS_VERSIONS, weights=TLS_WEIGHTS, k=1)[0]
|
||||
cipher_hex = random.choice(CIPHER_HEX)
|
||||
cipher_name = random.choice(CIPHER_NAMES)
|
||||
curve_hex = random.choice(NAMED_CURVES_HEX)
|
||||
curve_name = random.choice(NAMED_CURVES_TEXT)
|
||||
|
||||
# Timestamps: spread across 30 days
|
||||
time_str = row_time.strftime('%Y-%m-%d %H:%M:%S.') + f'{row_id % 1000000:06d}'
|
||||
ts = round(row_time.timestamp(), 6)
|
||||
# Advance time unevenly to create time patterns (more traffic during business hours)
|
||||
time_delta = random.choices([1, 5, 10, 30, 60, 300, 600, 1800],
|
||||
weights=[0.25, 0.20, 0.15, 0.10, 0.10, 0.10, 0.05, 0.05])[0]
|
||||
row_time += timedelta(seconds=time_delta)
|
||||
|
||||
# Row
|
||||
row = {
|
||||
':ips': src_ip,
|
||||
':ipd': dst_ip,
|
||||
':prs': random.choice([443, 8443, 465, 993, 80, 8080]),
|
||||
':prd': srv_info['port'],
|
||||
'time': time_str,
|
||||
'timestamp': ts,
|
||||
'0ver': tls_ver,
|
||||
'0cph': cipher_hex,
|
||||
'cipher-suite': cipher_name,
|
||||
'ecdhe-named-curve': curve_name,
|
||||
'0crv': curve_hex,
|
||||
'0rnd': rand_hex(28),
|
||||
'0rnt': rand_hex(4),
|
||||
'4ksz': str(random.choice([128, 256, 384])),
|
||||
'snam': snam,
|
||||
'cnam': cnam,
|
||||
'server-ip': dst_ip,
|
||||
'client-ip': client_ip,
|
||||
'4dur': str(round(random.uniform(0.1, 60.0), 2)),
|
||||
'8ack': str(random.randint(100, 20000)),
|
||||
'8pak': str(random.randint(1, 500)),
|
||||
'8ppk': str(random.randint(1, 200)),
|
||||
'8seq': str(round(random.uniform(1, 1000), 4)),
|
||||
'8ses': str(round(random.uniform(1, 100), 4)),
|
||||
'8byt': str(random.randint(200, 500000)),
|
||||
'2tmo': str(round(random.uniform(0, 500), 4)),
|
||||
'1ipp': str(random.randint(1, 50)),
|
||||
'tabl': random.choice(['TlsC', 'TlsS']),
|
||||
'name': random.choice(NODE_NAMES),
|
||||
'source-node': random.choice(SRC_NODES),
|
||||
'cnrs': '+',
|
||||
'isrs': '+' if random.random() < 0.8 else '',
|
||||
'4dbn': str(random.randint(1, 100)),
|
||||
'8dbd': str(random.randint(1, 200)),
|
||||
'8did': str(random.randint(1000, 9999)),
|
||||
'4srs': str(random.randint(10000, 99999)),
|
||||
':ips.latd': str(round(src_geo['lat'] + random.uniform(-0.5, 0.5), 6)),
|
||||
':ips.lond': str(round(src_geo['lon'] + random.uniform(-0.5, 0.5), 6)),
|
||||
':ipd.latd': str(round(srv_geo['lat'] + random.uniform(-0.1, 0.1), 6)),
|
||||
':ipd.lond': str(round(srv_geo['lon'] + random.uniform(-0.1, 0.1), 6)),
|
||||
':ips.ispn': src_geo['isp'],
|
||||
':ipd.ispn': srv_geo['isp'],
|
||||
':ips.orgn': src_geo['org'],
|
||||
':ipd.orgn': srv_geo['org'],
|
||||
':ips.city': src_geo['city'],
|
||||
':ipd.city': srv_geo['city'],
|
||||
':ips.anon': '',
|
||||
':ipd.anon': '',
|
||||
':ips.doma': snam,
|
||||
':ipd.doma': snam,
|
||||
'scnt': src_geo['country'].lower(),
|
||||
'dcnt': srv_geo['country'].lower(),
|
||||
'crcc': random.choice(['OK', 'OK', 'OK', 'ER']),
|
||||
'@iph': random.choice(['A', 'B', 'C']),
|
||||
'eiph': '+',
|
||||
'orga': src_geo['org'],
|
||||
'orgu': str(random.randint(1000, 99999)),
|
||||
}
|
||||
rows.append(row)
|
||||
|
||||
# For via_proxy pattern, also generate the proxy→server connection row
|
||||
if pattern == 'via_proxy' and random.random() < 0.7:
|
||||
proxy_region = PROXY_NODES[proxy_ip]['region']
|
||||
proxy_geo = REGIONS.get(proxy_region, REGIONS['US-West'])
|
||||
proxy_row = dict(row)
|
||||
proxy_row[':ips'] = proxy_ip
|
||||
proxy_row['client-ip'] = proxy_ip
|
||||
proxy_row[':ips.latd'] = str(round(proxy_geo['lat'] + random.uniform(-0.3, 0.3), 6))
|
||||
proxy_row[':ips.lond'] = str(round(proxy_geo['lon'] + random.uniform(-0.3, 0.3), 6))
|
||||
proxy_row[':ips.ispn'] = proxy_geo['isp']
|
||||
proxy_row[':ips.orgn'] = PROXY_NODES[proxy_ip]['org']
|
||||
proxy_row[':ips.city'] = proxy_geo['city']
|
||||
proxy_row['scnt'] = proxy_geo['country'].lower()
|
||||
proxy_row['orga'] = PROXY_NODES[proxy_ip]['org']
|
||||
rows.append(proxy_row)
|
||||
server_connection_counts[dst_ip] += 1
|
||||
|
||||
# Print topology stats
|
||||
print(f'\n Topology:')
|
||||
print(f' Servers: {len(SERVER_NODES)}')
|
||||
print(f' Proxies: {len(PROXY_NODES)}')
|
||||
print(f' Clients: {len(set(clients))}')
|
||||
print(f' Total rows: {len(rows)}')
|
||||
|
||||
# Degree distribution
|
||||
src_degrees = defaultdict(int)
|
||||
dst_degrees = defaultdict(int)
|
||||
for r in rows:
|
||||
src_degrees[r[':ips']] += 1
|
||||
dst_degrees[r[':ipd']] += 1
|
||||
|
||||
top_src = sorted(src_degrees.items(), key=lambda x: -x[1])[:10]
|
||||
top_dst = sorted(dst_degrees.items(), key=lambda x: -x[1])[:10]
|
||||
print(f' Top source IPs (out-degree):')
|
||||
for ip, deg in top_src:
|
||||
is_proxy = ' [proxy]' if ip in PROXY_NODES else (' [client]' if ip in CLIENTS_BY_REGION.get('US-West', []) or any(ip in v for v in CLIENTS_BY_REGION.values()) else '')
|
||||
print(f' {ip}: {deg}{is_proxy}')
|
||||
print(f' Top dest IPs (in-degree):')
|
||||
for ip, deg in top_dst:
|
||||
print(f' {ip}: {deg}{" [server]" if ip in SERVER_NODES else ""}')
|
||||
|
||||
return rows
|
||||
return row
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description='Generate realistic network topology test data')
|
||||
parser = argparse.ArgumentParser(description='Generate topology-aware TLS test data')
|
||||
parser.add_argument('--files', type=int, default=200, help='Number of CSV files')
|
||||
parser.add_argument('--rows', type=int, default=80, help='Rows per file (approximate)')
|
||||
parser.add_argument('--rows', type=int, default=100, help='Approximate rows per file')
|
||||
parser.add_argument('--seed', type=int, default=2026, help='Random seed')
|
||||
args = parser.parse_args()
|
||||
|
||||
random.seed(args.seed)
|
||||
out_dir = Path('data/gen_multi2')
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
print(f'Generating {args.files} files to {out_dir}...')
|
||||
print(f'Generating {args.files} files × ~{args.rows} rows → {out_dir}')
|
||||
print(f'Seed: {args.seed}')
|
||||
|
||||
# Initialize clients
|
||||
clients = init_clients()
|
||||
total_clients = len(set(clients))
|
||||
print(f'\n Regions: {len(REGIONS)}, Clients per region: {total_clients // len(REGIONS)}-{total_clients // len(REGIONS) + 5}')
|
||||
# Init topology
|
||||
all_clients, client_servers = init_topology(args.seed)
|
||||
servers = ALL_SERVERS
|
||||
proxies = ALL_PROXIES
|
||||
|
||||
random.seed(args.seed + 1)
|
||||
base_time = datetime(2026, 6, 1, 0, 0, 0)
|
||||
row_counter = 0
|
||||
|
||||
# Generate all rows at once so topology is consistent across files
|
||||
total_rows_target = args.files * args.rows
|
||||
all_rows = gen_rows(clients, total_rows_target, base_time)
|
||||
|
||||
# Distribute rows across files
|
||||
random.shuffle(all_rows)
|
||||
rows_per_file = len(all_rows) // args.files
|
||||
|
||||
# Column coverage (which columns to include per file)
|
||||
_keep_cols = {':ips', ':ipd', ':prs', ':prd', 'server-ip', 'client-ip', 'time', 'timestamp', 'snam', 'cnam'}
|
||||
# Identity columns that must never be dropped
|
||||
_keep_cols = {':ips', ':ipd', ':prs', ':prd', 'server-ip', 'client-ip'}
|
||||
_droppable = [c for c in COLUMNS if c not in _keep_cols]
|
||||
|
||||
total_written = 0
|
||||
for fi in range(args.files):
|
||||
start_idx = fi * rows_per_file
|
||||
end_idx = start_idx + rows_per_file if fi < args.files - 1 else len(all_rows)
|
||||
file_rows = all_rows[start_idx:end_idx]
|
||||
total_rows = 0
|
||||
degree_src = defaultdict(int)
|
||||
degree_dst = defaultdict(int)
|
||||
|
||||
# Randomly drop some columns per file for variety
|
||||
for fi in range(args.files):
|
||||
# Column dropping per file (matching gen_multi.py logic)
|
||||
dropped = set()
|
||||
for col in _droppable:
|
||||
if random.random() < 0.30: # 30% drop rate per column
|
||||
cov = COVERAGE.get(col, 1.0)
|
||||
drop_prob = max(0.10, min(0.90, 1.0 - cov))
|
||||
if random.random() < drop_prob:
|
||||
dropped.add(col)
|
||||
if len(dropped) < 3:
|
||||
extra = random.sample([c for c in _droppable if c not in dropped],
|
||||
k=min(3 - len(dropped), len(_droppable) - len(dropped)))
|
||||
dropped.update(extra)
|
||||
if len(dropped) == len(_droppable):
|
||||
spared = set(random.sample(list(dropped), k=min(3, len(dropped))))
|
||||
dropped -= spared
|
||||
file_columns = [c for c in COLUMNS if c not in dropped]
|
||||
|
||||
n_rows = random.randint(max(50, args.rows - 10), args.rows + 10)
|
||||
|
||||
# Generate rows
|
||||
file_rows = []
|
||||
file_base = base_time + timedelta(seconds=random.randint(0, 86400 * 30))
|
||||
for ri in range(n_rows):
|
||||
# Pick client and server using topology
|
||||
client_ip = random.choice(all_clients)
|
||||
client_region = None
|
||||
for rk, cl in CLIENTS_BY_REGION.items():
|
||||
if client_ip in cl:
|
||||
client_region = rk
|
||||
break
|
||||
|
||||
# Pattern: 70% direct, 20% via proxy, 10% proxy only
|
||||
pattern = random.choices(['direct', 'via_proxy', 'proxy_only'],
|
||||
weights=[0.70, 0.20, 0.10])[0]
|
||||
|
||||
if pattern == 'proxy_only':
|
||||
src_ip = random.choice(proxies)
|
||||
proxy_region = PROXY_NODES[src_ip]['region']
|
||||
dst_ip = random.choice(servers)
|
||||
row = make_row(row_counter, file_base, src_ip, proxy_region, dst_ip, pattern)
|
||||
file_rows.append(row)
|
||||
degree_src[src_ip] += 1
|
||||
degree_dst[dst_ip] += 1
|
||||
row_counter += 1
|
||||
elif pattern == 'via_proxy':
|
||||
# Client → server (direct connection row)
|
||||
dst_ip = random.choice(client_servers.get(client_ip, servers))
|
||||
row = make_row(row_counter, file_base, client_ip, client_region, dst_ip, 'direct')
|
||||
file_rows.append(row)
|
||||
degree_src[client_ip] += 1
|
||||
degree_dst[dst_ip] += 1
|
||||
row_counter += 1
|
||||
# Also generate proxy→server row if lucky
|
||||
if random.random() < 0.7:
|
||||
proxy_ip = random.choice(proxies)
|
||||
proxy_region = PROXY_NODES[proxy_ip]['region']
|
||||
row2 = make_row(row_counter, file_base, proxy_ip, proxy_region, dst_ip, 'via_proxy')
|
||||
file_rows.append(row2)
|
||||
degree_src[proxy_ip] += 1
|
||||
degree_dst[dst_ip] += 1
|
||||
row_counter += 1
|
||||
else:
|
||||
dst_ip = random.choice(client_servers.get(client_ip, servers))
|
||||
row = make_row(row_counter, file_base, client_ip, client_region, dst_ip, 'direct')
|
||||
file_rows.append(row)
|
||||
degree_src[client_ip] += 1
|
||||
degree_dst[dst_ip] += 1
|
||||
row_counter += 1
|
||||
|
||||
# Shuffle column order per file (matching gen_test_data)
|
||||
shuffled = file_columns[:]
|
||||
random.shuffle(shuffled)
|
||||
|
||||
# Write
|
||||
out_path = out_dir / f'{fi}.csv'
|
||||
with open(out_path, 'w', newline='', encoding='utf-8') as f:
|
||||
writer = csv.DictWriter(f, fieldnames=file_columns, extrasaction='ignore')
|
||||
writer = csv.DictWriter(f, fieldnames=shuffled, extrasaction='ignore')
|
||||
writer.writeheader()
|
||||
for row in file_rows:
|
||||
writer.writerow({k: v for k, v in row.items() if k in file_columns})
|
||||
writer.writerow(row)
|
||||
|
||||
total_written += len(file_rows)
|
||||
total_rows += len(file_rows)
|
||||
|
||||
if (fi + 1) % 50 == 0:
|
||||
print(f' [{fi + 1}/{args.files}] {total_written} rows written...')
|
||||
print(f' [{fi + 1}/{args.files}] {total_rows} rows...')
|
||||
|
||||
print(f'\nDone: {args.files} files, {total_written} total rows → {out_dir.resolve()}')
|
||||
# Stats
|
||||
top_src = sorted(degree_src.items(), key=lambda x: -x[1])[:10]
|
||||
top_dst = sorted(degree_dst.items(), key=lambda x: -x[1])[:10]
|
||||
print(f'\nDone: {args.files} files, {total_rows} rows → {out_dir.resolve()}')
|
||||
print(f' Servers: {len(servers)}, Proxies: {len(proxies)}, Clients: {len(set(all_clients))}')
|
||||
print(f' Top source (out-degree):')
|
||||
for ip, deg in top_src:
|
||||
label = ' [proxy]' if ip in PROXY_NODES else ' [client]'
|
||||
print(f' {ip}: {deg}{label}')
|
||||
print(f' Top dest (in-degree):')
|
||||
for ip, deg in top_dst:
|
||||
print(f' {ip}: {deg} [server]')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
Reference in New Issue
Block a user