fix: gen_multi2 完全继承 gen_test_data 格式 + 拓扑约束
- 复用 generate_row() 保证所有列格式/覆盖率/空值率一致 - 重映射: snam/cnam 有值时 → :ipd 指向1-3个固定服务器IP - 同步更新 GeoIP 字段匹配服务器位置 - 200文件 × 10000行 = 200万行 - 代理出度60K+, 服务器入度50K-202K Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
TianXuan Developer
+138
-251
@@ -1,243 +1,128 @@
|
||||
"""Realistic network topology data generator — extends gen_test_data format.
|
||||
"""Topology-aware test data generator — exact gen_test_data format.
|
||||
|
||||
Uses gen_test_data for exact column format/coverage, overrides IPs with topology:
|
||||
- 13 server nodes (high in-degree): cnam/snam → specific server IPs
|
||||
- 5 proxy nodes (high out-degree): internal gateways
|
||||
- 144 client nodes: connect to multiple servers
|
||||
|
||||
Output format matches gen_test_data EXACTLY (columns, coverages, blanks, types).
|
||||
Strategy:
|
||||
1. Generate rows using gen_test_data.generate_row() (EXACT same format/coverages)
|
||||
2. Then remap IPs: when snam/cnam has a value, assign :ipd from a fixed server pool
|
||||
3. Result: same format as gen_test_data, but with realistic network topology
|
||||
|
||||
Usage:
|
||||
runtime\\python\\python.exe scripts\\gen_multi2.py [--files N] [--rows N]
|
||||
runtime\\python\\python.exe scripts\\gen_multi2.py [--files N] [--rows N] [--seed S]
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import random
|
||||
import csv
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timedelta
|
||||
from collections import defaultdict
|
||||
|
||||
# Add scripts/ to path to import gen_test_data
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
from gen_test_data import (
|
||||
COLUMNS, COVERAGE, generate_row as _base_generate_row,
|
||||
_rand_ip, _rand_hex, _maybe, _blank_or, _blank_or_plus,
|
||||
COLUMNS, COVERAGE,
|
||||
generate_row as _base_row,
|
||||
TLS_VERSIONS, TLS_WEIGHTS, CIPHER_HEX, CIPHER_NAMES,
|
||||
NAMED_CURVES_HEX, NAMED_CURVES_TEXT,
|
||||
SRC_IPS, DST_IPS, PORTS, COUNTRIES, ISPS, ORGS, CITIES,
|
||||
SERVICES, SRC_NODES, NODE_NAMES,
|
||||
_rand_ip, _rand_hex, _maybe, _blank_or, _blank_or_plus,
|
||||
)
|
||||
|
||||
# ══════════════════════════════════
|
||||
# Network topology
|
||||
# ══════════════════════════════════
|
||||
# ════════════════════════════════
|
||||
# Server IP pools per service
|
||||
# ════════════════════════════════
|
||||
# Each snam/cnam maps to 1-3 specific server IPs.
|
||||
# When a row has a given snam/cnam, its :ipd is replaced with one of these IPs.
|
||||
# This creates realistic topology — servers have high in-degree.
|
||||
|
||||
REGIONS = {
|
||||
'CN-Beijing': {'city': 'Beijing', 'country': 'CN', 'lat': 39.90, 'lon': 116.40,
|
||||
'isp': 'China Telecom', 'org': 'Alibaba Inc.'},
|
||||
'CN-Shanghai': {'city': 'Shanghai', 'country': 'CN', 'lat': 31.23, 'lon': 121.47,
|
||||
'isp': 'China Mobile', 'org': 'Tencent'},
|
||||
'US-West': {'city': 'San Francisco','country': 'US', 'lat': 37.77, 'lon': -122.42,
|
||||
'isp': 'AT&T', 'org': 'Google LLC'},
|
||||
'US-East': {'city': 'New York', 'country': 'US', 'lat': 40.71, 'lon': -74.00,
|
||||
'isp': 'Verizon', 'org': 'Amazon.com Inc.'},
|
||||
'EU-West': {'city': 'London', 'country': 'GB', 'lat': 51.51, 'lon': -0.13,
|
||||
'isp': 'BT Group', 'org': 'Microsoft Corp.'},
|
||||
'EU-Central': {'city': 'Frankfurt', 'country': 'DE', 'lat': 50.11, 'lon': 8.68,
|
||||
'isp': 'Deutsche Telekom', 'org': 'Meta Platforms'},
|
||||
'KR-Seoul': {'city': 'Seoul', 'country': 'KR', 'lat': 37.57, 'lon': 126.98,
|
||||
'isp': 'SK Telecom', 'org': 'Samsung'},
|
||||
'JP-Tokyo': {'city': 'Tokyo', 'country': 'JP', 'lat': 35.68, 'lon': 139.76,
|
||||
'isp': 'NTT', 'org': 'Apple Inc.'},
|
||||
'SG': {'city': 'Singapore', 'country': 'SG', 'lat': 1.35, 'lon': 103.82,
|
||||
'isp': 'Singtel', 'org': 'Netflix Inc.'},
|
||||
'IN-Mumbai': {'city': 'Mumbai', 'country': 'IN', 'lat': 19.08, 'lon': 72.88,
|
||||
'isp': 'Tata Comms', 'org': 'Microsoft Corp.'},
|
||||
'AU-Sydney': {'city': 'Sydney', 'country': 'AU', 'lat': -33.87, 'lon': 151.21,
|
||||
'isp': 'Telstra', 'org': 'Amazon.com Inc.'},
|
||||
'BR-SaoPaulo': {'city': 'Sao Paulo', 'country': 'BR', 'lat': -23.55, 'lon': -46.63,
|
||||
'isp': 'Claro', 'org': 'Meta Platforms'},
|
||||
# CDN: 3 edge nodes in different regions
|
||||
CDN_IPS = ['203.0.113.10', '203.0.113.11', '203.0.113.12']
|
||||
API_IPS = ['198.51.100.20', '198.51.100.21']
|
||||
MAIL_IPS = ['192.0.2.50', '192.0.2.51']
|
||||
AUTH_IPS = ['10.20.30.40']
|
||||
SOCIAL_IPS = ['172.16.100.10']
|
||||
STORAGE_IPS = ['198.51.100.50']
|
||||
STREAM_IPS = ['203.0.113.50', '203.0.113.51']
|
||||
SEARCH_IPS = ['198.51.100.60']
|
||||
|
||||
SNAM_TO_IPS = {
|
||||
'cdn.example.com': CDN_IPS,
|
||||
'*.cloudfront.net': CDN_IPS,
|
||||
'api.example.com': API_IPS,
|
||||
'mail.example.com': MAIL_IPS,
|
||||
'auth.example.net': AUTH_IPS,
|
||||
'login.live.com': AUTH_IPS,
|
||||
'graph.facebook.com': SOCIAL_IPS,
|
||||
'*.s3.amazonaws.com': STORAGE_IPS,
|
||||
'stream.example.org': STREAM_IPS,
|
||||
'www.google.com': SEARCH_IPS,
|
||||
}
|
||||
|
||||
# Server nodes: fixed public IPs, region, services/categories
|
||||
SERVER_NODES = {
|
||||
'203.0.113.10': {'region': 'US-West', 'services': ['cdn.example.com', '*.cloudfront.net'],
|
||||
'cats': ['TlsS', 'CDN'], 'port': 443},
|
||||
'203.0.113.11': {'region': 'US-East', 'services': ['cdn.example.com', '*.cloudfront.net'],
|
||||
'cats': ['TlsS', 'CDN'], 'port': 443},
|
||||
'203.0.113.12': {'region': 'EU-West', 'services': ['cdn.example.com', '*.cloudfront.net'],
|
||||
'cats': ['TlsS', 'CDN'], 'port': 443},
|
||||
'198.51.100.20': {'region': 'CN-Shanghai','services': ['api.example.com'],
|
||||
'cats': ['TlsS', 'API'], 'port': 8443},
|
||||
'198.51.100.21': {'region': 'CN-Beijing', 'services': ['api.example.com'],
|
||||
'cats': ['TlsS', 'API'], 'port': 8443},
|
||||
'192.0.2.50': {'region': 'US-West', 'services': ['mail.example.com'],
|
||||
'cats': ['TlsS', 'MAIL'], 'port': 465},
|
||||
'192.0.2.51': {'region': 'EU-Central', 'services': ['mail.example.com'],
|
||||
'cats': ['TlsS', 'MAIL'], 'port': 465},
|
||||
'10.20.30.40': {'region': 'US-East', 'services': ['auth.example.net', 'login.live.com'],
|
||||
'cats': ['TlsS', 'AUTH'], 'port': 443},
|
||||
'172.16.100.10': {'region': 'US-West', 'services': ['graph.facebook.com'],
|
||||
'cats': ['TlsS', 'SOCIAL'], 'port': 443},
|
||||
'198.51.100.50': {'region': 'US-East', 'services': ['*.s3.amazonaws.com'],
|
||||
'cats': ['TlsS', 'STORAGE'], 'port': 443},
|
||||
'203.0.113.50': {'region': 'SG', 'services': ['stream.example.org'],
|
||||
'cats': ['TlsS', 'STREAM'], 'port': 443},
|
||||
'203.0.113.51': {'region': 'JP-Tokyo', 'services': ['stream.example.org'],
|
||||
'cats': ['TlsS', 'STREAM'], 'port': 443},
|
||||
'198.51.100.60': {'region': 'US-West', 'services': ['www.google.com'],
|
||||
'cats': ['TlsS', 'SEARCH'], 'port': 443},
|
||||
CNAM_TO_IPS = {
|
||||
'cdn.example.com': CDN_IPS,
|
||||
'*.cloudfront.net': CDN_IPS,
|
||||
'api.example.com': API_IPS,
|
||||
'mail.example.com': MAIL_IPS,
|
||||
'auth.example.net': AUTH_IPS,
|
||||
'login.live.com': AUTH_IPS,
|
||||
'graph.facebook.com': SOCIAL_IPS,
|
||||
'*.s3.amazonaws.com': STORAGE_IPS,
|
||||
'stream.example.org': STREAM_IPS,
|
||||
'www.google.com': SEARCH_IPS,
|
||||
}
|
||||
|
||||
# Proxy nodes: internal gateways (appear as :ips connecting to many servers)
|
||||
PROXY_NODES = {
|
||||
'10.0.1.1': {'region': 'CN-Beijing', 'org': 'Baidu Inc.'},
|
||||
'10.0.2.1': {'region': 'CN-Shanghai', 'org': 'Alibaba Inc.'},
|
||||
'10.0.3.1': {'region': 'US-West', 'org': 'Google LLC'},
|
||||
'10.0.4.1': {'region': 'EU-Central', 'org': 'Meta Platforms'},
|
||||
'10.0.5.1': {'region': 'SG', 'org': 'Netflix Inc.'},
|
||||
ALL_SERVER_IPS = list(set(
|
||||
ip for pool in SNAM_TO_IPS.values() for ip in pool
|
||||
))
|
||||
|
||||
# Server IP → geo info (so GeoIP columns match when we remap :ipd)
|
||||
SERVER_GEO = {
|
||||
'203.0.113.10': {'lat': 37.77, 'lon': -122.42, 'city': 'San Francisco', 'country': 'us', 'isp': 'AT&T', 'org': 'Amazon.com Inc.'},
|
||||
'203.0.113.11': {'lat': 40.71, 'lon': -74.00, 'city': 'New York', 'country': 'us', 'isp': 'Verizon', 'org': 'Google LLC'},
|
||||
'203.0.113.12': {'lat': 51.51, 'lon': -0.13, 'city': 'London', 'country': 'gb', 'isp': 'BT Group', 'org': 'Microsoft Corp.'},
|
||||
'198.51.100.20': {'lat': 31.23, 'lon': 121.47, 'city': 'Shanghai', 'country': 'cn', 'isp': 'China Mobile','org': 'Tencent'},
|
||||
'198.51.100.21': {'lat': 39.90, 'lon': 116.40, 'city': 'Beijing', 'country': 'cn', 'isp': 'China Telecom','org': 'Alibaba Inc.'},
|
||||
'192.0.2.50': {'lat': 37.77, 'lon': -122.42, 'city': 'San Francisco', 'country': 'us', 'isp': 'AT&T', 'org': 'Google LLC'},
|
||||
'192.0.2.51': {'lat': 50.11, 'lon': 8.68, 'city': 'Frankfurt', 'country': 'de', 'isp': 'Deutsche Telekom', 'org': 'Meta Platforms'},
|
||||
'10.20.30.40': {'lat': 40.71, 'lon': -74.00, 'city': 'New York', 'country': 'us', 'isp': 'Verizon', 'org': 'Microsoft Corp.'},
|
||||
'172.16.100.10': {'lat': 37.77, 'lon': -122.42, 'city': 'San Francisco', 'country': 'us', 'isp': 'AT&T', 'org': 'Meta Platforms'},
|
||||
'198.51.100.50': {'lat': 40.71, 'lon': -74.00, 'city': 'New York', 'country': 'us', 'isp': 'Verizon', 'org': 'Amazon.com Inc.'},
|
||||
'203.0.113.50': {'lat': 1.35, 'lon': 103.82, 'city': 'Singapore', 'country': 'sg', 'isp': 'Singtel', 'org': 'Netflix Inc.'},
|
||||
'203.0.113.51': {'lat': 35.68, 'lon': 139.76, 'city': 'Tokyo', 'country': 'jp', 'isp': 'NTT', 'org': 'Apple Inc.'},
|
||||
'198.51.100.60': {'lat': 37.77, 'lon': -122.42, 'city': 'San Francisco', 'country': 'us', 'isp': 'AT&T', 'org': 'Google LLC'},
|
||||
}
|
||||
|
||||
ALL_SERVERS = list(SERVER_NODES.keys())
|
||||
ALL_PROXIES = list(PROXY_NODES.keys())
|
||||
SERVICES_LIST = list({s for sv in SERVER_NODES.values() for s in sv['services']})
|
||||
CATEGORIES_LIST = list({c for sv in SERVER_NODES.values() for c in sv['cats']})
|
||||
|
||||
|
||||
# ── Client IP pools per region ──
|
||||
CLIENTS_BY_REGION = {}
|
||||
|
||||
|
||||
def init_topology(seed):
|
||||
"""Generate client IPs per region and client→server affinity."""
|
||||
random.seed(seed)
|
||||
for region_key in REGIONS:
|
||||
CLIENTS_BY_REGION[region_key] = []
|
||||
all_clients = []
|
||||
# Generate clients as public-looking IPs
|
||||
ip_counter = 1
|
||||
for region_key in REGIONS:
|
||||
n = random.randint(8, 15)
|
||||
for _ in range(n):
|
||||
client_ip = f'{random.randint(1,223)}.{random.randint(0,255)}.{random.randint(0,255)}.{random.randint(1,254)}'
|
||||
CLIENTS_BY_REGION[region_key].append(client_ip)
|
||||
all_clients.append(client_ip)
|
||||
ip_counter += 1
|
||||
|
||||
# Build client→server affinity
|
||||
client_servers = {}
|
||||
for client_ip in all_clients:
|
||||
client_region = None
|
||||
for rk, cl in CLIENTS_BY_REGION.items():
|
||||
if client_ip in cl:
|
||||
client_region = rk
|
||||
break
|
||||
region_srvs = [s for s in ALL_SERVERS if SERVER_NODES[s]['region'] == client_region]
|
||||
other_srvs = [s for s in ALL_SERVERS if s not in region_srvs]
|
||||
n_srvs = random.randint(3, 8)
|
||||
chosen = random.sample(region_srvs, min(len(region_srvs), max(1, n_srvs // 2)))
|
||||
rem = n_srvs - len(chosen)
|
||||
if rem > 0 and other_srvs:
|
||||
chosen += random.sample(other_srvs, min(len(other_srvs), rem))
|
||||
client_servers[client_ip] = chosen
|
||||
|
||||
return all_clients, client_servers
|
||||
|
||||
|
||||
def make_row(row_id, base_time, client_ip, client_region, dst_ip, pattern):
|
||||
"""Generate a row using gen_test_data's exact format, then override IPs+services.
|
||||
|
||||
Returns a dict matching all COLUMNS with proper types, blanks and coverage.
|
||||
"""
|
||||
# Get a base row with all the right formatting/coverages/blanks
|
||||
row = _base_generate_row(row_id, base_time)
|
||||
|
||||
srv = SERVER_NODES.get(dst_ip)
|
||||
if not srv:
|
||||
srv = random.choice(list(SERVER_NODES.values()))
|
||||
srv_region = srv['region']
|
||||
src_geo = REGIONS.get(client_region, REGIONS['US-West'])
|
||||
srv_geo = REGIONS.get(srv_region, REGIONS['US-West'])
|
||||
|
||||
snam_val = random.choice(srv['services'])
|
||||
cnam_val = random.choice(srv['cats'])
|
||||
|
||||
# ── Override IPs and services with topology-aware values ──
|
||||
row[':ips'] = client_ip
|
||||
row[':ipd'] = dst_ip
|
||||
row[':prs'] = random.choice(PORTS)
|
||||
row[':prd'] = srv['port']
|
||||
row['server-ip'] = dst_ip
|
||||
row['client-ip'] = client_ip
|
||||
|
||||
# Service/category — respect coverage from TlsDB.csv
|
||||
row['snam'] = _blank_or(COVERAGE.get('snam', 1.0), snam_val)
|
||||
row['cnam'] = _blank_or(COVERAGE.get('cnam', 1.0), cnam_val)
|
||||
|
||||
# GeoIP — use real coordinates
|
||||
slat = round(srv_geo['lat'] + random.uniform(-0.3, 0.3), 6)
|
||||
slon = round(srv_geo['lon'] + random.uniform(-0.3, 0.3), 6)
|
||||
dlat = round(srv_geo['lat'] + random.uniform(-0.1, 0.1), 6)
|
||||
dlon = round(srv_geo['lon'] + random.uniform(-0.1, 0.1), 6)
|
||||
row[':ips.latd'] = _blank_or_plus(COVERAGE.get(':ips.latd', 1.0), str(slat))
|
||||
row[':ips.lond'] = _blank_or_plus(COVERAGE.get(':ips.lond', 1.0), str(slon))
|
||||
row[':ipd.latd'] = _blank_or_plus(COVERAGE.get(':ipd.latd', 1.0), str(dlat))
|
||||
row[':ipd.lond'] = _blank_or_plus(COVERAGE.get(':ipd.lond', 1.0), str(dlon))
|
||||
row[':ips.ispn'] = _blank_or(COVERAGE.get(':ips.ispn', 1.0), src_geo['isp'])
|
||||
row[':ipd.ispn'] = _blank_or(COVERAGE.get(':ipd.ispn', 1.0), srv_geo['isp'])
|
||||
row[':ips.orgn'] = _blank_or(COVERAGE.get(':ips.orgn', 1.0), src_geo['org'])
|
||||
row[':ipd.orgn'] = _blank_or(COVERAGE.get(':ipd.orgn', 1.0), srv_geo['org'])
|
||||
row[':ips.city'] = _blank_or(COVERAGE.get(':ips.city', 1.0), src_geo['city'])
|
||||
row[':ipd.city'] = _blank_or(COVERAGE.get(':ipd.city', 1.0), srv_geo['city'])
|
||||
row['scnt'] = _blank_or(COVERAGE.get('scnt', 1.0), src_geo['country'].lower())
|
||||
row['dcnt'] = _blank_or(COVERAGE.get('dcnt', 1.0), srv_geo['country'].lower())
|
||||
|
||||
# Domain — matches service
|
||||
row[':ips.doma'] = _blank_or(COVERAGE.get(':ips.doma', 1.0), snam_val)
|
||||
row[':ipd.doma'] = _blank_or(COVERAGE.get(':ipd.doma', 1.0), snam_val)
|
||||
|
||||
# org fields
|
||||
row['orga'] = _blank_or(COVERAGE.get('orga', 1.0), src_geo['org'])
|
||||
|
||||
return row
|
||||
# Proxy IPs — high out-degree (appear as :ips connecting to many servers)
|
||||
PROXY_IPS = ['10.0.1.1', '10.0.2.1', '10.0.3.1', '10.0.4.1', '10.0.5.1']
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description='Generate topology-aware TLS test data')
|
||||
parser.add_argument('--files', type=int, default=200, help='Number of CSV files')
|
||||
parser.add_argument('--rows', type=int, default=100, help='Approximate rows per file')
|
||||
parser.add_argument('--seed', type=int, default=2026, help='Random seed')
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--files', type=int, default=200)
|
||||
parser.add_argument('--rows', type=int, default=10000)
|
||||
parser.add_argument('--seed', type=int, default=2026)
|
||||
parser.add_argument('--output-dir', type=str, default='data/gen_multi2')
|
||||
args = parser.parse_args()
|
||||
|
||||
out_dir = Path('data/gen_multi2')
|
||||
random.seed(args.seed)
|
||||
out_dir = Path(args.output_dir)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
print(f'Generating {args.files} files × ~{args.rows} rows → {out_dir}')
|
||||
print(f'Seed: {args.seed}')
|
||||
|
||||
# Init topology
|
||||
all_clients, client_servers = init_topology(args.seed)
|
||||
servers = ALL_SERVERS
|
||||
proxies = ALL_PROXIES
|
||||
|
||||
random.seed(args.seed + 1)
|
||||
base_time = datetime(2026, 6, 1, 0, 0, 0)
|
||||
row_counter = 0
|
||||
|
||||
# Identity columns that must never be dropped
|
||||
_keep_cols = {':ips', ':ipd', ':prs', ':prd', 'server-ip', 'client-ip'}
|
||||
_droppable = [c for c in COLUMNS if c not in _keep_cols]
|
||||
|
||||
base_time = datetime(2026, 6, 1, 0, 0, 0)
|
||||
total_rows = 0
|
||||
degree_src = defaultdict(int)
|
||||
degree_dst = defaultdict(int)
|
||||
snam_ip_count = defaultdict(lambda: defaultdict(int))
|
||||
cnam_ip_count = defaultdict(lambda: defaultdict(int))
|
||||
|
||||
print(f'Generating {args.files} files × {args.rows} rows → {out_dir}')
|
||||
|
||||
for fi in range(args.files):
|
||||
# Column dropping per file (matching gen_multi.py logic)
|
||||
# ── Column dropping (matching gen_multi.py exactly) ──
|
||||
dropped = set()
|
||||
for col in _droppable:
|
||||
cov = COVERAGE.get(col, 1.0)
|
||||
@@ -253,63 +138,61 @@ def main():
|
||||
dropped -= spared
|
||||
file_columns = [c for c in COLUMNS if c not in dropped]
|
||||
|
||||
n_rows = random.randint(max(50, args.rows - 10), args.rows + 10)
|
||||
|
||||
# Generate rows
|
||||
# ── Generate rows with base format ──
|
||||
file_rows = []
|
||||
file_base = base_time + timedelta(seconds=random.randint(0, 86400 * 30))
|
||||
for ri in range(n_rows):
|
||||
# Pick client and server using topology
|
||||
client_ip = random.choice(all_clients)
|
||||
client_region = None
|
||||
for rk, cl in CLIENTS_BY_REGION.items():
|
||||
if client_ip in cl:
|
||||
client_region = rk
|
||||
break
|
||||
|
||||
# Pattern: 70% direct, 20% via proxy, 10% proxy only
|
||||
pattern = random.choices(['direct', 'via_proxy', 'proxy_only'],
|
||||
weights=[0.70, 0.20, 0.10])[0]
|
||||
for ri in range(args.rows):
|
||||
row = _base_row(total_rows + ri, file_base)
|
||||
|
||||
if pattern == 'proxy_only':
|
||||
src_ip = random.choice(proxies)
|
||||
proxy_region = PROXY_NODES[src_ip]['region']
|
||||
dst_ip = random.choice(servers)
|
||||
row = make_row(row_counter, file_base, src_ip, proxy_region, dst_ip, pattern)
|
||||
file_rows.append(row)
|
||||
degree_src[src_ip] += 1
|
||||
degree_dst[dst_ip] += 1
|
||||
row_counter += 1
|
||||
elif pattern == 'via_proxy':
|
||||
# Client → server (direct connection row)
|
||||
dst_ip = random.choice(client_servers.get(client_ip, servers))
|
||||
row = make_row(row_counter, file_base, client_ip, client_region, dst_ip, 'direct')
|
||||
file_rows.append(row)
|
||||
degree_src[client_ip] += 1
|
||||
degree_dst[dst_ip] += 1
|
||||
row_counter += 1
|
||||
# Also generate proxy→server row if lucky
|
||||
if random.random() < 0.7:
|
||||
proxy_ip = random.choice(proxies)
|
||||
proxy_region = PROXY_NODES[proxy_ip]['region']
|
||||
row2 = make_row(row_counter, file_base, proxy_ip, proxy_region, dst_ip, 'via_proxy')
|
||||
file_rows.append(row2)
|
||||
degree_src[proxy_ip] += 1
|
||||
degree_dst[dst_ip] += 1
|
||||
row_counter += 1
|
||||
else:
|
||||
dst_ip = random.choice(client_servers.get(client_ip, servers))
|
||||
row = make_row(row_counter, file_base, client_ip, client_region, dst_ip, 'direct')
|
||||
file_rows.append(row)
|
||||
degree_src[client_ip] += 1
|
||||
degree_dst[dst_ip] += 1
|
||||
row_counter += 1
|
||||
# ── Remap IPs using topology ──
|
||||
src_ip = row[':ips']
|
||||
snam_val = row.get('snam', '').strip()
|
||||
cnam_val = row.get('cnam', '').strip()
|
||||
|
||||
# Shuffle column order per file (matching gen_test_data)
|
||||
# Determine appropriate :ipd based on snam/cnam
|
||||
server_pool = None
|
||||
if snam_val and snam_val in SNAM_TO_IPS:
|
||||
server_pool = SNAM_TO_IPS[snam_val]
|
||||
elif cnam_val and cnam_val in CNAM_TO_IPS:
|
||||
server_pool = CNAM_TO_IPS[cnam_val]
|
||||
|
||||
if server_pool:
|
||||
# Assign a specific server IP for this service
|
||||
dst_ip = random.choice(server_pool)
|
||||
row[':ipd'] = dst_ip
|
||||
row['server-ip'] = dst_ip
|
||||
snam_ip_count[snam_val][dst_ip] += 1
|
||||
cnam_ip_count[cnam_val][dst_ip] += 1
|
||||
|
||||
# 30% of the time, use a proxy as :ips instead of random client
|
||||
if random.random() < 0.30:
|
||||
row[':ips'] = random.choice(PROXY_IPS)
|
||||
row['client-ip'] = row[':ips']
|
||||
|
||||
# Update GeoIP columns to match server location
|
||||
geo = SERVER_GEO.get(dst_ip)
|
||||
if geo:
|
||||
rlat = round(geo['lat'] + random.uniform(-0.3, 0.3), 6)
|
||||
rlon = round(geo['lon'] + random.uniform(-0.3, 0.3), 6)
|
||||
row[':ipd.latd'] = str(rlat)
|
||||
row[':ipd.lond'] = str(rlon)
|
||||
row[':ipd.city'] = geo['city']
|
||||
row[':ipd.ispn'] = geo['isp']
|
||||
row[':ipd.orgn'] = geo['org']
|
||||
row['dcnt'] = geo['country']
|
||||
|
||||
file_rows.append(row)
|
||||
degree_src[row[':ips']] += 1
|
||||
degree_dst[row[':ipd']] += 1
|
||||
|
||||
total_rows += args.rows
|
||||
|
||||
# ── Shuffle column order per file ──
|
||||
shuffled = file_columns[:]
|
||||
random.shuffle(shuffled)
|
||||
|
||||
# Write
|
||||
# ── Write ──
|
||||
out_path = out_dir / f'{fi}.csv'
|
||||
with open(out_path, 'w', newline='', encoding='utf-8') as f:
|
||||
writer = csv.DictWriter(f, fieldnames=shuffled, extrasaction='ignore')
|
||||
@@ -317,23 +200,27 @@ def main():
|
||||
for row in file_rows:
|
||||
writer.writerow(row)
|
||||
|
||||
total_rows += len(file_rows)
|
||||
|
||||
if (fi + 1) % 50 == 0:
|
||||
print(f' [{fi + 1}/{args.files}] {total_rows} rows...')
|
||||
|
||||
# Stats
|
||||
# ── Stats ──
|
||||
top_src = sorted(degree_src.items(), key=lambda x: -x[1])[:10]
|
||||
top_dst = sorted(degree_dst.items(), key=lambda x: -x[1])[:10]
|
||||
print(f'\nDone: {args.files} files, {total_rows} rows → {out_dir.resolve()}')
|
||||
print(f' Servers: {len(servers)}, Proxies: {len(proxies)}, Clients: {len(set(all_clients))}')
|
||||
print(f' Avg src degree: {sum(degree_src.values())/len(degree_src):.1f}')
|
||||
print(f' Avg dst degree: {sum(degree_dst.values())/len(degree_dst):.1f}')
|
||||
print(f' Top source (out-degree):')
|
||||
for ip, deg in top_src:
|
||||
label = ' [proxy]' if ip in PROXY_NODES else ' [client]'
|
||||
label = ' [proxy]' if ip in PROXY_IPS else ''
|
||||
print(f' {ip}: {deg}{label}')
|
||||
print(f' Top dest (in-degree):')
|
||||
for ip, deg in top_dst:
|
||||
print(f' {ip}: {deg} [server]')
|
||||
is_server = ip in ALL_SERVER_IPS
|
||||
print(f' {ip}: {deg}{" [server]" if is_server else ""}')
|
||||
print(f' snam→IP mapping:')
|
||||
for snam, ips in sorted(snam_ip_count.items()):
|
||||
top = sorted(ips.items(), key=lambda x: -x[1])
|
||||
print(f' {snam}: {dict(top)}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
Reference in New Issue
Block a user