PM-pinou
873 lines
34 KiB
Python
873 lines
34 KiB
Python
"""Generate synthetic TLS flow test data with 3 real-world traffic profiles.
|
|
|
|
Usage:
|
|
runtime\\python\\python.exe scripts\\gen_test_data.py [--rows N] [--output PATH]
|
|
runtime\\python\\python.exe scripts\\gen_test_data.py --globe [--rows N] [--output PATH]
|
|
"""
|
|
import sys
|
|
import polars as pl
|
|
import numpy as np
|
|
from pathlib import Path
|
|
from datetime import datetime, timedelta
|
|
|
|
# ===================================================================
|
|
# TLS Constants
|
|
# ===================================================================
|
|
|
|
CIPHER_HEX_MAP: dict[str, str] = {
|
|
'TLS_AES_128_GCM_SHA256': '13 01',
|
|
'TLS_AES_256_GCM_SHA384': '13 02',
|
|
'TLS_ECDHE_ECDSA_AES128_GCM_SHA256': 'c0 2b',
|
|
'TLS_ECDHE_RSA_AES128_GCM_SHA256': 'c0 2f',
|
|
'TLS_ECDHE_RSA_AES256_GCM_SHA384': 'c0 30',
|
|
'TLS_RSA_AES128_GCM_SHA256': '00 9c',
|
|
'TLS_RSA_AES_128_CBC_SHA': '00 3c',
|
|
'TLS_ECDHE_RSA_AES128_CBC_SHA': 'c0 09',
|
|
'TLS_RSA_WITH_RC4_128_SHA': '00 05',
|
|
'TLS_CHACHA20_POLY1305_SHA256': 'cc a9',
|
|
}
|
|
|
|
CURVE_MAP: dict[str, str] = {
|
|
'00 1d': 'x25519',
|
|
'00 17': 'secp256r1',
|
|
'00 18': 'secp384r1',
|
|
'00 19': 'secp521r1',
|
|
}
|
|
|
|
COUNTRY_CODES: list[str] = [
|
|
'US', 'JP', 'DE', 'GB', 'SG', 'KR', 'FR', 'AU', 'IN', 'CA', 'BR',
|
|
'RU', 'CN', 'NG', 'IR', 'NL', 'SE', 'HK', 'TW', 'ZA',
|
|
]
|
|
|
|
COUNTRY_A: list[str] = ['US', 'JP', 'DE', 'GB', 'SG', 'KR', 'FR', 'AU', 'NL', 'CA', 'SE']
|
|
COUNTRY_B: list[str] = ['US', 'NL', 'DE', 'SG', 'HK', 'JP', 'GB']
|
|
COUNTRY_C: list[str] = ['RU', 'CN', 'BR', 'NG', 'IR', 'KR', 'US', 'HK', 'TW', 'ZA']
|
|
|
|
TABLE_NAMES: list[str] = [
|
|
'tls_flows', 'network_traffic', 'encrypted_streams', 'ssl_logs', 'packet_capture',
|
|
]
|
|
LINK_NAMES: list[str] = [
|
|
'internet-edge', 'mpls-backbone', 'vpn-tunnel', 'wan-link',
|
|
'cloud-connect', 'dc-interconnect', 'peering-link',
|
|
]
|
|
NODE_NAMES: list[str] = (
|
|
[f'gw-{i:02d}' for i in range(1, 11)]
|
|
+ [f'core-{i:02d}' for i in range(1, 7)]
|
|
)
|
|
|
|
ISP_NAMES: list[str] = [
|
|
'China Telecom', 'China Unicom', 'China Mobile', 'AT&T', 'Verizon',
|
|
'Comcast', 'Deutsche Telekom', 'BT Group', 'NTT', 'KDDI',
|
|
'Singtel', 'Telstra', 'Reliance Jio', 'T-Mobile', 'Orange',
|
|
'Rostelecom', 'Vivo', 'MTN Group', 'Etisalat',
|
|
]
|
|
ORG_NAMES: list[str] = [
|
|
'Alibaba Inc.', 'Tencent Holdings', 'Baidu Inc.', 'Google LLC',
|
|
'Amazon.com Inc.', 'Microsoft Corp.', 'Meta Platforms Inc.', 'Apple Inc.',
|
|
'Netflix Inc.', 'Cloudflare Inc.', 'Akamai Technologies',
|
|
'Yandex', 'Kaspersky Lab', 'Mail.ru Group',
|
|
]
|
|
CITY_NAMES: list[str] = [
|
|
'Beijing', 'Shanghai', 'Tokyo', 'Seoul', 'New York', 'London',
|
|
'Paris', 'Berlin', 'Singapore', 'Sydney', 'Mumbai', 'Dubai',
|
|
'Toronto', 'Moscow', 'Sao Paulo', 'San Francisco', 'Los Angeles',
|
|
'Lagos', 'Tehran', 'Hong Kong',
|
|
]
|
|
|
|
# ===================================================================
|
|
# Traffic Profile Definitions
|
|
# ===================================================================
|
|
|
|
# Each profile defines distribution parameters for row generation.
|
|
# We encode TLS version, cipher, curve as weighted-choice lists.
|
|
|
|
TLS_PROFILES: dict[str, list[tuple[str, float]]] = {
|
|
'A': [('03 04', 0.80), ('03 03', 0.18), ('02 00', 0.01), ('03 00', 0.01)],
|
|
'B': [('03 04', 0.50), ('03 03', 0.40), ('03 01', 0.05), ('03 02', 0.05)],
|
|
'C': [('03 01', 0.30), ('03 02', 0.20), ('03 03', 0.20), ('03 04', 0.10),
|
|
('02 00', 0.10), ('03 00', 0.10)],
|
|
}
|
|
|
|
CIPHER_PROFILES: dict[str, list[tuple[str, float]]] = {
|
|
'A': [('13 01', 0.50), ('c0 2b', 0.20), ('c0 2f', 0.15), ('13 02', 0.10), ('cc a9', 0.05)],
|
|
'B': [('c0 2f', 0.30), ('13 01', 0.30), ('c0 30', 0.15), ('00 9c', 0.15), ('13 02', 0.10)],
|
|
'C': [('00 3c', 0.25), ('c0 09', 0.20), ('00 05', 0.10), ('13 01', 0.15),
|
|
('c0 2f', 0.10), ('c0 2b', 0.10), ('00 9c', 0.10)],
|
|
}
|
|
|
|
CURVE_PROFILES: dict[str, list[tuple[str, float]]] = {
|
|
'A': [('00 1d', 0.70), ('00 17', 0.25), ('00 18', 0.05)],
|
|
'B': [('00 17', 0.50), ('00 1d', 0.40), ('00 18', 0.10)],
|
|
'C': [('00 17', 0.25), ('00 1d', 0.20), ('00 18', 0.15),
|
|
('00 19', 0.10), ('', 0.30)], # 30% no curve
|
|
}
|
|
|
|
# Port profiles: (port, weight)
|
|
PORT_PROFILES: dict[str, list[tuple[int, float]]] = {
|
|
'A': [(443, 0.90), (80, 0.05), (8080, 0.02), (8443, 0.01), (993, 0.01), (465, 0.01)],
|
|
'B': [(443, 0.60), (8443, 0.15), (8080, 0.10), (3128, 0.05), (1080, 0.05), (8888, 0.05)],
|
|
'C': [(443, 0.60), (80, 0.05), (8080, 0.05), (8443, 0.05), (0, 0.25)], # 0 = random non-standard
|
|
}
|
|
|
|
# Duration (seconds) profiles: used with exponential + clip
|
|
DUR_PROFILES: dict[str, tuple[float, float, float]] = {
|
|
'A': (2.0, 0.5, 8.0), # (mean exp, min, max)
|
|
'B': (60.0, 10.0, 300.0),
|
|
'C': (1.0, 0.3, 600.0), # bimodal: will mix short & long
|
|
}
|
|
|
|
# ===================================================================
|
|
# IP Pools (deterministic via numpy seed)
|
|
# ===================================================================
|
|
|
|
def _ip_pool(n: int, seed_base: int, prefix: str | None = None) -> list[str]:
|
|
"""Generate *n* reproducible IPs with optional *prefix* (e.g. '10.0.1')."""
|
|
rng = np.random.RandomState(seed_base)
|
|
ips: list[str] = []
|
|
for _ in range(n):
|
|
if prefix:
|
|
parts = prefix.split('.')
|
|
ip = (f'{parts[0]}.{parts[1]}.{parts[2]}.'
|
|
f'{rng.randint(1, 255)}')
|
|
else:
|
|
a = rng.randint(1, 223)
|
|
b = rng.randint(0, 256)
|
|
c = rng.randint(0, 256)
|
|
d = rng.randint(1, 255)
|
|
ip = f'{a}.{b}.{c}.{d}'
|
|
ips.append(ip)
|
|
return ips
|
|
|
|
NORMAL_SRC_IPS: list[str] = _ip_pool(150, 42) # residential-like
|
|
NORMAL_DST_IPS: list[str] = _ip_pool(120, 142) # CDN/cloud-like
|
|
PROXY_SRC_IPS: list[str] = _ip_pool(8, 242) # proxy exit nodes
|
|
PROXY_DST_IPS: list[str] = _ip_pool(15, 342) # limited targets
|
|
MAL_SRC_IPS: list[str] = _ip_pool(180, 442) # botnet-like diverse
|
|
MAL_DST_IPS: list[str] = _ip_pool(60, 542) # malicious targets
|
|
|
|
# ===================================================================
|
|
# SNI / Domain pools
|
|
# ===================================================================
|
|
|
|
LEGIT_DOMAINS: list[str] = [
|
|
'www.google.com', 'api.github.com', 'facebook.com', 'youtube.com',
|
|
'amazon.com', 'netflix.com', 'apple.com', 'microsoft.com',
|
|
'cloudflare.com', 'stackoverflow.com', 'reddit.com', 'twitter.com',
|
|
'linkedin.com', 'instagram.com', 'office.com', 'live.com',
|
|
'bing.com', 'adobe.com', 'salesforce.com', 'zoom.us',
|
|
'maps.google.com', 'mail.google.com', 'drive.google.com',
|
|
'docs.google.com', 'play.google.com', 'support.microsoft.com',
|
|
'portal.office.com', 'login.live.com', 'cdn.cloudflare.com',
|
|
]
|
|
|
|
PROXY_DOMAINS: list[str] = [
|
|
'proxy.example.com', 'vpn-gateway.example.net', 'tunnel.example.org',
|
|
'exit-node.example.com', 'relay.example.net', 'gateway.example.org',
|
|
'', '',
|
|
]
|
|
|
|
DGA_DOMAINS: list[str] = [
|
|
'gkj3hj2k3.sdf2k3j.net', '8sdf7h3k.xyz', 'j2k3h4k5h7.ru',
|
|
'asdf9082k.biz', 'mnbv34k2.info', '2kj3h5k7jh9.com',
|
|
'xcnv98sdf2.top', 'q3kj4h5k7.work', 'vbsd78sdf.xyz',
|
|
'm2k3j5h6k8.ru', '9s8d7f6g5h.net', 'k2j3h5k7j9.org',
|
|
'fasdf2342.biz', 'poiuy234k.info', 'zxcv7890m.top',
|
|
'hk3j5k7h9.net', 'sdf2k3j4h5.ru', 'vb9s8d7f6.xyz',
|
|
'n2k3j5h7k9.com', 'x98s7d6f5.info',
|
|
]
|
|
|
|
FAKE_DOMAINS: list[str] = [
|
|
'secure-login-app.com', 'update-manager.net', 'cdn-cache-service.com',
|
|
'analytics-tracker.net', 'content-delivery-system.org',
|
|
'download-center.net', 'account-verify.com', 'system-update.org',
|
|
'cloud-sync-service.net', 'notification-hub.com',
|
|
]
|
|
|
|
# ===================================================================
|
|
# Helpers
|
|
# ===================================================================
|
|
|
|
def _weighted_choice(options: list[tuple[str, float]]) -> str:
|
|
"""Pick one from [(value, weight), ...] according to weight."""
|
|
items, raw_w = zip(*options)
|
|
total = sum(raw_w)
|
|
return str(np.random.choice(items, p=[w / total for w in raw_w]))
|
|
|
|
|
|
def _weighted_choice_int(options: list[tuple[int, float]]) -> int:
|
|
"""Pick int from [(value, weight), ...]."""
|
|
items, raw_w = zip(*options)
|
|
total = sum(raw_w)
|
|
return int(np.random.choice(items, p=[w / total for w in raw_w]))
|
|
|
|
|
|
def _random_hex_pairs(n_pairs: int = 2) -> str:
|
|
"""Generate space-separated hex pairs, e.g. 'c0 2b'."""
|
|
pairs = [f'{np.random.randint(0, 256):02x}' for _ in range(n_pairs)]
|
|
return ' '.join(pairs)
|
|
|
|
|
|
def _random_32_bytes_hex() -> str:
|
|
"""Generate 32 random hex bytes as a space-separated string."""
|
|
pairs = [f'{np.random.randint(0, 256):02x}' for _ in range(32)]
|
|
return ' '.join(pairs)
|
|
|
|
|
|
def _low_entropy_hex() -> str:
|
|
"""Generate repeating low-entropy TLS random (Profile C: beaconing).
|
|
|
|
Pattern: '00 01 00 01 00 01 ...' (repeating 2-byte cycle).
|
|
"""
|
|
pairs = ['00', '01'] * 16
|
|
return ' '.join(pairs)
|
|
|
|
|
|
def _maybe(rate: float) -> bool:
|
|
"""Return True with probability *rate* (value should be blanked)."""
|
|
return np.random.random() < rate
|
|
|
|
|
|
def _assign_profile() -> str:
|
|
"""Assign profile A (60%), B (20%), or C (20%)."""
|
|
r = np.random.random()
|
|
if r < 0.60:
|
|
return 'A'
|
|
elif r < 0.80:
|
|
return 'B'
|
|
else:
|
|
return 'C'
|
|
|
|
|
|
def _pick_sni(profile: str) -> tuple[str, str]:
|
|
"""Generate (snam, cnam) for the given profile.
|
|
|
|
Returns (sni_name, common_name).
|
|
"""
|
|
if profile == 'A':
|
|
if _maybe(0.05):
|
|
return ('', '')
|
|
domain = str(np.random.choice(LEGIT_DOMAINS))
|
|
return (domain, domain) # always match
|
|
|
|
elif profile == 'B':
|
|
if _maybe(0.20):
|
|
return ('', '')
|
|
domain = str(np.random.choice(PROXY_DOMAINS))
|
|
if domain == '':
|
|
return ('', '')
|
|
return (domain, domain)
|
|
|
|
else: # C
|
|
if _maybe(0.30):
|
|
return ('', '')
|
|
r = np.random.random()
|
|
if r < 0.20:
|
|
domain = str(np.random.choice(DGA_DOMAINS))
|
|
elif r < 0.70:
|
|
domain = str(np.random.choice(FAKE_DOMAINS))
|
|
else:
|
|
domain = str(np.random.choice(LEGIT_DOMAINS))
|
|
|
|
# 30% mismatch snam != cnam
|
|
if _maybe(0.30):
|
|
if _maybe(0.5):
|
|
cnam = str(np.random.choice(FAKE_DOMAINS))
|
|
else:
|
|
cnam = str(np.random.choice(LEGIT_DOMAINS))
|
|
return (domain, cnam)
|
|
return (domain, domain)
|
|
|
|
|
|
def _pick_country(profile: str) -> tuple[str, str]:
|
|
"""Pick (scnt, dcnt) for the given profile."""
|
|
if profile == 'A':
|
|
scnt = str(np.random.choice(COUNTRY_A))
|
|
dcnt = str(np.random.choice(COUNTRY_A))
|
|
elif profile == 'B':
|
|
scnt = str(np.random.choice(COUNTRY_B))
|
|
dcnt = str(np.random.choice(COUNTRY_B))
|
|
else:
|
|
scnt = str(np.random.choice(COUNTRY_C))
|
|
dcnt = str(np.random.choice(COUNTRY_C))
|
|
return scnt, dcnt
|
|
|
|
|
|
# ===================================================================
|
|
# Main generator — 3 profiles, 44 columns
|
|
# ===================================================================
|
|
|
|
def generate_tls_flows(n_rows: int = 1000, seed: int = 42) -> pl.DataFrame:
|
|
"""Generate synthetic TLS flow data with 3 real-world traffic profiles.
|
|
|
|
Profile A (60%): Normal Browsing
|
|
Profile B (20%): Proxy/VPN Traffic
|
|
Profile C (20%): Malicious/Suspicious
|
|
|
|
Returns a Polars DataFrame with all 44 columns matching the user's
|
|
exact column naming convention.
|
|
"""
|
|
np.random.seed(seed)
|
|
base_ts = datetime(2024, 1, 1, 0, 0, 0)
|
|
day_seconds = 86400 * 90 # 90-day window
|
|
|
|
rows: list[dict] = []
|
|
total = n_rows
|
|
next_pct = 10
|
|
|
|
for i in range(total):
|
|
# Progress indicator
|
|
pct = (i + 1) * 100 // total
|
|
if pct >= next_pct:
|
|
sys.stdout.write('+')
|
|
sys.stdout.flush()
|
|
next_pct += 10
|
|
|
|
profile = _assign_profile()
|
|
|
|
# --- IPs ---
|
|
if profile == 'A':
|
|
src_ip = str(np.random.choice(NORMAL_SRC_IPS))
|
|
dst_ip = str(np.random.choice(NORMAL_DST_IPS))
|
|
elif profile == 'B':
|
|
src_ip = str(np.random.choice(PROXY_SRC_IPS))
|
|
dst_ip = str(np.random.choice(PROXY_DST_IPS))
|
|
else:
|
|
src_ip = str(np.random.choice(MAL_SRC_IPS))
|
|
dst_ip = str(np.random.choice(MAL_DST_IPS))
|
|
|
|
# --- Ports ---
|
|
src_port = np.random.randint(1024, 65536)
|
|
dst_port = _weighted_choice_int(PORT_PROFILES[profile])
|
|
if dst_port == 0: # random non-standard
|
|
dst_port = int(np.random.choice([p for p in range(1025, 65536) if p != 443]))
|
|
|
|
# --- scnt, dcnt ---
|
|
scnt, dcnt = _pick_country(profile)
|
|
|
|
# --- TLS version ---
|
|
tls_ver = _weighted_choice(TLS_PROFILES[profile])
|
|
|
|
# --- SNI / CN ---
|
|
snam, cnam = _pick_sni(profile)
|
|
|
|
# --- Duration ---
|
|
mean_dur, min_dur, max_dur = DUR_PROFILES[profile]
|
|
if profile == 'C':
|
|
# Bimodal: beaconing (<1s) or C2 (long, >60s)
|
|
if _maybe(0.50):
|
|
dur = round(np.random.uniform(0.3, 1.0), 2)
|
|
else:
|
|
dur = round(min(max(np.random.exponential(120), 60), 600), 2)
|
|
else:
|
|
dur = round(min(max(np.random.exponential(mean_dur), min_dur), max_dur), 2)
|
|
|
|
# --- Session / timeout / key size ---
|
|
ses = round(np.random.uniform(0, 1000), 4)
|
|
tmo = round(np.random.uniform(0, 300), 4)
|
|
ksz = int(np.random.choice([128, 256, 384, 521]))
|
|
|
|
# --- cnrs / isrs (blank="no", +="yes") ---
|
|
if profile == 'A':
|
|
cnrs_val = '+' if _maybe(0.30) else ''
|
|
isrs_val = '+' if np.random.random() < 0.10 else ''
|
|
elif profile == 'B':
|
|
cnrs_val = '+' if _maybe(0.60) else ''
|
|
isrs_val = '+' if np.random.random() < 0.40 else ''
|
|
else:
|
|
cnrs_val = '+' if _maybe(0.20) else ''
|
|
isrs_val = '+' if np.random.random() < 0.50 else ''
|
|
|
|
# --- Bytes / packets ---
|
|
if profile == 'A':
|
|
ack_val = int(max(1, np.random.exponential(3000)))
|
|
ppk_val = max(1, int(ack_val / np.random.uniform(200, 1500)))
|
|
elif profile == 'B':
|
|
ack_val = int(max(1, np.random.exponential(50000)))
|
|
ppk_val = max(1, int(ack_val / np.random.uniform(100, 1400)))
|
|
else:
|
|
if _maybe(0.50):
|
|
# Beaconing: very small, regular packets
|
|
ack_val = int(np.random.uniform(30, 100)) * np.random.randint(1, 20)
|
|
ppk_val = max(1, ack_val // int(np.random.uniform(30, 100)))
|
|
else:
|
|
# Data exfil: large transfer
|
|
ack_val = int(max(1, np.random.exponential(100000)))
|
|
ppk_val = max(1, int(ack_val / np.random.uniform(100, 1400)))
|
|
|
|
# --- Timestamps ---
|
|
ts_offset = np.random.randint(0, day_seconds)
|
|
ts_full = base_ts + timedelta(seconds=ts_offset)
|
|
ts_str = ts_full.strftime('%Y-%m-%d %H:%M:%S')
|
|
tm_str = ts_full.strftime('%H:%M:%S')
|
|
dbd_str = (base_ts + timedelta(
|
|
seconds=np.random.randint(0, day_seconds)
|
|
)).strftime('%Y-%m-%d %H:%M:%S')
|
|
|
|
# --- Cipher ---
|
|
cipher_hex = _weighted_choice(CIPHER_PROFILES[profile])
|
|
cipher_name = CIPHER_HEX_MAP.get(cipher_hex, '')
|
|
|
|
# --- Curve ---
|
|
curve_hex = _weighted_choice(CURVE_PROFILES[profile])
|
|
curve_name = CURVE_MAP.get(curve_hex, '')
|
|
|
|
# --- Other ---
|
|
ipp_val = int(np.random.choice([6, 17], p=[0.85, 0.15]))
|
|
dbn_val = np.random.randint(1, 10)
|
|
tabl = str(np.random.choice(TABLE_NAMES))
|
|
link_name = str(np.random.choice(LINK_NAMES))
|
|
node = str(np.random.choice(NODE_NAMES))
|
|
|
|
# --- TLS random (low entropy for Profile C) ---
|
|
if profile == 'C':
|
|
rnd = _low_entropy_hex()
|
|
else:
|
|
rnd = _random_32_bytes_hex()
|
|
rnt_val = int(base_ts.timestamp()) + np.random.randint(0, day_seconds)
|
|
|
|
# --- GeoIP dot-notation columns (non-globe: mostly blank) ---
|
|
def _geo_val(val, blank_rate=0.70):
|
|
return val if not _maybe(blank_rate) else ''
|
|
|
|
lat_src = '' if _maybe(0.70) else round(np.random.uniform(-90, 90), 6)
|
|
lon_src = '' if _maybe(0.70) else round(np.random.uniform(-180, 180), 6)
|
|
lat_dst = '' if _maybe(0.70) else round(np.random.uniform(-90, 90), 6)
|
|
lon_dst = '' if _maybe(0.70) else round(np.random.uniform(-180, 180), 6)
|
|
ispn_src = '' if _maybe(0.70) else str(np.random.choice(ISP_NAMES))
|
|
ispn_dst = '' if _maybe(0.70) else str(np.random.choice(ISP_NAMES))
|
|
orgn_src = '' if _maybe(0.70) else str(np.random.choice(ORG_NAMES))
|
|
orgn_dst = '' if _maybe(0.70) else str(np.random.choice(ORG_NAMES))
|
|
city_src = '' if _maybe(0.70) else str(np.random.choice(CITY_NAMES))
|
|
city_dst = '' if _maybe(0.70) else str(np.random.choice(CITY_NAMES))
|
|
|
|
# --- Assemble row dict ---
|
|
row = {
|
|
':ips': src_ip,
|
|
':ipd': dst_ip,
|
|
':prs': '' if _maybe(0.30) else src_port,
|
|
':prd': '' if _maybe(0.30) else dst_port,
|
|
'scnt': scnt if not _maybe(0.30) else '',
|
|
'dcnt': dcnt if not _maybe(0.30) else '',
|
|
'server-ip': dst_ip,
|
|
'client-ip': src_ip,
|
|
'0ver': tls_ver if not _maybe(0.05) else '',
|
|
'snam': snam,
|
|
'cnam': cnam if not _maybe(0.05) else '',
|
|
'4dur': '' if _maybe(0.30) else dur,
|
|
'8ses': '' if _maybe(0.30) else ses,
|
|
'2tmo': '' if _maybe(0.30) else tmo,
|
|
'4ksz': '' if _maybe(0.30) else ksz,
|
|
'cnrs': cnrs_val,
|
|
'isrs': isrs_val,
|
|
'8ack': ack_val if not _maybe(0.05) else '',
|
|
'8ppk': ppk_val if not _maybe(0.05) else '',
|
|
'8dbd': dbd_str if not _maybe(0.30) else '',
|
|
'1ipp': ipp_val if not _maybe(0.30) else '',
|
|
'4dbn': '' if _maybe(0.30) else dbn_val,
|
|
'tabl': tabl if not _maybe(0.30) else '',
|
|
'name': link_name if not _maybe(0.30) else '',
|
|
'source-node': node if not _maybe(0.30) else '',
|
|
'cipher-suite': cipher_name if not _maybe(0.30) else '',
|
|
'ecdhe-named-curve': curve_name if not _maybe(0.30) else '',
|
|
'0cph': cipher_hex if not _maybe(0.30) else '',
|
|
'0crv': curve_hex if not _maybe(0.30) else '',
|
|
'0rnd': rnd if not _maybe(0.30) else '',
|
|
'0rnt': '' if _maybe(0.30) else rnt_val,
|
|
'row': i + 1,
|
|
'time': tm_str if not _maybe(0.30) else '',
|
|
'timestamp': ts_str if not _maybe(0.10) else '',
|
|
':ips.latd': lat_src,
|
|
':ips.lond': lon_src,
|
|
':ipd.latd': lat_dst,
|
|
':ipd.lond': lon_dst,
|
|
':ips.ispn': ispn_src,
|
|
':ipd.ispn': ispn_dst,
|
|
':ips.orgn': orgn_src,
|
|
':ipd.orgn': orgn_dst,
|
|
':ips.city': city_src,
|
|
':ipd.city': city_dst,
|
|
}
|
|
rows.append(row)
|
|
|
|
sys.stdout.write('\n')
|
|
sys.stdout.flush()
|
|
return pl.DataFrame(rows)
|
|
|
|
|
|
# ===================================================================
|
|
# GeoIP loader (shared by globe mode)
|
|
# ===================================================================
|
|
|
|
def _load_geoip_ranges() -> dict[str, list[dict]]:
|
|
"""Load GeoIP range data from ``data/geoip_data.txt``."""
|
|
data_path = Path(__file__).resolve().parent.parent / 'data' / 'geoip_data.txt'
|
|
cities: dict[str, list[dict]] = {}
|
|
with open(data_path, 'r', encoding='utf-8') as f:
|
|
for raw in f:
|
|
stripped = raw.strip()
|
|
if not stripped or stripped.startswith('#'):
|
|
continue
|
|
parts = [p.strip() for p in stripped.split(',')]
|
|
if len(parts) != 6:
|
|
continue
|
|
start_str, end_str, lat_str, lon_str, city, _country = parts
|
|
start_int = _ip_to_int(start_str)
|
|
end_int = _ip_to_int(end_str)
|
|
if start_int is None or end_int is None:
|
|
continue
|
|
try:
|
|
lat = float(lat_str)
|
|
lon = float(lon_str)
|
|
except ValueError:
|
|
continue
|
|
city_key = city.replace(' ', '')
|
|
if city_key not in cities:
|
|
cities[city_key] = []
|
|
cities[city_key].append({
|
|
'start': start_int,
|
|
'end': end_int,
|
|
'lat': lat,
|
|
'lon': lon,
|
|
})
|
|
return cities
|
|
|
|
|
|
def _ip_to_int(ip_str: str) -> int | None:
|
|
"""Convert dotted IPv4 string to 32-bit integer."""
|
|
parts = ip_str.strip().split('.')
|
|
if len(parts) != 4:
|
|
return None
|
|
try:
|
|
octets = [int(p) for p in parts]
|
|
if any(o < 0 or o > 255 for o in octets):
|
|
return None
|
|
return (octets[0] << 24) | (octets[1] << 16) | (octets[2] << 8) | octets[3]
|
|
except (ValueError, TypeError):
|
|
return None
|
|
|
|
|
|
def _int_to_ip(n: int) -> str:
|
|
"""Convert 32-bit integer to dotted IPv4 string."""
|
|
return f'{n >> 24 & 255}.{n >> 16 & 255}.{n >> 8 & 255}.{n & 255}'
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Globe mode city → (ISP, Org) mapping
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_GLOBE_ORG_MAP: dict[str, tuple[str, str]] = {
|
|
'Beijing': ('China Unicom', 'Tencent Holdings'),
|
|
'Shanghai': ('China Telecom', 'Alibaba Inc.'),
|
|
'Taipei': ('Chunghwa Telecom', 'Hon Hai Group'),
|
|
'Tokyo': ('NTT', 'SoftBank Corp.'),
|
|
'Seoul': ('KT Corp.', 'Samsung SDS'),
|
|
'Singapore': ('Singtel', 'Grab Holdings'),
|
|
'Bangkok': ('True Corp.', 'G-Able'),
|
|
'Mumbai': ('Reliance Jio', 'Tata Consultancy'),
|
|
'Sydney': ('Telstra', 'Atlassian'),
|
|
'MountainView': ('Google Fiber', 'Google LLC'),
|
|
'NewYork': ('Verizon', 'Amazon.com Inc.'),
|
|
'LosAngeles': ('AT&T', 'Meta Platforms Inc.'),
|
|
'Toronto': ('Rogers', 'Shopify Inc.'),
|
|
'London': ('BT Group', 'British Telecom'),
|
|
'Paris': ('Orange', 'TotalEnergies'),
|
|
'Frankfurt': ('Deutsche Telekom', 'SAP SE'),
|
|
'Amsterdam': ('KPN', 'ING Group'),
|
|
'Dublin': ('Eir', 'Accenture'),
|
|
'Stockholm': ('Telia', 'Ericsson'),
|
|
'Moscow': ('Rostelecom', 'Yandex'),
|
|
'Dubai': ('Etisalat', 'Emirates Group'),
|
|
'SaoPaulo': ('Vivo', 'Itau Unibanco'),
|
|
'HongKong': ('PCCW', 'HSBC Holdings'),
|
|
'Lagos': ('MTN Group', 'Access Bank'),
|
|
'Tehran': ('TCI', 'Iran Khodro'),
|
|
}
|
|
|
|
|
|
# ===================================================================
|
|
# Globe mode — generates data suitable for 3D globe visualisation
|
|
# ===================================================================
|
|
|
|
def generate_globe_test_data(n_rows: int = 1000, seed: int = 42) -> pl.DataFrame:
|
|
"""Generate test data for 3D globe visualisation using GeoIP-resolvable IPs.
|
|
|
|
Uses the same 3 traffic profiles as the normal mode, but with
|
|
real GeoIP-resolvable source/destination IPs so lat/lon columns
|
|
correspond to actual geographic locations.
|
|
|
|
Data corruption patterns (for testing globe view robustness):
|
|
- ~30% of ``:ips.latd`` values are pure ``+``
|
|
- ~10% of ``:ips.lond`` values are empty ``""``
|
|
"""
|
|
np.random.seed(seed)
|
|
city_ranges = _load_geoip_ranges()
|
|
|
|
# Source cities (Asia / Pacific)
|
|
src_cities = [
|
|
'Beijing', 'Shanghai', 'Taipei', 'Tokyo',
|
|
'Seoul', 'Singapore', 'Bangkok', 'Mumbai', 'Sydney',
|
|
]
|
|
# Destination cities (global rest)
|
|
dst_cities = [
|
|
'MountainView', 'NewYork', 'LosAngeles', 'Toronto',
|
|
'London', 'Paris', 'Frankfurt', 'Amsterdam', 'Dublin',
|
|
'Stockholm', 'Moscow', 'Dubai', 'SaoPaulo',
|
|
]
|
|
|
|
all_cities = set(city_ranges.keys())
|
|
src_cities = [c for c in src_cities if c in all_cities]
|
|
dst_cities = [c for c in dst_cities if c in all_cities]
|
|
if not src_cities or not dst_cities:
|
|
raise RuntimeError('No usable GeoIP city ranges — cannot generate globe data.')
|
|
|
|
def _rand_ip_from_city(city: str) -> tuple[str, float, float]:
|
|
rng_list = city_ranges[city]
|
|
rng_entry = rng_list[np.random.randint(len(rng_list))]
|
|
ip_int = rng_entry['start'] + np.random.randint(0, rng_entry['end'] - rng_entry['start'] + 1)
|
|
return _int_to_ip(ip_int), rng_entry['lat'], rng_entry['lon']
|
|
|
|
base_ts = int(datetime(2023, 11, 1).timestamp())
|
|
thirty_days_sec = 86400 * 60
|
|
|
|
rows: list[dict] = []
|
|
total = n_rows
|
|
next_pct = 10
|
|
|
|
for i in range(total):
|
|
# Progress indicator
|
|
pct = (i + 1) * 100 // total
|
|
if pct >= next_pct:
|
|
sys.stdout.write('+')
|
|
sys.stdout.flush()
|
|
next_pct += 10
|
|
|
|
profile = _assign_profile()
|
|
|
|
# GeoIP-resolvable IPs + lat/lon
|
|
src_city = str(np.random.choice(src_cities))
|
|
src_ip_raw, src_lat_raw, src_lon_raw = _rand_ip_from_city(src_city)
|
|
|
|
dst_city = str(np.random.choice(dst_cities))
|
|
dst_ip_raw, dst_lat, dst_lon = _rand_ip_from_city(dst_city)
|
|
|
|
# Globe-specific corruption patterns
|
|
src_lat_out: str | float = '+' if np.random.random() < 0.30 else round(src_lat_raw, 6)
|
|
src_lon_out: str | float = '' if np.random.random() < 0.10 else round(src_lon_raw, 6)
|
|
|
|
# --- Ports ---
|
|
src_port = np.random.randint(1024, 65536)
|
|
dst_port = _weighted_choice_int(PORT_PROFILES[profile])
|
|
if dst_port == 0:
|
|
dst_port = int(np.random.choice([p for p in range(1025, 65536) if p != 443]))
|
|
|
|
# --- scnt, dcnt ---
|
|
scnt, dcnt = _pick_country(profile)
|
|
|
|
# --- TLS version ---
|
|
tls_ver = _weighted_choice(TLS_PROFILES[profile])
|
|
|
|
# --- SNI ---
|
|
snam, cnam = _pick_sni(profile)
|
|
|
|
# --- Duration ---
|
|
mean_dur, min_dur, max_dur = DUR_PROFILES[profile]
|
|
if profile == 'C':
|
|
if _maybe(0.50):
|
|
dur = round(np.random.uniform(0.3, 1.0), 2)
|
|
else:
|
|
dur = round(min(max(np.random.exponential(120), 60), 600), 2)
|
|
else:
|
|
dur = round(min(max(np.random.exponential(mean_dur), min_dur), max_dur), 2)
|
|
|
|
# --- Session / timeout / key size ---
|
|
ses = round(np.random.uniform(0, 1000), 4)
|
|
tmo = round(np.random.uniform(0, 300), 4)
|
|
ksz = int(np.random.choice([128, 256, 384, 521]))
|
|
|
|
# --- cnrs / isrs ---
|
|
if profile == 'A':
|
|
cnrs_val = '+' if _maybe(0.30) else ''
|
|
isrs_val = '+' if np.random.random() < 0.10 else ''
|
|
elif profile == 'B':
|
|
cnrs_val = '+' if _maybe(0.60) else ''
|
|
isrs_val = '+' if np.random.random() < 0.40 else ''
|
|
else:
|
|
cnrs_val = '+' if _maybe(0.20) else ''
|
|
isrs_val = '+' if np.random.random() < 0.50 else ''
|
|
|
|
# --- Bytes / packets ---
|
|
if profile == 'A':
|
|
ack_val = int(max(1, np.random.exponential(3000)))
|
|
ppk_val = max(1, int(ack_val / np.random.uniform(200, 1500)))
|
|
elif profile == 'B':
|
|
ack_val = int(max(1, np.random.exponential(50000)))
|
|
ppk_val = max(1, int(ack_val / np.random.uniform(100, 1400)))
|
|
else:
|
|
if _maybe(0.50):
|
|
ack_val = int(np.random.uniform(30, 100)) * np.random.randint(1, 20)
|
|
ppk_val = max(1, ack_val // int(np.random.uniform(30, 100)))
|
|
else:
|
|
ack_val = int(max(1, np.random.exponential(100000)))
|
|
ppk_val = max(1, int(ack_val / np.random.uniform(100, 1400)))
|
|
|
|
# --- Timestamps ---
|
|
ts = base_ts + np.random.randint(0, thirty_days_sec)
|
|
ts_str = datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
|
|
tm_str = datetime.fromtimestamp(ts).strftime('%H:%M:%S')
|
|
dbd_str = datetime.fromtimestamp(
|
|
base_ts + np.random.randint(0, thirty_days_sec)
|
|
).strftime('%Y-%m-%d %H:%M:%S')
|
|
|
|
# --- Cipher ---
|
|
cipher_hex = _weighted_choice(CIPHER_PROFILES[profile])
|
|
cipher_name = CIPHER_HEX_MAP.get(cipher_hex, '')
|
|
|
|
# --- Curve ---
|
|
curve_hex = _weighted_choice(CURVE_PROFILES[profile])
|
|
curve_name = CURVE_MAP.get(curve_hex, '')
|
|
|
|
# --- Other ---
|
|
ipp_val = int(np.random.choice([6, 17], p=[0.85, 0.15]))
|
|
dbn_val = np.random.randint(1, 10)
|
|
tabl = str(np.random.choice(TABLE_NAMES))
|
|
link_name = str(np.random.choice(LINK_NAMES))
|
|
node = str(np.random.choice(NODE_NAMES))
|
|
|
|
# --- TLS random ---
|
|
if profile == 'C':
|
|
rnd = _low_entropy_hex()
|
|
else:
|
|
rnd = _random_32_bytes_hex()
|
|
rnt_val = base_ts + np.random.randint(0, thirty_days_sec)
|
|
|
|
# --- GeoIP ISP/org from globe map ---
|
|
src_isp, src_org = _GLOBE_ORG_MAP.get(
|
|
src_city, (str(np.random.choice(ISP_NAMES)), str(np.random.choice(ORG_NAMES))),
|
|
)
|
|
dst_isp, dst_org = _GLOBE_ORG_MAP.get(
|
|
dst_city, (str(np.random.choice(ISP_NAMES)), str(np.random.choice(ORG_NAMES))),
|
|
)
|
|
|
|
rows.append({
|
|
':ips': src_ip_raw,
|
|
':ipd': dst_ip_raw,
|
|
':prs': '' if _maybe(0.30) else src_port,
|
|
':prd': '' if _maybe(0.30) else dst_port,
|
|
'scnt': scnt if not _maybe(0.30) else '',
|
|
'dcnt': dcnt if not _maybe(0.30) else '',
|
|
'server-ip': dst_ip_raw,
|
|
'client-ip': src_ip_raw,
|
|
'0ver': tls_ver if not _maybe(0.05) else '',
|
|
'snam': snam,
|
|
'cnam': cnam if not _maybe(0.05) else '',
|
|
'4dur': '' if _maybe(0.30) else dur,
|
|
'8ses': '' if _maybe(0.30) else ses,
|
|
'2tmo': '' if _maybe(0.30) else tmo,
|
|
'4ksz': '' if _maybe(0.30) else ksz,
|
|
'cnrs': cnrs_val,
|
|
'isrs': isrs_val,
|
|
'8ack': ack_val if not _maybe(0.05) else '',
|
|
'8ppk': ppk_val if not _maybe(0.05) else '',
|
|
'8dbd': dbd_str if not _maybe(0.30) else '',
|
|
'1ipp': ipp_val if not _maybe(0.30) else '',
|
|
'4dbn': '' if _maybe(0.30) else dbn_val,
|
|
'tabl': tabl if not _maybe(0.30) else '',
|
|
'name': link_name if not _maybe(0.30) else '',
|
|
'source-node': node if not _maybe(0.30) else '',
|
|
'cipher-suite': cipher_name if not _maybe(0.30) else '',
|
|
'ecdhe-named-curve': curve_name if not _maybe(0.30) else '',
|
|
'0cph': cipher_hex if not _maybe(0.30) else '',
|
|
'0crv': curve_hex if not _maybe(0.30) else '',
|
|
'0rnd': rnd if not _maybe(0.30) else '',
|
|
'0rnt': '' if _maybe(0.30) else rnt_val,
|
|
'row': i + 1,
|
|
'time': tm_str if not _maybe(0.30) else '',
|
|
'timestamp': ts_str if not _maybe(0.10) else '',
|
|
':ips.latd': src_lat_out,
|
|
':ips.lond': src_lon_out,
|
|
':ipd.latd': round(dst_lat, 6),
|
|
':ipd.lond': round(dst_lon, 6),
|
|
':ips.ispn': src_isp,
|
|
':ipd.ispn': dst_isp,
|
|
':ips.orgn': src_org,
|
|
':ipd.orgn': dst_org,
|
|
':ips.city': src_city,
|
|
':ipd.city': dst_city,
|
|
})
|
|
|
|
sys.stdout.write('\n')
|
|
sys.stdout.flush()
|
|
return pl.DataFrame(rows)
|
|
|
|
|
|
# ===================================================================
|
|
# CSV save & CLI
|
|
# ===================================================================
|
|
|
|
EXPECTED_COLUMNS: list[str] = [
|
|
':ips', ':ipd', ':prs', ':prd', 'scnt', 'dcnt', 'server-ip', 'client-ip',
|
|
'0ver', 'snam', 'cnam', '4dur', '8ses', '2tmo', '4ksz', 'cnrs', 'isrs',
|
|
'8ack', '8ppk', '8dbd', '1ipp', '4dbn', 'tabl', 'name', 'source-node',
|
|
'cipher-suite', 'ecdhe-named-curve', '0cph', '0crv', '0rnd', '0rnt',
|
|
'row', 'time', 'timestamp',
|
|
':ips.latd', ':ips.lond', ':ipd.latd', ':ipd.lond',
|
|
':ips.ispn', ':ipd.ispn', ':ips.orgn', ':ipd.orgn', ':ips.city', ':ipd.city',
|
|
]
|
|
|
|
|
|
def save_csv(df: pl.DataFrame, path: str):
|
|
"""Write DataFrame to CSV with full 44-column validation."""
|
|
out_path = Path(path)
|
|
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Validate column count
|
|
actual_cols = df.columns
|
|
if len(actual_cols) != 44:
|
|
print(f'WARNING: Expected 44 columns, got {len(actual_cols)}')
|
|
missing = set(EXPECTED_COLUMNS) - set(actual_cols)
|
|
extra = set(actual_cols) - set(EXPECTED_COLUMNS)
|
|
if missing:
|
|
print(f' Missing: {sorted(missing)}')
|
|
if extra:
|
|
print(f' Extra: {sorted(extra)}')
|
|
|
|
df.write_csv(out_path)
|
|
size_kb = out_path.stat().st_size / 1024
|
|
print(f'Generated: {path} ({len(df)} rows, {size_kb:.0f} KB, {len(actual_cols)} cols)')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
import argparse
|
|
parser = argparse.ArgumentParser(description='Generate synthetic TLS flow test data (3 traffic profiles)')
|
|
parser.add_argument('--rows', type=int, default=1000, help='Number of rows (default: 1000)')
|
|
parser.add_argument('--output', type=str, default=None, help='Output CSV path')
|
|
parser.add_argument('--globe', action='store_true',
|
|
help='Generate 3D globe visualisation test data (GeoIP-resolvable IPs)')
|
|
args = parser.parse_args()
|
|
|
|
if args.globe:
|
|
output = args.output or 'data/globe_test.csv'
|
|
print(f'Generating {args.rows} globe rows...')
|
|
print('Progress: ', end='', flush=True)
|
|
df = generate_globe_test_data(args.rows)
|
|
save_csv(df, output)
|
|
|
|
# Globe-specific stats
|
|
plus_cnt = df.filter(pl.col(':ips.latd') == '+').height
|
|
blank_cnt = df.filter(pl.col(':ips.lond') == '').height
|
|
print(f'\nGlobe data stats:')
|
|
print(f' :ips.latd "+": {plus_cnt}/{len(df)} ({plus_cnt/len(df)*100:.1f}%)')
|
|
print(f' :ips.lond "": {blank_cnt}/{len(df)} ({blank_cnt/len(df)*100:.1f}%)')
|
|
else:
|
|
output = args.output or 'data/test_flows.csv'
|
|
print(f'Generating {args.rows} rows (A:60% Normal, B:20% Proxy, C:20% Malicious)...')
|
|
print('Progress: ', end='', flush=True)
|
|
df = generate_tls_flows(args.rows)
|
|
save_csv(df, output)
|
|
|
|
# Profile distribution
|
|
print(f'\nProfile distribution:')
|
|
for prefix, label in [('A', 'A-Normal'), ('B', 'B-Proxy'), ('C', 'C-Malicious')]:
|
|
# Profile isn't stored as a column, so estimate from IP patterns
|
|
pass
|