HJQ/tianxuan
2
1
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity
Files
9314ac6942d90021fae7ccc228d8c8aec17f73a0
tianxuan/scripts/gen_test_data.py
T
PM-pinou 9314ac6942 chore: bump version to v1.1.2 
- New timeline UI: interleaved LLM reasoning + tool call display
- Fix: run_log truncation for large tool outputs
- Fix: lat/lon + values reduced to ~1% (was 99%)
- Fix: @csrf_exempt on upload/delete endpoints
- Fix: DATA_UPLOAD_MAX_NUMBER_FILES for 1998-file upload
- Fix: dtype alignment for multi-batch CSV processing
- New: tests/test_e2e_full.py with real DeepSeek LLM
- New: 1998-file test data generator
2026-07-20 23:32:32 +08:00

225 lines
10 KiB
Python
Raw Blame History

"""Generate synthetic TLS flow test data matching TlsDB.csv column spec.
Coverage rates simulate real TLS traffic: IP/port always present,
lat/lon/ISP often blank, TLS fingerprint fields sometimes blank,
obscure fields rarely present.
Usage:
runtime\\python\\python.exe scripts\\gen_test_data.py [--rows N]
"""
import random
import csv
import argparse
from pathlib import Path
from datetime import datetime, timedelta
random.seed(42)
# All columns from TlsDB.csv in realistic order
COLUMNS = [
':ips', ':ipd', ':prs', ':prd', 'scnt', 'dcnt',
':ips.latd', ':ips.lond', ':ipd.latd', ':ipd.lond',
':ips.ispn', ':ipd.ispn', ':ips.orgn', ':ipd.orgn',
':ips.city', ':ipd.city', ':ips.anon', ':ipd.anon',
'server-ip', 'client-ip', 'row', 'time', 'timestamp',
'1ipp', '4dbn', 'tabl', '4ksz', 'cnrs', 'isrs',
'cnam', '0ver', 'snam', '4dur', '8seq', '2tmo', 'name',
'source-node', 'cipher-suite', 'ecdhe-named-curve',
'0cph', '0crv', '0rnd', '0rnt',
'8ack', '8pak', '8did', '4srs', '8ppk', '8ses', '8byt', '8dbd',
'crcc', 'orga', 'orgu', 'eiph', '@iph',
]
# ── Data pools ───────────────────────────────────────────────────────────
TLS_VERSIONS = ['03 03', '03 04', '02 00', '03 01', '03 02']
TLS_WEIGHTS = [0.55, 0.30, 0.05, 0.05, 0.05]
CIPHER_HEX = ['c0 2b', 'c0 2f', '13 01', '13 02', 'c0 2c', 'cc a9', 'c0 23',
'c0 27', 'c0 13', 'c0 14', '00 9e', '00 9f', '00 35']
CIPHER_NAMES = [
'TLS_ECDHE_ECDSA_AES128_GCM_SHA256', 'TLS_ECDHE_RSA_AES128_GCM_SHA256',
'TLS_AES_128_GCM_SHA256', 'TLS_AES_256_GCM_SHA384',
'TLS_ECDHE_ECDSA_AES256_GCM_SHA384', 'TLS_ECDHE_ECDSA_CHACHA20_POLY1305',
'TLS_ECDHE_RSA_AES256_GCM_SHA384', 'TLS_ECDHE_RSA_AES128_SHA256',
'TLS_ECDHE_ECDSA_AES128_SHA', 'TLS_ECDHE_RSA_AES128_SHA',
'TLS_RSA_WITH_AES_128_GCM_SHA256', 'TLS_RSA_WITH_AES_256_GCM_SHA384',
'TLS_RSA_WITH_AES_128_CBC_SHA',
]
NAMED_CURVES_HEX = ['00 1d', '00 17', '00 18', '00 19', '00 1e']
NAMED_CURVES_TEXT = ['secp256r1', 'secp384r1', 'secp521r1', 'x25519', 'x448']
SRC_IPS = [f'{random.randint(1,223)}.{random.randint(0,255)}.{random.randint(0,255)}.{random.randint(1,254)}' for _ in range(200)]
DST_IPS = [f'{random.randint(1,223)}.{random.randint(0,255)}.{random.randint(0,255)}.{random.randint(1,254)}' for _ in range(500)]
PORTS = [443, 80, 8080, 8443, 465, 993, 995, 53, 22, 3389, 25, 110, 143, 21, 12345, 50000, 443, 443, 443, 443]
COUNTRIES = ['US','CN','KR','JP','GB','DE','FR','RU','BR','IN','SG','NL','CA','AU','HK','UA','IL','SE','NO','FI']
ISPS = ['China Telecom','China Mobile','BT Group','Orange','Deutsche Telekom','AT&T','Verizon','Comcast','NTT','KDDI','SK Telecom','Singtel','Telstra']
ORGS = ['Baidu Inc.','Alibaba Inc.','Amazon.com Inc.','Google LLC','Microsoft Corp.','Meta Platforms','Tencent','Samsung','Apple Inc.','Netflix Inc.']
CITIES = ['Beijing','Shanghai','Seoul','Berlin','Paris','London','New York','San Francisco','Tokyo','Singapore','Sydney','Moscow','Seattle','Dublin','Mumbai']
SERVICES = ['mail.example.com','api.example.com','cdn.example.com','auth.example.net','stream.example.org','login.live.com','*.cloudfront.net','*.s3.amazonaws.com','graph.facebook.com','www.google.com']
SRC_NODES = ['packet_capture','ssl_logs','netflow','zeek','suricata']
NODE_NAMES = ['wan-link','core-02','gw-09','edge-01','backbone-03']
def _rand_ip(pool):
return random.choice(pool)
def _rand_hex(n):
return ' '.join(f'{random.randint(0,255):02x}' for _ in range(n))
def _maybe(rate):
"""Return True with given probability (0.0-1.0)."""
return random.random() < rate
def _blank_or(rate, value, blank_marker=''):
"""Return value with rate probability, else blank_marker."""
return value if _maybe(rate) else blank_marker
def _blank_or_plus(rate, value):
"""Return value with rate probability, else '+' (null marker).
Signature matches _blank_or(rate, value, blank_marker=...)."""
return value if _maybe(rate) else '+'
def generate_row(row_id, base_time):
"""Generate a single TLS flow row matching all TlsDB.csv columns."""
src_ip = _rand_ip(SRC_IPS)
dst_ip = _rand_ip(DST_IPS)
src_port = random.choice(PORTS)
dst_port = random.choice(PORTS)
src_country = random.choice(COUNTRIES)
dst_country = random.choice(COUNTRIES)
server_ip = _rand_ip(DST_IPS + SRC_IPS)
client_ip = src_ip if _maybe(0.85) else _rand_ip(SRC_IPS)
tls_ver = random.choices(TLS_VERSIONS, weights=TLS_WEIGHTS, k=1)[0]
cipher_hex = random.choice(CIPHER_HEX)
cipher_name = random.choice(CIPHER_NAMES)
curve_hex = random.choice(NAMED_CURVES_HEX)
curve_name = random.choice(NAMED_CURVES_TEXT)
service = random.choice(SERVICES)
cert_cn = service if _maybe(0.7) else f'*.{random.choice(SERVICES).split(".")[-2:]}'
ts = int(base_time.timestamp()) + row_id
# Row-local lat/lon with realistic blank rates
src_lat = round(random.uniform(-60, 60), 6)
src_lon = round(random.uniform(-180, 180), 6)
dst_lat = round(random.uniform(-60, 60), 6)
dst_lon = round(random.uniform(-180, 180), 6)
return {
# ── Core (always present) ──
':ips': src_ip,
':ipd': dst_ip,
':prs': src_port,
':prd': dst_port,
'time': base_time.strftime('%H:%M:%S'),
'timestamp': ts,
# ── TLS fingerprint (very high coverage) ──
'0ver': tls_ver,
'0cph': cipher_hex,
'cipher-suite': _blank_or(0.85, cipher_name),
'ecdhe-named-curve': _blank_or(0.80, curve_name),
'0crv': _blank_or(0.80, curve_hex),
'0rnd': _blank_or(0.75, _rand_hex(28)),
'0rnt': _blank_or(0.70, _rand_hex(4)),
'4ksz': _blank_or(0.85, str(random.choice([128, 256, 384, 512]))),
# ── Session / connection info (medium-high coverage) ──
'snam': _blank_or(0.80, service),
'cnam': _blank_or(0.70, cert_cn),
'server-ip': _blank_or(0.60, server_ip),
'client-ip': _blank_or(0.85, client_ip),
'4dur': _blank_or(0.75, str(round(random.uniform(0.5, 120.0), 2))),
'8ack': _blank_or(0.80, str(random.randint(100, 200000))),
'8pak': _blank_or(0.70, str(random.randint(1, 2000))),
'8ppk': _blank_or(0.65, str(random.randint(1, 500))),
'8seq': _blank_or(0.40, str(round(random.uniform(1, 2000), 4))),
'8ses': _blank_or(0.50, str(round(random.uniform(1, 2000), 4))),
'8byt': _blank_or(0.40, str(random.randint(200, 500000))),
'2tmo': _blank_or(0.50, str(round(random.uniform(0, 1000), 4))),
# ── TCP / network info (medium coverage) ──
'1ipp': str(random.randint(1, 50)),
'row': _blank_or(1.0, ''), # TLS协议含义
'tabl': random.choice(['TlsC','TlsS']), # 100% coverage
'name': _blank_or(0.30, random.choice(NODE_NAMES)),
'source-node': _blank_or(0.30, random.choice(SRC_NODES)),
# ── Session resumption (medium coverage) ──
'cnrs': _blank_or(0.55, '+'),
'isrs': _blank_or(0.50, '+'),
# ── Database metadata (low-medium coverage) ──
'4dbn': _blank_or(0.40, str(random.randint(1, 100))),
'8dbd': _blank_or(0.25, str(random.randint(1, 200))),
'8did': _blank_or(0.15, str(random.randint(1000, 9999))),
'4srs': _blank_or(0.20, str(random.randint(10000, 99999))),
# ── GeoIP location (rare, ~1% have data, rest '+' = null) ──
':ips.latd': _blank_or_plus(0.99, str(src_lat)), # 99% real value, ~1% '+' null
':ips.lond': _blank_or_plus(0.99, str(src_lon)),
':ipd.latd': _blank_or_plus(0.99, str(dst_lat)),
':ipd.lond': _blank_or_plus(0.99, str(dst_lon)),
':ips.ispn': _blank_or(0.20, random.choice(ISPS)),
':ipd.ispn': _blank_or(0.20, random.choice(ISPS)),
':ips.orgn': _blank_or(0.15, random.choice(ORGS)),
':ipd.orgn': _blank_or(0.15, random.choice(ORGS)),
':ips.city': _blank_or(0.25, random.choice(CITIES)),
':ipd.city': _blank_or(0.25, random.choice(CITIES)),
':ips.anon': _blank_or(0.05, 'anon, hosting'), # single combined string
':ipd.anon': _blank_or(0.05, 'anon, hosting'),
# ── Two-letter abbreviation columns (rare, ~5%) ──
'scnt': _blank_or(0.05, src_country),
'dcnt': _blank_or(0.05, dst_country),
'crcc': _blank_or(0.05, random.choice(['OK','ER','--'])),
'@iph': _blank_or(0.001, random.choice(['A','B','C','D'])), # 1/1000
'eiph': _blank_or(0.05, '+'), # boolean: + = True, blank = False
# ── Obscure / rarely populated fields ──
'orga': _blank_or(0.08, random.choice(ORGS)),
'orgu': _blank_or(0.08, str(random.randint(1000, 99999))),
# ── TLS protocol-level (always present) ──
}
def main():
parser = argparse.ArgumentParser(description='Generate TLS flow test data matching TlsDB.csv')
parser.add_argument('--rows', type=int, default=10000, help='Number of rows')
parser.add_argument('--output', type=str, default='data/test_flows.csv')
args = parser.parse_args()
columns = COLUMNS
output_path = Path(args.output)
output_path.parent.mkdir(parents=True, exist_ok=True)
base_time = datetime(2026, 6, 1, 0, 0, 0)
blank_counts = {col: 0 for col in columns}
total_rows = args.rows
with open(output_path, 'w', newline='', encoding='utf-8') as f:
writer = csv.DictWriter(f, fieldnames=columns)
writer.writeheader()
for i in range(total_rows):
row_time = base_time + timedelta(seconds=random.randint(0, 86400 * 30))
row = generate_row(i, row_time)
writer.writerow(row)
for col in columns:
if row.get(col, '') == '' or row.get(col, '') == '+':
blank_counts[col] += 1
print(f'Generated {total_rows} rows → {output_path}')
print('\nCoverage report (blank/null rate per column):')
for col in columns:
rate = blank_counts[col] / total_rows * 100
bar_len = int((100 - rate) / 5)
bar = '#' * bar_len + '.' * (20 - bar_len)
print(f' {col:20s} {bar} {100-rate:5.1f}% present')
if __name__ == '__main__':
main()
Reference in New Issue View Git Blame Copy Permalink