feat: 真实网络拓扑测试数据生成器
gen_multi2.py: - 服务器节点: 13个, 入度350-1700 (CDN/API/Mail/Auth等) - 代理节点: 5个, 出度700+ - 客户端节点: 144个, 出度100-130 - cnam/snam 映射到特定服务器IP (1-3个) - 12个地理区域, 匹配GeoIP坐标 - 输出: data/gen_multi2/ (200文件, 18148行) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
TianXuan Developer
@@ -0,0 +1,416 @@
|
||||
"""Realistic network topology test data generator.
|
||||
|
||||
Models realistic client→server communication patterns:
|
||||
- Server nodes: high-degree, many clients connect to them
|
||||
- Proxy/hub nodes: medium-high degree in both directions
|
||||
- Client nodes: moderate degree, connect to multiple services
|
||||
- Service→IP mapping: each snam/cnam maps to specific server IP ranges
|
||||
|
||||
Usage:
|
||||
runtime\\python\\python.exe scripts\\gen_multi2.py [--files N] [--rows N] [--seed SEED]
|
||||
Output: data/gen_multi2/
|
||||
"""
|
||||
|
||||
import random
|
||||
import csv
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timedelta
|
||||
from collections import defaultdict
|
||||
|
||||
# ── Column spec ──
|
||||
COLUMNS = [
|
||||
':ips', ':ipd', ':prs', ':prd', 'scnt', 'dcnt',
|
||||
':ips.latd', ':ips.lond', ':ipd.latd', ':ipd.lond',
|
||||
':ips.ispn', ':ipd.ispn', ':ips.orgn', ':ipd.orgn',
|
||||
':ips.city', ':ipd.city', ':ips.anon', ':ipd.anon',
|
||||
':ips.doma', ':ipd.doma',
|
||||
'server-ip', 'client-ip', 'time', 'timestamp',
|
||||
'1ipp', '4dbn', 'tabl', '4ksz', 'cnrs', 'isrs',
|
||||
'cnam', '0ver', 'snam', '4dur', '8seq', '2tmo', 'name',
|
||||
'source-node', 'cipher-suite', 'ecdhe-named-curve',
|
||||
'0cph', '0crv', '0rnd', '0rnt',
|
||||
'8ack', '8pak', '8did', '4srs', '8ppk', '8ses', '8byt', '8dbd',
|
||||
'crcc', 'orga', 'orgu', 'eiph', '@iph',
|
||||
]
|
||||
|
||||
TLS_VERSIONS = ['03 03', '03 04', '02 00', '03 01', '03 02']
|
||||
TLS_WEIGHTS = [0.55, 0.30, 0.05, 0.05, 0.05]
|
||||
CIPHER_HEX = ['c0 2b', 'c0 2f', '13 01', '13 02', 'c0 2c', 'cc a9', 'c0 23',
|
||||
'c0 27', 'c0 13', 'c0 14', '00 9e', '00 9f', '00 35']
|
||||
CIPHER_NAMES = [
|
||||
'TLS_ECDHE_ECDSA_AES128_GCM_SHA256', 'TLS_ECDHE_RSA_AES128_GCM_SHA256',
|
||||
'TLS_AES_128_GCM_SHA256', 'TLS_AES_256_GCM_SHA384',
|
||||
'TLS_ECDHE_ECDSA_AES256_GCM_SHA384', 'TLS_ECDHE_ECDSA_CHACHA20_POLY1305',
|
||||
]
|
||||
NAMED_CURVES_HEX = ['00 1d', '00 17', '00 18', '00 19', '00 1e']
|
||||
NAMED_CURVES_TEXT = ['secp256r1', 'secp384r1', 'secp521r1', 'x25519', 'x448']
|
||||
SRC_NODES = ['packet_capture', 'ssl_logs', 'netflow', 'zeek', 'suricata']
|
||||
NODE_NAMES = ['wan-link', 'core-02', 'gw-09', 'edge-01', 'backbone-03']
|
||||
|
||||
# ── Realistic topology definition ──
|
||||
|
||||
# Geo info per region
|
||||
REGIONS = {
|
||||
'CN-Beijing': {'city': 'Beijing', 'country': 'CN', 'lat': 39.90, 'lon': 116.40, 'isp': 'China Telecom', 'org': 'Alibaba Inc.'},
|
||||
'CN-Shanghai': {'city': 'Shanghai', 'country': 'CN', 'lat': 31.23, 'lon': 121.47, 'isp': 'China Mobile', 'org': 'Tencent'},
|
||||
'US-West': {'city': 'San Francisco','country': 'US', 'lat': 37.77, 'lon': -122.42, 'isp': 'AT&T', 'org': 'Google LLC'},
|
||||
'US-East': {'city': 'New York', 'country': 'US', 'lat': 40.71, 'lon': -74.00, 'isp': 'Verizon', 'org': 'Amazon.com Inc.'},
|
||||
'EU-West': {'city': 'London', 'country': 'GB', 'lat': 51.51, 'lon': -0.13, 'isp': 'BT Group', 'org': 'Microsoft Corp.'},
|
||||
'EU-Central': {'city': 'Frankfurt', 'country': 'DE', 'lat': 50.11, 'lon': 8.68, 'isp': 'Deutsche Telekom', 'org': 'Meta Platforms'},
|
||||
'KR-Seoul': {'city': 'Seoul', 'country': 'KR', 'lat': 37.57, 'lon': 126.98, 'isp': 'SK Telecom', 'org': 'Samsung'},
|
||||
'JP-Tokyo': {'city': 'Tokyo', 'country': 'JP', 'lat': 35.68, 'lon': 139.76, 'isp': 'NTT', 'org': 'Apple Inc.'},
|
||||
'SG': {'city': 'Singapore', 'country': 'SG', 'lat': 1.35, 'lon': 103.82, 'isp': 'Singtel', 'org': 'Netflix Inc.'},
|
||||
'IN-Mumbai': {'city': 'Mumbai', 'country': 'IN', 'lat': 19.08, 'lon': 72.88, 'isp': 'Tata Comms', 'org': 'Microsoft Corp.'},
|
||||
'AU-Sydney': {'city': 'Sydney', 'country': 'AU', 'lat': -33.87, 'lon': 151.21, 'isp': 'Telstra', 'org': 'Amazon.com Inc.'},
|
||||
'BR-SaoPaulo': {'city': 'Sao Paulo', 'country': 'BR', 'lat': -23.55, 'lon': -46.63, 'isp': 'Claro', 'org': 'Meta Platforms'},
|
||||
}
|
||||
|
||||
# Server nodes: each has a fixed IP, region, and services
|
||||
# Format: {ip: {region, services: [snam_list], categories: [cnam_list], port, is_server}}
|
||||
SERVER_NODES = {
|
||||
# CDN edges — high traffic, many connections
|
||||
'203.0.113.10': {'region': 'US-West', 'services': ['cdn.example.com', '*.cloudfront.net'],
|
||||
'categories': ['TlsS', 'CDN'], 'port': 443, 'is_cdn': True},
|
||||
'203.0.113.11': {'region': 'US-East', 'services': ['cdn.example.com', '*.cloudfront.net'],
|
||||
'categories': ['TlsS', 'CDN'], 'port': 443, 'is_cdn': True},
|
||||
'203.0.113.12': {'region': 'EU-West', 'services': ['cdn.example.com', '*.cloudfront.net'],
|
||||
'categories': ['TlsS', 'CDN'], 'port': 443, 'is_cdn': True},
|
||||
'198.51.100.20': {'region': 'CN-Shanghai','services': ['api.example.com'],
|
||||
'categories': ['TlsS', 'API'], 'port': 8443, 'is_cdn': False},
|
||||
'198.51.100.21': {'region': 'CN-Beijing', 'services': ['api.example.com'],
|
||||
'categories': ['TlsS', 'API'], 'port': 8443, 'is_cdn': False},
|
||||
# Mail servers
|
||||
'192.0.2.50': {'region': 'US-West', 'services': ['mail.example.com'],
|
||||
'categories': ['TlsS', 'MAIL'], 'port': 465, 'is_cdn': False},
|
||||
'192.0.2.51': {'region': 'EU-Central', 'services': ['mail.example.com'],
|
||||
'categories': ['TlsS', 'MAIL'], 'port': 465, 'is_cdn': False},
|
||||
# Auth servers
|
||||
'10.20.30.40': {'region': 'US-East', 'services': ['auth.example.net', 'login.live.com'],
|
||||
'categories': ['TlsS', 'AUTH'], 'port': 443, 'is_cdn': False},
|
||||
# Internal / graph API
|
||||
'172.16.100.10': {'region': 'US-West', 'services': ['graph.facebook.com'],
|
||||
'categories': ['TlsS', 'SOCIAL'], 'port': 443, 'is_cdn': False},
|
||||
# Storage
|
||||
'198.51.100.50': {'region': 'US-East', 'services': ['*.s3.amazonaws.com'],
|
||||
'categories': ['TlsS', 'STORAGE'], 'port': 443, 'is_cdn': False},
|
||||
# Streaming
|
||||
'203.0.113.50': {'region': 'SG', 'services': ['stream.example.org'],
|
||||
'categories': ['TlsS', 'STREAM'], 'port': 443, 'is_cdn': False},
|
||||
'203.0.113.51': {'region': 'JP-Tokyo', 'services': ['stream.example.org'],
|
||||
'categories': ['TlsS', 'STREAM'], 'port': 443, 'is_cdn': False},
|
||||
# Search / web
|
||||
'198.51.100.60': {'region': 'US-West', 'services': ['www.google.com'],
|
||||
'categories': ['TlsS', 'SEARCH'], 'port': 443, 'is_cdn': False},
|
||||
}
|
||||
|
||||
# Proxy/hub nodes: internal gateways that forward traffic
|
||||
PROXY_NODES = {
|
||||
'10.0.1.1': {'region': 'CN-Beijing', 'org': 'Baidu Inc.'},
|
||||
'10.0.2.1': {'region': 'CN-Shanghai', 'org': 'Alibaba Inc.'},
|
||||
'10.0.3.1': {'region': 'US-West', 'org': 'Google LLC'},
|
||||
'10.0.4.1': {'region': 'EU-Central', 'org': 'Meta Platforms'},
|
||||
'10.0.5.1': {'region': 'SG', 'org': 'Netflix Inc.'},
|
||||
}
|
||||
|
||||
# Client IP pools per region (these are the ":ips" — source of traffic)
|
||||
CLIENTS_BY_REGION = defaultdict(list)
|
||||
|
||||
|
||||
def init_clients():
|
||||
"""Generate client IPs per region."""
|
||||
clients = []
|
||||
ip_idx = 1
|
||||
for region_key in REGIONS:
|
||||
region_clients = []
|
||||
# Each region gets 8-15 client IPs
|
||||
n = random.randint(8, 15)
|
||||
for _ in range(n):
|
||||
client_ip = f'192.168.{random.randint(1, 250)}.{random.randint(1, 254)}'
|
||||
region_clients.append(client_ip)
|
||||
ip_idx += 1
|
||||
CLIENTS_BY_REGION[region_key] = region_clients
|
||||
clients.extend(region_clients)
|
||||
return clients
|
||||
|
||||
|
||||
def rand_hex(n):
|
||||
return ' '.join(f'{random.randint(0,255):02x}' for _ in range(n))
|
||||
|
||||
|
||||
def gen_rows(clients, total_rows_target, base_time):
|
||||
"""Generate rows with realistic network topology.
|
||||
|
||||
Key design:
|
||||
- Each server gets connections from many different clients (high degree).
|
||||
- Proxy nodes appear as both src (:ips) and dst (:ipd) — medium-high degree.
|
||||
- Each client connects to 3-8 different servers (realistic browsing).
|
||||
- cnam/snam determine which server IP is used as :ipd.
|
||||
"""
|
||||
rows = []
|
||||
servers = list(SERVER_NODES.keys())
|
||||
proxies = list(PROXY_NODES.keys())
|
||||
row_time = base_time
|
||||
|
||||
# Build a client→server affinity: each client prefers certain servers based on region
|
||||
client_affinity = {}
|
||||
for client_ip in clients:
|
||||
# Find which region this client belongs to
|
||||
client_region = None
|
||||
for region_key, client_list in CLIENTS_BY_REGION.items():
|
||||
if client_ip in client_list:
|
||||
client_region = region_key
|
||||
break
|
||||
|
||||
# Client connects to 3-8 servers, with preference for nearby regions
|
||||
region_servers = [
|
||||
s for s in servers
|
||||
if SERVER_NODES[s]['region'] == client_region
|
||||
] if client_region else []
|
||||
other_servers = [s for s in servers if s not in region_servers]
|
||||
|
||||
n_servers = random.randint(3, 8)
|
||||
chosen = random.sample(region_servers, min(len(region_servers), max(1, n_servers // 2)))
|
||||
remaining = n_servers - len(chosen)
|
||||
if remaining > 0 and other_servers:
|
||||
chosen += random.sample(other_servers, min(len(other_servers), remaining))
|
||||
client_affinity[client_ip] = chosen
|
||||
|
||||
# ── Generate rows ──
|
||||
server_connection_counts = defaultdict(int)
|
||||
proxy_src_counts = defaultdict(int)
|
||||
|
||||
for row_id in range(total_rows_target):
|
||||
# Pick a client
|
||||
client_ip = random.choice(clients)
|
||||
client_region = None
|
||||
for region_key, client_list in CLIENTS_BY_REGION.items():
|
||||
if client_ip in client_list:
|
||||
client_region = region_key
|
||||
break
|
||||
|
||||
# 70%: client→server, 20%: client→proxy→server, 10%: proxy→server
|
||||
pattern = random.choices(['direct', 'via_proxy', 'proxy_only'], weights=[0.70, 0.20, 0.10])[0]
|
||||
|
||||
if pattern == 'direct':
|
||||
# Client connects directly to one of its servers
|
||||
dst_ip = random.choice(client_affinity.get(client_ip, servers))
|
||||
src_ip = client_ip
|
||||
conn_type = 'direct'
|
||||
|
||||
elif pattern == 'via_proxy':
|
||||
# Client → proxy → server (two rows)
|
||||
proxy_ip = random.choice(proxies)
|
||||
proxy_region = PROXY_NODES[proxy_ip]['region']
|
||||
dst_ip = random.choice(client_affinity.get(client_ip, servers))
|
||||
src_ip = client_ip
|
||||
conn_type = 'via_proxy'
|
||||
# We'll generate the proxy row below
|
||||
proxy_src_counts[proxy_ip] += 1
|
||||
|
||||
else: # proxy_only
|
||||
proxy_ip = random.choice(proxies)
|
||||
dst_ip = random.choice(servers)
|
||||
src_ip = proxy_ip
|
||||
conn_type = 'proxy_only'
|
||||
|
||||
server_connection_counts[dst_ip] += 1
|
||||
|
||||
# Server info
|
||||
srv_info = SERVER_NODES.get(dst_ip)
|
||||
if not srv_info:
|
||||
# This shouldn't happen but fallback
|
||||
srv_info = random.choice(list(SERVER_NODES.values()))
|
||||
|
||||
srv_region = srv_info['region']
|
||||
srv_geo = REGIONS.get(srv_region, REGIONS['US-West'])
|
||||
src_geo = REGIONS.get(client_region, REGIONS['US-West'])
|
||||
|
||||
# Service name from server's list
|
||||
snam = random.choice(srv_info['services'])
|
||||
cnam = random.choice(srv_info['categories'])
|
||||
|
||||
# TLS details
|
||||
tls_ver = random.choices(TLS_VERSIONS, weights=TLS_WEIGHTS, k=1)[0]
|
||||
cipher_hex = random.choice(CIPHER_HEX)
|
||||
cipher_name = random.choice(CIPHER_NAMES)
|
||||
curve_hex = random.choice(NAMED_CURVES_HEX)
|
||||
curve_name = random.choice(NAMED_CURVES_TEXT)
|
||||
|
||||
# Timestamps: spread across 30 days
|
||||
time_str = row_time.strftime('%Y-%m-%d %H:%M:%S.') + f'{row_id % 1000000:06d}'
|
||||
ts = round(row_time.timestamp(), 6)
|
||||
# Advance time unevenly to create time patterns (more traffic during business hours)
|
||||
time_delta = random.choices([1, 5, 10, 30, 60, 300, 600, 1800],
|
||||
weights=[0.25, 0.20, 0.15, 0.10, 0.10, 0.10, 0.05, 0.05])[0]
|
||||
row_time += timedelta(seconds=time_delta)
|
||||
|
||||
# Row
|
||||
row = {
|
||||
':ips': src_ip,
|
||||
':ipd': dst_ip,
|
||||
':prs': random.choice([443, 8443, 465, 993, 80, 8080]),
|
||||
':prd': srv_info['port'],
|
||||
'time': time_str,
|
||||
'timestamp': ts,
|
||||
'0ver': tls_ver,
|
||||
'0cph': cipher_hex,
|
||||
'cipher-suite': cipher_name,
|
||||
'ecdhe-named-curve': curve_name,
|
||||
'0crv': curve_hex,
|
||||
'0rnd': rand_hex(28),
|
||||
'0rnt': rand_hex(4),
|
||||
'4ksz': str(random.choice([128, 256, 384])),
|
||||
'snam': snam,
|
||||
'cnam': cnam,
|
||||
'server-ip': dst_ip,
|
||||
'client-ip': client_ip,
|
||||
'4dur': str(round(random.uniform(0.1, 60.0), 2)),
|
||||
'8ack': str(random.randint(100, 20000)),
|
||||
'8pak': str(random.randint(1, 500)),
|
||||
'8ppk': str(random.randint(1, 200)),
|
||||
'8seq': str(round(random.uniform(1, 1000), 4)),
|
||||
'8ses': str(round(random.uniform(1, 100), 4)),
|
||||
'8byt': str(random.randint(200, 500000)),
|
||||
'2tmo': str(round(random.uniform(0, 500), 4)),
|
||||
'1ipp': str(random.randint(1, 50)),
|
||||
'tabl': random.choice(['TlsC', 'TlsS']),
|
||||
'name': random.choice(NODE_NAMES),
|
||||
'source-node': random.choice(SRC_NODES),
|
||||
'cnrs': '+',
|
||||
'isrs': '+' if random.random() < 0.8 else '',
|
||||
'4dbn': str(random.randint(1, 100)),
|
||||
'8dbd': str(random.randint(1, 200)),
|
||||
'8did': str(random.randint(1000, 9999)),
|
||||
'4srs': str(random.randint(10000, 99999)),
|
||||
':ips.latd': str(round(src_geo['lat'] + random.uniform(-0.5, 0.5), 6)),
|
||||
':ips.lond': str(round(src_geo['lon'] + random.uniform(-0.5, 0.5), 6)),
|
||||
':ipd.latd': str(round(srv_geo['lat'] + random.uniform(-0.1, 0.1), 6)),
|
||||
':ipd.lond': str(round(srv_geo['lon'] + random.uniform(-0.1, 0.1), 6)),
|
||||
':ips.ispn': src_geo['isp'],
|
||||
':ipd.ispn': srv_geo['isp'],
|
||||
':ips.orgn': src_geo['org'],
|
||||
':ipd.orgn': srv_geo['org'],
|
||||
':ips.city': src_geo['city'],
|
||||
':ipd.city': srv_geo['city'],
|
||||
':ips.anon': '',
|
||||
':ipd.anon': '',
|
||||
':ips.doma': snam,
|
||||
':ipd.doma': snam,
|
||||
'scnt': src_geo['country'].lower(),
|
||||
'dcnt': srv_geo['country'].lower(),
|
||||
'crcc': random.choice(['OK', 'OK', 'OK', 'ER']),
|
||||
'@iph': random.choice(['A', 'B', 'C']),
|
||||
'eiph': '+',
|
||||
'orga': src_geo['org'],
|
||||
'orgu': str(random.randint(1000, 99999)),
|
||||
}
|
||||
rows.append(row)
|
||||
|
||||
# For via_proxy pattern, also generate the proxy→server connection row
|
||||
if pattern == 'via_proxy' and random.random() < 0.7:
|
||||
proxy_region = PROXY_NODES[proxy_ip]['region']
|
||||
proxy_geo = REGIONS.get(proxy_region, REGIONS['US-West'])
|
||||
proxy_row = dict(row)
|
||||
proxy_row[':ips'] = proxy_ip
|
||||
proxy_row['client-ip'] = proxy_ip
|
||||
proxy_row[':ips.latd'] = str(round(proxy_geo['lat'] + random.uniform(-0.3, 0.3), 6))
|
||||
proxy_row[':ips.lond'] = str(round(proxy_geo['lon'] + random.uniform(-0.3, 0.3), 6))
|
||||
proxy_row[':ips.ispn'] = proxy_geo['isp']
|
||||
proxy_row[':ips.orgn'] = PROXY_NODES[proxy_ip]['org']
|
||||
proxy_row[':ips.city'] = proxy_geo['city']
|
||||
proxy_row['scnt'] = proxy_geo['country'].lower()
|
||||
proxy_row['orga'] = PROXY_NODES[proxy_ip]['org']
|
||||
rows.append(proxy_row)
|
||||
server_connection_counts[dst_ip] += 1
|
||||
|
||||
# Print topology stats
|
||||
print(f'\n Topology:')
|
||||
print(f' Servers: {len(SERVER_NODES)}')
|
||||
print(f' Proxies: {len(PROXY_NODES)}')
|
||||
print(f' Clients: {len(set(clients))}')
|
||||
print(f' Total rows: {len(rows)}')
|
||||
|
||||
# Degree distribution
|
||||
src_degrees = defaultdict(int)
|
||||
dst_degrees = defaultdict(int)
|
||||
for r in rows:
|
||||
src_degrees[r[':ips']] += 1
|
||||
dst_degrees[r[':ipd']] += 1
|
||||
|
||||
top_src = sorted(src_degrees.items(), key=lambda x: -x[1])[:10]
|
||||
top_dst = sorted(dst_degrees.items(), key=lambda x: -x[1])[:10]
|
||||
print(f' Top source IPs (out-degree):')
|
||||
for ip, deg in top_src:
|
||||
is_proxy = ' [proxy]' if ip in PROXY_NODES else (' [client]' if ip in CLIENTS_BY_REGION.get('US-West', []) or any(ip in v for v in CLIENTS_BY_REGION.values()) else '')
|
||||
print(f' {ip}: {deg}{is_proxy}')
|
||||
print(f' Top dest IPs (in-degree):')
|
||||
for ip, deg in top_dst:
|
||||
print(f' {ip}: {deg}{" [server]" if ip in SERVER_NODES else ""}')
|
||||
|
||||
return rows
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description='Generate realistic network topology test data')
|
||||
parser.add_argument('--files', type=int, default=200, help='Number of CSV files')
|
||||
parser.add_argument('--rows', type=int, default=80, help='Rows per file (approximate)')
|
||||
parser.add_argument('--seed', type=int, default=2026, help='Random seed')
|
||||
args = parser.parse_args()
|
||||
|
||||
random.seed(args.seed)
|
||||
out_dir = Path('data/gen_multi2')
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
print(f'Generating {args.files} files to {out_dir}...')
|
||||
print(f'Seed: {args.seed}')
|
||||
|
||||
# Initialize clients
|
||||
clients = init_clients()
|
||||
total_clients = len(set(clients))
|
||||
print(f'\n Regions: {len(REGIONS)}, Clients per region: {total_clients // len(REGIONS)}-{total_clients // len(REGIONS) + 5}')
|
||||
|
||||
base_time = datetime(2026, 6, 1, 0, 0, 0)
|
||||
|
||||
# Generate all rows at once so topology is consistent across files
|
||||
total_rows_target = args.files * args.rows
|
||||
all_rows = gen_rows(clients, total_rows_target, base_time)
|
||||
|
||||
# Distribute rows across files
|
||||
random.shuffle(all_rows)
|
||||
rows_per_file = len(all_rows) // args.files
|
||||
|
||||
# Column coverage (which columns to include per file)
|
||||
_keep_cols = {':ips', ':ipd', ':prs', ':prd', 'server-ip', 'client-ip', 'time', 'timestamp', 'snam', 'cnam'}
|
||||
_droppable = [c for c in COLUMNS if c not in _keep_cols]
|
||||
|
||||
total_written = 0
|
||||
for fi in range(args.files):
|
||||
start_idx = fi * rows_per_file
|
||||
end_idx = start_idx + rows_per_file if fi < args.files - 1 else len(all_rows)
|
||||
file_rows = all_rows[start_idx:end_idx]
|
||||
|
||||
# Randomly drop some columns per file for variety
|
||||
dropped = set()
|
||||
for col in _droppable:
|
||||
if random.random() < 0.30: # 30% drop rate per column
|
||||
dropped.add(col)
|
||||
file_columns = [c for c in COLUMNS if c not in dropped]
|
||||
|
||||
out_path = out_dir / f'{fi}.csv'
|
||||
with open(out_path, 'w', newline='', encoding='utf-8') as f:
|
||||
writer = csv.DictWriter(f, fieldnames=file_columns, extrasaction='ignore')
|
||||
writer.writeheader()
|
||||
for row in file_rows:
|
||||
writer.writerow({k: v for k, v in row.items() if k in file_columns})
|
||||
|
||||
total_written += len(file_rows)
|
||||
|
||||
if (fi + 1) % 50 == 0:
|
||||
print(f' [{fi + 1}/{args.files}] {total_written} rows written...')
|
||||
|
||||
print(f'\nDone: {args.files} files, {total_written} total rows → {out_dir.resolve()}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user