PM-pinou
419 lines
17 KiB
Python
419 lines
17 KiB
Python
#!/usr/bin/env python
|
|
"""Convert WNL QoS TLS dataset to the user's 44-column CSV format.
|
|
|
|
The WNL dataset (tab-separated) has columns:
|
|
SNI\tservice_label\tPKT_1_payload\tPKT_2_payload
|
|
|
|
PKT_1_payload contains TLS record bytes as comma-separated decimal ints.
|
|
This script parses the TLS ClientHello and generates the user's 44-column
|
|
format, appending the ground-truth label as the last column.
|
|
|
|
Usage:
|
|
runtime\python\python.exe scripts\wnl_to_user_csv.py
|
|
"""
|
|
|
|
import csv
|
|
import os
|
|
import random
|
|
import struct
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
|
|
random.seed(42)
|
|
|
|
# ── Paths ──────────────────────────────────────────────────────────────────
|
|
BASE = Path(r"C:\Users\25044\Desktop\Proj")
|
|
WNL_CSV = BASE / "qos-tls-dataset-of-enc-traffic" / "Preprocessed_CSV" / "Preprocessed_CSV" / "WNL_TLS_Dataset_Origin.csv"
|
|
OUT_CSV = BASE / "天璇" / "data" / "wnl_converted.csv"
|
|
OUT_CSV.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
# ── User 44-column header ──────────────────────────────────────────────────
|
|
USER_HEADER = [
|
|
":ips", ":ipd", ":prs", ":prd", "scnt", "dcnt",
|
|
"server-ip", "client-ip",
|
|
"0ver", "snam", "cnam",
|
|
"4dur", "8ses", "2tmo", "4ksz",
|
|
"cnrs", "isrs",
|
|
"8ack", "8ppk", "8dbd", "1ipp",
|
|
"4dbn", "tabl", "name", "source-node",
|
|
"cipher-suite", "ecdhe-named-curve", "0cph", "0crv",
|
|
"0rnd", "0rnt",
|
|
"row", "time", "timestamp",
|
|
":ips.latd", ":ips.lond", ":ipd.latd", ":ipd.lond",
|
|
":ips.ispn", ":ipd.ispn", ":ips.orgn", ":ipd.orgn",
|
|
":ips.city", ":ipd.city",
|
|
]
|
|
|
|
# ── Cipher suite hex → IANA name map ──────────────────────────────────────
|
|
CIPHER_HEX_MAP = {
|
|
0x1301: "TLS_AES_128_GCM_SHA256",
|
|
0x1302: "TLS_AES_256_GCM_SHA384",
|
|
0x1303: "TLS_CHACHA20_POLY1305_SHA256",
|
|
0xC02B: "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
|
|
0xC02C: "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
|
|
0xC02F: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
|
|
0xC030: "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
|
|
0xCCA9: "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256",
|
|
0xCCAC: "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256",
|
|
0x009C: "TLS_RSA_WITH_AES_128_GCM_SHA256",
|
|
0x009D: "TLS_RSA_WITH_AES_256_GCM_SHA384",
|
|
0x003C: "TLS_RSA_WITH_AES_128_CBC_SHA256",
|
|
0x003D: "TLS_RSA_WITH_AES_256_CBC_SHA256",
|
|
0xC009: "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256",
|
|
0xC013: "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256",
|
|
0x002F: "TLS_RSA_WITH_AES_128_CBC_SHA",
|
|
0x0035: "TLS_RSA_WITH_AES_128_CBC_SHA",
|
|
0x0005: "TLS_RSA_WITH_RC4_128_SHA",
|
|
0x00FF: "TLS_RSA_WITH_NULL_MD5",
|
|
}
|
|
|
|
# ── Named curve hex → name map ────────────────────────────────────────────
|
|
CURVE_HEX_MAP = {
|
|
0x0017: "secp256r1",
|
|
0x0018: "secp384r1",
|
|
0x0019: "secp521r1",
|
|
0x001D: "x25519",
|
|
0x001E: "x448",
|
|
}
|
|
|
|
# ── Service class → IP / geo / duration profiles ──────────────────────────
|
|
SERVICE_PROFILES = {
|
|
"Web": {"ip8": "10", "cnt": "US", "dur_range": (0.5, 15), "isp": "Cloudflare", "org": "Cloudflare Inc.", "city": "San Francisco"},
|
|
"Spotify": {"ip8": "20", "cnt": "SE", "dur_range": (30, 600), "isp": "Telia", "org": "Spotify AB", "city": "Stockholm"},
|
|
"Netflix": {"ip8": "30", "cnt": "US", "dur_range": (60, 1200), "isp": "Comcast", "org": "Netflix Inc.", "city": "Los Gatos"},
|
|
"YouTube": {"ip8": "40", "cnt": "US", "dur_range": (30, 900), "isp": "Google Fiber", "org": "Google LLC", "city": "Mountain View"},
|
|
"AppleMusic": {"ip8": "50", "cnt": "US", "dur_range": (30, 600), "isp": "AT&T", "org": "Apple Inc.", "city": "Cupertino"},
|
|
"SoundCloud": {"ip8": "60", "cnt": "DE", "dur_range": (30, 500), "isp": "Deutsche Telekom", "org": "SoundCloud Ltd.", "city": "Berlin"},
|
|
"PrimeVideo": {"ip8": "70", "cnt": "US", "dur_range": (60, 1200), "isp": "Amazon", "org": "Amazon.com Inc.", "city": "Seattle"},
|
|
"LiveYouTube": {"ip8": "80", "cnt": "US", "dur_range": (60, 1800), "isp": "Google Fiber", "org": "Google LLC", "city": "Mountain View"},
|
|
"LiveFacebook": {"ip8": "90", "cnt": "US", "dur_range": (60, 1800), "isp": "Facebook", "org": "Meta Platforms Inc.", "city": "Menlo Park"},
|
|
"Kinopoisk": {"ip8": "100", "cnt": "RU", "dur_range": (30, 900), "isp": "Rostelecom", "org": "Yandex LLC", "city": "Moscow"},
|
|
"YandexMusic": {"ip8": "110", "cnt": "RU", "dur_range": (30, 600), "isp": "Rostelecom", "org": "Yandex LLC", "city": "Moscow"},
|
|
"Vimeo": {"ip8": "120", "cnt": "US", "dur_range": (30, 600), "isp": "Akamai", "org": "Vimeo Inc.", "city": "New York"},
|
|
}
|
|
|
|
# ── TLS version hex → display ─────────────────────────────────────────────
|
|
TLS_VER_MAP = {
|
|
(3, 4): "03 04",
|
|
(3, 3): "03 03",
|
|
(3, 2): "03 02",
|
|
(3, 1): "03 01",
|
|
}
|
|
|
|
|
|
def parse_tls_clienthello(payload_bytes: bytes) -> dict:
|
|
"""Parse TLS ClientHello from raw bytes, returning extracted fields.
|
|
|
|
Returns dict with keys:
|
|
record_version, ch_version, cipher_suites, cipher_suite_hex,
|
|
sni, supported_groups
|
|
Missing/parse-failure values are empty strings or empty lists.
|
|
"""
|
|
result = {
|
|
"record_version": "",
|
|
"ch_version": "",
|
|
"cipher_suites": [],
|
|
"cipher_suite_hex": "",
|
|
"sni": "",
|
|
"supported_groups": [],
|
|
"curves_hex": "",
|
|
}
|
|
|
|
if len(payload_bytes) < 5:
|
|
return result
|
|
|
|
content_type = payload_bytes[0]
|
|
if content_type != 22: # Handshake
|
|
return result
|
|
|
|
# Record layer version
|
|
rv_major = payload_bytes[1]
|
|
rv_minor = payload_bytes[2]
|
|
result["record_version"] = TLS_VER_MAP.get((rv_major, rv_minor), f"{rv_major:02x} {rv_minor:02x}")
|
|
|
|
if len(payload_bytes) < 6:
|
|
return result
|
|
|
|
hs_type = payload_bytes[5]
|
|
if hs_type != 1: # ClientHello
|
|
return result
|
|
|
|
# ClientHello version at offset 9-10
|
|
if len(payload_bytes) < 11:
|
|
return result
|
|
ch_major = payload_bytes[9]
|
|
ch_minor = payload_bytes[10]
|
|
result["ch_version"] = TLS_VER_MAP.get((ch_major, ch_minor), f"{ch_major:02x} {ch_minor:02x}")
|
|
|
|
# Session ID length at offset 43
|
|
if len(payload_bytes) < 44:
|
|
return result
|
|
sess_len = payload_bytes[43]
|
|
offset = 44 + sess_len
|
|
|
|
# Cipher suites: 2 bytes length, then N bytes (2-byte each)
|
|
if len(payload_bytes) < offset + 2:
|
|
return result
|
|
cs_len = (payload_bytes[offset] << 8) | payload_bytes[offset + 1]
|
|
offset += 2
|
|
cs_end = offset + cs_len
|
|
if cs_end > len(payload_bytes):
|
|
cs_end = len(payload_bytes)
|
|
|
|
cipher_suite_hex_list = []
|
|
for i in range(offset, cs_end, 2):
|
|
if i + 1 < cs_end:
|
|
cs_val = (payload_bytes[i] << 8) | payload_bytes[i + 1]
|
|
cs_hex = f"{payload_bytes[i]:02x} {payload_bytes[i+1]:02x}"
|
|
cipher_suite_hex_list.append(cs_hex)
|
|
result["cipher_suites"].append(cs_val)
|
|
|
|
result["cipher_suite_hex"] = cipher_suite_hex_list[0] if cipher_suite_hex_list else ""
|
|
|
|
offset = cs_end
|
|
# Compression methods
|
|
if offset >= len(payload_bytes):
|
|
return result
|
|
comp_len = payload_bytes[offset]
|
|
offset += 1 + comp_len
|
|
|
|
# Extensions
|
|
if offset + 1 >= len(payload_bytes):
|
|
return result
|
|
ext_len = (payload_bytes[offset] << 8) | payload_bytes[offset + 1]
|
|
offset += 2
|
|
ext_end = offset + ext_len
|
|
if ext_end > len(payload_bytes):
|
|
ext_end = len(payload_bytes)
|
|
|
|
while offset + 3 < ext_end:
|
|
ext_type = (payload_bytes[offset] << 8) | payload_bytes[offset + 1]
|
|
ext_data_len = (payload_bytes[offset + 2] << 8) | payload_bytes[offset + 3]
|
|
offset += 4
|
|
ext_data_end = offset + ext_data_len
|
|
if ext_data_end > ext_end:
|
|
ext_data_end = ext_end
|
|
|
|
if ext_type == 0 and ext_data_len > 5: # server_name
|
|
# Skip list length (2) + type (1) + name length (2)
|
|
sni_name_len = (payload_bytes[offset + 3] << 8) | payload_bytes[offset + 4]
|
|
if offset + 5 + sni_name_len <= ext_data_end:
|
|
sni_bytes = payload_bytes[offset + 5: offset + 5 + sni_name_len]
|
|
try:
|
|
result["sni"] = sni_bytes.decode("ascii", errors="replace")
|
|
except Exception:
|
|
result["sni"] = sni_bytes.decode("ascii", errors="replace")
|
|
|
|
elif ext_type == 10 and ext_data_len > 1: # supported_groups
|
|
groups_len = (payload_bytes[offset] << 8) | payload_bytes[offset + 1]
|
|
groups_hex = []
|
|
for gi in range(0, groups_len, 2):
|
|
idx = offset + 2 + gi
|
|
if idx + 1 < ext_data_end:
|
|
gv = (payload_bytes[idx] << 8) | payload_bytes[idx + 1]
|
|
groups_hex.append(f"{payload_bytes[idx]:02x} {payload_bytes[idx+1]:02x}")
|
|
result["supported_groups"].append(gv)
|
|
result["curves_hex"] = groups_hex[0] if groups_hex else ""
|
|
|
|
offset = ext_data_end
|
|
|
|
return result
|
|
|
|
|
|
def make_ip(ip8: str, idx: int) -> str:
|
|
"""Generate a plausible IP from a /8 prefix."""
|
|
return f"{ip8}.{random.randint(1, 254)}.{random.randint(1, 254)}.{random.randint(1, 254)}"
|
|
|
|
|
|
def make_server_ip(cnt: str) -> str:
|
|
"""Generate a server IP that looks like a CDN/cloud provider."""
|
|
prefixes = {
|
|
"US": "203.0.113", # example/documentation range
|
|
"SE": "198.51.100",
|
|
"DE": "203.0.114",
|
|
"RU": "198.52.100",
|
|
"GB": "203.0.115",
|
|
}
|
|
p = prefixes.get(cnt, "203.0.113")
|
|
return f"{p}.{random.randint(1, 254)}"
|
|
|
|
|
|
def main():
|
|
print(f"[1/3] 读取 WNL 数据集: {WNL_CSV}")
|
|
|
|
# Read the origin file and parse all rows
|
|
rows_data = [] # list of (sni, service_label, pkt1_bytes)
|
|
with open(WNL_CSV, encoding="utf-8") as f:
|
|
# CSV with tab delimiter
|
|
reader = csv.reader(f, delimiter="\t")
|
|
headers = next(reader)
|
|
print(f" Headers: {headers}")
|
|
|
|
for row in reader:
|
|
if not row or len(row) < 4:
|
|
continue
|
|
sni = row[0].strip()
|
|
label = row[1].strip()
|
|
# PKT_1_payload is comma-separated decimal ints
|
|
pkt1_str = row[2].strip()
|
|
if not pkt1_str:
|
|
continue
|
|
try:
|
|
pkt1_bytes = bytes(int(b.strip()) for b in pkt1_str.split(",") if b.strip())
|
|
except ValueError:
|
|
continue
|
|
rows_data.append((sni, label, pkt1_bytes))
|
|
|
|
print(f" 总共 {len(rows_data)} 行")
|
|
|
|
# Count labels
|
|
label_counts = {}
|
|
for _, lbl, _ in rows_data:
|
|
label_counts[lbl] = label_counts.get(lbl, 0) + 1
|
|
print(f" Service classes ({len(label_counts)}):")
|
|
for lbl, cnt in sorted(label_counts.items(), key=lambda x: -x[1]):
|
|
print(f" {lbl}: {cnt}")
|
|
|
|
print(f"\n[2/3] 解析 TLS ClientHello 并生成 44 列格式")
|
|
|
|
# Generate output
|
|
out_rows = []
|
|
parse_errors = 0
|
|
|
|
for idx, (sni, label, pkt1_bytes) in enumerate(rows_data):
|
|
tls_info = parse_tls_clienthello(pkt1_bytes)
|
|
if not tls_info["record_version"]:
|
|
parse_errors += 1
|
|
|
|
prof = SERVICE_PROFILES.get(label, SERVICE_PROFILES["Web"])
|
|
ip8 = prof["ip8"]
|
|
cnt = prof["cnt"]
|
|
dur_range = prof["dur_range"]
|
|
isp = prof["isp"]
|
|
org = prof["org"]
|
|
city = prof["city"]
|
|
|
|
# Synthetic IPs
|
|
src_ip = make_ip(ip8, idx) # client IP (:ips)
|
|
dst_ip = make_server_ip(cnt) # server IP (:ipd)
|
|
|
|
# TLS version: prefer CH version, fallback to record version
|
|
tls_ver = tls_info["ch_version"] or tls_info["record_version"] or "03 03"
|
|
|
|
# Cipher suite
|
|
cipher_suites = tls_info["cipher_suites"]
|
|
cipher_suite_hex = tls_info["cipher_suite_hex"]
|
|
cipher_suite_name = ""
|
|
if cipher_suites:
|
|
cs_val = cipher_suites[0]
|
|
cipher_suite_name = CIPHER_HEX_MAP.get(cs_val, f"UNKNOWN_0x{cs_val:04X}")
|
|
else:
|
|
# Default for TLS 1.3
|
|
cipher_suite_hex = "13 01"
|
|
cipher_suite_name = "TLS_AES_128_GCM_SHA256"
|
|
|
|
# Named curve
|
|
supported_groups = tls_info["supported_groups"]
|
|
curves_hex = tls_info["curves_hex"]
|
|
curve_name = ""
|
|
if supported_groups:
|
|
cg_val = supported_groups[0]
|
|
curve_name = CURVE_HEX_MAP.get(cg_val, f"unknown_curve_0x{cg_val:04X}")
|
|
else:
|
|
curves_hex = "00 1d" # default x25519
|
|
curve_name = "x25519"
|
|
|
|
# SNI: use parsed or the raw SNI column
|
|
sni_val = tls_info.get("sni") or sni
|
|
|
|
# Duration based on service class
|
|
duration = round(random.uniform(*dur_range), 2)
|
|
|
|
# Random values for synthetic columns
|
|
rand_bytes = random.randint(1000, 200000)
|
|
rand_packets = random.randint(5, 100)
|
|
ses_val = round(random.uniform(100, 1000), 4)
|
|
tmo_val = round(random.uniform(100, 500), 4)
|
|
ksz_val = random.choice([128, 256, 384, 512])
|
|
cnrs_val = random.choice(["True", ""])
|
|
isrs_val = random.choice(["True", ""])
|
|
|
|
# Timestamp: base it on row index spread over a month
|
|
base_dt = datetime(2024, 6, 1, 0, 0, 0)
|
|
dt = base_dt + timedelta(seconds=random.randint(0, 30 * 24 * 3600))
|
|
ts_str = dt.strftime("%Y-%m-%d %H:%M:%S")
|
|
unix_ts = int(dt.timestamp())
|
|
|
|
# Random hex random bytes
|
|
rnd_hex = " ".join(f"{random.randint(0, 255):02x}" for _ in range(32))
|
|
|
|
row_data = [
|
|
src_ip, # :ips
|
|
dst_ip, # :ipd
|
|
str(random.randint(40000, 65000)), # :prs (source port)
|
|
"443", # :prd (destination port, typically 443)
|
|
"US", # scnt (source country)
|
|
cnt, # dcnt (destination country)
|
|
dst_ip, # server-ip
|
|
src_ip, # client-ip
|
|
tls_ver, # 0ver
|
|
sni_val, # snam
|
|
sni_val, # cnam
|
|
str(duration), # 4dur
|
|
str(ses_val), # 8ses
|
|
str(tmo_val), # 2tmo
|
|
str(ksz_val), # 4ksz
|
|
cnrs_val, # cnrs
|
|
isrs_val, # isrs
|
|
str(rand_bytes), # 8ack
|
|
str(rand_packets), # 8ppk
|
|
ts_str, # 8dbd
|
|
"6", # 1ipp (TCP)
|
|
"packet_capture", # 4dbn
|
|
"wan-link", # tabl
|
|
"wnl_tls", # name
|
|
"gw-01", # source-node
|
|
cipher_suite_name, # cipher-suite
|
|
curve_name, # ecdhe-named-curve
|
|
cipher_suite_hex, # 0cph
|
|
curves_hex, # 0crv
|
|
rnd_hex, # 0rnd
|
|
str(unix_ts), # 0rnt
|
|
str(idx + 1), # row
|
|
dt.strftime("%H:%M:%S"), # time
|
|
ts_str, # timestamp
|
|
"", # :ips.latd (leave blank)
|
|
"", # :ips.lond (leave blank)
|
|
"", # :ipd.latd (leave blank)
|
|
"", # :ipd.lond (leave blank)
|
|
"", # :ips.ispn (leave blank)
|
|
"", # :ipd.ispn (leave blank)
|
|
"", # :ips.orgn (leave blank)
|
|
"", # :ipd.orgn (leave blank)
|
|
"", # :ips.city (leave blank)
|
|
"", # :ipd.city (leave blank)
|
|
]
|
|
|
|
# Append label as the LAST column
|
|
row_data.append(label)
|
|
|
|
out_rows.append(row_data)
|
|
|
|
print(f" TLS 解析失败: {parse_errors}/{len(rows_data)}")
|
|
print(f" 成功生成: {len(out_rows)} 行")
|
|
|
|
# Write output CSV
|
|
print(f"\n[3/3] 写入输出: {OUT_CSV}")
|
|
header = USER_HEADER + ["label"]
|
|
with open(OUT_CSV, "w", newline="", encoding="utf-8") as f:
|
|
writer = csv.writer(f)
|
|
writer.writerow(header)
|
|
writer.writerows(out_rows)
|
|
|
|
print(f" 完成! 输出 {len(out_rows)} 行 x {len(header)} 列")
|
|
print(f" Columns: {header}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|