PM-pinou
309 lines
11 KiB
Python
309 lines
11 KiB
Python
"""TLS value normalization — maps hex wire format, IANA names, and display
|
|
names to unified underscore-connected enum names.
|
|
|
|
Usage::
|
|
|
|
from analysis.value_normalizer import normalize_lf
|
|
|
|
lf = pl.LazyFrame({...})
|
|
type_map = classify_schema(lf) # {col_name: DataType}
|
|
lf = normalize_lf(lf, type_map)
|
|
# -> LazyFrame with added {col}_norm columns
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import polars as pl
|
|
|
|
# ═════════════════════════════════════════════════════════════════════════════
|
|
# TLS Version map — hex wire format & display names
|
|
# ═════════════════════════════════════════════════════════════════════════════
|
|
|
|
TLS_VERSION_MAP: dict[str, str] = {
|
|
# hex wire format (space-separated hex pairs)
|
|
'03 04': 'tls_1_3',
|
|
'03 03': 'tls_1_2',
|
|
'03 02': 'tls_1_1',
|
|
'03 01': 'tls_1_0',
|
|
'02 00': 'ssl_2_0',
|
|
'03 00': 'ssl_3_0',
|
|
# display formats
|
|
'tlsv1.3': 'tls_1_3',
|
|
'tls 1.3': 'tls_1_3',
|
|
'1.3': 'tls_1_3',
|
|
'tlsv1.2': 'tls_1_2',
|
|
'tls 1.2': 'tls_1_2',
|
|
'1.2': 'tls_1_2',
|
|
'tlsv1.1': 'tls_1_1',
|
|
'tls 1.1': 'tls_1_1',
|
|
'1.1': 'tls_1_1',
|
|
'tlsv1.0': 'tls_1_0',
|
|
'tls 1.0': 'tls_1_0',
|
|
'1.0': 'tls_1_0',
|
|
'tlsv1': 'tls_1_0',
|
|
'tls 1': 'tls_1_0',
|
|
'1': 'tls_1_0',
|
|
'sslv2': 'ssl_2_0',
|
|
'ssl 2': 'ssl_2_0',
|
|
'2': 'ssl_2_0',
|
|
'sslv3': 'ssl_3_0',
|
|
'ssl 3': 'ssl_3_0',
|
|
'3': 'ssl_3_0',
|
|
}
|
|
"""TLS version mapping.
|
|
|
|
Keys are hex wire format (``'03 04'``) or display names (``'TLSv1.3'``,
|
|
``'tls 1.3'``, etc.). Matching is case-insensitive. Unmatched values
|
|
produce ``'unknown_version'``.
|
|
"""
|
|
|
|
# ═════════════════════════════════════════════════════════════════════════════
|
|
# Cipher Suite maps
|
|
# ═════════════════════════════════════════════════════════════════════════════
|
|
|
|
CIPHER_SUITE_MAP: dict[str, str] = {
|
|
# hex wire format → canonical underscore name
|
|
'13 01': 'tls_aes_128_gcm_sha256',
|
|
'13 02': 'tls_aes_256_gcm_sha384',
|
|
'13 03': 'tls_chacha20_poly1305_sha256',
|
|
'c0 2b': 'tls_ecdhe_ecdsa_aes_128_gcm_sha256',
|
|
'c0 2c': 'tls_ecdhe_ecdsa_aes_256_gcm_sha384',
|
|
'c0 2d': 'tls_ecdhe_ecdsa_aes_128_cbc_sha256',
|
|
'c0 2f': 'tls_ecdhe_rsa_aes_128_gcm_sha256',
|
|
'c0 30': 'tls_ecdhe_rsa_aes_256_gcm_sha384',
|
|
'cc a9': 'tls_ecdhe_ecdsa_chacha20_poly1305_sha256',
|
|
'cc ac': 'tls_ecdhe_rsa_chacha20_poly1305_sha256',
|
|
'00 9c': 'tls_rsa_aes_128_gcm_sha256',
|
|
'00 9d': 'tls_rsa_aes_256_gcm_sha384',
|
|
'00 3c': 'tls_rsa_aes_128_cbc_sha256',
|
|
'00 3d': 'tls_rsa_aes_256_cbc_sha256',
|
|
'c0 09': 'tls_ecdhe_ecdsa_aes_128_cbc_sha256',
|
|
'c0 13': 'tls_ecdhe_rsa_aes_128_cbc_sha256',
|
|
'00 35': 'tls_rsa_aes_128_cbc_sha',
|
|
'00 2f': 'tls_rsa_aes_128_cbc_sha',
|
|
'00 05': 'tls_rsa_rc4_128_sha',
|
|
'00 ff': 'tls_rsa_null_md5',
|
|
}
|
|
"""Cipher suite hex → canonical name. Unmatched hex produces
|
|
``'unknown_cipher_0x{hex}'``."""
|
|
|
|
_CIPHER_DISPLAY_NORMALIZED: dict[str, str] = {
|
|
# Normalized key (lowercase, separators removed) → canonical name
|
|
'tlsaes128gcmsha256': 'tls_aes_128_gcm_sha256',
|
|
'tlsaes256gcmsha384': 'tls_aes_256_gcm_sha384',
|
|
'aes128gcm': 'tls_aes_128_gcm_sha256',
|
|
'aes256gcm': 'tls_aes_256_gcm_sha384',
|
|
'chacha20poly1305': 'tls_chacha20_poly1305_sha256',
|
|
'ecdheecdsaaes128gcm': 'tls_ecdhe_ecdsa_aes_128_gcm_sha256',
|
|
'ecdhersaaes128gcm': 'tls_ecdhe_rsa_aes_128_gcm_sha256',
|
|
'ecdhersaaes256gcm': 'tls_ecdhe_rsa_aes_256_gcm_sha384',
|
|
'rsaaes128gcm': 'tls_rsa_aes_128_gcm_sha256',
|
|
'rsaaes256gcm': 'tls_rsa_aes_256_gcm_sha384',
|
|
}
|
|
"""Cipher suite display-name normalisation lookup.
|
|
|
|
Keys are IANA/display names with separators removed and lowercased
|
|
(e.g. ``'TLS_AES_128_GCM_SHA256'`` → ``'tlsaes128gmcsha256'``).
|
|
"""
|
|
|
|
# ═════════════════════════════════════════════════════════════════════════════
|
|
# Named Curve maps
|
|
# ═════════════════════════════════════════════════════════════════════════════
|
|
|
|
CURVE_MAP: dict[str, str] = {
|
|
# hex wire format → canonical name
|
|
'00 1d': 'x25519',
|
|
'00 17': 'secp256r1',
|
|
'00 18': 'secp384r1',
|
|
'00 19': 'secp521r1',
|
|
'00 1e': 'x448',
|
|
# display names
|
|
'x25519': 'x25519',
|
|
'curve25519': 'x25519',
|
|
'secp256r1': 'secp256r1',
|
|
'prime256v1': 'secp256r1',
|
|
'p256': 'secp256r1',
|
|
'p-256': 'secp256r1',
|
|
'p_256': 'secp256r1',
|
|
'secp384r1': 'secp384r1',
|
|
'p384': 'secp384r1',
|
|
'p-384': 'secp384r1',
|
|
'p_384': 'secp384r1',
|
|
'secp521r1': 'secp521r1',
|
|
'p521': 'secp521r1',
|
|
'p-521': 'secp521r1',
|
|
'p_521': 'secp521r1',
|
|
'x448': 'x448',
|
|
'curve448': 'x448',
|
|
}
|
|
"""Named curve mapping.
|
|
|
|
Keys are hex wire format (``'00 1d'``) or display names (``'X25519'``,
|
|
``'prime256v1'``, etc.). Matching is case-insensitive.
|
|
"""
|
|
|
|
|
|
def _build_when_chain(
|
|
v: pl.Expr,
|
|
mapping: dict[str, str],
|
|
fallback: pl.Expr,
|
|
) -> pl.Expr:
|
|
"""Build a ``pl.when()...then()...otherwise()`` chain from *mapping*.
|
|
|
|
Parameters
|
|
----------
|
|
v:
|
|
Polars expression representing the column value (already
|
|
lowercased and stripped).
|
|
mapping:
|
|
``{lookup_key: canonical_value}`` dict.
|
|
fallback:
|
|
Expression for unmatched values.
|
|
|
|
Returns
|
|
-------
|
|
pl.Expr
|
|
A chain ``pl.when(...).then(...)...otherwise(fallback)``.
|
|
"""
|
|
expr = pl.when(v.is_null()).then(pl.lit(None, pl.Utf8))
|
|
for key, val in mapping.items():
|
|
expr = expr.when(v == key).then(pl.lit(val))
|
|
return expr.otherwise(fallback)
|
|
|
|
|
|
# ═════════════════════════════════════════════════════════════════════════════
|
|
# Per-column normalisation functions
|
|
# ═════════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
def _norm_tls_version(col: str) -> pl.Expr:
|
|
"""Normalise a TLS version column (``0ver``).
|
|
|
|
Accepts hex wire format (``'03 04'``) or display names
|
|
(``'TLSv1.3'``, ``'tls 1.3'``, ``'1.3'``, ``'SSLv3'``, etc.).
|
|
Unmatched values produce ``'unknown_version'``.
|
|
"""
|
|
v = pl.col(col).cast(pl.Utf8).str.strip_chars().str.to_lowercase()
|
|
return _build_when_chain(v, TLS_VERSION_MAP, pl.lit('unknown_version'))
|
|
|
|
|
|
def _norm_cipher(col: str) -> pl.Expr:
|
|
"""Normalise a cipher-suite hex column (``0cph``).
|
|
|
|
Unmatched hex values produce ``'unknown_cipher_0x{hex}'`` where
|
|
``{hex}`` is the original (stripped, lowercased) value.
|
|
"""
|
|
v = pl.col(col).cast(pl.Utf8).str.strip_chars().str.to_lowercase()
|
|
fallback = pl.lit('unknown_cipher_0x') + v
|
|
return _build_when_chain(v, CIPHER_SUITE_MAP, fallback)
|
|
|
|
|
|
def _norm_cipher_display(col: str) -> pl.Expr:
|
|
"""Normalise a cipher-suite display-name column (``cipher-suite``).
|
|
|
|
The column value is lowercased and all separators (spaces, hyphens,
|
|
underscores) are removed before lookup.
|
|
|
|
Unmatched names produce ``'{normalised}_unknown'``.
|
|
"""
|
|
raw = pl.col(col).cast(pl.Utf8).str.strip_chars()
|
|
v = raw.str.to_lowercase().str.replace_all(r'[\s\-_]', '')
|
|
fallback = v + pl.lit('_unknown')
|
|
return _build_when_chain(v, _CIPHER_DISPLAY_NORMALIZED, fallback)
|
|
|
|
|
|
def _norm_curve(col: str) -> pl.Expr:
|
|
"""Normalise a named-curve hex column (``0crv``).
|
|
|
|
Unmatched hex values produce ``'unknown_curve_0x{hex}'``.
|
|
"""
|
|
v = pl.col(col).cast(pl.Utf8).str.strip_chars().str.to_lowercase()
|
|
fallback = pl.lit('unknown_curve_0x') + v
|
|
return _build_when_chain(v, CURVE_MAP, fallback)
|
|
|
|
|
|
def _norm_curve_display(col: str) -> pl.Expr:
|
|
"""Normalise a named-curve display-name column (``ecdhe-named-curve``).
|
|
|
|
Accepts names like ``'X25519'``, ``'prime256v1'``, ``'P-256'``, etc.
|
|
Unmatched names produce ``'unknown_curve_{name}'`` where ``{name}``
|
|
is the original value lowercased.
|
|
"""
|
|
v = pl.col(col).cast(pl.Utf8).str.strip_chars().str.to_lowercase()
|
|
fallback = pl.lit('unknown_curve_') + v
|
|
return _build_when_chain(v, CURVE_MAP, fallback)
|
|
|
|
|
|
def _norm_cnrs(col: str) -> pl.Expr:
|
|
"""Normalise a CNRS / ISRS column.
|
|
|
|
``'+'`` (optionally surrounded by whitespace) → ``'yes'``.
|
|
Blank, null, or any other value → ``'no'``.
|
|
|
|
These columns are **never** missing; blank always means ``'no'``.
|
|
"""
|
|
c = pl.col(col).cast(pl.Utf8).str.strip_chars()
|
|
return (
|
|
pl.when(c.is_null() | (c == ''))
|
|
.then(pl.lit('no'))
|
|
.when(c.str.to_lowercase() == '+')
|
|
.then(pl.lit('yes'))
|
|
.otherwise(pl.lit('no'))
|
|
)
|
|
|
|
|
|
# ═════════════════════════════════════════════════════════════════════════════
|
|
# Public API
|
|
# ═════════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
def normalize_lf(lf: pl.LazyFrame, type_map: dict) -> pl.LazyFrame:
|
|
"""Add ``{col}_norm`` columns with normalised values.
|
|
|
|
Parameters
|
|
----------
|
|
lf:
|
|
A Polars :class:`LazyFrame` containing TLS-related columns.
|
|
type_map:
|
|
A ``{column_name: ...}`` dict (e.g. from
|
|
:func:`~analysis.type_classifier.classify_schema`). Column
|
|
names present in the map receive normalisation.
|
|
|
|
Returns
|
|
-------
|
|
pl.LazyFrame
|
|
A new :class:`LazyFrame` with ``{col}_norm`` columns appended.
|
|
"""
|
|
exprs: list[pl.Expr] = []
|
|
|
|
if '0ver' in type_map:
|
|
exprs.append(_norm_tls_version('0ver').alias('0ver_norm'))
|
|
|
|
if '0cph' in type_map:
|
|
exprs.append(_norm_cipher('0cph').alias('0cph_norm'))
|
|
|
|
if 'cipher-suite' in type_map:
|
|
exprs.append(
|
|
_norm_cipher_display('cipher-suite').alias('cipher_suite_norm')
|
|
)
|
|
|
|
if '0crv' in type_map:
|
|
exprs.append(_norm_curve('0crv').alias('0crv_norm'))
|
|
|
|
if 'ecdhe-named-curve' in type_map:
|
|
exprs.append(
|
|
_norm_curve_display('ecdhe-named-curve').alias(
|
|
'ecdhe_named_curve_norm'
|
|
)
|
|
)
|
|
|
|
if 'cnrs' in type_map:
|
|
exprs.append(_norm_cnrs('cnrs').alias('cnrs_norm'))
|
|
|
|
if 'isrs' in type_map:
|
|
exprs.append(_norm_cnrs('isrs').alias('isrs_norm'))
|
|
|
|
if exprs:
|
|
lf = lf.with_columns(exprs)
|
|
return lf
|