"""Tests for analysis.value_normalizer — TLS value normalisation. Covers: TLS version (hex & display), cipher suite (hex & display), named curve (hex & display), CNRS/ISRS, and the top-level :func:`normalize_lf` integration. """ import pytest import polars as pl from analysis.value_normalizer import ( TLS_VERSION_MAP, CIPHER_SUITE_MAP, CURVE_MAP, normalize_lf, ) # ═════════════════════════════════════════════════════════════════════════ # TLS Version # ═════════════════════════════════════════════════════════════════════════ class TestNormTlsVersion: """_norm_tls_version — hex & display → canonical enum name.""" def test_hex_03_04_tls_1_3(self): lf = pl.LazyFrame({'0ver': ['03 04']}) result = normalize_lf(lf, {'0ver': None}).collect() assert result['0ver_norm'][0] == 'tls_1_3' def test_hex_03_03_tls_1_2(self): lf = pl.LazyFrame({'0ver': ['03 03']}) result = normalize_lf(lf, {'0ver': None}).collect() assert result['0ver_norm'][0] == 'tls_1_2' def test_hex_03_02_tls_1_1(self): lf = pl.LazyFrame({'0ver': ['03 02']}) result = normalize_lf(lf, {'0ver': None}).collect() assert result['0ver_norm'][0] == 'tls_1_1' def test_hex_03_01_tls_1_0(self): lf = pl.LazyFrame({'0ver': ['03 01']}) result = normalize_lf(lf, {'0ver': None}).collect() assert result['0ver_norm'][0] == 'tls_1_0' def test_hex_02_00_ssl_2_0(self): lf = pl.LazyFrame({'0ver': ['02 00']}) result = normalize_lf(lf, {'0ver': None}).collect() assert result['0ver_norm'][0] == 'ssl_2_0' def test_hex_03_00_ssl_3_0(self): lf = pl.LazyFrame({'0ver': ['03 00']}) result = normalize_lf(lf, {'0ver': None}).collect() assert result['0ver_norm'][0] == 'ssl_3_0' def test_display_tlsv1_3(self): lf = pl.LazyFrame({'0ver': ['TLSv1.3']}) result = normalize_lf(lf, {'0ver': None}).collect() assert result['0ver_norm'][0] == 'tls_1_3' def test_display_tlsv1_2(self): lf = pl.LazyFrame({'0ver': ['tlsv1.2']}) result = normalize_lf(lf, {'0ver': None}).collect() assert result['0ver_norm'][0] == 'tls_1_2' def test_display_tls_1_1(self): lf = pl.LazyFrame({'0ver': ['TLS 1.1']}) result = normalize_lf(lf, {'0ver': None}).collect() assert result['0ver_norm'][0] == 'tls_1_1' def test_display_bare_1_3(self): lf = pl.LazyFrame({'0ver': ['1.3']}) result = normalize_lf(lf, {'0ver': None}).collect() assert result['0ver_norm'][0] == 'tls_1_3' def test_display_bare_1_2(self): lf = pl.LazyFrame({'0ver': ['1.2']}) result = normalize_lf(lf, {'0ver': None}).collect() assert result['0ver_norm'][0] == 'tls_1_2' def test_display_tlsv1(self): lf = pl.LazyFrame({'0ver': ['TLSv1']}) result = normalize_lf(lf, {'0ver': None}).collect() assert result['0ver_norm'][0] == 'tls_1_0' def test_display_sslv2(self): lf = pl.LazyFrame({'0ver': ['SSLv2']}) result = normalize_lf(lf, {'0ver': None}).collect() assert result['0ver_norm'][0] == 'ssl_2_0' def test_display_sslv3(self): lf = pl.LazyFrame({'0ver': ['sslv3']}) result = normalize_lf(lf, {'0ver': None}).collect() assert result['0ver_norm'][0] == 'ssl_3_0' def test_display_bare_2(self): lf = pl.LazyFrame({'0ver': ['2']}) result = normalize_lf(lf, {'0ver': None}).collect() assert result['0ver_norm'][0] == 'ssl_2_0' def test_display_bare_3(self): lf = pl.LazyFrame({'0ver': ['3']}) result = normalize_lf(lf, {'0ver': None}).collect() assert result['0ver_norm'][0] == 'ssl_3_0' def test_case_insensitive(self): lf = pl.LazyFrame({'0ver': ['tlsv1.3', 'TLSv1.3', 'TlsV1.3']}) result = normalize_lf(lf, {'0ver': None}).collect() assert result['0ver_norm'].to_list() == ['tls_1_3', 'tls_1_3', 'tls_1_3'] def test_unknown_version(self): lf = pl.LazyFrame({'0ver': ['ff ff']}) result = normalize_lf(lf, {'0ver': None}).collect() assert result['0ver_norm'][0] == 'unknown_version' def test_unknown_display(self): lf = pl.LazyFrame({'0ver': ['TLSv9.9']}) result = normalize_lf(lf, {'0ver': None}).collect() assert result['0ver_norm'][0] == 'unknown_version' def test_null_remains_null(self): lf = pl.LazyFrame({'0ver': [None]}) result = normalize_lf(lf, {'0ver': None}).collect() assert result['0ver_norm'][0] is None def test_mixed_values(self): lf = pl.LazyFrame({'0ver': ['03 04', 'tlsv1.2', '1.3', 'ssl 2', None]}) result = normalize_lf(lf, {'0ver': None}).collect() assert result['0ver_norm'].to_list() == [ 'tls_1_3', 'tls_1_2', 'tls_1_3', 'ssl_2_0', None, ] # ═════════════════════════════════════════════════════════════════════════ # Cipher Suite — hex # ═════════════════════════════════════════════════════════════════════════ class TestNormCipher: """_norm_cipher — hex cipher suite → canonical name.""" def test_hex_13_01(self): lf = pl.LazyFrame({'0cph': ['13 01']}) result = normalize_lf(lf, {'0cph': None}).collect() assert result['0cph_norm'][0] == 'tls_aes_128_gcm_sha256' def test_hex_13_02(self): lf = pl.LazyFrame({'0cph': ['13 02']}) result = normalize_lf(lf, {'0cph': None}).collect() assert result['0cph_norm'][0] == 'tls_aes_256_gcm_sha384' def test_hex_13_03(self): lf = pl.LazyFrame({'0cph': ['13 03']}) result = normalize_lf(lf, {'0cph': None}).collect() assert result['0cph_norm'][0] == 'tls_chacha20_poly1305_sha256' def test_hex_c0_2b(self): lf = pl.LazyFrame({'0cph': ['c0 2b']}) result = normalize_lf(lf, {'0cph': None}).collect() assert result['0cph_norm'][0] == 'tls_ecdhe_ecdsa_aes_128_gcm_sha256' def test_hex_00_35(self): lf = pl.LazyFrame({'0cph': ['00 35']}) result = normalize_lf(lf, {'0cph': None}).collect() assert result['0cph_norm'][0] == 'tls_rsa_aes_128_cbc_sha' def test_hex_00_2f(self): """00 2f also maps to tls_rsa_aes_128_cbc_sha (same canonical).""" lf = pl.LazyFrame({'0cph': ['00 2f']}) result = normalize_lf(lf, {'0cph': None}).collect() assert result['0cph_norm'][0] == 'tls_rsa_aes_128_cbc_sha' def test_unknown_hex(self): lf = pl.LazyFrame({'0cph': ['ab cd']}) result = normalize_lf(lf, {'0cph': None}).collect() assert result['0cph_norm'][0] == 'unknown_cipher_0xab cd' def test_null_remains_null(self): lf = pl.LazyFrame({'0cph': [None]}) result = normalize_lf(lf, {'0cph': None}).collect() assert result['0cph_norm'][0] is None def test_case_insensitive_hex(self): lf = pl.LazyFrame({'0cph': ['C0 2B']}) result = normalize_lf(lf, {'0cph': None}).collect() assert result['0cph_norm'][0] == 'tls_ecdhe_ecdsa_aes_128_gcm_sha256' # ═════════════════════════════════════════════════════════════════════════ # Cipher Suite — display name # ═════════════════════════════════════════════════════════════════════════ class TestNormCipherDisplay: """_norm_cipher_display — IANA/display name → canonical name.""" def test_iana_full_name(self): lf = pl.LazyFrame({'cipher-suite': ['TLS_AES_128_GCM_SHA256']}) result = normalize_lf(lf, {'cipher-suite': None}).collect() assert result['cipher_suite_norm'][0] == 'tls_aes_128_gcm_sha256' def test_iana_full_name_lowercase(self): lf = pl.LazyFrame({'cipher-suite': ['tls_aes_256_gcm_sha384']}) result = normalize_lf(lf, {'cipher-suite': None}).collect() assert result['cipher_suite_norm'][0] == 'tls_aes_256_gcm_sha384' def test_display_aes128gcm(self): lf = pl.LazyFrame({'cipher-suite': ['AES 128 GCM']}) result = normalize_lf(lf, {'cipher-suite': None}).collect() assert result['cipher_suite_norm'][0] == 'tls_aes_128_gcm_sha256' def test_display_aes256gcm(self): lf = pl.LazyFrame({'cipher-suite': ['AES 256 GCM']}) result = normalize_lf(lf, {'cipher-suite': None}).collect() assert result['cipher_suite_norm'][0] == 'tls_aes_256_gcm_sha384' def test_display_chacha20(self): lf = pl.LazyFrame({'cipher-suite': ['CHACHA20-POLY1305']}) result = normalize_lf(lf, {'cipher-suite': None}).collect() assert result['cipher_suite_norm'][0] == 'tls_chacha20_poly1305_sha256' def test_display_ecdhe_ecdsa_aes128gcm(self): lf = pl.LazyFrame({'cipher-suite': ['ECDHE-ECDSA-AES128-GCM']}) result = normalize_lf(lf, {'cipher-suite': None}).collect() assert result['cipher_suite_norm'][0] == 'tls_ecdhe_ecdsa_aes_128_gcm_sha256' def test_display_ecdhe_rsa_aes256gcm(self): lf = pl.LazyFrame({'cipher-suite': ['ECDHE-RSA-AES256-GCM']}) result = normalize_lf(lf, {'cipher-suite': None}).collect() assert result['cipher_suite_norm'][0] == 'tls_ecdhe_rsa_aes_256_gcm_sha384' def test_display_rsa_aes128gcm(self): lf = pl.LazyFrame({'cipher-suite': ['RSA AES 128 GCM']}) result = normalize_lf(lf, {'cipher-suite': None}).collect() assert result['cipher_suite_norm'][0] == 'tls_rsa_aes_128_gcm_sha256' def test_display_hyphen_variants(self): lf = pl.LazyFrame({'cipher-suite': ['ECDHE-RSA-AES128-GCM']}) result = normalize_lf(lf, {'cipher-suite': None}).collect() assert result['cipher_suite_norm'][0] == 'tls_ecdhe_rsa_aes_128_gcm_sha256' def test_unknown_display(self): lf = pl.LazyFrame({'cipher-suite': ['CUSTOM_CIPHER_999']}) result = normalize_lf(lf, {'cipher-suite': None}).collect() assert result['cipher_suite_norm'][0] == 'customcipher999_unknown' def test_null_remains_null(self): lf = pl.LazyFrame({'cipher-suite': [None]}) result = normalize_lf(lf, {'cipher-suite': None}).collect() assert result['cipher_suite_norm'][0] is None def test_mixed_display_values(self): lf = pl.LazyFrame({ 'cipher-suite': [ 'TLS_AES_128_GCM_SHA256', 'AES 256 GCM', 'ECDHE-RSA-AES128-GCM', 'bogus', None, ], }) result = normalize_lf(lf, {'cipher-suite': None}).collect() expected = [ 'tls_aes_128_gcm_sha256', 'tls_aes_256_gcm_sha384', 'tls_ecdhe_rsa_aes_128_gcm_sha256', 'bogus_unknown', None, ] assert result['cipher_suite_norm'].to_list() == expected # ═════════════════════════════════════════════════════════════════════════ # Named Curve — hex # ═════════════════════════════════════════════════════════════════════════ class TestNormCurve: """_norm_curve — hex curve → canonical name.""" def test_hex_00_1d_x25519(self): lf = pl.LazyFrame({'0crv': ['00 1d']}) result = normalize_lf(lf, {'0crv': None}).collect() assert result['0crv_norm'][0] == 'x25519' def test_hex_00_17_secp256r1(self): lf = pl.LazyFrame({'0crv': ['00 17']}) result = normalize_lf(lf, {'0crv': None}).collect() assert result['0crv_norm'][0] == 'secp256r1' def test_hex_00_18_secp384r1(self): lf = pl.LazyFrame({'0crv': ['00 18']}) result = normalize_lf(lf, {'0crv': None}).collect() assert result['0crv_norm'][0] == 'secp384r1' def test_hex_00_19_secp521r1(self): lf = pl.LazyFrame({'0crv': ['00 19']}) result = normalize_lf(lf, {'0crv': None}).collect() assert result['0crv_norm'][0] == 'secp521r1' def test_hex_00_1e_x448(self): lf = pl.LazyFrame({'0crv': ['00 1e']}) result = normalize_lf(lf, {'0crv': None}).collect() assert result['0crv_norm'][0] == 'x448' def test_unknown_hex_curve(self): lf = pl.LazyFrame({'0crv': ['ff ff']}) result = normalize_lf(lf, {'0crv': None}).collect() assert result['0crv_norm'][0] == 'unknown_curve_0xff ff' def test_null_remains_null(self): lf = pl.LazyFrame({'0crv': [None]}) result = normalize_lf(lf, {'0crv': None}).collect() assert result['0crv_norm'][0] is None def test_case_insensitive_hex_curve(self): lf = pl.LazyFrame({'0crv': ['00 1D']}) result = normalize_lf(lf, {'0crv': None}).collect() assert result['0crv_norm'][0] == 'x25519' # ═════════════════════════════════════════════════════════════════════════ # Named Curve — display name # ═════════════════════════════════════════════════════════════════════════ class TestNormCurveDisplay: """_norm_curve_display — display name → canonical name.""" def test_x25519(self): lf = pl.LazyFrame({'ecdhe-named-curve': ['X25519']}) result = normalize_lf(lf, {'ecdhe-named-curve': None}).collect() assert result['ecdhe_named_curve_norm'][0] == 'x25519' def test_curve25519(self): lf = pl.LazyFrame({'ecdhe-named-curve': ['Curve25519']}) result = normalize_lf(lf, {'ecdhe-named-curve': None}).collect() assert result['ecdhe_named_curve_norm'][0] == 'x25519' def test_secp256r1(self): lf = pl.LazyFrame({'ecdhe-named-curve': ['secp256r1']}) result = normalize_lf(lf, {'ecdhe-named-curve': None}).collect() assert result['ecdhe_named_curve_norm'][0] == 'secp256r1' def test_prime256v1(self): lf = pl.LazyFrame({'ecdhe-named-curve': ['prime256v1']}) result = normalize_lf(lf, {'ecdhe-named-curve': None}).collect() assert result['ecdhe_named_curve_norm'][0] == 'secp256r1' def test_p256(self): lf = pl.LazyFrame({'ecdhe-named-curve': ['P256']}) result = normalize_lf(lf, {'ecdhe-named-curve': None}).collect() assert result['ecdhe_named_curve_norm'][0] == 'secp256r1' def test_p_256(self): lf = pl.LazyFrame({'ecdhe-named-curve': ['P-256']}) result = normalize_lf(lf, {'ecdhe-named-curve': None}).collect() assert result['ecdhe_named_curve_norm'][0] == 'secp256r1' def test_p_256_underscore(self): lf = pl.LazyFrame({'ecdhe-named-curve': ['P_256']}) result = normalize_lf(lf, {'ecdhe-named-curve': None}).collect() assert result['ecdhe_named_curve_norm'][0] == 'secp256r1' def test_secp384r1(self): lf = pl.LazyFrame({'ecdhe-named-curve': ['secp384r1']}) result = normalize_lf(lf, {'ecdhe-named-curve': None}).collect() assert result['ecdhe_named_curve_norm'][0] == 'secp384r1' def test_p384(self): lf = pl.LazyFrame({'ecdhe-named-curve': ['P384']}) result = normalize_lf(lf, {'ecdhe-named-curve': None}).collect() assert result['ecdhe_named_curve_norm'][0] == 'secp384r1' def test_p_384(self): lf = pl.LazyFrame({'ecdhe-named-curve': ['p-384']}) result = normalize_lf(lf, {'ecdhe-named-curve': None}).collect() assert result['ecdhe_named_curve_norm'][0] == 'secp384r1' def test_secp521r1(self): lf = pl.LazyFrame({'ecdhe-named-curve': ['Secp521r1']}) result = normalize_lf(lf, {'ecdhe-named-curve': None}).collect() assert result['ecdhe_named_curve_norm'][0] == 'secp521r1' def test_p521(self): lf = pl.LazyFrame({'ecdhe-named-curve': ['P521']}) result = normalize_lf(lf, {'ecdhe-named-curve': None}).collect() assert result['ecdhe_named_curve_norm'][0] == 'secp521r1' def test_x448(self): lf = pl.LazyFrame({'ecdhe-named-curve': ['x448']}) result = normalize_lf(lf, {'ecdhe-named-curve': None}).collect() assert result['ecdhe_named_curve_norm'][0] == 'x448' def test_curve448(self): lf = pl.LazyFrame({'ecdhe-named-curve': ['Curve448']}) result = normalize_lf(lf, {'ecdhe-named-curve': None}).collect() assert result['ecdhe_named_curve_norm'][0] == 'x448' def test_unknown_curve_display(self): lf = pl.LazyFrame({'ecdhe-named-curve': ['brainpoolP256r1']}) result = normalize_lf(lf, {'ecdhe-named-curve': None}).collect() assert result['ecdhe_named_curve_norm'][0] == 'unknown_curve_brainpoolp256r1' def test_null_remains_null(self): lf = pl.LazyFrame({'ecdhe-named-curve': [None]}) result = normalize_lf(lf, {'ecdhe-named-curve': None}).collect() assert result['ecdhe_named_curve_norm'][0] is None def test_case_insensitive_p256(self): lf = pl.LazyFrame({'ecdhe-named-curve': ['P-256', 'p_256', 'P256']}) result = normalize_lf(lf, {'ecdhe-named-curve': None}).collect() assert result['ecdhe_named_curve_norm'].to_list() == [ 'secp256r1', 'secp256r1', 'secp256r1', ] # ═════════════════════════════════════════════════════════════════════════ # CNRS / ISRS # ═════════════════════════════════════════════════════════════════════════ class TestNormCnrs: """_norm_cnrs — '+' → 'yes', blank/null → 'no'.""" def test_plus_yes(self): lf = pl.LazyFrame({'cnrs': ['+']}) result = normalize_lf(lf, {'cnrs': None}).collect() assert result['cnrs_norm'][0] == 'yes' def test_blank_no(self): lf = pl.LazyFrame({'cnrs': ['']}) result = normalize_lf(lf, {'cnrs': None}).collect() assert result['cnrs_norm'][0] == 'no' def test_null_no(self): lf = pl.LazyFrame({'cnrs': [None]}) result = normalize_lf(lf, {'cnrs': None}).collect() assert result['cnrs_norm'][0] == 'no' def test_whitespace_plus(self): lf = pl.LazyFrame({'cnrs': [' + ']}) result = normalize_lf(lf, {'cnrs': None}).collect() assert result['cnrs_norm'][0] == 'yes' def test_anything_else_no(self): lf = pl.LazyFrame({'cnrs': ['-', 'true', 'maybe', '0', '1']}) result = normalize_lf(lf, {'cnrs': None}).collect() assert result['cnrs_norm'].to_list() == ['no', 'no', 'no', 'no', 'no'] def test_isrs_plus_yes(self): lf = pl.LazyFrame({'isrs': ['+']}) result = normalize_lf(lf, {'isrs': None}).collect() assert result['isrs_norm'][0] == 'yes' def test_isrs_blank_no(self): lf = pl.LazyFrame({'isrs': ['']}) result = normalize_lf(lf, {'isrs': None}).collect() assert result['isrs_norm'][0] == 'no' # ═════════════════════════════════════════════════════════════════════════ # Integration — normalize_lf # ═════════════════════════════════════════════════════════════════════════ class TestNormalizeLf: """Top-level normalize_lf — column routing & no-op cases.""" def test_all_columns_at_once(self): lf = pl.LazyFrame({ '0ver': ['03 04'], '0cph': ['13 01'], 'cipher-suite': ['TLS_AES_128_GCM_SHA256'], '0crv': ['00 1d'], 'ecdhe-named-curve': ['X25519'], 'cnrs': ['+'], 'isrs': [''], }) type_map = { '0ver': None, '0cph': None, 'cipher-suite': None, '0crv': None, 'ecdhe-named-curve': None, 'cnrs': None, 'isrs': None, } result = normalize_lf(lf, type_map).collect() assert result['0ver_norm'][0] == 'tls_1_3' assert result['0cph_norm'][0] == 'tls_aes_128_gcm_sha256' assert result['cipher_suite_norm'][0] == 'tls_aes_128_gcm_sha256' assert result['0crv_norm'][0] == 'x25519' assert result['ecdhe_named_curve_norm'][0] == 'x25519' assert result['cnrs_norm'][0] == 'yes' assert result['isrs_norm'][0] == 'no' def test_no_matching_columns(self): """Empty type_map → no _norm columns added.""" lf = pl.LazyFrame({'0ver': ['03 04']}) result = normalize_lf(lf, {}).collect() assert '0ver_norm' not in result.columns assert result['0ver'][0] == '03 04' def test_subset_of_columns(self): """Only columns present in type_map get normalised.""" lf = pl.LazyFrame({ '0ver': ['03 04'], '0cph': ['13 01'], 'cnrs': ['+'], }) result = normalize_lf(lf, {'0ver': None, 'cnrs': None}).collect() assert '0ver_norm' in result.columns assert 'cnrs_norm' in result.columns assert '0cph_norm' not in result.columns def test_preserves_original_columns(self): """Original columns are kept unchanged.""" lf = pl.LazyFrame({ '0ver': ['03 04'], 'src_ip': ['10.0.0.1'], }) result = normalize_lf(lf, {'0ver': None}).collect() assert '0ver' in result.columns assert 'src_ip' in result.columns assert result['src_ip'][0] == '10.0.0.1' def test_duplicate_normalized_column_exists(self): """If a _norm column already exists, Polars appends suffixed.""" lf = pl.LazyFrame({ '0ver': ['03 04'], '0ver_norm': ['old'], }) result = normalize_lf(lf, {'0ver': None}).collect() # Polars will create 0ver_norm with new value assert result['0ver_norm'][0] == 'tls_1_3' def test_lazyness_preserved(self): """normalize_lf returns a LazyFrame, not materialised.""" lf = pl.LazyFrame({'0ver': ['03 04']}) result = normalize_lf(lf, {'0ver': None}) assert isinstance(result, pl.LazyFrame)