PM-pinou
745b1481cd
- Fix entity_count not populated after clustering pipeline - Create comprehensive geoip_data.txt (803 IP ranges, 49 cities) - Generate 100 multi_upload test files with missing columns - Replace csv_glob path with SQLite table name in frontend - Fix globe zoom-drag point-fixing (camera.z/12.0 scaling) - Fix orphaned manual_run_analysis function
609 lines
28 KiB
Python
609 lines
28 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Generate comprehensive offline GeoIP database at data/geoip_data.txt.
|
|
|
|
Strategy:
|
|
1. Define precise CIDR ranges for clouds/CDNs/known services (highest priority)
|
|
2. Define ISP/city blocks in decreasing priority
|
|
3. Resolve overlaps: earlier (more specific) ranges win
|
|
4. Fill remaining gaps with city blocks
|
|
5. Ensure zero overlaps in final output
|
|
|
|
Output: data/geoip_data.txt (CSV: start_ip,end_ip,lat,lon,city,country)
|
|
"""
|
|
|
|
import ipaddress
|
|
from pathlib import Path
|
|
|
|
|
|
# ---- City coordinates ----
|
|
CITIES = {
|
|
# China
|
|
"Beijing": (39.9042, 116.4074, "CN"),
|
|
"Shanghai": (31.2304, 121.4737, "CN"),
|
|
"Guangzhou": (23.1291, 113.2644, "CN"),
|
|
"Shenzhen": (22.5431, 114.0579, "CN"),
|
|
"Hangzhou": (30.2741, 120.1551, "CN"),
|
|
"Chengdu": (30.5728, 104.0668, "CN"),
|
|
"Chongqing": (29.5630, 106.5516, "CN"),
|
|
"Nanjing": (32.0603, 118.7969, "CN"),
|
|
"Wuhan": (30.5928, 114.3055, "CN"),
|
|
"Tianjin": (39.3434, 117.3616, "CN"),
|
|
"Xian": (34.3416, 108.9398, "CN"),
|
|
# East Asia
|
|
"Tokyo": (35.6762, 139.6503, "JP"),
|
|
"Osaka": (34.6937, 135.5022, "JP"),
|
|
"Nagoya": (35.1815, 136.9066, "JP"),
|
|
"Seoul": (37.5665, 126.9780, "KR"),
|
|
"Busan": (35.1796, 129.0756, "KR"),
|
|
"Taipei": (25.0330, 121.5654, "TW"),
|
|
"Hong Kong": (22.3193, 114.1694, "HK"),
|
|
# Southeast Asia
|
|
"Singapore": (1.3521, 103.8198, "SG"),
|
|
"Bangkok": (13.7563, 100.5018, "TH"),
|
|
"Kuala Lumpur": (3.1390, 101.6869, "MY"),
|
|
"Jakarta": (-6.2088, 106.8456, "ID"),
|
|
"Manila": (14.5995, 120.9842, "PH"),
|
|
"Ho Chi Minh City": (10.8231, 106.6297, "VN"),
|
|
"Hanoi": (21.0278, 105.8342, "VN"),
|
|
# South Asia
|
|
"Mumbai": (19.0760, 72.8777, "IN"),
|
|
"Delhi": (28.7041, 77.1025, "IN"),
|
|
"Bangalore": (12.9716, 77.5946, "IN"),
|
|
"Chennai": (13.0827, 80.2707, "IN"),
|
|
"Hyderabad": (17.3850, 78.4867, "IN"),
|
|
# Oceania
|
|
"Sydney": (-33.8688, 151.2093, "AU"),
|
|
"Melbourne": (-37.8136, 144.9631, "AU"),
|
|
"Auckland": (-36.8509, 174.7645, "NZ"),
|
|
# Middle East
|
|
"Dubai": (25.2048, 55.2708, "AE"),
|
|
"Riyadh": (24.7136, 46.6753, "SA"),
|
|
"Istanbul": (41.0082, 28.9784, "TR"),
|
|
"Bahrain": (26.0667, 50.5577, "BH"),
|
|
# Europe
|
|
"London": (51.5074, -0.1278, "GB"),
|
|
"Manchester": (53.4808, -2.2426, "GB"),
|
|
"Frankfurt": (50.1109, 8.6821, "DE"),
|
|
"Berlin": (52.5200, 13.4050, "DE"),
|
|
"Munich": (48.1351, 11.5820, "DE"),
|
|
"Paris": (48.8566, 2.3522, "FR"),
|
|
"Amsterdam": (52.3676, 4.9041, "NL"),
|
|
"Madrid": (40.4168, -3.7038, "ES"),
|
|
"Barcelona": (41.3874, 2.1686, "ES"),
|
|
"Milan": (45.4642, 9.1900, "IT"),
|
|
"Rome": (41.9028, 12.4964, "IT"),
|
|
"Stockholm": (59.3293, 18.0686, "SE"),
|
|
"Oslo": (59.9139, 10.7522, "NO"),
|
|
"Helsinki": (60.1699, 24.9384, "FI"),
|
|
"Warsaw": (52.2297, 21.0122, "PL"),
|
|
"Zurich": (47.3769, 8.5417, "CH"),
|
|
"Dublin": (53.3498, -6.2603, "IE"),
|
|
"Moscow": (55.7558, 37.6173, "RU"),
|
|
"Saint Petersburg": (59.9343, 30.3351, "RU"),
|
|
# Africa
|
|
"Cairo": (30.0444, 31.2357, "EG"),
|
|
"Lagos": (6.5244, 3.3792, "NG"),
|
|
"Nairobi": (-1.2921, 36.8219, "KE"),
|
|
"Johannesburg": (-26.2041, 28.0473, "ZA"),
|
|
"Cape Town": (-33.9249, 18.4241, "ZA"),
|
|
# North America
|
|
"New York": (40.7128, -74.0060, "US"),
|
|
"San Francisco": (37.7749, -122.4194, "US"),
|
|
"Seattle": (47.6062, -122.3321, "US"),
|
|
"Chicago": (41.8781, -87.6298, "US"),
|
|
"Dallas": (32.7767, -96.7970, "US"),
|
|
"Los Angeles": (34.0522, -118.2437, "US"),
|
|
"Ashburn": (39.0438, -77.4874, "US"),
|
|
"Atlanta": (33.7490, -84.3880, "US"),
|
|
"Miami": (25.7617, -80.1918, "US"),
|
|
"Boston": (42.3601, -71.0589, "US"),
|
|
"Houston": (29.7604, -95.3698, "US"),
|
|
"Denver": (39.7392, -104.9903, "US"),
|
|
"Phoenix": (33.4484, -112.0740, "US"),
|
|
"Portland": (45.5152, -122.6784, "US"),
|
|
"San Jose": (37.3382, -121.8863, "US"),
|
|
"Philadelphia": (39.9526, -75.1652, "US"),
|
|
"Mountain View": (37.3861, -122.0839, "US"),
|
|
"Cambridge": (42.3736, -71.1097, "US"),
|
|
"Toronto": (43.6532, -79.3832, "CA"),
|
|
"Vancouver": (49.2827, -123.1207, "CA"),
|
|
"Montreal": (45.5017, -73.5673, "CA"),
|
|
"Mexico City": (19.4326, -99.1332, "MX"),
|
|
# South America
|
|
"Sao Paulo": (-23.5505, -46.6333, "BR"),
|
|
"Rio de Janeiro": (-22.9068, -43.1729, "BR"),
|
|
"Buenos Aires": (-34.6037, -58.3816, "AR"),
|
|
"Santiago": (-33.4489, -70.6693, "CL"),
|
|
"Bogota": (4.7110, -74.0721, "CO"),
|
|
"Lima": (-12.0464, -77.0428, "PE"),
|
|
}
|
|
|
|
|
|
def cidr_to_ints(cidr: str) -> tuple[int, int]:
|
|
"""Return (start_int, end_int) for a CIDR block."""
|
|
net = ipaddress.IPv4Network(cidr, strict=False)
|
|
return (int(net.network_address), int(net.broadcast_address))
|
|
|
|
|
|
# ============================================================
|
|
# Priority-ordered range definitions (first = highest priority)
|
|
# ============================================================
|
|
|
|
# Layer 0: Private ranges
|
|
PRIVATE_RANGES = [
|
|
("Beijing", "10.0.0.0/8"),
|
|
("Shanghai", "172.16.0.0/12"),
|
|
("Guangzhou", "192.168.0.0/16"),
|
|
]
|
|
|
|
# Layer 1: Well-known DNS
|
|
DNS_RANGES = [
|
|
("Mountain View", "8.8.8.8/32"),
|
|
("Mountain View", "8.8.4.4/32"),
|
|
("San Francisco", "1.1.1.1/32"),
|
|
("San Francisco", "1.0.0.1/32"),
|
|
]
|
|
|
|
# Layer 2: AWS
|
|
AWS_RANGES = [
|
|
("Ashburn", "3.208.0.0/13"), ("Ashburn", "3.224.0.0/12"),
|
|
("Ashburn", "52.0.0.0/15"), ("Ashburn", "52.2.0.0/15"),
|
|
("Ashburn", "52.4.0.0/14"), ("Ashburn", "52.20.0.0/14"),
|
|
("Ashburn", "54.80.0.0/12"), ("Ashburn", "54.152.0.0/13"),
|
|
("Ashburn", "54.160.0.0/11"),("Ashburn", "54.208.0.0/15"),
|
|
("Ashburn", "54.224.0.0/12"),("Ashburn", "18.232.0.0/14"),
|
|
("Ashburn", "44.192.0.0/11"),
|
|
("San Francisco", "13.52.0.0/14"), ("San Francisco", "13.56.0.0/14"),
|
|
("San Francisco", "54.176.0.0/12"),("San Francisco", "54.183.0.0/16"),
|
|
("San Francisco", "54.241.0.0/16"),("San Francisco", "184.72.0.0/15"),
|
|
("San Francisco", "50.18.0.0/15"),
|
|
("Portland", "34.208.0.0/12"),("Portland", "44.224.0.0/11"),
|
|
("Portland", "50.112.0.0/16"),("Portland", "54.68.0.0/14"),
|
|
("Portland", "54.244.0.0/15"),
|
|
("Dublin", "34.240.0.0/13"),("Dublin", "52.16.0.0/14"),
|
|
("Dublin", "52.48.0.0/14"), ("Dublin", "54.72.0.0/13"),
|
|
("Dublin", "54.154.0.0/15"),("Dublin", "63.32.0.0/14"),
|
|
("London", "3.8.0.0/13"), ("London", "18.168.0.0/14"),
|
|
("London", "18.170.0.0/15"),("London", "35.176.0.0/15"),
|
|
("London", "35.178.0.0/15"),
|
|
("Frankfurt", "3.64.0.0/12"), ("Frankfurt", "18.184.0.0/14"),
|
|
("Frankfurt", "18.194.0.0/15"),("Frankfurt", "35.156.0.0/14"),
|
|
("Frankfurt", "52.28.0.0/14"), ("Frankfurt", "52.57.0.0/16"),
|
|
("Paris", "15.188.0.0/16"),("Paris", "35.180.0.0/14"),
|
|
("Paris", "52.47.0.0/16"),
|
|
("Stockholm", "13.48.0.0/14"), ("Stockholm", "16.16.0.0/16"),
|
|
("Stockholm", "16.170.0.0/15"),
|
|
("Milan", "15.160.0.0/16"),("Milan", "35.152.0.0/16"),
|
|
("Singapore", "3.0.0.0/15"), ("Singapore", "13.212.0.0/14"),
|
|
("Singapore", "13.228.0.0/15"),("Singapore", "18.136.0.0/14"),
|
|
("Singapore", "46.137.0.0/17"),("Singapore", "52.74.0.0/16"),
|
|
("Singapore", "54.179.0.0/16"),
|
|
("Tokyo", "3.112.0.0/14"), ("Tokyo", "13.112.0.0/14"),
|
|
("Tokyo", "13.230.0.0/15"),("Tokyo", "18.176.0.0/15"),
|
|
("Tokyo", "54.64.0.0/15"), ("Tokyo", "54.92.0.0/16"),
|
|
("Tokyo", "54.168.0.0/15"),("Tokyo", "54.238.0.0/15"),
|
|
("Osaka", "13.208.0.0/16"),("Osaka", "15.152.0.0/16"),
|
|
("Seoul", "3.34.0.0/15"), ("Seoul", "13.124.0.0/16"),
|
|
("Seoul", "13.209.0.0/16"),("Seoul", "15.164.0.0/16"),
|
|
("Seoul", "52.78.0.0/16"),
|
|
("Mumbai", "3.6.0.0/15"), ("Mumbai", "13.126.0.0/15"),
|
|
("Mumbai", "15.206.0.0/16"),("Mumbai", "35.154.0.0/15"),
|
|
("Mumbai", "52.66.0.0/16"),
|
|
("Sydney", "3.24.0.0/14"), ("Sydney", "13.54.0.0/15"),
|
|
("Sydney", "13.210.0.0/15"),("Sydney", "52.62.0.0/16"),
|
|
("Sydney", "54.66.0.0/16"), ("Sydney", "54.206.0.0/16"),
|
|
("Sao Paulo", "15.228.0.0/15"),("Sao Paulo", "18.228.0.0/15"),
|
|
("Sao Paulo", "54.94.0.0/16"), ("Sao Paulo", "54.232.0.0/15"),
|
|
("Montreal", "3.96.0.0/14"), ("Montreal", "15.222.0.0/15"),
|
|
("Montreal", "35.182.0.0/14"),("Montreal", "52.60.0.0/16"),
|
|
("Montreal", "99.79.0.0/16"),
|
|
("Hong Kong", "18.162.0.0/15"),
|
|
("Bahrain", "15.184.0.0/15"),("Bahrain", "157.175.0.0/16"),
|
|
("Cape Town", "13.244.0.0/15"),
|
|
]
|
|
|
|
# Layer 3: Azure
|
|
AZURE_RANGES = [
|
|
("Ashburn", "20.0.0.0/11"), ("Ashburn", "40.64.0.0/10"),
|
|
("Ashburn", "52.224.0.0/11"),("Ashburn", "52.145.0.0/16"),
|
|
("Ashburn", "104.40.0.0/13"),
|
|
("San Francisco", "13.64.0.0/11"), ("San Francisco", "40.112.0.0/13"),
|
|
("San Francisco", "104.42.0.0/16"),("San Francisco", "138.91.0.0/16"),
|
|
("Chicago", "23.96.0.0/13"), ("Chicago", "40.80.0.0/12"),
|
|
("Dallas", "13.84.0.0/15"), ("Dallas", "23.100.0.0/15"),
|
|
("Dallas", "40.117.0.0/16"),
|
|
("London", "20.49.0.0/16"), ("London", "51.140.0.0/14"),
|
|
("London", "51.105.0.0/16"),
|
|
("Frankfurt", "20.52.0.0/16"), ("Frankfurt", "40.115.0.0/16"),
|
|
("Frankfurt", "51.103.0.0/16"),
|
|
("Amsterdam", "20.56.0.0/16"), ("Amsterdam", "40.119.0.0/16"),
|
|
("Amsterdam", "52.232.0.0/13"),
|
|
("Paris", "20.43.0.0/16"), ("Paris", "40.89.0.0/16"),
|
|
("Tokyo", "20.44.0.0/15"), ("Tokyo", "40.74.0.0/15"),
|
|
("Tokyo", "52.185.0.0/16"),
|
|
("Singapore", "20.184.0.0/15"),("Singapore", "40.82.0.0/16"),
|
|
("Singapore", "52.139.0.0/16"),
|
|
("Seoul", "20.41.0.0/16"), ("Seoul", "52.141.0.0/16"),
|
|
("Mumbai", "20.192.0.0/16"),("Mumbai", "104.211.0.0/16"),
|
|
("Sydney", "20.36.0.0/14"), ("Sydney", "40.126.0.0/16"),
|
|
("Sydney", "52.147.0.0/16"),
|
|
("Sao Paulo", "191.232.0.0/15"),("Sao Paulo", "20.197.0.0/16"),
|
|
("Toronto", "20.38.0.0/15"),
|
|
("Hong Kong", "20.196.0.0/16"),
|
|
]
|
|
|
|
# Layer 4: Google Cloud
|
|
GCP_RANGES = [
|
|
("Ashburn", "34.0.0.0/8"), ("Ashburn", "35.224.0.0/12"),
|
|
("Ashburn", "104.196.0.0/14"),
|
|
("San Francisco", "35.230.0.0/15"),("San Francisco", "104.198.0.0/14"),
|
|
("Chicago", "35.222.0.0/15"),("Chicago", "104.154.0.0/15"),
|
|
("Dallas", "35.220.0.0/15"),
|
|
("London", "35.240.0.0/13"),("London", "104.199.0.0/14"),
|
|
("Frankfurt", "35.234.0.0/15"),("Frankfurt", "104.155.0.0/14"),
|
|
("Amsterdam", "35.204.0.0/14"),
|
|
("Tokyo", "35.200.0.0/13"),
|
|
("Singapore", "34.80.0.0/15"), ("Singapore", "34.87.0.0/16"),
|
|
("Seoul", "35.216.0.0/14"),("Seoul", "34.64.0.0/11"),
|
|
("Mumbai", "34.93.0.0/16"),
|
|
("Sydney", "35.201.0.0/16"),
|
|
("Sao Paulo", "35.199.0.0/16"),("Sao Paulo", "34.95.0.0/16"),
|
|
("Taipei", "35.194.0.0/14"),
|
|
("Hong Kong", "34.92.0.0/16"),
|
|
("Zurich", "34.65.0.0/16"),
|
|
("Helsinki", "34.88.0.0/15"),
|
|
("Warsaw", "34.118.0.0/16"),
|
|
]
|
|
|
|
# Layer 5: Alibaba Cloud
|
|
ALIBABA_RANGES = [
|
|
("Beijing", "39.96.0.0/13"), ("Beijing", "47.92.0.0/14"),
|
|
("Beijing", "101.200.0.0/15"),("Beijing", "120.24.0.0/14"),
|
|
("Beijing", "120.48.0.0/15"),("Beijing", "182.92.0.0/16"),
|
|
("Beijing", "59.110.0.0/16"),
|
|
("Shanghai", "47.100.0.0/14"),("Shanghai", "101.132.0.0/16"),
|
|
("Shanghai", "106.14.0.0/15"),("Shanghai", "120.55.0.0/16"),
|
|
("Shanghai", "139.196.0.0/14"),("Shanghai", "47.103.0.0/16"),
|
|
("Hangzhou", "47.96.0.0/15"), ("Hangzhou", "120.26.0.0/16"),
|
|
("Hangzhou", "120.76.0.0/15"),("Hangzhou", "121.40.0.0/14"),
|
|
("Hangzhou", "121.196.0.0/14"),("Hangzhou", "47.98.0.0/15"),
|
|
("Shenzhen", "47.106.0.0/16"),("Shenzhen", "120.78.0.0/15"),
|
|
("Shenzhen", "47.115.0.0/16"),
|
|
("Guangzhou", "120.80.0.0/15"),
|
|
("Chengdu", "47.108.0.0/15"),
|
|
("Hong Kong", "47.52.0.0/14"), ("Hong Kong", "47.89.0.0/16"),
|
|
("Hong Kong", "47.56.0.0/15"),
|
|
("Singapore", "47.74.0.0/15"), ("Singapore", "47.88.0.0/16"),
|
|
("Singapore", "149.129.0.0/16"),
|
|
("Tokyo", "47.91.0.0/16"), ("Tokyo", "47.117.0.0/16"),
|
|
("Frankfurt", "47.254.0.0/16"),("Frankfurt", "47.116.0.0/16"),
|
|
("London", "47.118.0.0/16"),
|
|
("San Francisco", "47.57.0.0/16"), ("San Francisco", "47.76.0.0/15"),
|
|
("San Francisco", "47.253.0.0/16"),
|
|
("Sydney", "47.114.0.0/16"),
|
|
("Kuala Lumpur", "47.120.0.0/16"),
|
|
("Jakarta", "47.122.0.0/16"),
|
|
("Mumbai", "47.112.0.0/16"),
|
|
("Dubai", "47.119.0.0/16"),
|
|
]
|
|
|
|
# Layer 6: Cloudflare CDN
|
|
CLOUDFLARE_RANGES = [
|
|
("San Francisco", "173.245.48.0/20"), ("San Francisco", "103.21.244.0/22"),
|
|
("San Francisco", "103.22.200.0/22"), ("San Francisco", "103.31.4.0/22"),
|
|
("San Francisco", "141.101.64.0/18"), ("San Francisco", "108.162.192.0/18"),
|
|
("San Francisco", "190.93.240.0/20"), ("San Francisco", "188.114.96.0/20"),
|
|
("San Francisco", "197.234.240.0/22"), ("San Francisco", "198.41.128.0/17"),
|
|
("San Francisco", "162.158.0.0/15"), ("San Francisco", "104.16.0.0/13"),
|
|
("San Francisco", "104.24.0.0/14"), ("San Francisco", "172.64.0.0/13"),
|
|
("San Francisco", "131.0.72.0/22"),
|
|
]
|
|
|
|
# Layer 7: Akamai CDN
|
|
AKAMAI_RANGES = [
|
|
("Cambridge", "23.0.0.0/12"), ("Cambridge", "23.32.0.0/11"),
|
|
("Cambridge", "23.64.0.0/14"), ("Cambridge", "23.72.0.0/13"),
|
|
("Cambridge", "23.192.0.0/11"), ("Cambridge", "23.208.0.0/12"),
|
|
("Cambridge", "72.246.0.0/15"), ("Cambridge", "92.122.0.0/15"),
|
|
("Cambridge", "95.100.0.0/15"), ("Cambridge", "96.6.0.0/15"),
|
|
("Cambridge", "96.16.0.0/15"), ("Cambridge", "104.64.0.0/10"),
|
|
("Cambridge", "184.24.0.0/13"), ("Cambridge", "184.84.0.0/14"),
|
|
("Cambridge", "2.16.0.0/13"), ("Cambridge", "2.20.0.0/14"),
|
|
("Cambridge", "2.22.0.0/16"),
|
|
]
|
|
|
|
# Layer 8: Fastly CDN
|
|
FASTLY_RANGES = [
|
|
("San Francisco", "151.101.0.0/16"), ("San Francisco", "157.52.64.0/18"),
|
|
("San Francisco", "157.52.128.0/17"), ("San Francisco","167.82.0.0/17"),
|
|
("San Francisco", "167.82.128.0/18"), ("San Francisco","172.111.64.0/18"),
|
|
("San Francisco", "185.31.16.0/22"), ("San Francisco","199.27.72.0/21"),
|
|
("San Francisco", "199.232.0.0/16"),
|
|
]
|
|
|
|
# Layer 9: Chinese ISPs
|
|
CN_ISP_RANGES = [
|
|
("Shanghai", "1.0.0.0/8"), ("Beijing", "14.0.0.0/8"),
|
|
("Shanghai", "27.0.0.0/8"), ("Guangzhou","36.0.0.0/8"),
|
|
("Beijing", "42.0.0.0/8"), ("Shenzhen", "58.0.0.0/8"),
|
|
("Shanghai", "59.0.0.0/8"), ("Beijing", "60.0.0.0/8"),
|
|
("Shanghai", "61.0.0.0/8"), ("Beijing", "101.0.0.0/8"),
|
|
("Guangzhou","110.0.0.0/8"), ("Shanghai", "111.0.0.0/8"),
|
|
("Beijing", "112.0.0.0/8"), ("Shanghai", "113.0.0.0/8"),
|
|
("Beijing", "114.0.0.0/8"), ("Shanghai", "115.0.0.0/8"),
|
|
("Beijing", "116.0.0.0/8"), ("Shanghai", "117.0.0.0/8"),
|
|
("Beijing", "118.0.0.0/8"), ("Shanghai", "119.0.0.0/8"),
|
|
("Beijing", "120.0.0.0/8"), ("Shanghai", "121.0.0.0/8"),
|
|
("Beijing", "122.0.0.0/8"), ("Shanghai", "123.0.0.0/8"),
|
|
("Beijing", "124.0.0.0/8"), ("Shanghai", "125.0.0.0/8"),
|
|
("Guangzhou","139.0.0.0/8"), ("Shanghai", "180.0.0.0/8"),
|
|
("Beijing", "182.0.0.0/8"), ("Shanghai", "183.0.0.0/8"),
|
|
("Beijing", "202.0.0.0/8"), ("Beijing", "203.0.0.0/8"),
|
|
("Shanghai", "210.0.0.0/8"), ("Beijing", "211.0.0.0/8"),
|
|
("Shanghai", "218.0.0.0/8"), ("Beijing", "219.0.0.0/8"),
|
|
("Guangzhou","220.0.0.0/8"), ("Shanghai", "221.0.0.0/8"),
|
|
("Beijing", "222.0.0.0/8"), ("Beijing", "223.0.0.0/8"),
|
|
]
|
|
|
|
# Layer 10: US ISPs
|
|
US_ISP_RANGES = [
|
|
("New York", "24.0.0.0/8"), ("Chicago", "50.128.0.0/9"),
|
|
("Denver", "50.160.0.0/11"),("Philadelphia", "67.160.0.0/11"),
|
|
("Boston", "69.136.0.0/13"),("Seattle", "69.248.0.0/12"),
|
|
("San Francisco", "71.192.0.0/10"),("Atlanta", "73.0.0.0/8"),
|
|
("Houston", "76.0.0.0/8"), ("New York", "96.224.0.0/11"),
|
|
("Chicago", "98.192.0.0/10"),
|
|
("Dallas", "12.0.0.0/8"), ("Atlanta", "32.0.0.0/8"),
|
|
("New York", "45.0.0.0/8"), ("Chicago", "64.0.0.0/8"),
|
|
("San Francisco", "65.0.0.0/8"), ("Dallas", "66.0.0.0/8"),
|
|
("Miami", "67.0.0.0/8"), ("New York", "69.0.0.0/8"),
|
|
("Chicago", "70.0.0.0/8"), ("San Francisco","75.0.0.0/8"),
|
|
("New York", "99.0.0.0/8"), ("Dallas", "104.0.0.0/8"),
|
|
("Atlanta", "107.0.0.0/8"), ("Chicago", "108.0.0.0/8"),
|
|
("New York", "71.160.0.0/12"),("Dallas", "72.64.0.0/10"),
|
|
("Atlanta", "74.96.0.0/11"), ("San Francisco","96.240.0.0/12"),
|
|
("Chicago", "100.0.0.0/10"), ("New York", "108.16.0.0/12"),
|
|
("Dallas", "141.144.0.0/12"),("Miami", "173.48.0.0/12"),
|
|
]
|
|
|
|
# Layer 11: European ISPs
|
|
EU_ISP_RANGES = [
|
|
("Frankfurt", "79.192.0.0/10"), ("Berlin", "80.128.0.0/11"),
|
|
("Munich", "87.128.0.0/10"), ("Frankfurt", "91.0.0.0/10"),
|
|
("Berlin", "93.192.0.0/10"), ("Munich", "217.80.0.0/12"),
|
|
("Frankfurt", "217.224.0.0/11"),
|
|
("London", "2.0.0.0/8"), ("London", "5.0.0.0/8"),
|
|
("London", "25.0.0.0/8"), ("London", "31.0.0.0/8"),
|
|
("London", "51.0.0.0/8"), ("London", "81.0.0.0/8"),
|
|
("London", "109.0.0.0/8"), ("London", "146.0.0.0/8"),
|
|
("London", "176.0.0.0/8"), ("London", "185.0.0.0/8"),
|
|
("London", "188.0.0.0/8"), ("London", "212.0.0.0/8"),
|
|
("Paris", "37.0.0.0/8"), ("Paris", "128.0.0.0/8"),
|
|
("Paris", "129.0.0.0/8"), ("Paris", "137.0.0.0/8"),
|
|
("Paris", "145.0.0.0/8"), ("Paris", "147.0.0.0/8"),
|
|
("Paris", "156.0.0.0/8"), ("Paris", "159.0.0.0/8"),
|
|
("Paris", "161.0.0.0/8"), ("Paris", "194.0.0.0/8"),
|
|
("Amsterdam", "46.0.0.0/8"), ("Amsterdam", "62.0.0.0/8"),
|
|
("Amsterdam", "77.0.0.0/8"), ("Amsterdam", "80.0.0.0/8"),
|
|
("Amsterdam", "82.0.0.0/8"), ("Amsterdam", "83.0.0.0/8"),
|
|
("Amsterdam", "84.0.0.0/8"), ("Amsterdam", "85.0.0.0/8"),
|
|
("Amsterdam", "86.0.0.0/8"), ("Amsterdam", "88.0.0.0/8"),
|
|
("Amsterdam", "89.0.0.0/8"), ("Amsterdam", "90.0.0.0/8"),
|
|
("Amsterdam", "195.0.0.0/8"), ("Amsterdam", "213.0.0.0/8"),
|
|
("Berlin", "78.0.0.0/8"), ("Berlin", "79.0.0.0/8"),
|
|
("Berlin", "87.0.0.0/8"), ("Berlin", "95.0.0.0/8"),
|
|
("Berlin", "130.0.0.0/8"), ("Berlin", "134.0.0.0/8"),
|
|
("Berlin", "141.0.0.0/8"), ("Berlin", "149.0.0.0/8"),
|
|
("Berlin", "193.0.0.0/8"), ("Berlin", "217.0.0.0/8"),
|
|
("Stockholm", "131.0.0.0/8"), ("Stockholm", "155.0.0.0/8"),
|
|
("Stockholm", "158.0.0.0/8"), ("Stockholm", "171.0.0.0/8"),
|
|
("Milan", "132.0.0.0/8"), ("Milan", "138.0.0.0/8"),
|
|
("Milan", "143.0.0.0/8"), ("Milan", "151.0.0.0/8"),
|
|
("Madrid", "5.0.0.0/8"), ("Madrid", "65.16.0.0/12"),
|
|
("Madrid", "150.192.0.0/10"),("Madrid", "161.64.0.0/12"),
|
|
("Moscow", "92.0.0.0/8"), ("Moscow", "94.0.0.0/8"),
|
|
]
|
|
|
|
# Layer 12: Japanese ISPs
|
|
JP_ISP_RANGES = [
|
|
("Tokyo", "126.0.0.0/8"), ("Tokyo", "133.0.0.0/8"),
|
|
("Tokyo", "150.0.0.0/8"), ("Osaka", "153.128.0.0/10"),
|
|
("Tokyo", "157.0.0.0/8"), ("Osaka", "160.0.0.0/8"),
|
|
("Tokyo", "163.0.0.0/8"), ("Osaka", "164.0.0.0/8"),
|
|
("Nagoya", "192.50.0.0/16"),("Tokyo", "202.208.0.0/12"),
|
|
("Osaka", "210.128.0.0/11"),("Nagoya","210.160.0.0/12"),
|
|
("Tokyo", "210.224.0.0/12"),("Osaka", "211.0.0.0/12"),
|
|
("Tokyo", "59.128.0.0/12"), ("Osaka", "60.56.0.0/13"),
|
|
("Tokyo", "61.112.0.0/12"), ("Osaka", "111.64.0.0/11"),
|
|
("Tokyo", "118.0.0.0/11"), ("Osaka", "121.80.0.0/12"),
|
|
("Tokyo", "122.16.0.0/12"), ("Osaka", "125.0.0.0/14"),
|
|
]
|
|
|
|
# ============================================================
|
|
# Build non-overlapping range list
|
|
# ============================================================
|
|
|
|
def ip_to_int(ip: str) -> int:
|
|
parts = [int(p) for p in ip.split(".")]
|
|
return (parts[0] << 24) | (parts[1] << 16) | (parts[2] << 8) | parts[3]
|
|
|
|
|
|
def int_to_ip(val: int) -> str:
|
|
return f"{(val >> 24) & 0xFF}.{(val >> 16) & 0xFF}.{(val >> 8) & 0xFF}.{val & 0xFF}"
|
|
|
|
|
|
# Collect all priority ranges
|
|
all_layers = [
|
|
("Private", PRIVATE_RANGES),
|
|
("DNS", DNS_RANGES),
|
|
("AWS", AWS_RANGES),
|
|
("Azure", AZURE_RANGES),
|
|
("GCP", GCP_RANGES),
|
|
("Alibaba", ALIBABA_RANGES),
|
|
("Cloudflare", CLOUDFLARE_RANGES),
|
|
("Akamai", AKAMAI_RANGES),
|
|
("Fastly", FASTLY_RANGES),
|
|
("CN_ISP", CN_ISP_RANGES),
|
|
("US_ISP", US_ISP_RANGES),
|
|
("EU_ISP", EU_ISP_RANGES),
|
|
("JP_ISP", JP_ISP_RANGES),
|
|
]
|
|
|
|
# Reserved ranges to skip
|
|
RESERVED = [
|
|
(ip_to_int("0.0.0.0"), ip_to_int("0.255.255.255")),
|
|
(ip_to_int("127.0.0.0"), ip_to_int("127.255.255.255")),
|
|
(ip_to_int("169.254.0.0"), ip_to_int("169.254.255.255")),
|
|
(ip_to_int("224.0.0.0"), ip_to_int("255.255.255.255")),
|
|
]
|
|
|
|
def is_reserved(s: int, e: int) -> bool:
|
|
for rs, re in RESERVED:
|
|
if s <= re and e >= rs:
|
|
return True
|
|
return False
|
|
|
|
|
|
# Process layers in priority order — higher priority wins on overlap
|
|
# merged is kept sorted; new ranges are inserted, trimming overlaps
|
|
merged: list[tuple[int, int, str, float, float, str]] = []
|
|
|
|
def insert_range(s: int, e: int, city: str, lat: float, lon: float, cc: str) -> None:
|
|
"""Insert range into merged, trimming parts that overlap existing (higher-priority) ranges."""
|
|
# Find position using bisect
|
|
lo, hi = 0, len(merged)
|
|
while lo < hi:
|
|
mid = (lo + hi) // 2
|
|
if merged[mid][0] < s:
|
|
lo = mid + 1
|
|
else:
|
|
hi = mid
|
|
idx = lo
|
|
|
|
# Check for overlap with previous entry
|
|
if idx > 0 and merged[idx-1][1] >= s:
|
|
s = merged[idx-1][1] + 1
|
|
|
|
# Trim against next entries
|
|
while idx < len(merged) and s <= e:
|
|
if merged[idx][0] > e:
|
|
break
|
|
if s < merged[idx][0]:
|
|
# Gap: insert [s, merged[idx][0]-1]
|
|
merged.insert(idx, (s, merged[idx][0] - 1, city, lat, lon, cc))
|
|
idx += 1
|
|
s = merged[idx-1][1] + 1 if idx > 0 else s
|
|
s = max(s, merged[idx][1] + 1)
|
|
idx += 1
|
|
|
|
if s <= e:
|
|
merged.insert(idx, (s, e, city, lat, lon, cc))
|
|
|
|
for layer_name, layer_ranges in all_layers:
|
|
for city, cidr in layer_ranges:
|
|
s, e = cidr_to_ints(cidr)
|
|
if is_reserved(s, e):
|
|
continue
|
|
lat, lon, cc = CITIES[city]
|
|
insert_range(s, e, city, lat, lon, cc)
|
|
|
|
# ---- Gap filling cities (rotating) ----
|
|
gap_cities = [
|
|
("Shanghai", 31.2304, 121.4737, "CN"),
|
|
("Beijing", 39.9042, 116.4074, "CN"),
|
|
("Tokyo", 35.6762, 139.6503, "JP"),
|
|
("Seoul", 37.5665, 126.9780, "KR"),
|
|
("Singapore", 1.3521, 103.8198, "SG"),
|
|
("London", 51.5074, -0.1278, "GB"),
|
|
("Frankfurt", 50.1109, 8.6821, "DE"),
|
|
("New York", 40.7128, -74.0060, "US"),
|
|
("San Francisco", 37.7749, -122.4194, "US"),
|
|
("Sydney", -33.8688, 151.2093, "AU"),
|
|
("Mumbai", 19.0760, 72.8777, "IN"),
|
|
("Paris", 48.8566, 2.3522, "FR"),
|
|
("Amsterdam", 52.3676, 4.9041, "NL"),
|
|
("Sao Paulo", -23.5505, -46.6333, "BR"),
|
|
("Moscow", 55.7558, 37.6173, "RU"),
|
|
("Chicago", 41.8781, -87.6298, "US"),
|
|
("Dallas", 32.7767, -96.7970, "US"),
|
|
("Atlanta", 33.7490, -84.3880, "US"),
|
|
("Toronto", 43.6532, -79.3832, "CA"),
|
|
("Hong Kong", 22.3193, 114.1694, "HK"),
|
|
("Taipei", 25.0330, 121.5654, "TW"),
|
|
("Madrid", 40.4168, -3.7038, "ES"),
|
|
("Milan", 45.4642, 9.1900, "IT"),
|
|
("Stockholm", 59.3293, 18.0686, "SE"),
|
|
("Osaka", 34.6937, 135.5022, "JP"),
|
|
("Guangzhou", 23.1291, 113.2644, "CN"),
|
|
("Shenzhen", 22.5431, 114.0579, "CN"),
|
|
("Berlin", 52.5200, 13.4050, "DE"),
|
|
("Dubai", 25.2048, 55.2708, "AE"),
|
|
("Bangkok", 13.7563, 100.5018, "TH"),
|
|
]
|
|
|
|
# Fill gaps — use /9 blocks (8,388,608 IPs) for large gaps, /12 (1,048,576) for smaller
|
|
idx = 0
|
|
final_ranges: list[tuple[int, int, str, float, float, str]] = []
|
|
last_end = ip_to_int("1.0.0.0") - 1 # start from 1.0.0.0
|
|
MAX_IP = ip_to_int("223.255.255.255")
|
|
CHUNK_BIG = 0x800000 # /9 = 8,388,608 IPs
|
|
CHUNK_SMALL = 0x100000 # /12 = 1,048,576 IPs
|
|
|
|
for s, e, city, lat, lon, cc in merged:
|
|
gap_size = s - last_end - 1
|
|
if gap_size > 0:
|
|
gap_start = last_end + 1
|
|
gap_end = s - 1
|
|
chunk = CHUNK_BIG if gap_size > 0x800000 else CHUNK_SMALL
|
|
curr = gap_start
|
|
while curr <= gap_end:
|
|
chunk_end = min(curr + chunk - 1, gap_end)
|
|
gcity, glat, glon, gcc = gap_cities[idx % len(gap_cities)]
|
|
final_ranges.append((curr, chunk_end, gcity, glat, glon, gcc))
|
|
idx += 1
|
|
curr = chunk_end + 1
|
|
final_ranges.append((s, e, city, lat, lon, cc))
|
|
last_end = e
|
|
|
|
# Trailing gap
|
|
if last_end < MAX_IP:
|
|
gap_size = MAX_IP - last_end
|
|
curr = last_end + 1
|
|
chunk = CHUNK_BIG if gap_size > 0x800000 else CHUNK_SMALL
|
|
while curr <= MAX_IP:
|
|
chunk_end = min(curr + chunk - 1, MAX_IP)
|
|
gcity, glat, glon, gcc = gap_cities[idx % len(gap_cities)]
|
|
final_ranges.append((curr, chunk_end, gcity, glat, glon, gcc))
|
|
idx += 1
|
|
curr = chunk_end + 1
|
|
|
|
# ============================================================
|
|
# Verify
|
|
# ============================================================
|
|
final_ranges.sort(key=lambda x: x[0])
|
|
overlaps = sum(1 for i in range(len(final_ranges)-1) if final_ranges[i][1] >= final_ranges[i+1][0])
|
|
has_gaps = sum(1 for i in range(len(final_ranges)-1) if final_ranges[i][1]+1 < final_ranges[i+1][0])
|
|
|
|
print(f"Overlap check: {'OK (0)' if overlaps == 0 else f'FAIL ({overlaps})'}")
|
|
print(f"Gap check: {'OK (0)' if has_gaps == 0 else f'HAS GAPS ({has_gaps})'}")
|
|
|
|
# ============================================================
|
|
# Write output
|
|
# ============================================================
|
|
out_path = Path(__file__).parent / "geoip_data.txt"
|
|
with open(out_path, "w", encoding="utf-8") as f:
|
|
f.write("# ===== Offline GeoIP Database — 天璇 (TianXuan) =====\n")
|
|
f.write("# Format: start_ip,end_ip,lat,lon,city,country\n")
|
|
f.write("# Layers: Private, DNS, AWS, Azure, GCP, Alibaba, Cloudflare, Akamai, Fastly, CN_ISP, US_ISP, EU_ISP, JP_ISP\n")
|
|
f.write("# Strategy: priority-ordered, overlap-resolved (first wins), gaps filled with city blocks\n\n")
|
|
for s, e, city, lat, lon, cc in final_ranges:
|
|
f.write(f"{int_to_ip(s)},{int_to_ip(e)},{lat},{lon},{city},{cc}\n")
|
|
|
|
cities_used = sorted(set(r[2] for r in final_ranges))
|
|
print(f"Total entries: {len(final_ranges)}")
|
|
print(f"Unique cities: {len(cities_used)} ({', '.join(cities_used)})")
|
|
print(f"File size: {out_path.stat().st_size:,} bytes")
|
|
print(f"First: {int_to_ip(final_ranges[0][0])} ({final_ranges[0][2]})")
|
|
print(f"Last: {int_to_ip(final_ranges[-1][1])} ({final_ranges[-1][2]})")
|