From c09d0704af593be7396208e493d2649fe5125210 Mon Sep 17 00:00:00 2001 From: Philip Prindeville Date: Mon, 25 May 2020 14:05:42 -0600 Subject: [PATCH] geoip: re-add Maxmind scripts --- geoip/Makefile.am | 4 +- geoip/xt_geoip_build_maxmind | 268 +++++++++++++++++++++++++++++++++++ geoip/xt_geoip_dl.1 | 5 +- geoip/xt_geoip_dl_maxmind | 7 + geoip/xt_geoip_fetch_maxmind | 95 +++++++++++++ 5 files changed, 375 insertions(+), 4 deletions(-) create mode 100755 geoip/xt_geoip_build_maxmind create mode 100755 geoip/xt_geoip_dl_maxmind create mode 100755 geoip/xt_geoip_fetch_maxmind diff --git a/geoip/Makefile.am b/geoip/Makefile.am index ec3ba51..7bbf3bc 100644 --- a/geoip/Makefile.am +++ b/geoip/Makefile.am @@ -1,7 +1,7 @@ # -*- Makefile -*- -bin_SCRIPTS = xt_geoip_fetch +bin_SCRIPTS = xt_geoip_fetch xt_geoip_fetch_maxmind -pkglibexec_SCRIPTS = xt_geoip_build xt_geoip_dl +pkglibexec_SCRIPTS = xt_geoip_build xt_geoip_build_maxmind xt_geoip_dl xt_geoip_dl_maxmind man1_MANS = xt_geoip_build.1 xt_geoip_dl.1 xt_geoip_fetch.1 diff --git a/geoip/xt_geoip_build_maxmind b/geoip/xt_geoip_build_maxmind new file mode 100755 index 0000000..feb9607 --- /dev/null +++ b/geoip/xt_geoip_build_maxmind @@ -0,0 +1,268 @@ +#!/usr/bin/perl +# +# Converter for MaxMind (GeoLite2) CSV database to binary, for xt_geoip +# Copyright Jan Engelhardt, 2008-2011 +# Copyright Philip Prindeville, 2018 +# +use Getopt::Long; +use Net::CIDR::Lite; +use Socket qw(AF_INET AF_INET6 inet_pton); +use warnings; +use Text::CSV_XS; # or trade for Text::CSV +use strict; + +my $csv = Text::CSV_XS->new({ + allow_whitespace => 1, + binary => 1, + eol => $/, +}); # or Text::CSV +my $source_dir = "."; +my $quiet = 0; +my $target_dir = "."; + +&Getopt::Long::Configure(qw(bundling)); +&GetOptions( + "D=s" => \$target_dir, + "S=s" => \$source_dir, + "q" => \$quiet, + "s" => sub { $target_dir = "/usr/share/xt_geoip"; }, +); + +if (!-d $source_dir) { + print STDERR "Source directory \"$source_dir\" does not exist.\n"; + exit 1; +} +if (!-d $target_dir) { + print STDERR "Target directory \"$target_dir\" does not exist.\n"; + exit 1; +} + +my %countryId; +my %countryName; +&loadCountries(); +&dump(&collect()); + +sub loadCountries +{ + sub id; sub cc; sub long; sub ct; sub cn; + + %countryId = (); + %countryName = (); + + my $file = "$source_dir/GeoLite2-Country-Locations-en.csv"; + open(my $fh, '<', $file) || die "Couldn't open list country names\n"; + + # first line is headers + my $row = $csv->getline($fh); + + my %header = map { ($row->[$_], $_); } (0..$#{$row}); + + my %pairs = ( + country_iso_code => 'ISO Country Code', + geoname_id => 'ID', + country_name => 'Country Name', + continent_code => 'Continent Code', + continent_name => 'Continent Name', + ); + + # verify that the columns we need are present + map { die "Table has no $pairs{$_} column\n" unless (exists $header{$_}); } keys %pairs; + + my %remapping = ( + id => 'geoname_id', + cc => 'country_iso_code', + long => 'country_name', + ct => 'continent_code', + cn => 'continent_name', + ); + + # now create a function which returns the value of that column # + map { eval "sub $_ () { \$header{\$remapping{$_}}; }" ; } keys %remapping; + + while (my $row = $csv->getline($fh)) { + if ($row->[cc] eq '' && $row->[long] eq '') { + $countryId{$row->[id]} = $row->[ct]; + $countryName{$row->[ct]} = $row->[cn]; + } else { + $countryId{$row->[id]} = $row->[cc]; + $countryName{$row->[cc]} = $row->[long]; + } + } + + $countryName{A1} = 'Anonymous Proxy'; + $countryName{A2} = 'Satellite Provider'; + $countryName{O1} = 'Other Country'; + + close($fh); + + # clean up the namespace + undef &id; undef &cc; undef &long; undef &ct; undef &cn; +} + +sub lookupCountry +{ + my ($id, $rid, $proxy, $sat) = @_; + + if ($proxy) { + return 'A1'; + } elsif ($sat) { + return 'A2'; + } + $id ||= $rid; + if ($id eq '') { + return 'O1'; + } + die "Unknown id: $id line $.\n" unless (exists $countryId{$id}); + return $countryId{$id}; +} + +sub collect +{ + my ($file, $fh, $row); + my (%country, %header); + + sub net; sub id; sub rid; sub proxy; sub sat; + + my %pairs = ( + network => 'Network', + registered_country_geoname_id => 'Registered Country ID', + geoname_id => 'Country ID', + is_anonymous_proxy => 'Anonymous Proxy', + is_satellite_provider => 'Satellite', + ); + + foreach (sort keys %countryName) { + $country{$_} = { + name => $countryName{$_}, + pool_v4 => Net::CIDR::Lite->new(), + pool_v6 => Net::CIDR::Lite->new(), + }; + } + + $file = "$source_dir/GeoLite2-Country-Blocks-IPv4.csv"; + open($fh, '<', $file) || die "Can't open IPv4 database\n"; + + # first line is headers + $row = $csv->getline($fh); + + %header = map { ($row->[$_], $_); } (0..$#{$row}); + + # verify that the columns we need are present + map { die "Table has no %pairs{$_} column\n" unless (exists $header{$_}); } keys %pairs; + + my %remapping = ( + net => 'network', + id => 'geoname_id', + rid => 'registered_country_geoname_id', + proxy => 'is_anonymous_proxy', + sat => 'is_satellite_provider', + ); + + # now create a function which returns the value of that column # + map { eval "sub $_ () { \$header{\$remapping{$_}}; }" ; } keys %remapping; + + while ($row = $csv->getline($fh)) { + my ($cc, $cidr); + + $cc = lookupCountry($row->[id], $row->[rid], $row->[proxy], $row->[sat]); + $cidr = $row->[net]; + $country{$cc}->{pool_v4}->add($cidr); + + if ($. % 4096 == 0) { + print STDERR "\r\e[2K$. entries"; + } + } + + print STDERR "\r\e[2K$. entries total\n"; + + close($fh); + + # clean up the namespace + undef &net; undef &id; undef &rid; undef &proxy; undef &sat; + + $file = "$source_dir/GeoLite2-Country-Blocks-IPv6.csv"; + open($fh, '<', $file) || die "Can't open IPv6 database\n"; + + # first line is headers + $row = $csv->getline($fh); + + %header = map { ($row->[$_], $_); } (0..$#{$row}); + + # verify that the columns we need are present + map { die "Table has no %pairs{$_} column\n" unless (exists $header{$_}); } keys %pairs; + + # unlikely the IPv6 table has different columns, but just to be sure + # create a function which returns the value of that column # + map { eval "sub $_ () { \$header{\$remapping{$_}}; }" ; } keys %remapping; + + while ($row = $csv->getline($fh)) { + my ($cc, $cidr); + + $cc = lookupCountry($row->[id], $row->[rid], $row->[proxy], $row->[sat]); + $cidr = $row->[net]; + $country{$cc}->{pool_v6}->add($cidr); + + if (!$quiet && $. % 4096 == 0) { + print STDERR "\r\e[2K$. entries"; + } + } + + print STDERR "\r\e[2K$. entries total\n" unless ($quiet); + + close($fh); + + # clean up the namespace + undef &net; undef &id; undef &rid; undef &proxy; undef &sat; + + return \%country; +} + +sub dump +{ + my $country = shift @_; + + foreach my $iso_code (sort keys %{$country}) { + &dump_one($iso_code, $country->{$iso_code}); + } +} + +sub dump_one +{ + my($iso_code, $country) = @_; + my @ranges; + + @ranges = $country->{pool_v4}->list_range(); + + writeCountry($iso_code, $country->{name}, AF_INET, @ranges); + + @ranges = $country->{pool_v6}->list_range(); + + writeCountry($iso_code, $country->{name}, AF_INET6, @ranges); +} + +sub writeCountry +{ + my ($iso_code, $name, $family, @ranges) = @_; + my $fh; + + printf "%5u IPv%s ranges for %s %s\n", + scalar(@ranges), + ($family == AF_INET ? '4' : '6'), + $iso_code, $name unless ($quiet); + + my $file = "$target_dir/".uc($iso_code).".iv".($family == AF_INET ? '4' : '6'); + if (!open($fh, '>', $file)) { + print STDERR "Error opening $file: $!\n"; + exit 1; + } + + binmode($fh); + + foreach my $range (@ranges) { + my ($start, $end) = split('-', $range); + $start = inet_pton($family, $start); + $end = inet_pton($family, $end); + print $fh $start, $end; + } + close $fh; +} diff --git a/geoip/xt_geoip_dl.1 b/geoip/xt_geoip_dl.1 index 3934335..76cb5b0 100644 --- a/geoip/xt_geoip_dl.1 +++ b/geoip/xt_geoip_dl.1 @@ -7,8 +7,9 @@ xt_geoip_dl \(em download GeoIP database files \fI/usr/libexec/xt_geoip/\fP\fBxt_geoip_dl\fP .SH Description .PP -Downloads and unpacks the MaxMind GeoIP Country Lite databases for IPv4 and -IPv6 and unpacks them to the current directory. +Downloads the DB-IP Country Lite databases for IPv4 and IPv6 and unpacks them +to the current directory. The alternate \fBxt_geoip_dl_maxmind\fP script can be +used for MaxMind formatted CSV databases. .PP Since the script is usually installed to the libexec directory of the xtables-addons package and this is outside $PATH (on purpose), invoking the diff --git a/geoip/xt_geoip_dl_maxmind b/geoip/xt_geoip_dl_maxmind new file mode 100755 index 0000000..1de6044 --- /dev/null +++ b/geoip/xt_geoip_dl_maxmind @@ -0,0 +1,7 @@ +#!/bin/sh + +rm -rf GeoLite2-Country-CSV_* + +wget -q http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country-CSV.zip +unzip -q GeoLite2-Country-CSV.zip +rm -f GeoLite2-Country-CSV.zip diff --git a/geoip/xt_geoip_fetch_maxmind b/geoip/xt_geoip_fetch_maxmind new file mode 100755 index 0000000..0624519 --- /dev/null +++ b/geoip/xt_geoip_fetch_maxmind @@ -0,0 +1,95 @@ +#!/usr/bin/perl +# +# Utility to query GeoIP database +# Copyright Philip Prindeville, 2018 +# +use Getopt::Long; +use Socket qw(AF_INET AF_INET6 inet_ntop); +use warnings; +use strict; + +sub AF_INET_SIZE() { 4 } +sub AF_INET6_SIZE() { 16 } + +my $target_dir = "."; +my $ipv4 = 0; +my $ipv6 = 0; + +&Getopt::Long::Configure(qw(bundling)); +&GetOptions( + "D=s" => \$target_dir, + "4" => \$ipv4, + "6" => \$ipv6, +); + +if (!-d $target_dir) { + print STDERR "Target directory $target_dir does not exit.\n"; + exit 1; +} + +# if neither specified, assume both +if (! $ipv4 && ! $ipv6) { + $ipv4 = $ipv6 = 1; +} + +foreach my $cc (@ARGV) { + if ($cc !~ m/^([a-z]{2}|a[12]|o1)$/i) { + print STDERR "Invalid country code '$cc'\n"; + exit 1; + } + + my $file = $target_dir . '/' . uc($cc) . '.iv4'; + + if (! -f $file) { + printf STDERR "Can't find data for country '$cc'\n"; + exit 1; + } + + my ($contents, $buffer, $bytes, $fh); + + if ($ipv4) { + open($fh, '<', $file) || die "Couldn't open file for '$cc'\n"; + + binmode($fh); + + while (($bytes = read($fh, $buffer, AF_INET_SIZE * 2)) == AF_INET_SIZE * 2) { + my ($start, $end) = unpack('a4a4', $buffer); + $start = inet_ntop(AF_INET, $start); + $end = inet_ntop(AF_INET, $end); + print $start, '-', $end, "\n"; + } + close($fh); + if (! defined $bytes) { + printf STDERR "Error reading file for '$cc'\n"; + exit 1; + } elsif ($bytes != 0) { + printf STDERR "Short read on file for '$cc'\n"; + exit 1; + } + } + + substr($file, -1) = '6'; + + if ($ipv6) { + open($fh, '<', $file) || die "Couldn't open file for '$cc'\n"; + + binmode($fh); + + while (($bytes = read($fh, $buffer, AF_INET6_SIZE * 2)) == AF_INET6_SIZE * 2) { + my ($start, $end) = unpack('a16a16', $buffer); + $start = inet_ntop(AF_INET6, $start); + $end = inet_ntop(AF_INET6, $end); + print $start, '-', $end, "\n"; + } + close($fh); + if (! defined $bytes) { + printf STDERR "Error reading file for '$cc'\n"; + exit 1; + } elsif ($bytes != 0) { + printf STDERR "Short read on file for '$cc'\n"; + exit 1; + } + } +} + +exit 0;