mirror of
git://git.code.sf.net/p/xtables-addons/xtables-addons
synced 2025-09-05 20:26:38 +02:00
geoip: re-add Maxmind scripts
This commit is contained in:
Philip Prindeville
committed by
Jan Engelhardt
Jan Engelhardt
parent
0021003dc7
commit
c09d0704af
@@ -1,7 +1,7 @@
|
||||
# -*- Makefile -*-
|
||||
|
||||
bin_SCRIPTS = xt_geoip_fetch
|
||||
bin_SCRIPTS = xt_geoip_fetch xt_geoip_fetch_maxmind
|
||||
|
||||
pkglibexec_SCRIPTS = xt_geoip_build xt_geoip_dl
|
||||
pkglibexec_SCRIPTS = xt_geoip_build xt_geoip_build_maxmind xt_geoip_dl xt_geoip_dl_maxmind
|
||||
|
||||
man1_MANS = xt_geoip_build.1 xt_geoip_dl.1 xt_geoip_fetch.1
|
||||
|
||||
268
geoip/xt_geoip_build_maxmind
Executable file
268
geoip/xt_geoip_build_maxmind
Executable file
@@ -0,0 +1,268 @@
|
||||
#!/usr/bin/perl
|
||||
#
|
||||
# Converter for MaxMind (GeoLite2) CSV database to binary, for xt_geoip
|
||||
# Copyright Jan Engelhardt, 2008-2011
|
||||
# Copyright Philip Prindeville, 2018
|
||||
#
|
||||
use Getopt::Long;
|
||||
use Net::CIDR::Lite;
|
||||
use Socket qw(AF_INET AF_INET6 inet_pton);
|
||||
use warnings;
|
||||
use Text::CSV_XS; # or trade for Text::CSV
|
||||
use strict;
|
||||
|
||||
my $csv = Text::CSV_XS->new({
|
||||
allow_whitespace => 1,
|
||||
binary => 1,
|
||||
eol => $/,
|
||||
}); # or Text::CSV
|
||||
my $source_dir = ".";
|
||||
my $quiet = 0;
|
||||
my $target_dir = ".";
|
||||
|
||||
&Getopt::Long::Configure(qw(bundling));
|
||||
&GetOptions(
|
||||
"D=s" => \$target_dir,
|
||||
"S=s" => \$source_dir,
|
||||
"q" => \$quiet,
|
||||
"s" => sub { $target_dir = "/usr/share/xt_geoip"; },
|
||||
);
|
||||
|
||||
if (!-d $source_dir) {
|
||||
print STDERR "Source directory \"$source_dir\" does not exist.\n";
|
||||
exit 1;
|
||||
}
|
||||
if (!-d $target_dir) {
|
||||
print STDERR "Target directory \"$target_dir\" does not exist.\n";
|
||||
exit 1;
|
||||
}
|
||||
|
||||
my %countryId;
|
||||
my %countryName;
|
||||
&loadCountries();
|
||||
&dump(&collect());
|
||||
|
||||
sub loadCountries
|
||||
{
|
||||
sub id; sub cc; sub long; sub ct; sub cn;
|
||||
|
||||
%countryId = ();
|
||||
%countryName = ();
|
||||
|
||||
my $file = "$source_dir/GeoLite2-Country-Locations-en.csv";
|
||||
open(my $fh, '<', $file) || die "Couldn't open list country names\n";
|
||||
|
||||
# first line is headers
|
||||
my $row = $csv->getline($fh);
|
||||
|
||||
my %header = map { ($row->[$_], $_); } (0..$#{$row});
|
||||
|
||||
my %pairs = (
|
||||
country_iso_code => 'ISO Country Code',
|
||||
geoname_id => 'ID',
|
||||
country_name => 'Country Name',
|
||||
continent_code => 'Continent Code',
|
||||
continent_name => 'Continent Name',
|
||||
);
|
||||
|
||||
# verify that the columns we need are present
|
||||
map { die "Table has no $pairs{$_} column\n" unless (exists $header{$_}); } keys %pairs;
|
||||
|
||||
my %remapping = (
|
||||
id => 'geoname_id',
|
||||
cc => 'country_iso_code',
|
||||
long => 'country_name',
|
||||
ct => 'continent_code',
|
||||
cn => 'continent_name',
|
||||
);
|
||||
|
||||
# now create a function which returns the value of that column #
|
||||
map { eval "sub $_ () { \$header{\$remapping{$_}}; }" ; } keys %remapping;
|
||||
|
||||
while (my $row = $csv->getline($fh)) {
|
||||
if ($row->[cc] eq '' && $row->[long] eq '') {
|
||||
$countryId{$row->[id]} = $row->[ct];
|
||||
$countryName{$row->[ct]} = $row->[cn];
|
||||
} else {
|
||||
$countryId{$row->[id]} = $row->[cc];
|
||||
$countryName{$row->[cc]} = $row->[long];
|
||||
}
|
||||
}
|
||||
|
||||
$countryName{A1} = 'Anonymous Proxy';
|
||||
$countryName{A2} = 'Satellite Provider';
|
||||
$countryName{O1} = 'Other Country';
|
||||
|
||||
close($fh);
|
||||
|
||||
# clean up the namespace
|
||||
undef &id; undef &cc; undef &long; undef &ct; undef &cn;
|
||||
}
|
||||
|
||||
sub lookupCountry
|
||||
{
|
||||
my ($id, $rid, $proxy, $sat) = @_;
|
||||
|
||||
if ($proxy) {
|
||||
return 'A1';
|
||||
} elsif ($sat) {
|
||||
return 'A2';
|
||||
}
|
||||
$id ||= $rid;
|
||||
if ($id eq '') {
|
||||
return 'O1';
|
||||
}
|
||||
die "Unknown id: $id line $.\n" unless (exists $countryId{$id});
|
||||
return $countryId{$id};
|
||||
}
|
||||
|
||||
sub collect
|
||||
{
|
||||
my ($file, $fh, $row);
|
||||
my (%country, %header);
|
||||
|
||||
sub net; sub id; sub rid; sub proxy; sub sat;
|
||||
|
||||
my %pairs = (
|
||||
network => 'Network',
|
||||
registered_country_geoname_id => 'Registered Country ID',
|
||||
geoname_id => 'Country ID',
|
||||
is_anonymous_proxy => 'Anonymous Proxy',
|
||||
is_satellite_provider => 'Satellite',
|
||||
);
|
||||
|
||||
foreach (sort keys %countryName) {
|
||||
$country{$_} = {
|
||||
name => $countryName{$_},
|
||||
pool_v4 => Net::CIDR::Lite->new(),
|
||||
pool_v6 => Net::CIDR::Lite->new(),
|
||||
};
|
||||
}
|
||||
|
||||
$file = "$source_dir/GeoLite2-Country-Blocks-IPv4.csv";
|
||||
open($fh, '<', $file) || die "Can't open IPv4 database\n";
|
||||
|
||||
# first line is headers
|
||||
$row = $csv->getline($fh);
|
||||
|
||||
%header = map { ($row->[$_], $_); } (0..$#{$row});
|
||||
|
||||
# verify that the columns we need are present
|
||||
map { die "Table has no %pairs{$_} column\n" unless (exists $header{$_}); } keys %pairs;
|
||||
|
||||
my %remapping = (
|
||||
net => 'network',
|
||||
id => 'geoname_id',
|
||||
rid => 'registered_country_geoname_id',
|
||||
proxy => 'is_anonymous_proxy',
|
||||
sat => 'is_satellite_provider',
|
||||
);
|
||||
|
||||
# now create a function which returns the value of that column #
|
||||
map { eval "sub $_ () { \$header{\$remapping{$_}}; }" ; } keys %remapping;
|
||||
|
||||
while ($row = $csv->getline($fh)) {
|
||||
my ($cc, $cidr);
|
||||
|
||||
$cc = lookupCountry($row->[id], $row->[rid], $row->[proxy], $row->[sat]);
|
||||
$cidr = $row->[net];
|
||||
$country{$cc}->{pool_v4}->add($cidr);
|
||||
|
||||
if ($. % 4096 == 0) {
|
||||
print STDERR "\r\e[2K$. entries";
|
||||
}
|
||||
}
|
||||
|
||||
print STDERR "\r\e[2K$. entries total\n";
|
||||
|
||||
close($fh);
|
||||
|
||||
# clean up the namespace
|
||||
undef &net; undef &id; undef &rid; undef &proxy; undef &sat;
|
||||
|
||||
$file = "$source_dir/GeoLite2-Country-Blocks-IPv6.csv";
|
||||
open($fh, '<', $file) || die "Can't open IPv6 database\n";
|
||||
|
||||
# first line is headers
|
||||
$row = $csv->getline($fh);
|
||||
|
||||
%header = map { ($row->[$_], $_); } (0..$#{$row});
|
||||
|
||||
# verify that the columns we need are present
|
||||
map { die "Table has no %pairs{$_} column\n" unless (exists $header{$_}); } keys %pairs;
|
||||
|
||||
# unlikely the IPv6 table has different columns, but just to be sure
|
||||
# create a function which returns the value of that column #
|
||||
map { eval "sub $_ () { \$header{\$remapping{$_}}; }" ; } keys %remapping;
|
||||
|
||||
while ($row = $csv->getline($fh)) {
|
||||
my ($cc, $cidr);
|
||||
|
||||
$cc = lookupCountry($row->[id], $row->[rid], $row->[proxy], $row->[sat]);
|
||||
$cidr = $row->[net];
|
||||
$country{$cc}->{pool_v6}->add($cidr);
|
||||
|
||||
if (!$quiet && $. % 4096 == 0) {
|
||||
print STDERR "\r\e[2K$. entries";
|
||||
}
|
||||
}
|
||||
|
||||
print STDERR "\r\e[2K$. entries total\n" unless ($quiet);
|
||||
|
||||
close($fh);
|
||||
|
||||
# clean up the namespace
|
||||
undef &net; undef &id; undef &rid; undef &proxy; undef &sat;
|
||||
|
||||
return \%country;
|
||||
}
|
||||
|
||||
sub dump
|
||||
{
|
||||
my $country = shift @_;
|
||||
|
||||
foreach my $iso_code (sort keys %{$country}) {
|
||||
&dump_one($iso_code, $country->{$iso_code});
|
||||
}
|
||||
}
|
||||
|
||||
sub dump_one
|
||||
{
|
||||
my($iso_code, $country) = @_;
|
||||
my @ranges;
|
||||
|
||||
@ranges = $country->{pool_v4}->list_range();
|
||||
|
||||
writeCountry($iso_code, $country->{name}, AF_INET, @ranges);
|
||||
|
||||
@ranges = $country->{pool_v6}->list_range();
|
||||
|
||||
writeCountry($iso_code, $country->{name}, AF_INET6, @ranges);
|
||||
}
|
||||
|
||||
sub writeCountry
|
||||
{
|
||||
my ($iso_code, $name, $family, @ranges) = @_;
|
||||
my $fh;
|
||||
|
||||
printf "%5u IPv%s ranges for %s %s\n",
|
||||
scalar(@ranges),
|
||||
($family == AF_INET ? '4' : '6'),
|
||||
$iso_code, $name unless ($quiet);
|
||||
|
||||
my $file = "$target_dir/".uc($iso_code).".iv".($family == AF_INET ? '4' : '6');
|
||||
if (!open($fh, '>', $file)) {
|
||||
print STDERR "Error opening $file: $!\n";
|
||||
exit 1;
|
||||
}
|
||||
|
||||
binmode($fh);
|
||||
|
||||
foreach my $range (@ranges) {
|
||||
my ($start, $end) = split('-', $range);
|
||||
$start = inet_pton($family, $start);
|
||||
$end = inet_pton($family, $end);
|
||||
print $fh $start, $end;
|
||||
}
|
||||
close $fh;
|
||||
}
|
||||
@@ -7,8 +7,9 @@ xt_geoip_dl \(em download GeoIP database files
|
||||
\fI/usr/libexec/xt_geoip/\fP\fBxt_geoip_dl\fP
|
||||
.SH Description
|
||||
.PP
|
||||
Downloads and unpacks the MaxMind GeoIP Country Lite databases for IPv4 and
|
||||
IPv6 and unpacks them to the current directory.
|
||||
Downloads the DB-IP Country Lite databases for IPv4 and IPv6 and unpacks them
|
||||
to the current directory. The alternate \fBxt_geoip_dl_maxmind\fP script can be
|
||||
used for MaxMind formatted CSV databases.
|
||||
.PP
|
||||
Since the script is usually installed to the libexec directory of the
|
||||
xtables-addons package and this is outside $PATH (on purpose), invoking the
|
||||
|
||||
7
geoip/xt_geoip_dl_maxmind
Executable file
7
geoip/xt_geoip_dl_maxmind
Executable file
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
|
||||
rm -rf GeoLite2-Country-CSV_*
|
||||
|
||||
wget -q http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country-CSV.zip
|
||||
unzip -q GeoLite2-Country-CSV.zip
|
||||
rm -f GeoLite2-Country-CSV.zip
|
||||
95
geoip/xt_geoip_fetch_maxmind
Executable file
95
geoip/xt_geoip_fetch_maxmind
Executable file
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/perl
|
||||
#
|
||||
# Utility to query GeoIP database
|
||||
# Copyright Philip Prindeville, 2018
|
||||
#
|
||||
use Getopt::Long;
|
||||
use Socket qw(AF_INET AF_INET6 inet_ntop);
|
||||
use warnings;
|
||||
use strict;
|
||||
|
||||
sub AF_INET_SIZE() { 4 }
|
||||
sub AF_INET6_SIZE() { 16 }
|
||||
|
||||
my $target_dir = ".";
|
||||
my $ipv4 = 0;
|
||||
my $ipv6 = 0;
|
||||
|
||||
&Getopt::Long::Configure(qw(bundling));
|
||||
&GetOptions(
|
||||
"D=s" => \$target_dir,
|
||||
"4" => \$ipv4,
|
||||
"6" => \$ipv6,
|
||||
);
|
||||
|
||||
if (!-d $target_dir) {
|
||||
print STDERR "Target directory $target_dir does not exit.\n";
|
||||
exit 1;
|
||||
}
|
||||
|
||||
# if neither specified, assume both
|
||||
if (! $ipv4 && ! $ipv6) {
|
||||
$ipv4 = $ipv6 = 1;
|
||||
}
|
||||
|
||||
foreach my $cc (@ARGV) {
|
||||
if ($cc !~ m/^([a-z]{2}|a[12]|o1)$/i) {
|
||||
print STDERR "Invalid country code '$cc'\n";
|
||||
exit 1;
|
||||
}
|
||||
|
||||
my $file = $target_dir . '/' . uc($cc) . '.iv4';
|
||||
|
||||
if (! -f $file) {
|
||||
printf STDERR "Can't find data for country '$cc'\n";
|
||||
exit 1;
|
||||
}
|
||||
|
||||
my ($contents, $buffer, $bytes, $fh);
|
||||
|
||||
if ($ipv4) {
|
||||
open($fh, '<', $file) || die "Couldn't open file for '$cc'\n";
|
||||
|
||||
binmode($fh);
|
||||
|
||||
while (($bytes = read($fh, $buffer, AF_INET_SIZE * 2)) == AF_INET_SIZE * 2) {
|
||||
my ($start, $end) = unpack('a4a4', $buffer);
|
||||
$start = inet_ntop(AF_INET, $start);
|
||||
$end = inet_ntop(AF_INET, $end);
|
||||
print $start, '-', $end, "\n";
|
||||
}
|
||||
close($fh);
|
||||
if (! defined $bytes) {
|
||||
printf STDERR "Error reading file for '$cc'\n";
|
||||
exit 1;
|
||||
} elsif ($bytes != 0) {
|
||||
printf STDERR "Short read on file for '$cc'\n";
|
||||
exit 1;
|
||||
}
|
||||
}
|
||||
|
||||
substr($file, -1) = '6';
|
||||
|
||||
if ($ipv6) {
|
||||
open($fh, '<', $file) || die "Couldn't open file for '$cc'\n";
|
||||
|
||||
binmode($fh);
|
||||
|
||||
while (($bytes = read($fh, $buffer, AF_INET6_SIZE * 2)) == AF_INET6_SIZE * 2) {
|
||||
my ($start, $end) = unpack('a16a16', $buffer);
|
||||
$start = inet_ntop(AF_INET6, $start);
|
||||
$end = inet_ntop(AF_INET6, $end);
|
||||
print $start, '-', $end, "\n";
|
||||
}
|
||||
close($fh);
|
||||
if (! defined $bytes) {
|
||||
printf STDERR "Error reading file for '$cc'\n";
|
||||
exit 1;
|
||||
} elsif ($bytes != 0) {
|
||||
printf STDERR "Short read on file for '$cc'\n";
|
||||
exit 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
exit 0;
|
||||
Reference in New Issue
Block a user