Compare commits

..

11 Commits
v3.0 ... v3.1

Author SHA1 Message Date
Jan Engelhardt
30fb410003 Xtables-addons 3.1 2018-08-14 14:31:10 +02:00
Jan Engelhardt
3ea761a1ed build: add support for Linux 4.18 2018-08-14 14:29:30 +02:00
Jan Engelhardt
4603d3e0f4 build: add support for Linux 4.17 2018-08-14 14:23:04 +02:00
Jan Engelhardt
b0b2b5a74c build: fix 4.16 warning 2018-08-14 14:22:40 +02:00
Jan Engelhardt
082d42fb21 build: match documented and coded build requirements 2018-08-14 14:22:17 +02:00
Jan Engelhardt
b1f0e118a0 doc: add 3.0 headline in changelog 2018-08-14 14:15:07 +02:00
Philip Prindeville
56fba3ecff geoip: simplify handling table column names
Signed-off-by: Philip Prindeville <philipp@redfish-solutions.com>
2018-04-30 09:41:29 +02:00
Philip Prindeville
9057fb48f3 geoip: add database query tool for use with ipsets
Add a tool for retrieiving the IPv4 or IPv6 (or both!) CIDR ranges
for a given country, which can then be injected into an ipset if
one doesn't want to use (or have available) the xt_geoip extension.

Signed-off-by: Philip Prindeville <philipp@redfish-solutions.com>
2018-04-30 09:41:21 +02:00
Philip Prindeville
e19f91ddb4 geoip: update man page for xt_geoip_build
Signed-off-by: Philip Prindeville <philipp@redfish-solutions.com>
2018-04-30 09:40:54 +02:00
Philip Prindeville
256ac1a4f6 geoip: adapt to GeoLite2 database
Requires Net::CIDR::Lite for manipulating CIDR blocks, aggregation, etc.
since database is stored as subnet/mask pairs and may require compaction
into ranges (which can combine adjacent subnets).

We don't use Net::CIDR because it's a clunkier interface.

Signed-off-by: Philip Prindeville <philipp@redfish-solutions.com>
2018-04-30 09:40:51 +02:00
Philip Prindeville
b91dbd03c7 geoip: store database in network byte order
This allows a single database to be built and distributed as a
package that is accepted by both big- and little-endian hosts.

Signed-off-by: Philip Prindeville <philipp@redfish-solutions.com>
2018-02-19 01:42:29 +01:00
10 changed files with 406 additions and 140 deletions

1
.gitignore vendored
View File

@@ -3,6 +3,7 @@
*.lo
*.loT
*.o
.cache.mk
.deps/
.dirstamp
.libs/

View File

@@ -1,4 +1,4 @@
AC_INIT([xtables-addons], [3.0])
AC_INIT([xtables-addons], [3.1])
AC_CONFIG_AUX_DIR([build-aux])
AC_CONFIG_HEADERS([config.h])
AC_CONFIG_MACRO_DIR([m4])
@@ -26,7 +26,7 @@ fi
AC_CHECK_HEADERS([linux/netfilter/x_tables.h], [],
[AC_MSG_ERROR([You need to have linux/netfilter/x_tables.h, see INSTALL file for details])])
PKG_CHECK_MODULES([libxtables], [xtables >= 1.4.5])
PKG_CHECK_MODULES([libxtables], [xtables >= 1.6.0])
xtlibdir="$(pkg-config --variable=xtlibdir xtables)"
AC_ARG_WITH([xtlibdir],
@@ -57,9 +57,9 @@ if test -n "$kbuilddir"; then
echo "WARNING: Version detection did not succeed. Continue at own luck.";
else
echo "$kmajor.$kminor.$kmicro.$kstable in $kbuilddir";
if test "$kmajor" -gt 4 -o "$kmajor" -eq 4 -a "$kminor" -gt 16; then
if test "$kmajor" -gt 4 -o "$kmajor" -eq 4 -a "$kminor" -gt 18; then
echo "WARNING: That kernel version is not officially supported yet. Continue at own luck.";
elif test "$kmajor" -eq 4 -a "$kminor" -ge 15; then
elif test "$kmajor" -eq 4 -a "$kminor" -ge 18; then
:
else
echo "WARNING: That kernel version is not officially supported.";

View File

@@ -1,6 +1,16 @@
HEAD
====
v3.1 (2018-08-14)
=================
Enhancements:
- support for Linux 4.17, 4.18
v3.0 (2018-02-12)
=================
Enhancements:
- support for Linux 4.15, 4.16
Changes:

View File

@@ -49,6 +49,38 @@ static struct option geoip_opts[] = {
{NULL},
};
#if __BYTE_ORDER == __LITTLE_ENDIAN
static void geoip_swap_le16(uint16_t *buf)
{
unsigned char *p = (void *)buf;
uint16_t n= p[0] + (p[1] << 8);
p[0] = (n >> 8) & 0xff;
p[1] = n & 0xff;
}
static void geoip_swap_in6(struct in6_addr *in6)
{
geoip_swap_le16(&in6->s6_addr16[0]);
geoip_swap_le16(&in6->s6_addr16[1]);
geoip_swap_le16(&in6->s6_addr16[2]);
geoip_swap_le16(&in6->s6_addr16[3]);
geoip_swap_le16(&in6->s6_addr16[4]);
geoip_swap_le16(&in6->s6_addr16[5]);
geoip_swap_le16(&in6->s6_addr16[6]);
geoip_swap_le16(&in6->s6_addr16[7]);
}
static void geoip_swap_le32(uint32_t *buf)
{
unsigned char *p = (void *)buf;
uint32_t n = p[0] + (p[1] << 8) + (p[2] << 16) + (p[3] << 24);
p[0] = (n >> 24) & 0xff;
p[1] = (n >> 16) & 0xff;
p[2] = (n >> 8) & 0xff;
p[3] = n & 0xff;
}
#endif
static void *
geoip_get_subnets(const char *code, uint32_t *count, uint8_t nfproto)
{
@@ -56,21 +88,15 @@ geoip_get_subnets(const char *code, uint32_t *count, uint8_t nfproto)
struct stat sb;
char buf[256];
int fd;
#if __BYTE_ORDER == __LITTLE_ENDIAN
unsigned int n;
#endif
/* Use simple integer vector files */
if (nfproto == NFPROTO_IPV6) {
#if __BYTE_ORDER == _BIG_ENDIAN
snprintf(buf, sizeof(buf), GEOIP_DB_DIR "/BE/%s.iv6", code);
#else
snprintf(buf, sizeof(buf), GEOIP_DB_DIR "/LE/%s.iv6", code);
#endif
} else {
#if __BYTE_ORDER == _BIG_ENDIAN
snprintf(buf, sizeof(buf), GEOIP_DB_DIR "/BE/%s.iv4", code);
#else
snprintf(buf, sizeof(buf), GEOIP_DB_DIR "/LE/%s.iv4", code);
#endif
}
if (nfproto == NFPROTO_IPV6)
snprintf(buf, sizeof(buf), GEOIP_DB_DIR "/%s.iv6", code);
else
snprintf(buf, sizeof(buf), GEOIP_DB_DIR "/%s.iv4", code);
if ((fd = open(buf, O_RDONLY)) < 0) {
fprintf(stderr, "Could not open %s: %s\n", buf, strerror(errno));
@@ -98,6 +124,25 @@ geoip_get_subnets(const char *code, uint32_t *count, uint8_t nfproto)
xtables_error(OTHER_PROBLEM, "geoip: insufficient memory");
read(fd, subnets, sb.st_size);
close(fd);
#if __BYTE_ORDER == __LITTLE_ENDIAN
for (n = 0; n < *count; ++n) {
switch (nfproto) {
case NFPROTO_IPV6: {
struct geoip_subnet6 *gs6 = &(((struct geoip_subnet6 *)subnets)[n]);
geoip_swap_in6(&gs6->begin);
geoip_swap_in6(&gs6->end);
break;
}
case NFPROTO_IPV4: {
struct geoip_subnet4 *gs4 = &(((struct geoip_subnet4 *)subnets)[n]);
geoip_swap_le32(&gs4->begin);
geoip_swap_le32(&gs4->end);
break;
}
}
}
#endif
return subnets;
}

View File

@@ -363,7 +363,11 @@ dnetmap_tg(struct sk_buff *skb, const struct xt_action_param *par)
__be32 prenat_ip, postnat_ip, prenat_ip_prev;
const struct xt_DNETMAP_tginfo *tginfo = par->targinfo;
const struct nf_nat_range *mr = &tginfo->prefix;
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 18, 0)
struct nf_nat_range2 newrange;
#else
struct nf_nat_range newrange;
#endif
struct dnetmap_entry *e;
struct dnetmap_prefix *p;
__s32 jttl;

View File

@@ -1,86 +1,234 @@
#!/usr/bin/perl
#
# Converter for MaxMind CSV database to binary, for xt_geoip
# Copyright © Jan Engelhardt, 2008-2011
# Copyright Jan Engelhardt, 2008-2011
# Copyright Philip Prindeville, 2018
#
use Getopt::Long;
use IO::Handle;
use Net::CIDR::Lite;
use Socket qw(AF_INET AF_INET6 inet_pton);
use warnings;
use Text::CSV_XS; # or trade for Text::CSV
use strict;
my $le32 = pack("V", 0x10000000);
my $be32 = pack("N", 0x10000000);
my $u32 = undef;
sub wantBE { return !$u32 || $u32 eq $be32; }
sub wantLE { return !$u32 || $u32 eq $le32; }
my $csv = Text::CSV_XS->new({
allow_whitespace => 1,
binary => 1,
eol => $/,
}); # or Text::CSV
my $target_dir = ".";
my $native_only = 0;
&Getopt::Long::Configure(qw(bundling));
&GetOptions(
"D=s" => \$target_dir,
"n" => \$native_only,
);
if (!-d $target_dir) {
print STDERR "Target directory $target_dir does not exist.\n";
exit 1;
}
my @dbs = qw(LE BE);
if ($native_only) {
$u32 = pack("L", 0x10000000);
if ($u32 eq $le32) {
@dbs = qw(LE);
} elsif ($u32 eq $be32) {
@dbs = qw(BE);
} else {
print STDERRR "Cannot determine endianness.\n";
exit 1;
}
}
foreach (@dbs) {
my $dir = "$target_dir/$_";
if (!-e $dir && !mkdir($dir)) {
print STDERR "Could not mkdir $dir: $!\n";
exit 1;
}
}
my %countryId;
my %countryName;
my $dir = findVersion();
&loadCountries();
&dump(&collect());
sub findVersion
{
my @dirs = ();
opendir(my $dh, '.') || die "Can't open .: $!\n";
while (readdir $dh) {
if ($_ =~ m/^GeoLite2-Country-CSV_\d{8}$/) {
push(@dirs, $_);
}
}
closedir $dh;
@dirs = sort @dirs;
return pop(@dirs);
}
sub loadCountries
{
my $file = "$dir/GeoLite2-Country-Locations-en.csv";
sub id; sub cc; sub long; sub ct; sub cn;
%countryId = ();
%countryName = ();
open(my $fh, '<', $file) || die "Couldn't open list country names\n";
# first line is headers
my $row = $csv->getline($fh);
my %header = map { ($row->[$_], $_); } (0..$#{$row});
my %pairs = (
country_iso_code => 'ISO Country Code',
geoname_id => 'ID',
country_name => 'Country Name',
continent_code => 'Continent Code',
continent_name => 'Continent Name',
);
# verify that the columns we need are present
map { die "Table has no $pairs{$_} column\n" unless (exists $header{$_}); } keys %pairs;
my %remapping = (
id => 'geoname_id',
cc => 'country_iso_code',
long => 'country_name',
ct => 'continent_code',
cn => 'continent_name',
);
# now create a function which returns the value of that column #
map { eval "sub $_ () { \$header{\$remapping{$_}}; }" ; } keys %remapping;
while (my $row = $csv->getline($fh)) {
if ($row->[cc] eq '' && $row->[long] eq '') {
$countryId{$row->[id]} = $row->[ct];
$countryName{$row->[ct]} = $row->[cn];
} else {
$countryId{$row->[id]} = $row->[cc];
$countryName{$row->[cc]} = $row->[long];
}
}
$countryName{A1} = 'Anonymous Proxy';
$countryName{A2} = 'Satellite Provider';
$countryName{O1} = 'Other Country';
close($fh);
# clean up the namespace
undef &id; undef &cc; undef &long; undef &ct; undef &cn;
}
sub lookupCountry
{
my ($id, $rid, $proxy, $sat) = @_;
if ($proxy) {
return 'A1';
} elsif ($sat) {
return 'A2';
}
$id ||= $rid;
if ($id eq '') {
return 'O1';
}
die "Unknown id: $id line $.\n" unless (exists $countryId{$id});
return $countryId{$id};
}
sub collect
{
my %country;
my ($file, $fh, $row);
my (%country, %header);
sub net; sub id; sub rid; sub proxy; sub sat;
my %pairs = (
network => 'Network',
registered_country_geoname_id => 'Registered Country ID',
geoname_id => 'Country ID',
is_anonymous_proxy => 'Anonymous Proxy',
is_satellite_provider => 'Satellite',
);
foreach (sort keys %countryName) {
$country{$_} = {
name => $countryName{$_},
pool_v4 => Net::CIDR::Lite->new(),
pool_v6 => Net::CIDR::Lite->new(),
};
}
$file = "$dir/GeoLite2-Country-Blocks-IPv4.csv";
open($fh, '<', $file) || die "Can't open IPv4 database\n";
# first line is headers
$row = $csv->getline($fh);
%header = map { ($row->[$_], $_); } (0..$#{$row});
# verify that the columns we need are present
map { die "Table has no %pairs{$_} column\n" unless (exists $header{$_}); } keys %pairs;
my %remapping = (
net => 'network',
id => 'geoname_id',
rid => 'registered_country_geoname_id',
proxy => 'is_anonymous_proxy',
sat => 'is_satellite_provider',
);
# now create a function which returns the value of that column #
map { eval "sub $_ () { \$header{\$remapping{$_}}; }" ; } keys %remapping;
while ($row = $csv->getline($fh)) {
my ($cc, $cidr);
$cc = lookupCountry($row->[id], $row->[rid], $row->[proxy], $row->[sat]);
$cidr = $row->[net];
$country{$cc}->{pool_v4}->add($cidr);
while (my $row = $csv->getline(*ARGV)) {
if (!defined($country{$row->[4]})) {
$country{$row->[4]} = {
name => $row->[5],
pool_v4 => [],
pool_v6 => [],
};
}
my $c = $country{$row->[4]};
if ($row->[0] =~ /:/) {
push(@{$c->{pool_v6}},
[&ip6_pack($row->[0]), &ip6_pack($row->[1])]);
} else {
push(@{$c->{pool_v4}}, [$row->[2], $row->[3]]);
}
if ($. % 4096 == 0) {
print STDERR "\r\e[2K$. entries";
}
}
print STDERR "\r\e[2K$. entries total\n";
close($fh);
# clean up the namespace
undef &net; undef &id; undef &rid; undef &proxy; undef &sat;
$file = "$dir/GeoLite2-Country-Blocks-IPv6.csv";
open($fh, '<', $file) || die "Can't open IPv6 database\n";
# first line is headers
$row = $csv->getline($fh);
%header = map { ($row->[$_], $_); } (0..$#{$row});
# verify that the columns we need are present
map { die "Table has no %pairs{$_} column\n" unless (exists $header{$_}); } keys %pairs;
# unlikely the IPv6 table has different columns, but just to be sure
# create a function which returns the value of that column #
map { eval "sub $_ () { \$header{\$remapping{$_}}; }" ; } keys %remapping;
while ($row = $csv->getline($fh)) {
my ($cc, $cidr);
$cc = lookupCountry($row->[id], $row->[rid], $row->[proxy], $row->[sat]);
$cidr = $row->[net];
$country{$cc}->{pool_v6}->add($cidr);
if ($. % 4096 == 0) {
print STDERR "\r\e[2K$. entries";
}
}
print STDERR "\r\e[2K$. entries total\n";
close($fh);
# clean up the namespace
undef &net; undef &id; undef &rid; undef &proxy; undef &sat;
return \%country;
}
@@ -88,7 +236,7 @@ sub dump
{
my $country = shift @_;
foreach my $iso_code (sort keys %$country) {
foreach my $iso_code (sort keys %{$country}) {
&dump_one($iso_code, $country->{$iso_code});
}
}
@@ -96,80 +244,41 @@ sub dump
sub dump_one
{
my($iso_code, $country) = @_;
my($file, $fh_le, $fh_be);
my @ranges;
printf "%5u IPv6 ranges for %s %s\n",
scalar(@{$country->{pool_v6}}),
$iso_code, $country->{name};
@ranges = $country->{pool_v4}->list_range();
if (wantLE) {
$file = "$target_dir/LE/".uc($iso_code).".iv6";
if (!open($fh_le, "> $file")) {
print STDERR "Error opening $file: $!\n";
exit 1;
}
foreach my $range (@{$country->{pool_v6}}) {
print $fh_le &ip6_swap($range->[0]), &ip6_swap($range->[1]);
}
close $fh_le;
}
if (wantBE) {
$file = "$target_dir/BE/".uc($iso_code).".iv6";
if (!open($fh_be, "> $file")) {
print STDERR "Error opening $file: $!\n";
exit 1;
}
foreach my $range (@{$country->{pool_v6}}) {
print $fh_be $range->[0], $range->[1];
}
close $fh_be;
}
writeCountry($iso_code, $country->{name}, AF_INET, @ranges);
printf "%5u IPv4 ranges for %s %s\n",
scalar(@{$country->{pool_v4}}),
$iso_code, $country->{name};
@ranges = $country->{pool_v6}->list_range();
if (wantLE) {
$file = "$target_dir/LE/".uc($iso_code).".iv4";
if (!open($fh_le, "> $file")) {
print STDERR "Error opening $file: $!\n";
exit 1;
}
foreach my $range (@{$country->{pool_v4}}) {
print $fh_le pack("VV", $range->[0], $range->[1]);
}
close $fh_le;
}
if (wantBE) {
$file = "$target_dir/BE/".uc($iso_code).".iv4";
if (!open($fh_be, "> $file")) {
print STDERR "Error opening $file: $!\n";
exit 1;
}
foreach my $range (@{$country->{pool_v4}}) {
print $fh_be pack("NN", $range->[0], $range->[1]);
}
close $fh_be;
}
writeCountry($iso_code, $country->{name}, AF_INET6, @ranges);
}
sub ip6_pack
sub writeCountry
{
my $addr = shift @_;
$addr =~ s{::}{:!:};
my @addr = split(/:/, $addr);
my @e = (0) x 8;
foreach (@addr) {
if ($_ eq "!") {
$_ = join(':', @e[0..(8-scalar(@addr))]);
}
my ($iso_code, $name, $family, @ranges) = @_;
my $fh;
printf "%5u IPv%s ranges for %s %s\n",
scalar(@ranges),
($family == AF_INET ? '4' : '6'),
$iso_code, $name;
my $file = "$target_dir/".uc($iso_code).".iv".($family == AF_INET ? '4' : '6');
if (!open($fh, '>', $file)) {
print STDERR "Error opening $file: $!\n";
exit 1;
}
@addr = split(/:/, join(':', @addr));
$_ = hex($_) foreach @addr;
return pack("n*", @addr);
binmode($fh);
foreach my $range (@ranges) {
my ($start, $end) = split('-', $range);
$start = inet_pton($family, $start);
$end = inet_pton($family, $end);
print $fh $start, $end;
}
close $fh;
}
sub ip6_swap
{
return pack("V*", unpack("N*", shift @_));
}

View File

@@ -5,7 +5,7 @@ xt_geoip_build \(em convert GeoIP.csv to packed format for xt_geoip
.SH Syntax
.PP
\fI/usr/libexec/xt_geoip/\fP\fBxt_geoip_build\fP [\fB\-D\fP
\fItarget_dir\fP] [\fIfile\fP...]
\fItarget_dir\fP]
.SH Description
.PP
xt_geoip_build is used to build packed raw representations of the range
@@ -16,7 +16,12 @@ required to be loaded into memory. The ranges in the packed database files are
also ordered, as xt_geoip relies on this property for its bisection approach to
work.
.PP
Input is processed from the listed files, or if none is given, from stdin.
It expects to find a directory named
.IR GeoLite2-Country-CSV_YYYYMMDD
in the current directory, and will select the most recent if multiple
instances are found. The
.IR xt_geoip_dl
script can be used to populate this directory.
.PP
Since the script is usually installed to the libexec directory of the
xtables-addons package and this is outside $PATH (on purpose), invoking the

View File

@@ -1,8 +1,7 @@
#!/bin/sh
rm -f GeoIPv6.csv GeoIPv6.csv.gz GeoIPCountryCSV.zip GeoIPCountryWhois.csv;
wget \
http://geolite.maxmind.com/download/geoip/database/GeoIPv6.csv.gz \
http://geolite.maxmind.com/download/geoip/database/GeoIPCountryCSV.zip;
gzip -d GeoIPv6.csv.gz;
unzip GeoIPCountryCSV.zip;
rm -rf GeoLite2-Country-CSV_*
wget -q http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country-CSV.zip
unzip -q GeoLite2-Country-CSV.zip
rm -f GeoLite2-Country-CSV.zip

93
geoip/xt_geoip_fetch Executable file
View File

@@ -0,0 +1,93 @@
#!/usr/bin/perl
#
# Utility to query GeoIP database
# Copyright Philip Prindeville, 2018
#
use Getopt::Long;
use Socket qw(AF_INET AF_INET6 inet_ntop);
use warnings;
use strict;
sub AF_INET_SIZE() { 4 }
sub AF_INET6_SIZE() { 16 }
my $target_dir = ".";
my $ipv4 = 0;
my $ipv6 = 0;
&Getopt::Long::Configure(qw(bundling));
&GetOptions(
"D=s" => \$target_dir,
"4" => \$ipv4,
"6" => \$ipv6,
);
if (!-d $target_dir) {
print STDERR "Target directory $target_dir does not exit.\n";
exit 1;
}
# if neither specified, assume both
if (! $ipv4 && ! $ipv6) {
$ipv4 = $ipv6 = 1;
}
foreach my $cc (@ARGV) {
if ($cc !~ m/^([a-z]{2}|a[12]|o1)$/i) {
print STDERR "Invalid country code '$cc'\n";
exit 1;
}
my $file = $target_dir . '/' . uc($cc) . '.iv4';
if (! -f $file) {
printf STDERR "Can't find data for country '$cc'\n";
exit 1;
}
my ($contents, $buffer, $bytes, $fh);
if ($ipv4) {
open($fh, '<', $file) || die "Couldn't open file for '$cc'\n";
binmode($fh);
while (($bytes = read($fh, $buffer, AF_INET_SIZE * 2)) == AF_INET_SIZE * 2) {
my $start = inet_ntop(AF_INET, substr($buffer, 0, AF_INET_SIZE));
my $end = inet_ntop(AF_INET, substr($buffer, AF_INET_SIZE));
print $start, '-', $end, "\n";
}
close($fh);
if (! defined $bytes) {
printf STDERR "Error reading file for '$cc'\n";
exit 1;
} elsif ($bytes != 0) {
printf STDERR "Short read on file for '$cc'\n";
exit 1;
}
}
substr($file, -1) = '6';
if ($ipv6) {
open($fh, '<', $file) || die "Couldn't open file for '$cc'\n";
binmode($fh);
while (($bytes = read($fh, $buffer, AF_INET6_SIZE * 2)) == AF_INET6_SIZE * 2) {
my $start = inet_ntop(AF_INET6, substr($buffer, 0, AF_INET6_SIZE));
my $end = inet_ntop(AF_INET6, substr($buffer, AF_INET6_SIZE));
print $start, '-', $end, "\n";
}
close($fh);
if (! defined $bytes) {
printf STDERR "Error reading file for '$cc'\n";
exit 1;
} elsif ($bytes != 0) {
printf STDERR "Short read on file for '$cc'\n";
exit 1;
}
}
}
exit 0;

View File

@@ -1,4 +1,4 @@
.TH xtables-addons 8 "Lilac" "" "v3.0 (2018-02-12)"
.TH xtables-addons 8 "Windows" "" "v3.1 (2018-08-14)"
.SH Name
Xtables-addons \(em additional extensions for iptables, ip6tables, etc.
.SH Targets