geoip: add quiet flag to xt_geoip_build

Conceivably someone might want to run a refresh of the geoip database
from within a script, particularly an unattended script such as a cron
job. Do not generate output in that case.

Signed-off-by: Philip Prindeville <philipp@redfish-solutions.com>

configure.ac

@@ -1,4 +1,4 @@
-AC_INIT([xtables-addons], [3.7])
+AC_INIT([xtables-addons], [3.9])
 AC_CONFIG_AUX_DIR([build-aux])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_MACRO_DIR([m4])
@@ -57,11 +57,11 @@ if test -n "$kbuilddir"; then
 		echo "WARNING: Version detection did not succeed. Continue at own luck.";
 	else
 		echo "$kmajor.$kminor.$kmicro.$kstable in $kbuilddir";
-		if test "$kmajor" -gt 5 -o "$kmajor" -eq 5 -a "$kminor" -gt 4; then
+		if test "$kmajor" -gt 5 -o "$kmajor" -eq 5 -a "$kminor" -gt 6; then
 			echo "WARNING: That kernel version is not officially supported yet. Continue at own luck.";
 		elif test "$kmajor" -eq 5 -a "$kminor" -ge 0; then
 			:
-		elif test "$kmajor" -eq 4 -a "$kminor" -ge 18; then
+		elif test "$kmajor" -eq 4 -a "$kminor" -ge 15; then
 			:
 		else
 			echo "WARNING: That kernel version is not officially supported.";

doc/changelog.txt

@@ -1,3 +1,14 @@
+v3.9 (2020-02-25)
+=================
+- Support for Linux 5.6 procfs changes
+
+
+v3.8 (2020-02-03)
+=================
+- Support for Linux 5.5
+- xt_geoip_build now expects the DBIP format as input,
+  Maxmind is thrown out.
+
 
 v3.7 (2019-12-01)
 =================

extensions/compat_xtables.h

@@ -51,4 +51,13 @@ static inline struct net *par_net(const struct xt_action_param *par)
 #	define NF_CT_ASSERT(x)	WARN_ON(!(x))
 #endif
 
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0)
+#	define proc_ops file_operations
+#	define proc_open open
+#	define proc_read read
+#	define proc_write write
+#	define proc_lseek llseek
+#	define proc_release release
+#endif
+
 #endif /* _XTABLES_COMPAT_H */

extensions/pknock/xt_pknock.c

@@ -295,12 +295,11 @@ pknock_proc_open(struct inode *inode, struct file *file)
 	return ret;
 }
 
-static const struct file_operations pknock_proc_ops = {
+static const struct proc_ops pknock_proc_ops = {
-	.owner = THIS_MODULE,
+	.proc_open    = pknock_proc_open,
-	.open = pknock_proc_open,
+	.proc_read    = seq_read,
-	.read = seq_read,
+	.proc_lseek   = seq_lseek,
-	.llseek = seq_lseek,
+	.proc_release = seq_release,
-	.release = seq_release
 };
 
 /**

extensions/xt_DNETMAP.c

@@ -115,7 +115,7 @@ static DEFINE_SPINLOCK(dnetmap_lock);
 static DEFINE_MUTEX(dnetmap_mutex);
 
 #ifdef CONFIG_PROC_FS
-static const struct file_operations dnetmap_tg_fops, dnetmap_stat_proc_fops;
+static const struct proc_ops dnetmap_tg_fops, dnetmap_stat_proc_fops;
 #endif
 
 static inline unsigned int dnetmap_entry_hash(const __be32 addr)
@@ -768,12 +768,11 @@ dnetmap_tg_proc_write(struct file *file, const char __user *input,size_t size, l
 }
 
 
-static const struct file_operations dnetmap_tg_fops = {
+static const struct proc_ops dnetmap_tg_fops = {
-	.open = dnetmap_seq_open,
+	.proc_open    = dnetmap_seq_open,
-	.read = seq_read,
+	.proc_read    = seq_read,
-	.write   = dnetmap_tg_proc_write,
+	.proc_write   = dnetmap_tg_proc_write,
-	.release = seq_release_private,
+	.proc_release = seq_release_private,
-	.owner = THIS_MODULE,
 };
 
 /* for statistics */
@@ -817,11 +816,11 @@ static int dnetmap_stat_proc_open(struct inode *inode, struct file *file)
 	return single_open(file, dnetmap_stat_proc_show, PDE_DATA(inode));
 }
 
-static const struct file_operations dnetmap_stat_proc_fops = {
+static const struct proc_ops dnetmap_stat_proc_fops = {
-	.open    = dnetmap_stat_proc_open,
+	.proc_open    = dnetmap_stat_proc_open,
-	.read    = seq_read,
+	.proc_read    = seq_read,
-	.llseek  = seq_lseek,
+	.proc_lseek   = seq_lseek,
-	.release = single_release,
+	.proc_release = single_release,
 };
 
 static int __net_init dnetmap_proc_net_init(struct net *net)

extensions/xt_condition.c

@@ -111,12 +111,12 @@ condition_proc_write(struct file *file, const char __user *buffer,
 	return length;
 }
 
-static const struct file_operations condition_proc_fops = {
+static const struct proc_ops condition_proc_fops = {
-	.open    = condition_proc_open,
+	.proc_open    = condition_proc_open,
-	.read    = seq_read,
+	.proc_read    = seq_read,
-	.llseek  = seq_lseek,
+	.proc_write   = condition_proc_write,
-	.write   = condition_proc_write,
+	.proc_lseek   = seq_lseek,
-	.release = single_release,
+	.proc_release = single_release,
 };
 
 static bool

extensions/xt_quota2.c

@@ -117,12 +117,12 @@ quota_proc_write(struct file *file, const char __user *input,
 	return size;
 }
 
-static const struct file_operations quota_proc_fops = {
+static const struct proc_ops quota_proc_fops = {
-	.open    = quota_proc_open,
+	.proc_open    = quota_proc_open,
-	.read    = seq_read,
+	.proc_read    = seq_read,
-	.llseek  = seq_lseek,
+	.proc_write   = quota_proc_write,
-	.write   = quota_proc_write,
+	.proc_lseek   = seq_lseek,
-	.release = single_release,
+	.proc_release = single_release,
 };
 
 static struct xt_quota_counter *

geoip/Makefile.am

@@ -1,5 +1,7 @@
 # -*- Makefile -*-
 
+bin_SCRIPTS = xt_geoip_fetch
+
 pkglibexec_SCRIPTS = xt_geoip_build xt_geoip_dl
 
-man1_MANS = xt_geoip_build.1 xt_geoip_dl.1
+man1_MANS = xt_geoip_build.1 xt_geoip_dl.1 xt_geoip_fetch.1

geoip/xt_geoip_build

@@ -1,8 +1,9 @@
 #!/usr/bin/perl
 #
-#	Converter for MaxMind (GeoLite2) CSV database to binary, for xt_geoip
+#	Converter for DBIP (Country Lite) CSV database to binary, for xt_geoip
 #	Copyright Jan Engelhardt, 2008-2011
 #	Copyright Philip Prindeville, 2018
+#	Copyright Arjen de Korte, 2020
 #
 use Getopt::Long;
 use Net::CIDR::Lite;
@@ -16,201 +17,62 @@ my $csv = Text::CSV_XS->new({
 	binary => 1,
 	eol => $/,
 }); # or Text::CSV
-my $source_dir = ".";
+my $quiet = 0;
+my $input_file = "dbip-country-lite.csv";
 my $target_dir = ".";
 
 &Getopt::Long::Configure(qw(bundling));
 &GetOptions(
 	"D=s" => \$target_dir,
-	"S=s" => \$source_dir,
+	"i=s" => \$input_file,
+	"q" => \$quiet,
+	"s" => sub { $target_dir = "/usr/share/xt_geoip"; },
 );
 
-if (!-d $source_dir) {
-	print STDERR "Source directory \"$source_dir\" does not exist.\n";
-	exit 1;
-}
 if (!-d $target_dir) {
 	print STDERR "Target directory \"$target_dir\" does not exist.\n";
 	exit 1;
 }
 
-my %countryId;
-my %countryName;
-&loadCountries();
 &dump(&collect());
 
-sub loadCountries
-{
-	sub id; sub cc; sub long; sub ct; sub cn;
-
-	%countryId = ();
-	%countryName = ();
-
-	my $file = "$source_dir/GeoLite2-Country-Locations-en.csv";
-	open(my $fh, '<', $file) || die "Couldn't open list country names\n";
-
-	# first line is headers
-	my $row = $csv->getline($fh);
-
-	my %header = map { ($row->[$_], $_); } (0..$#{$row});
-
-	my %pairs = (
-		country_iso_code => 'ISO Country Code',
-		geoname_id => 'ID',
-		country_name => 'Country Name',
-		continent_code => 'Continent Code',
-		continent_name => 'Continent Name',
-	);
-
-	# verify that the columns we need are present
-	map { die "Table has no $pairs{$_} column\n" unless (exists $header{$_}); } keys %pairs;
-
-	my %remapping = (
-		id => 'geoname_id',
-		cc => 'country_iso_code',
-		long => 'country_name',
-		ct => 'continent_code',
-		cn => 'continent_name',
-	);
-
-	# now create a function which returns the value of that column #
-	map { eval "sub $_ () { \$header{\$remapping{$_}}; }" ; } keys %remapping;
-
-	while (my $row = $csv->getline($fh)) {
-		if ($row->[cc] eq '' && $row->[long] eq '') {
-			$countryId{$row->[id]} = $row->[ct];
-			$countryName{$row->[ct]} = $row->[cn];
-		} else {
-			$countryId{$row->[id]} = $row->[cc];
-			$countryName{$row->[cc]} = $row->[long];
-		}
-	}
-
-	$countryName{A1} = 'Anonymous Proxy';
-	$countryName{A2} = 'Satellite Provider';
-	$countryName{O1} = 'Other Country';
-
-	close($fh);
-
-	# clean up the namespace
-	undef &id; undef &cc; undef &long; undef &ct; undef &cn;
-}
-
-sub lookupCountry
-{
-	my ($id, $rid, $proxy, $sat) = @_;
-
-	if ($proxy) {
-		return 'A1';
-	} elsif ($sat) {
-		return 'A2';
-	}
-	$id ||= $rid;
-	if ($id eq '') {
-		return 'O1';
-	}
-	die "Unknown id: $id line $.\n" unless (exists $countryId{$id});
-	return $countryId{$id};
-}
-
 sub collect
 {
 	my ($file, $fh, $row);
-	my (%country, %header);
+	my (%country);
-
+	if ($input_file eq "-") {
-	sub net; sub id; sub rid; sub proxy; sub sat;
+		open($fh, "<&STDIN");
-
+	} else {
-	my %pairs = (
+		open($fh, "<", $input_file) || die "Cannot open $input_file: $!\n";
-		network => 'Network',
-		registered_country_geoname_id => 'Registered Country ID',
-		geoname_id => 'Country ID',
-		is_anonymous_proxy => 'Anonymous Proxy',
-		is_satellite_provider => 'Satellite',
-	);
-
-	foreach (sort keys %countryName) {
-		$country{$_} = {
-			name => $countryName{$_},
-			pool_v4 => Net::CIDR::Lite->new(),
-			pool_v6 => Net::CIDR::Lite->new(),
-		};
 	}
 
-	$file = "$source_dir/GeoLite2-Country-Blocks-IPv4.csv";
-	open($fh, '<', $file) || die "Can't open IPv4 database\n";
-
-	# first line is headers
-	$row = $csv->getline($fh);
-
-	%header = map { ($row->[$_], $_); } (0..$#{$row});
-
-	# verify that the columns we need are present
-	map { die "Table has no %pairs{$_} column\n" unless (exists $header{$_}); } keys %pairs;
-
-	my %remapping = (
-		net => 'network',
-		id => 'geoname_id',
-		rid => 'registered_country_geoname_id',
-		proxy => 'is_anonymous_proxy',
-		sat => 'is_satellite_provider',
-	);
-
-	# now create a function which returns the value of that column #
-	map { eval "sub $_ () { \$header{\$remapping{$_}}; }" ; } keys %remapping;
-
 	while ($row = $csv->getline($fh)) {
-		my ($cc, $cidr);
+		my ($cc, $range);
 
-		$cc = lookupCountry($row->[id], $row->[rid], $row->[proxy], $row->[sat]);
+		$cc = $row->[2];
-		$cidr = $row->[net];
+		$range = $row->[0] . "-" . $row->[1];
-		$country{$cc}->{pool_v4}->add($cidr);
 
-		if ($. % 4096 == 0) {
+		if (!exists($country{$cc})) {
+			$country{$cc} = { pool_v4 => Net::CIDR::Lite->new(), pool_v6 => Net::CIDR::Lite->new() };
+		}
+
+		if (index($range, '.') > 0) {
+			$country{$cc}->{pool_v4}->add_range($range);
+		}
+
+		if (index($range, ':') > 0) {
+			$country{$cc}->{pool_v6}->add_range($range);
+		}
+
+		if (!$quiet && $. % 4096 == 0) {
 			print STDERR "\r\e[2K$. entries";
 		}
 	}
 
-	print STDERR "\r\e[2K$. entries total\n";
+	print STDERR "\r\e[2K$. entries total\n" unless ($quiet);
 
 	close($fh);
 
-	# clean up the namespace
-	undef &net; undef &id; undef &rid; undef &proxy; undef &sat;
-
-	$file = "$source_dir/GeoLite2-Country-Blocks-IPv6.csv";
-	open($fh, '<', $file) || die "Can't open IPv6 database\n";
-
-	# first line is headers
-	$row = $csv->getline($fh);
-
-	%header = map { ($row->[$_], $_); } (0..$#{$row});
-
-	# verify that the columns we need are present
-	map { die "Table has no %pairs{$_} column\n" unless (exists $header{$_}); } keys %pairs;
-
-	# unlikely the IPv6 table has different columns, but just to be sure
-	# create a function which returns the value of that column #
-	map { eval "sub $_ () { \$header{\$remapping{$_}}; }" ; } keys %remapping;
-
-	while ($row = $csv->getline($fh)) {
-		my ($cc, $cidr);
-
-		$cc = lookupCountry($row->[id], $row->[rid], $row->[proxy], $row->[sat]);
-		$cidr = $row->[net];
-		$country{$cc}->{pool_v6}->add($cidr);
-
-		if ($. % 4096 == 0) {
-			print STDERR "\r\e[2K$. entries";
-		}
-	}
-
-	print STDERR "\r\e[2K$. entries total\n";
-
-	close($fh);
-
-	# clean up the namespace
-	undef &net; undef &id; undef &rid; undef &proxy; undef &sat;
-
 	return \%country;
 }
 
@@ -230,22 +92,22 @@ sub dump_one
 
 	@ranges = $country->{pool_v4}->list_range();
 
-	writeCountry($iso_code, $country->{name}, AF_INET, @ranges);
+	writeCountry($iso_code, AF_INET, @ranges);
 
 	@ranges = $country->{pool_v6}->list_range();
 
-	writeCountry($iso_code, $country->{name}, AF_INET6, @ranges);
+	writeCountry($iso_code, AF_INET6, @ranges);
 }
 
 sub writeCountry
 {
-	my ($iso_code, $name, $family, @ranges) = @_;
+	my ($iso_code, $family, @ranges) = @_;
 	my $fh;
 
-	printf "%5u IPv%s ranges for %s %s\n",
+	printf "%5u IPv%s ranges for %s\n",
 		scalar(@ranges),
 		($family == AF_INET ? '4' : '6'),
-		$iso_code, $name;
+		$iso_code unless ($quiet);
 
 	my $file = "$target_dir/".uc($iso_code).".iv".($family == AF_INET ? '4' : '6');
 	if (!open($fh, '>', $file)) {

geoip/xt_geoip_build.1

@@ -5,7 +5,7 @@ xt_geoip_build \(em convert GeoIP.csv to packed format for xt_geoip
 .SH Syntax
 .PP
 \fI/usr/libexec/xt_geoip/\fP\fBxt_geoip_build\fP [\fB\-D\fP
-\fItarget_dir\fP] [\fB\-S\fP \fIsource_dir\fP]
+\fItarget_dir\fP] [\fB\-i\fP \fIinput_file\fP]
 .SH Description
 .PP
 xt_geoip_build is used to build packed raw representations of the range
@@ -24,16 +24,18 @@ script requires it to be called with a path.
 \fB\-D\fP \fItarget_dir\fP
 Specifies the target directory into which the files are to be put. Defaults to ".".
 .TP
-\fB\-S\fP \fIsource_dir\fP
+\fB\-i\fP \fIinput_file\fP
-Specifies the source directory from which to read the three files by the name
+Specifies the source location of the DBIP CSV file. Defaults to
-of \fBGeoLite2\-Country\-Blocks\-IPv4.csv\fP,
+"dbip-country-lite.csv". Use "-" to read from stdin.
-\fBGeoLite2\-Country\-Blocks\-IPv6.csv\fP and
+.TP
-\fBGeoLite2\-Country\-Locations\-en.csv\fP. Defaults to ".".
+\fB\-s\fP
+"System mode". Equivalent to \fB\-D /usr/share/xt_geoip\fP.
 .SH Application
 .PP
-Shell commands to build the databases and put them to where they are expected:
+Shell commands to build the databases and put them to where they are expected
+(usually run as root):
 .PP
-xt_geoip_build \-D /usr/share/xt_geoip
+xt_geoip_build \-s
 .SH See also
 .PP
 xt_geoip_dl(1)

geoip/xt_geoip_dl

@@ -1,7 +1,5 @@
 #!/bin/sh
 
-rm -rf GeoLite2-Country-CSV_*
+timestamp=$(date "+%Y-%m")
-
+wget -q "https://download.db-ip.com/free/dbip-country-lite-$timestamp.csv.gz" -O- | \
-wget -q http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country-CSV.zip
+	gzip -cd >dbip-country-lite.csv
-unzip -q GeoLite2-Country-CSV.zip
-rm -f GeoLite2-Country-CSV.zip

geoip/xt_geoip_fetch

@@ -53,8 +53,9 @@ foreach my $cc (@ARGV) {
 		binmode($fh);
 
 		while (($bytes = read($fh, $buffer, AF_INET_SIZE * 2)) == AF_INET_SIZE * 2) {
-			my $start = inet_ntop(AF_INET, substr($buffer, 0, AF_INET_SIZE));
+			my ($start, $end) = unpack('a4a4', $buffer);
-			my $end = inet_ntop(AF_INET, substr($buffer, AF_INET_SIZE));
+			$start = inet_ntop(AF_INET, $start);
+			$end = inet_ntop(AF_INET, $end);
 			print $start, '-', $end, "\n";
 		}
 		close($fh);
@@ -75,8 +76,9 @@ foreach my $cc (@ARGV) {
 		binmode($fh);
 
 		while (($bytes = read($fh, $buffer, AF_INET6_SIZE * 2)) == AF_INET6_SIZE * 2) {
-			my $start = inet_ntop(AF_INET6, substr($buffer, 0, AF_INET6_SIZE));
+			my ($start, $end) = unpack('a16a16', $buffer);
-			my $end = inet_ntop(AF_INET6, substr($buffer, AF_INET6_SIZE));
+			$start = inet_ntop(AF_INET6, $start);
+			$end = inet_ntop(AF_INET6, $end);
 			print $start, '-', $end, "\n";
 		}
 		close($fh);

geoip/xt_geoip_fetch.1

@@ -0,0 +1,35 @@
+.TH xt_geoip_fetch 1 "2020-04-30" "xtables-addons" "xtables-addons"
+.SH Name
+.PP
+xt_geoip_fetch \(em dump a country database to stdout
+.SH Syntax
+.PP
+\fBxt_geoip_fetch\fP [\fB\-D\fP
+\fIdatabase_dir\fP] [\fB-4\fP] [\fB-6\fP] \fIcc\fP [ \fIcc\fP ... ]
+.SH Description
+.PP
+xt_geoip_fetch unpacks a country's IPv4 or IPv6 databases and dumps
+them to standard output as a sorted, non-overlaping list of ranges (which
+is how they're represented in the database) suitable for browsing or
+further processing.
+.PP Options
+.TP
+\fB\-D\fP \fIdatabase_dir\fP
+Specifies the directory into which the files have been put. Defaults to ".".
+.TP
+\fB-4\fP
+Specifies IPv4 data only.
+.TP
+\fB-6\fP
+Specifies IPv6 data only.
+.TP
+\fIcc\fP [ \fIcc\fP ... ]
+The ISO-3166 country code names of the desired countries' databases.
+.SH Application
+.PP
+Shell command to dump the list of Swiss IPv6 address ranges:
+.PP
+xt_geoip_fetch \-D /usr/share/xt_geoip \-6 ch
+.SH See also
+.PP
+xt_geoip_build(1)

xtables-addons.8.in

@@ -1,4 +1,4 @@
-.TH xtables-addons 8 "" "" "v3.7 (2019-12-01)"
+.TH xtables-addons 8 "" "" "v3.9 (2020-02-25)"
 .SH Name
 Xtables-addons \(em additional extensions for iptables, ip6tables, etc.
 .SH Targets