Xtables-addons 3.3

This mechanism has not seen any use in recent years (the "sources"
file is still the same) — drop it.

.gitignore

@@ -3,6 +3,7 @@
 *.lo
 *.loT
 *.o
+.cache.mk
 .deps/
 .dirstamp
 .libs/

README

@@ -1,57 +1,15 @@
 Xtables-addons
 ==============
 
-Xtables-addons is the proclaimed successor to patch-o-matic(-ng). It
-contains extensions that were not accepted in the main Xtables
-package.
+Xtables-addons is a set of extensions that were not accepted in the
+Linux kernel and/or main Xtables/iptables package.
 
-Xtables-addons is different from patch-o-matic in that you do not
-have to patch or recompile either kernel or Xtables(iptables). But
-please see the INSTALL file for the minimum requirements of this
-package.
-
-All code imported from patch-o-matic has been reviewed and all
-apparent bugs like binary stability across multiarches, missing
-sanity checks and incorrect endianess handling have been fixed,
-simplified, and sped up.
+It superseded the earlier patch-o-matic(-ng) package in that no
+patching and/or recompilation of either the kernel or
+Xtables/iptables is required. However, do see the INSTALL file for
+the minimum requirements of Xtables-addons.
 
 
 Included in this package
 ========================
 - xt_ACCOUNT 1.16, libxt_ACCOUNT 1.3
-
-
-Inclusion into a kernel tree
-============================
-
-
-
-
-External extensions
-===================
-
-The program "xa-download-more" can be used to download more
-extensions from 3rd parties into the source tree. The URLs are listed
-in the "sources" file. If the "sources" file contains an entry like
-
-	http://foobar.org/xa/
-
-xa-download-more will inspect http://foobar.org/xa/xa-index.txt for
-files to download. That file may contain
-
-	foobar.tar.bz2
-
-and xa-download-more will then retrieve and unpack
-http://foobar.org/xa/foobar.tar.bz2.
-
-Files that should be contained in the tarball are an mconfig and
-Kbuild files to control building the extension, libxt_foobar.c for
-the userspace extension and xt_foobar.c for the kernel extension.
-
-	mconfig.foobar
-	extensions/Kbuild.foobar
-	extensions/Mbuild.foobar
-	extensions/libxt_foobar.c
-	extensions/libxt_foobar.man
-	extensions/xt_foobar.c
-	extensions/xt_foobar.h

configure.ac

@@ -1,4 +1,4 @@
-AC_INIT([xtables-addons], [3.0])
+AC_INIT([xtables-addons], [3.3])
 AC_CONFIG_AUX_DIR([build-aux])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_MACRO_DIR([m4])
@@ -26,7 +26,7 @@ fi
 
 AC_CHECK_HEADERS([linux/netfilter/x_tables.h], [],
 	[AC_MSG_ERROR([You need to have linux/netfilter/x_tables.h, see INSTALL file for details])])
-PKG_CHECK_MODULES([libxtables], [xtables >= 1.4.5])
+PKG_CHECK_MODULES([libxtables], [xtables >= 1.6.0])
 xtlibdir="$(pkg-config --variable=xtlibdir xtables)"
 
 AC_ARG_WITH([xtlibdir],
@@ -57,9 +57,9 @@ if test -n "$kbuilddir"; then
 		echo "WARNING: Version detection did not succeed. Continue at own luck.";
 	else
 		echo "$kmajor.$kminor.$kmicro.$kstable in $kbuilddir";
-		if test "$kmajor" -gt 4 -o "$kmajor" -eq 4 -a "$kminor" -gt 16; then
+		if test "$kmajor" -gt 5 -o "$kmajor" -eq 5 -a "$kminor" -gt 0; then
 			echo "WARNING: That kernel version is not officially supported yet. Continue at own luck.";
-		elif test "$kmajor" -eq 4 -a "$kminor" -ge 15; then
+		elif test "$kmajor" -eq 4 -a "$kminor" -ge 18; then
 			:
 		else
 			echo "WARNING: That kernel version is not officially supported.";

doc/changelog.txt

@@ -1,6 +1,29 @@
 
 HEAD
 ====
+
+
+v3.3 (2019-03-07)
+=================
+Enhancements:
+- support for Linux 5.0
+
+
+v3.2 (2018-09-07)
+=================
+Changes:
+- rework xt_geoip_build to scan the immediate directory for .csv,
+  not to scan for GeoLite2-Country-CSV_\d+.
+
+
+v3.1 (2018-08-14)
+=================
+Enhancements:
+- support for Linux 4.17, 4.18
+
+
+v3.0 (2018-02-12)
+=================
 Enhancements:
 - support for Linux 4.15, 4.16
 Changes:

extensions/ACCOUNT/xt_ACCOUNT.c

@@ -40,6 +40,9 @@
 #error "ipt_ACCOUNT needs at least a PAGE_SIZE of 4096"
 #endif
 
+static unsigned int max_tables_limit = 128;
+module_param(max_tables_limit, uint, 0);
+
 /**
  * Internal table structure, generated by check_entry()
  * @name:	name of the table
@@ -185,7 +188,7 @@ static int ipt_acc_table_insert(struct ipt_acc_table *ipt_acc_tables,
 		name, NIPQUAD(ip), NIPQUAD(netmask));
 
 	/* Look for existing table */
-	for (i = 0; i < ACCOUNT_MAX_TABLES; i++) {
+	for (i = 0; i < max_tables_limit; i++) {
 		if (strncmp(ipt_acc_tables[i].name, name,
 		    ACCOUNT_TABLE_NAME_LEN) == 0) {
 			pr_debug("ACCOUNT: Found existing slot: %d - "
@@ -209,7 +212,7 @@ static int ipt_acc_table_insert(struct ipt_acc_table *ipt_acc_tables,
 	}
 
 	/* Insert new table */
-	for (i = 0; i < ACCOUNT_MAX_TABLES; i++) {
+	for (i = 0; i < max_tables_limit; i++) {
 		/* Found free slot */
 		if (ipt_acc_tables[i].name[0] == 0) {
 			unsigned int netsize = 0;
@@ -258,7 +261,7 @@ static int ipt_acc_table_insert(struct ipt_acc_table *ipt_acc_tables,
 
 	/* No free slot found */
 	printk("ACCOUNT: No free table slot found (max: %d). "
-		"Please increase ACCOUNT_MAX_TABLES.\n", ACCOUNT_MAX_TABLES);
+		"Please increase the \"max_tables_limit\" module parameter.\n", max_tables_limit);
 	return -1;
 }
 
@@ -299,7 +302,7 @@ static void ipt_acc_destroy(const struct xt_tgdtor_param *par)
 	info->table_nr = -1;	/* Set back to original state */
 
 	/* Look for table */
-	for (i = 0; i < ACCOUNT_MAX_TABLES; i++) {
+	for (i = 0; i < max_tables_limit; i++) {
 		if (strncmp(ian->ipt_acc_tables[i].name, info->table_name,
 		    ACCOUNT_TABLE_NAME_LEN) == 0) {
 			pr_debug("ACCOUNT: Found table at slot: %d\n", i);
@@ -604,12 +607,12 @@ static int ipt_acc_handle_prepare_read(struct ipt_acc_table *ipt_acc_tables,
 	int table_nr = -1;
 	uint8_t depth;
 
-	for (table_nr = 0; table_nr < ACCOUNT_MAX_TABLES; table_nr++)
+	for (table_nr = 0; table_nr < max_tables_limit; table_nr++)
 		if (strncmp(ipt_acc_tables[table_nr].name, tablename,
 		    ACCOUNT_TABLE_NAME_LEN) == 0)
 			break;
 
-	if (table_nr == ACCOUNT_MAX_TABLES) {
+	if (table_nr == max_tables_limit) {
 		printk("ACCOUNT: ipt_acc_handle_prepare_read(): "
 			"Table %s not found\n", tablename);
 		return -1;
@@ -707,12 +710,12 @@ static int ipt_acc_handle_prepare_read_flush(struct ipt_acc_table *ipt_acc_table
 	int table_nr;
 	void *new_data_page;
 
-	for (table_nr = 0; table_nr < ACCOUNT_MAX_TABLES; table_nr++)
+	for (table_nr = 0; table_nr < max_tables_limit; table_nr++)
 		if (strncmp(ipt_acc_tables[table_nr].name, tablename,
 		    ACCOUNT_TABLE_NAME_LEN) == 0)
 			break;
 
-	if (table_nr == ACCOUNT_MAX_TABLES) {
+	if (table_nr == max_tables_limit) {
 		printk("ACCOUNT: ipt_acc_handle_prepare_read_flush(): "
 			"Table %s not found\n", tablename);
 		return -1;
@@ -1052,7 +1055,7 @@ static int ipt_acc_get_ctl(struct sock *sk, int cmd, void *user, int *len)
 		spin_lock_bh(&ian->ipt_acc_lock);
 
 		/* Determine size of table names */
-		for (i = 0; i < ACCOUNT_MAX_TABLES; i++) {
+		for (i = 0; i < max_tables_limit; i++) {
 			if (ian->ipt_acc_tables[i].name[0] != 0)
 				size += strlen(ian->ipt_acc_tables[i].name) + 1;
 		}
@@ -1067,7 +1070,7 @@ static int ipt_acc_get_ctl(struct sock *sk, int cmd, void *user, int *len)
 		}
 		/* Copy table names to userspace */
 		tnames = ian->ipt_acc_tmpbuf;
-		for (i = 0; i < ACCOUNT_MAX_TABLES; i++) {
+		for (i = 0; i < max_tables_limit; i++) {
 			if (ian->ipt_acc_tables[i].name[0] != 0) {
 				name_len = strlen(ian->ipt_acc_tables[i].name) + 1;
 				memcpy(tnames, ian->ipt_acc_tables[i].name, name_len);
@@ -1100,7 +1103,7 @@ static int __net_init ipt_acc_net_init(struct net *net)
 	memset(ian, 0, sizeof(*ian));
 	sema_init(&ian->ipt_acc_userspace_mutex, 1);
 
-	ian->ipt_acc_tables = kcalloc(ACCOUNT_MAX_TABLES,
+	ian->ipt_acc_tables = kcalloc(max_tables_limit,
 		sizeof(struct ipt_acc_table), GFP_KERNEL);
 	if (ian->ipt_acc_tables == NULL) {
 		printk("ACCOUNT: Out of memory allocating account_tables structure");

extensions/ACCOUNT/xt_ACCOUNT.h

@@ -34,7 +34,6 @@
 #define IPT_SO_GET_ACCOUNT_GET_TABLE_NAMES (SO_ACCOUNT_BASE_CTL + 8)
 #define IPT_SO_GET_ACCOUNT_MAX	  IPT_SO_GET_ACCOUNT_GET_TABLE_NAMES
 
-#define ACCOUNT_MAX_TABLES 128
 #define ACCOUNT_TABLE_NAME_LEN 32
 #define ACCOUNT_MAX_HANDLES 10
 

extensions/libxt_geoip.c

@@ -49,6 +49,38 @@ static struct option geoip_opts[] = {
 	{NULL},
 };
 
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+static void geoip_swap_le16(uint16_t *buf)
+{
+	unsigned char *p = (void *)buf;
+	uint16_t n= p[0] + (p[1] << 8);
+	p[0] = (n >> 8) & 0xff;
+	p[1] = n & 0xff;
+}
+
+static void geoip_swap_in6(struct in6_addr *in6)
+{
+	geoip_swap_le16(&in6->s6_addr16[0]);
+	geoip_swap_le16(&in6->s6_addr16[1]);
+	geoip_swap_le16(&in6->s6_addr16[2]);
+	geoip_swap_le16(&in6->s6_addr16[3]);
+	geoip_swap_le16(&in6->s6_addr16[4]);
+	geoip_swap_le16(&in6->s6_addr16[5]);
+	geoip_swap_le16(&in6->s6_addr16[6]);
+	geoip_swap_le16(&in6->s6_addr16[7]);
+}
+
+static void geoip_swap_le32(uint32_t *buf)
+{
+	unsigned char *p = (void *)buf;
+	uint32_t n = p[0] + (p[1] << 8) + (p[2] << 16) + (p[3] << 24);
+	p[0] = (n >> 24) & 0xff;
+	p[1] = (n >> 16) & 0xff;
+	p[2] = (n >> 8) & 0xff;
+	p[3] = n & 0xff;
+}
+#endif
+
 static void *
 geoip_get_subnets(const char *code, uint32_t *count, uint8_t nfproto)
 {
@@ -56,21 +88,15 @@ geoip_get_subnets(const char *code, uint32_t *count, uint8_t nfproto)
 	struct stat sb;
 	char buf[256];
 	int fd;
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+	unsigned int n;
+#endif
 
 	/* Use simple integer vector files */
-	if (nfproto == NFPROTO_IPV6) {
-#if __BYTE_ORDER == _BIG_ENDIAN
-		snprintf(buf, sizeof(buf), GEOIP_DB_DIR "/BE/%s.iv6", code);
-#else
-		snprintf(buf, sizeof(buf), GEOIP_DB_DIR "/LE/%s.iv6", code);
-#endif
-	} else {
-#if __BYTE_ORDER == _BIG_ENDIAN
-		snprintf(buf, sizeof(buf), GEOIP_DB_DIR "/BE/%s.iv4", code);
-#else
-		snprintf(buf, sizeof(buf), GEOIP_DB_DIR "/LE/%s.iv4", code);
-#endif
-	}
+	if (nfproto == NFPROTO_IPV6)
+		snprintf(buf, sizeof(buf), GEOIP_DB_DIR "/%s.iv6", code);
+	else
+		snprintf(buf, sizeof(buf), GEOIP_DB_DIR "/%s.iv4", code);
 
 	if ((fd = open(buf, O_RDONLY)) < 0) {
 		fprintf(stderr, "Could not open %s: %s\n", buf, strerror(errno));
@@ -98,6 +124,25 @@ geoip_get_subnets(const char *code, uint32_t *count, uint8_t nfproto)
 		xtables_error(OTHER_PROBLEM, "geoip: insufficient memory");
 	read(fd, subnets, sb.st_size);
 	close(fd);
+
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+	for (n = 0; n < *count; ++n) {
+		switch (nfproto) {
+		case NFPROTO_IPV6: {
+			struct geoip_subnet6 *gs6 = &(((struct geoip_subnet6 *)subnets)[n]);
+			geoip_swap_in6(&gs6->begin);
+			geoip_swap_in6(&gs6->end);
+			break;
+		}
+		case NFPROTO_IPV4: {
+			struct geoip_subnet4 *gs4 = &(((struct geoip_subnet4 *)subnets)[n]);
+			geoip_swap_le32(&gs4->begin);
+			geoip_swap_le32(&gs4->end);
+			break;
+		}
+		}
+	}
+#endif
 	return subnets;
 }
 
@@ -135,7 +180,7 @@ check_geoip_cc(char *cc, u_int16_t cc_used[], u_int8_t count)
 			cc[i] = toupper(cc[i]);
 		else
 			xtables_error(PARAMETER_PROBLEM,
-				"geoip:  invalid country code '%s'", cc);
+				"geoip: invalid country code '%s'", cc);
 
 	/* Convert chars into a single 16 bit integer.
 	 * FIXME:	This assumes that a country code is

extensions/xt_DNETMAP.c

@@ -363,7 +363,11 @@ dnetmap_tg(struct sk_buff *skb, const struct xt_action_param *par)
 	__be32 prenat_ip, postnat_ip, prenat_ip_prev;
 	const struct xt_DNETMAP_tginfo *tginfo = par->targinfo;
 	const struct nf_nat_range *mr = &tginfo->prefix;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 18, 0)
+	struct nf_nat_range2 newrange;
+#else
 	struct nf_nat_range newrange;
+#endif
 	struct dnetmap_entry *e;
 	struct dnetmap_prefix *p;
 	__s32 jttl;

extensions/xt_SYSRQ.c

@@ -314,7 +314,7 @@ static void sysrq_crypto_exit(void)
 static int __init sysrq_crypto_init(void)
 {
 #if defined(WITH_CRYPTO)
-	struct timeval now;
+	struct timespec64 now;
 	int ret;
 
 	sysrq_tfm = crypto_alloc_shash(sysrq_hash, 0, 0);
@@ -339,7 +339,7 @@ static int __init sysrq_crypto_init(void)
 		    sizeof(sysrq_password), GFP_KERNEL);
 	if (sysrq_digest_password == NULL)
 		goto fail;
-	do_gettimeofday(&now);
+	ktime_get_real_ts64(&now);
 	sysrq_seqno = now.tv_sec;
 	return 0;
 

geoip/xt_geoip_build

@@ -1,86 +1,216 @@
 #!/usr/bin/perl
 #
-#	Converter for MaxMind CSV database to binary, for xt_geoip
-#	Copyright © Jan Engelhardt, 2008-2011
+#	Converter for MaxMind (GeoLite2) CSV database to binary, for xt_geoip
+#	Copyright Jan Engelhardt, 2008-2011
+#	Copyright Philip Prindeville, 2018
 #
 use Getopt::Long;
-use IO::Handle;
+use Net::CIDR::Lite;
+use Socket qw(AF_INET AF_INET6 inet_pton);
+use warnings;
 use Text::CSV_XS; # or trade for Text::CSV
 use strict;
 
-my $le32 = pack("V", 0x10000000);
-my $be32 = pack("N", 0x10000000);
-my $u32 = undef;
-
-sub wantBE { return !$u32 || $u32 eq $be32; }
-sub wantLE { return !$u32 || $u32 eq $le32; }
-
 my $csv = Text::CSV_XS->new({
 	allow_whitespace => 1,
 	binary => 1,
 	eol => $/,
 }); # or Text::CSV
+my $source_dir = ".";
 my $target_dir = ".";
-my $native_only = 0;
 
 &Getopt::Long::Configure(qw(bundling));
 &GetOptions(
 	"D=s" => \$target_dir,
-	"n"   => \$native_only,
+	"S=s" => \$source_dir,
 );
 
-if (!-d $target_dir) {
-	print STDERR "Target directory $target_dir does not exist.\n";
+if (!-d $source_dir) {
+	print STDERR "Source directory \"$source_dir\" does not exist.\n";
 	exit 1;
 }
-my @dbs = qw(LE BE);
-if ($native_only) {
-	$u32 = pack("L", 0x10000000);
-	if ($u32 eq $le32) {
-		@dbs = qw(LE);
-	} elsif ($u32 eq $be32) {
-		@dbs = qw(BE);
-	} else {
-		print STDERRR "Cannot determine endianness.\n";
-		exit 1;
-	}
-}
-
-foreach (@dbs) {
-	my $dir = "$target_dir/$_";
-	if (!-e $dir && !mkdir($dir)) {
-		print STDERR "Could not mkdir $dir: $!\n";
-		exit 1;
-	}
+if (!-d $target_dir) {
+	print STDERR "Target directory \"$target_dir\" does not exist.\n";
+	exit 1;
 }
 
+my %countryId;
+my %countryName;
+&loadCountries();
 &dump(&collect());
 
+sub loadCountries
+{
+	sub id; sub cc; sub long; sub ct; sub cn;
+
+	%countryId = ();
+	%countryName = ();
+
+	my $file = "$source_dir/GeoLite2-Country-Locations-en.csv";
+	open(my $fh, '<', $file) || die "Couldn't open list country names\n";
+
+	# first line is headers
+	my $row = $csv->getline($fh);
+
+	my %header = map { ($row->[$_], $_); } (0..$#{$row});
+
+	my %pairs = (
+		country_iso_code => 'ISO Country Code',
+		geoname_id => 'ID',
+		country_name => 'Country Name',
+		continent_code => 'Continent Code',
+		continent_name => 'Continent Name',
+	);
+
+	# verify that the columns we need are present
+	map { die "Table has no $pairs{$_} column\n" unless (exists $header{$_}); } keys %pairs;
+
+	my %remapping = (
+		id => 'geoname_id',
+		cc => 'country_iso_code',
+		long => 'country_name',
+		ct => 'continent_code',
+		cn => 'continent_name',
+	);
+
+	# now create a function which returns the value of that column #
+	map { eval "sub $_ () { \$header{\$remapping{$_}}; }" ; } keys %remapping;
+
+	while (my $row = $csv->getline($fh)) {
+		if ($row->[cc] eq '' && $row->[long] eq '') {
+			$countryId{$row->[id]} = $row->[ct];
+			$countryName{$row->[ct]} = $row->[cn];
+		} else {
+			$countryId{$row->[id]} = $row->[cc];
+			$countryName{$row->[cc]} = $row->[long];
+		}
+	}
+
+	$countryName{A1} = 'Anonymous Proxy';
+	$countryName{A2} = 'Satellite Provider';
+	$countryName{O1} = 'Other Country';
+
+	close($fh);
+
+	# clean up the namespace
+	undef &id; undef &cc; undef &long; undef &ct; undef &cn;
+}
+
+sub lookupCountry
+{
+	my ($id, $rid, $proxy, $sat) = @_;
+
+	if ($proxy) {
+		return 'A1';
+	} elsif ($sat) {
+		return 'A2';
+	}
+	$id ||= $rid;
+	if ($id eq '') {
+		return 'O1';
+	}
+	die "Unknown id: $id line $.\n" unless (exists $countryId{$id});
+	return $countryId{$id};
+}
+
 sub collect
 {
-	my %country;
+	my ($file, $fh, $row);
+	my (%country, %header);
+
+	sub net; sub id; sub rid; sub proxy; sub sat;
+
+	my %pairs = (
+		network => 'Network',
+		registered_country_geoname_id => 'Registered Country ID',
+		geoname_id => 'Country ID',
+		is_anonymous_proxy => 'Anonymous Proxy',
+		is_satellite_provider => 'Satellite',
+	);
+
+	foreach (sort keys %countryName) {
+		$country{$_} = {
+			name => $countryName{$_},
+			pool_v4 => Net::CIDR::Lite->new(),
+			pool_v6 => Net::CIDR::Lite->new(),
+		};
+	}
+
+	$file = "$source_dir/GeoLite2-Country-Blocks-IPv4.csv";
+	open($fh, '<', $file) || die "Can't open IPv4 database\n";
+
+	# first line is headers
+	$row = $csv->getline($fh);
+
+	%header = map { ($row->[$_], $_); } (0..$#{$row});
+
+	# verify that the columns we need are present
+	map { die "Table has no %pairs{$_} column\n" unless (exists $header{$_}); } keys %pairs;
+
+	my %remapping = (
+		net => 'network',
+		id => 'geoname_id',
+		rid => 'registered_country_geoname_id',
+		proxy => 'is_anonymous_proxy',
+		sat => 'is_satellite_provider',
+	);
+
+	# now create a function which returns the value of that column #
+	map { eval "sub $_ () { \$header{\$remapping{$_}}; }" ; } keys %remapping;
+
+	while ($row = $csv->getline($fh)) {
+		my ($cc, $cidr);
+
+		$cc = lookupCountry($row->[id], $row->[rid], $row->[proxy], $row->[sat]);
+		$cidr = $row->[net];
+		$country{$cc}->{pool_v4}->add($cidr);
 
-	while (my $row = $csv->getline(*ARGV)) {
-		if (!defined($country{$row->[4]})) {
-			$country{$row->[4]} = {
-				name => $row->[5],
-				pool_v4 => [],
-				pool_v6 => [],
-			};
-		}
-		my $c = $country{$row->[4]};
-		if ($row->[0] =~ /:/) {
-			push(@{$c->{pool_v6}},
-			     [&ip6_pack($row->[0]), &ip6_pack($row->[1])]);
-		} else {
-			push(@{$c->{pool_v4}}, [$row->[2], $row->[3]]);
-		}
 		if ($. % 4096 == 0) {
 			print STDERR "\r\e[2K$. entries";
 		}
 	}
 
 	print STDERR "\r\e[2K$. entries total\n";
+
+	close($fh);
+
+	# clean up the namespace
+	undef &net; undef &id; undef &rid; undef &proxy; undef &sat;
+
+	$file = "$source_dir/GeoLite2-Country-Blocks-IPv6.csv";
+	open($fh, '<', $file) || die "Can't open IPv6 database\n";
+
+	# first line is headers
+	$row = $csv->getline($fh);
+
+	%header = map { ($row->[$_], $_); } (0..$#{$row});
+
+	# verify that the columns we need are present
+	map { die "Table has no %pairs{$_} column\n" unless (exists $header{$_}); } keys %pairs;
+
+	# unlikely the IPv6 table has different columns, but just to be sure
+	# create a function which returns the value of that column #
+	map { eval "sub $_ () { \$header{\$remapping{$_}}; }" ; } keys %remapping;
+
+	while ($row = $csv->getline($fh)) {
+		my ($cc, $cidr);
+
+		$cc = lookupCountry($row->[id], $row->[rid], $row->[proxy], $row->[sat]);
+		$cidr = $row->[net];
+		$country{$cc}->{pool_v6}->add($cidr);
+
+		if ($. % 4096 == 0) {
+			print STDERR "\r\e[2K$. entries";
+		}
+	}
+
+	print STDERR "\r\e[2K$. entries total\n";
+
+	close($fh);
+
+	# clean up the namespace
+	undef &net; undef &id; undef &rid; undef &proxy; undef &sat;
+
 	return \%country;
 }
 
@@ -88,7 +218,7 @@ sub dump
 {
 	my $country = shift @_;
 
-	foreach my $iso_code (sort keys %$country) {
+	foreach my $iso_code (sort keys %{$country}) {
 		&dump_one($iso_code, $country->{$iso_code});
 	}
 }
@@ -96,80 +226,40 @@ sub dump
 sub dump_one
 {
 	my($iso_code, $country) = @_;
-	my($file, $fh_le, $fh_be);
+	my @ranges;
 
-	printf "%5u IPv6 ranges for %s %s\n",
-		scalar(@{$country->{pool_v6}}),
-		$iso_code, $country->{name};
+	@ranges = $country->{pool_v4}->list_range();
 
-	if (wantLE) {
-		$file = "$target_dir/LE/".uc($iso_code).".iv6";
-		if (!open($fh_le, "> $file")) {
-			print STDERR "Error opening $file: $!\n";
-			exit 1;
-		}
-		foreach my $range (@{$country->{pool_v6}}) {
-			print $fh_le &ip6_swap($range->[0]), &ip6_swap($range->[1]);
-		}
-		close $fh_le;
-	}
-	if (wantBE) {
-		$file = "$target_dir/BE/".uc($iso_code).".iv6";
-		if (!open($fh_be, "> $file")) {
-			print STDERR "Error opening $file: $!\n";
-			exit 1;
-		}
-		foreach my $range (@{$country->{pool_v6}}) {
-			print $fh_be $range->[0], $range->[1];
-		}
-		close $fh_be;
-	}
+	writeCountry($iso_code, $country->{name}, AF_INET, @ranges);
 
-	printf "%5u IPv4 ranges for %s %s\n",
-		scalar(@{$country->{pool_v4}}),
-		$iso_code, $country->{name};
+	@ranges = $country->{pool_v6}->list_range();
 
-	if (wantLE) {
-		$file = "$target_dir/LE/".uc($iso_code).".iv4";
-		if (!open($fh_le, "> $file")) {
-			print STDERR "Error opening $file: $!\n";
-			exit 1;
-		}
-		foreach my $range (@{$country->{pool_v4}}) {
-			print $fh_le pack("VV", $range->[0], $range->[1]);
-		}
-		close $fh_le;
-	}
-	if (wantBE) {
-		$file = "$target_dir/BE/".uc($iso_code).".iv4";
-		if (!open($fh_be, "> $file")) {
-			print STDERR "Error opening $file: $!\n";
-			exit 1;
-		}
-		foreach my $range (@{$country->{pool_v4}}) {
-			print $fh_be pack("NN", $range->[0], $range->[1]);
-		}
-		close $fh_be;
-	}
+	writeCountry($iso_code, $country->{name}, AF_INET6, @ranges);
 }
 
-sub ip6_pack
+sub writeCountry
 {
-	my $addr = shift @_;
-	$addr =~ s{::}{:!:};
-	my @addr = split(/:/, $addr);
-	my @e = (0) x 8;
-	foreach (@addr) {
-		if ($_ eq "!") {
-			$_ = join(':', @e[0..(8-scalar(@addr))]);
-		}
-	}
-	@addr = split(/:/, join(':', @addr));
-	$_ = hex($_) foreach @addr;
-	return pack("n*", @addr);
-}
+	my ($iso_code, $name, $family, @ranges) = @_;
+	my $fh;
 
-sub ip6_swap
-{
-	return pack("V*", unpack("N*", shift @_));
+	printf "%5u IPv%s ranges for %s %s\n",
+		scalar(@ranges),
+		($family == AF_INET ? '4' : '6'),
+		$iso_code, $name;
+
+	my $file = "$target_dir/".uc($iso_code).".iv".($family == AF_INET ? '4' : '6');
+	if (!open($fh, '>', $file)) {
+		print STDERR "Error opening $file: $!\n";
+		exit 1;
+	}
+
+	binmode($fh);
+
+	foreach my $range (@ranges) {
+		my ($start, $end) = split('-', $range);
+		$start = inet_pton($family, $start);
+		$end = inet_pton($family, $end);
+		print $fh $start, $end;
+	}
+	close $fh;
 }

geoip/xt_geoip_build.1

@@ -5,7 +5,7 @@ xt_geoip_build \(em convert GeoIP.csv to packed format for xt_geoip
 .SH Syntax
 .PP
 \fI/usr/libexec/xt_geoip/\fP\fBxt_geoip_build\fP [\fB\-D\fP
-\fItarget_dir\fP] [\fIfile\fP...]
+\fItarget_dir\fP] [\fB\-S\fP \fIsource_dir\fP]
 .SH Description
 .PP
 xt_geoip_build is used to build packed raw representations of the range
@@ -16,15 +16,19 @@ required to be loaded into memory. The ranges in the packed database files are
 also ordered, as xt_geoip relies on this property for its bisection approach to
 work.
 .PP
-Input is processed from the listed files, or if none is given, from stdin.
-.PP
 Since the script is usually installed to the libexec directory of the
 xtables-addons package and this is outside $PATH (on purpose), invoking the
 script requires it to be called with a path.
 .PP Options
 .TP
 \fB\-D\fP \fItarget_dir\fP
-Specify a target directory into which the files are to be put.
+Specifies the target directory into which the files are to be put. Defaults to ".".
+.TP
+\fB\-S\fP \fIsource_dir\fP
+Specifies the source directory from which to read the three files by the name
+of \fBGeoLite2\-Country\-Blocks\-IPv4.csv\fP,
+\fBGeoLite2\-Country\-Blocks\-IPv6.csv\fP and
+\fBGeoLite2\-Country\-Locations\-en.csv\fP. Defaults to ".".
 .SH Application
 .PP
 Shell commands to build the databases and put them to where they are expected:

geoip/xt_geoip_dl

@@ -1,8 +1,7 @@
 #!/bin/sh
 
-rm -f GeoIPv6.csv GeoIPv6.csv.gz GeoIPCountryCSV.zip GeoIPCountryWhois.csv;
-wget \
-	http://geolite.maxmind.com/download/geoip/database/GeoIPv6.csv.gz \
-	http://geolite.maxmind.com/download/geoip/database/GeoIPCountryCSV.zip;
-gzip -d GeoIPv6.csv.gz;
-unzip GeoIPCountryCSV.zip;
+rm -rf GeoLite2-Country-CSV_*
+
+wget -q http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country-CSV.zip
+unzip -q GeoLite2-Country-CSV.zip
+rm -f GeoLite2-Country-CSV.zip

geoip/xt_geoip_fetch

@@ -0,0 +1,93 @@
+#!/usr/bin/perl
+#
+#	Utility to query GeoIP database
+#	Copyright Philip Prindeville, 2018
+#
+use Getopt::Long;
+use Socket qw(AF_INET AF_INET6 inet_ntop);
+use warnings;
+use strict;
+
+sub AF_INET_SIZE() { 4 }
+sub AF_INET6_SIZE() { 16 }
+
+my $target_dir = ".";
+my $ipv4 = 0;
+my $ipv6 = 0;
+
+&Getopt::Long::Configure(qw(bundling));
+&GetOptions(
+	"D=s" => \$target_dir,
+	"4"   => \$ipv4,
+	"6"   => \$ipv6,
+);
+
+if (!-d $target_dir) {
+	print STDERR "Target directory $target_dir does not exit.\n";
+	exit 1;
+}
+
+# if neither specified, assume both
+if (! $ipv4 && ! $ipv6) {
+	$ipv4 = $ipv6 = 1;
+}
+
+foreach my $cc (@ARGV) {
+	if ($cc !~ m/^([a-z]{2}|a[12]|o1)$/i) {
+		print STDERR "Invalid country code '$cc'\n";
+		exit 1;
+	}
+
+	my $file = $target_dir . '/' . uc($cc) . '.iv4';
+
+	if (! -f $file) {
+		printf STDERR "Can't find data for country '$cc'\n";
+		exit 1;
+	}
+
+	my ($contents, $buffer, $bytes, $fh);
+
+	if ($ipv4) {
+		open($fh, '<', $file) || die "Couldn't open file for '$cc'\n";
+
+		binmode($fh);
+
+		while (($bytes = read($fh, $buffer, AF_INET_SIZE * 2)) == AF_INET_SIZE * 2) {
+			my $start = inet_ntop(AF_INET, substr($buffer, 0, AF_INET_SIZE));
+			my $end = inet_ntop(AF_INET, substr($buffer, AF_INET_SIZE));
+			print $start, '-', $end, "\n";
+		}
+		close($fh);
+		if (! defined $bytes) {
+			printf STDERR "Error reading file for '$cc'\n";
+			exit 1;
+		} elsif ($bytes != 0) {
+			printf STDERR "Short read on file for '$cc'\n";
+			exit 1;
+		}
+	}
+
+	substr($file, -1) = '6';
+
+	if ($ipv6) {
+		open($fh, '<', $file) || die "Couldn't open file for '$cc'\n";
+
+		binmode($fh);
+
+		while (($bytes = read($fh, $buffer, AF_INET6_SIZE * 2)) == AF_INET6_SIZE * 2) {
+			my $start = inet_ntop(AF_INET6, substr($buffer, 0, AF_INET6_SIZE));
+			my $end = inet_ntop(AF_INET6, substr($buffer, AF_INET6_SIZE));
+			print $start, '-', $end, "\n";
+		}
+		close($fh);
+		if (! defined $bytes) {
+			printf STDERR "Error reading file for '$cc'\n";
+			exit 1;
+		} elsif ($bytes != 0) {
+			printf STDERR "Short read on file for '$cc'\n";
+			exit 1;
+		}
+	}
+}
+
+exit 0;

sources

@@ -1,3 +0,0 @@
-#
-# Source URLs for external patchlets
-#

xa-download-more

@@ -1,83 +0,0 @@
-#!/usr/bin/perl -w
-
-use HTTP::Request;
-use LWP::UserAgent;
-use strict;
-
-&main(\@ARGV);
-
-sub main
-{
-	local *FH;
-
-	if (!-d "downloads") {
-		if (!mkdir("downloads")) {
-			die "Could not create downloads/ directory";
-		}
-	}
-
-	open(FH, "<sources");
-	while (defined($_ = <FH>)) {
-		chomp $_;
-		$_ =~ s/#.*//gs;
-		$_ =~ s/^\s+|\s+$//gs;
-		if (length($_) == 0) {
-			next;
-		}
-		&process_index($_);
-	}
-
-	close FH;
-}
-
-sub process_index
-{
-	my $top = shift @_;
-	my($agent, $res, $url);
-	local *FH;
-
-	$agent = LWP::UserAgent->new();
-	$agent->env_proxy();
-
-	$url = &slash_remove("$top/xa-index.txt");
-	print "  GET     $url\n";
-	$res = $agent->get($url);
-	if (!$res->is_success()) {
-		print STDERR "  `->     ", $res->status_line(), "\n";
-		return;
-	}
-
-	foreach my $ext (split(/\s+/, $res->content())) {
-		my($ex_url, $ex_res);
-
-		$ex_url = &slash_remove("$top/$ext");
-		print "  GET     $ex_url\n";
-
-		$ex_res = $agent->mirror($ex_url, "downloads/$ext");
-		if ($ex_res->code() == 304) {
-			# "Not modified" = up to date
-			next;
-		}
-		if (!$ex_res->is_success()) {
-			print STDERR "  `->     ", $ex_res->status_line(), "\n";
-			next;
-		}
-
-		print "  UNPACK  downloads/$ext\n";
-		system "tar", "-xjf", "downloads/$ext";
-	}
-}
-
-sub slash_remove
-{
-	my $s = shift @_;
-	$s =~ s{(\w+://)(.*)}{$1.&slash_remove2($2)}eg;
-	return $s;
-}
-
-sub slash_remove2
-{
-	my $s = shift @_;
-	$s =~ s{/+}{/}g;
-	return $s;
-}

xtables-addons.8.in

@@ -1,4 +1,4 @@
-.TH xtables-addons 8 "Lilac" "" "v3.0 (2018-02-12)"
+.TH xtables-addons 8 "" "" "v3.3 (2019-03-07)"
 .SH Name
 Xtables-addons \(em additional extensions for iptables, ip6tables, etc.
 .SH Targets