Merge pull request 'Don't call python3.8, but python.' (#6) from feature/fix-python into master

Reviewed-on: #6

.editorconfig

@@ -0,0 +1,5 @@
+root = true
+
+[*.sh]
+indent_size = 4
+indent_style = tab

ddos-mitigator.conf

@@ -0,0 +1,30 @@
+# Example configuration file for ddos-mitigator.sh.
+# PLEASE TAKE CARE not to put any whitespace around the '=' signs, as this file is directly sourced by the script
+# file and this needs to conform to the BASH syntax. Also, make sure to declare the COUNTRIES variable with the
+# correct array syntax: COUNTRIES=("AA" "BB" "CC"), or to comment it out altogether.
+
+# The path to the GeoIP2 database file (must be either country or city database). This parameter is mandatory. If it is
+# not specified here, it must be given on the command line (through the -d option).
+DATABASE_FILE="/path/to/geoip/country-or-city-database.mmdb"
+
+# Enable the autopilot for automatically banning IP addresses of the desired countries (see also COUNTRIES option).
+# Only ban IP addresses with at least AUTOPILOT current connections. If the value is not specified or 0, don't
+# automatically ban IP addresses, but run in interactive mode.
+AUTOPILOT="0"
+
+# Defines the subnet size in bytes to be analyzed. Valid values are:
+# -  8 for class A networks (X.0.0.0/8)
+# - 16 for class B networks (X.X.0.0/16)
+# - 24 for class C networks (X.X.X.0/24)
+# - 32 for class D networks (X.X.X.X/32)
+# If not specified, run in interactive mode and prompt for the netmask size.
+NETMASK="8"
+
+# The country-codes to block as an array. Defaults to "CN" (China).
+#COUNTRIES=("CN" "HK" "TW")
+
+# Specify the JAIL to use for banning the IP addresses. Defaults to 'apache-auth'.
+#JAIL="apache-auth"
+
+# The desired port to monitor. Defaults to 443 (https).
+#PORT="443"

ddos-mitigator.sh

@@ -3,93 +3,172 @@
 #                                                                              #
 # Try and prevent apache overloads by banning IP addresses that have (too)     #
 # many open connections.                                                       #
-# This script uses netstat to determine the connections to the HTTPS port of   #
-# the host machine and provides automated whois information retrieval based on #
-# the address or the /24- /16- or /8-subnet thereof. Addresses (or subnets)    #
-# are presented to the user in order of descending connection count. For each  #
-# address (or subnet), the user can choose to ban or ignore it. Addresses (or  #
-# subnets) chosen to be banned will be blocked by the apache-badbots jail of   #
-# fail2ban.                                                                    #
+# This script uses ss to determine the connections to a configurable port      #
+# on the host machine and provides automated GeoIP information retrieval based #
+# the address or the /24-, /16- or /8-subnet thereof. A GeoIP city- or country #
+# database must be installed separately and is provided to the script via a    #
+# command line parameter.                                                      #
+# Addresses (or subnets) are presented to the user in order of descending      #
+# connection count. For each address (or subnet), the user can choose to ban   #
+# or ignore it. Addresses (or subnets) chosen to be banned will be blocked by  #
+# a configurable jail of fail2ban.                                             #
 # Author: Manuel Friedli, <manuel@fritteli.ch>                                 #
 # This script is licenced under the GNU General Public Licence, version 3 or   #
 # later.                                                                       #
 #                                                                              #
 ################################################################################
 
-# Set the host's own IP address. So far, only an IPv4 address is supported.
-MY_IP="94.199.214.20"
-# Set the desired port to monitor.
-MY_PORT="443"
+################################################################################
+#                                                                              #
+# Prerequisites:                                                               #
+#   - app-admin/sudo (`sudo`)                                                  #
+#   - dev-lang/python:3.8 (`python`)                                           #
+#   - net-analyzer/fail2ban (`fail2ban-client`)                                #
+#   - sys-apps/coreutils (`cut`, `id`, `sort`, `touch`, `tr`, `uniq`)          #
+#   - sys-apps/grep (`grep`)                                                   #
+#   - sys-apps/iproute2 (`ss`)                                                 #
+#   - sys-apps/moreutils (`sponge`)                                            #
+#   - sys-apps/util-linux (`getopt`)                                           #
+#                                                                              #
+################################################################################
 
-# After this point, no editing is required.
-# Define the files that will contain the addresses an subnets.
-fileraw="raw-http.txt"
-filtered="filtered-http.txt"
-file8="sorted-http-8.txt"
-file16="sorted-http-16.txt"
-file24="sorted-http-24.txt"
-file32="sorted-http-32.txt"
-# This file will contain the addresses to be banned.
-banlist="banlist.txt"
-# This file contains the output of the last invocation of whois
-whoisoutput="whois.txt"
+# Store the start time; this enables us to output the total runtime at the end.
+start="$(date +%s)"
+
+# Dependencies of this script. Simple array with the following structure:
+# (command package [...])
+dependencies=(
+	"sudo" "app-admin/sudo"
+	"python" "dev-lang/python:3.8"
+	"fail2ban-client" "net-analyzer/fail2ban"
+	"cut" "sys-apps/coreutils"
+	"id" "sys-apps/coreutils"
+	"sort" "sys-apps/coreutils"
+	"touch" "sys-apps/coreutils"
+	"tr" "sys-apps/coreutils"
+	"uniq" "sys-apps/coreutils"
+	"grep" "sys-apps/grep"
+	"ss" "sys-apps/iproute2"
+	"sponge" "sys-apps/moreutils"
+	"getopt" "sys-apps/util-linux"
+)
 
 # These suffixes must be appended to the respective addresses and subnets.
-ext8=".0.0.0/8"
-ext16=".0.0/16"
-ext24=".0/24"
-ext32="/32"
+suffix8="/8"
+suffix16="/16"
+suffix24="/24"
+suffix32="/32"
+ext8=".0.0.0"
+ext16=".0.0"
+ext24=".0"
+ext32=""
 
 # Define some constants to format the output in a colorful way.
-red="\033[38;2;255;0;43m"
-yellow="\033[38;2;255;204;0m"
-green="\033[38;2;0;179;89m"
-blue="\033[38;2;0;85;255m"
-bold="\033[1m"
-reset="\033[0m"
+red="$(printf '\033[38;2;255;0;43m')"
+yellow="$(printf '\033[38;2;255;204;0m')"
+green="$(printf '\033[38;2;0;179;89m')"
+blue="$(printf '\033[38;2;0;85;255m')"
+bold="$(printf '\033[1m')"
+reset="$(printf '\033[0m')"
 
 # Clean up when the script exits.
-trap 'sudo -k; popd >/dev/null; rm -r ${tmpdir}' EXIT
+trap 'sudo -k 2>/dev/null >&2; rm -r ${tmpdir}' EXIT
 
-# Create a temp directory, chdir into it and create the (initially empty)
-# banlist file.
-tmpdir=$(mktemp -d)
-# Suppress output of dir stack.
-pushd "${tmpdir}" > /dev/null
-touch "${banlist}"
+function is_installed() {
+	which "${1}" 2>/dev/null >&2
+	return $?
+}
 
-function printHelp() {
+function print_missing_dependency() {
+	local command="${1}"
+	local package="${2}"
+
+	echo "${red}Command ${bold}${command}${reset}${red} not found.${reset} Please install package ${blue}${package}${reset}." >&2
+}
+
+function check_dependencies() {
+	local arraylength=${#dependencies[@]}
+	local res=
+	local command=
+	local package=
+	# 0: true, all installed; 1: false, at least one command/package missing
+	local all_installed=0
+
+	for ((i = 0; i < ${arraylength}; i += 2)); do
+		command="${dependencies[$i]}"
+		package="${dependencies[$i + 1]}"
+		is_installed "${command}" "${package}"
+		res=$?
+		if [[ ${res} -ne 0 ]]; then
+			print_missing_dependency "${command}" "${package}"
+			all_installed=1
+		fi
+	done
+
+	return ${all_installed}
+}
+
+function print_help() {
 	cat <<ENDOFHELP
-Usage: $(basename $0) [OPTION...]
+Usage: $(basename $0) -d FILE [OPTION...]
 
-  -a, --auto[=LIMIT]  Enable the autopilot for automatically banning chinese IP
-                      addresses (whois output must contain "source: APNIC" and
-                      "country: CN").
-                      When LIMIT is given, only auto-ban IP addresses with at
-                      least LIMIT current connections.
+  ${bold}Mandatory options:${reset}
+  -d, --database=FILE                 The path to the GeoIP2 database file (must
+                                      be either country or city database).
+
+  ${bold}Optional options:${reset}
+  -a, --auto[=LIMIT]                  Enable the autopilot for automatically
+                                      banning IP addresses of the desired
+                                      countries (see also -c option).
+                                      When LIMIT is given, only auto-ban IP
+                                      addresses with at least LIMIT current
+                                      connections.
                                       When LIMIT is omitted, assume LIMIT=1.
 
-  -j, --jail=JAIL     Specify the JAIL to use for banning the IP addresses. If
-                      not set, uses 'apache-auth'.
+  -c, --country=COUNTRY[,COUNTRY...]  The country-codes to block as a list of
+                                      comma-separated values; defaults to 'CN'
+                                      (China).
 
-  -n, --netmask=SIZE  SIZE defines the subnet size in bytes to be analyzed.
+  -e, --dependencies                  Check if all required dependencies are
+                                      installed. If all dependencies are found,
+                                      the program terminates with exit code 0.
+                                      Otherwise, missing dependencies are
+                                      printed to stderr and the program
+                                      terminates with exit code 1.
+
+  -f, --config-file=FILENAME          Specify the full path to the configuration
+                                      file. If omitted, all options are read
+                                      from the command line.
+                                      Any parameter specified on the command
+                                      line takes precedence over configuration
+                                      option read from the file.
+
+  -j, --jail=JAIL                     Specify the JAIL to use for banning the IP
+                                      addresses.
+                                      Defaults to 'apache-auth'.
+
+  -n, --netmask=SIZE                  SIZE defines the subnet size in bytes to
+                                      be analyzed.
                                       Valid values are:
                                       - 1 or 8 for class A networks (X.0.0.0/8)
                                       - 2 or 16 for class B networks (X.X.0.0/16)
                                       - 3 or 24 for class C networks (X.X.X.0/24)
                                       - 4 or 32 for class D networks (X.X.X.X/32)
 
+  -p, --port=PORT                     The desired port to monitor.
+                                      Defaults to 443 (https).
+
   -h, --help                          Show this help message
 
 Mandatory or optional arguments to long options are also mandatory or optional
 for any corresponding short options.
 
-When invoked without options, the autopilot is disabled and the netmask SIZE is
-inquired interactively.
+When invoked without optional options, the autopilot is disabled and the
+netmask SIZE is inquired interactively.
 ENDOFHELP
 }
 
-function execAsRoot() {
+function exec_as_root() {
 	if [[ $(id -un) == "root" ]]; then
 		"$@"
 	else
@@ -97,8 +176,70 @@ function execAsRoot() {
 	fi
 }
 
-function parseCommandline() {
-	TEMP=$(getopt -o 'a::,j:,n:,h' -l 'auto::,jail:,netmask:,help' -- "$@")
+function filter() {
+	# list of current connections
+	file="${1}"
+	# subnet extension, e.g. ".0.0"
+	ext="${2}"
+	# subnet suffix, e.g. "/16"
+	suffix="${3}"
+	rm -f "${filtered}"
+	touch "${filtered}"
+
+	# Reject already banned addresses
+	while read -r -u3 address; do
+		if [[ "${banned}" != *"${address}${ext}${suffix}"* ]]; then
+			echo "${address}" >>"${filtered}"
+		else
+			echo "IGNORING ${address}${ext}${suffix}, already banned."
+		fi
+	done 3<"${file}"
+
+	mv "${filtered}" "${file}"
+}
+
+function set_default_values() {
+	if [[ -z "${autopilot}" ]]; then
+		autopilot=0
+	fi
+	if [[ -z "${bancountries}" ]]; then
+		bancountries=("CN")
+	fi
+	if [[ -z "${jail}" ]]; then
+		jail="apache-auth"
+	fi
+	if [[ -z "${netmask}" ]]; then
+		netmask=0
+	fi
+	if [[ -z "${port}" ]]; then
+		port=443
+	fi
+}
+
+function parse_config_file() {
+	source "${configfile}"
+	if [[ -z "${autopilot}" ]]; then
+		autopilot="${AUTOPILOT}"
+	fi
+	if [[ -z "${bancountries}" ]]; then
+		bancountries=(${COUNTRIES[@]})
+	fi
+	if [[ -z "${database}" ]]; then
+		database="${DATABASE_FILE}"
+	fi
+	if [[ -z "${jail}" ]]; then
+		jail="${JAIL}"
+	fi
+	if [[ -z "${netmask}" ]]; then
+		netmask="${NETMASK}"
+	fi
+	if [[ -z "${port}" ]]; then
+		port="${PORT}"
+	fi
+}
+
+function parse_command_line_args() {
+	TEMP=$(getopt -o 'a::,c:,d:,e,f:,j:,n:,p:,h' -l 'auto::,country:,database:,dependencies,config-file:,jail:,netmask:,port:,help' -- "$@")
 
 	if [ $? -ne 0 ]; then
 		echo 'Error parsing command line options. Terminating. Invoke with --help for help.' >&2
@@ -109,49 +250,64 @@ function parseCommandline() {
 	unset TEMP
 
 	while true; do
-		case "$1" in
+		case "${1}" in
 		'-a' | '--auto')
-				case "$2" in
+			case "${2}" in
 			'')
 				autopilot=1
 				;;
-					*[!0-9]*)
-						echo "Invalid argument for parameter 'auto': '$2'. Invoke with --help for help." >&2
-						exit 1
-					;;
 			*)
 				autopilot="$2"
 				;;
 			esac
 			shift
 			;;
+		'-c' | '--country')
+			IFS=',' read -ra bancountries <<<"${2}"
+			shift
+			;;
+		'-d' | '--database')
+			database="${2}"
+			shift
+			;;
+		'-e' | '--dependencies')
+			check_dependencies
+			exit $?
+			;;
+		'-f' | '--config-file')
+			configfile="${2}"
+			shift
+			;;
 		'-j' | '--jail')
-				jail="$2"
+			jail="${2}"
 			shift
 			;;
 		'-n' | '--netmask')
-				case "$2" in
-					'1'|'8')
+			case "${2}" in
+			'1')
 				netmask=8
 				;;
-					'2'|'16')
+			'2')
 				netmask=16
 				;;
-					'3'|'24')
+			'3')
 				netmask=24
 				;;
-					'4'|'32')
+			'4')
 				netmask=32
 				;;
 			*)
-						echo "Invalid argument for parameter 'netmask': '$2'. Invoke with --help for help." >&2
-						exit 1
+				netmask="${2}"
 				;;
 			esac
 			shift
 			;;
+		'-p' | '--port')
+			port="${2}"
+			shift
+			;;
 		'-h' | '--help')
-				printHelp
+			print_help
 			exit
 			;;
 		'--')
@@ -159,60 +315,279 @@ function parseCommandline() {
 			break
 			;;
 		*)
-				echo "Unknown error on command line argument '$1'. Terminating." >&2
+			echo "Unknown error on command line argument '${1}'. Terminating." >&2
 			exit 1
 			;;
 		esac
 		shift
 	done
+
+	# If the config file option is set, parse the config file.
+	if [[ ! -z ${configfile+x} ]]; then
+		if [[ ! -f "${configfile}" || ! -r "${configfile}" ]]; then
+			echo "Can not read configuration file '${2}'. Invoke with --help for help." >&2
+			exit 1
+		fi
+
+		parse_config_file
+	fi
+
+	# Here, we set the default values for all options that have not been set yet.
+	set_default_values
 }
 
-# Parse the command line options
-autopilot=0
-netmask=0
-jail="apache-auth"
+function validate_parameter_values() {
+	# GeoIP-Database
+	if [[ -z "${database}" ]]; then
+		echo "No GeoIP database specified. Invoke with --help for more information." >&2
+		exit 1
+	fi
 
-parseCommandline "$@"
+	if [[ ! -f "${database}" || ! -r "${database}" ]]; then
+		echo "Database '${database}' is not accessible." >&2
+		exit 1
+	fi
+
+	# Autopilot
+	case "${autopilot}" in
+	*[!0-9]*)
+		echo "Invalid value for parameter 'auto' / 'AUTOPILOT': '${autopilot}'." >&2
+		echo "Invoke with --help for help." >&2
+		exit 1
+		;;
+	esac
+
+	# Countries
+	if [[ -z ${bancountries[@]// /} ]]; then
+		echo "Invalid value for parameter 'country' / 'COUNTRIES': '${bancountries[*]}'." >&2
+		echo "Invoke with --help for help." >&2
+		exit 1
+	fi
+
+	# Jail
+	if [[ -z "${jail}" ]]; then
+		echo "Invalid value for parameter 'jail' / 'JAIL': '${jail}'." >&2
+		echo "Invoke with --help for help." >&2
+		exit 1
+	fi
+
+	# Netmask
+	case "${netmask}" in
+	'0' | '8' | '16' | '24' | '32')
+		# Everything OK.
+		;;
+	*)
+		echo "Invalid value for parameter 'netmask': '${2}'." >&2
+		echo "Invoke with --help for help." >&2
+		exit 1
+		;;
+	esac
+
+	# Port
+	case "${port}" in
+	*[!0-9]*)
+		echo "Invalid value for parameter 'port' / 'PORT': '${autopilot}'." >&2
+		echo "Invoke with --help for help." >&2
+		exit 1
+		;;
+	esac
+	if [[ ${port} -lt 0 || ${port} -gt 65535 ]]; then
+		echo "Invalid value for parameter 'port' / 'PORT': '${autopilot}'." >&2
+		echo "Value must be between 0 ... 65535 (inclusive)." >&2
+		echo "Invoke with --help for help." >&2
+		exit 1
+	fi
+}
+
+################################################################################
+# Set the highlighting color for the address count. The color goes from green
+# (low count) over yellow (intermediate count) to red (high count). Depending
+# on the size of the subnet (/8, /16, /24 or /32), the transitions from one
+# color to the next happen at different values.
+################################################################################
+function set_highlight_color() {
+	local count=${1}
+	case "${choice}" in
+	"1")
+		# /32: 0 <= green < 3 <= yellow < 5 <= red
+		if [ ${count} -ge 5 ]; then
+			hilite="${red}"
+		elif [ ${count} -ge 3 ]; then
+			hilite="${yellow}"
+		else
+			hilite="${green}"
+		fi
+		;;
+	"2")
+		# /24: 0 <= green < 7 <= yellow < 13 <= red
+		if [ ${count} -ge 13 ]; then
+			hilite="${red}"
+		elif [ ${count} -ge 7 ]; then
+			hilite="${yellow}"
+		else
+			hilite="${green}"
+		fi
+		;;
+	"3")
+		# /16: 0 <= green < 13 <= yellow < 25 <= red
+		if [ ${count} -ge 25 ]; then
+			hilite="${red}"
+		elif [ ${count} -ge 13 ]; then
+			hilite="${yellow}"
+		else
+			hilite="${green}"
+		fi
+		;;
+	"4")
+		# /8: 0 <= green < 21 <= yellow < 49 <= red
+		if [ ${count} -ge 49 ]; then
+			hilite="${red}"
+		elif [ ${count} -ge 21 ]; then
+			hilite="${yellow}"
+		else
+			hilite="${green}"
+		fi
+		;;
+	*)
+		# ???: We should never get here. As a fall-back, just use no
+		# highlighting.
+		hilite=""
+		;;
+	esac
+}
+
+################################################################################
+# Process the file denoted by $1. For each line in the file, the count and the
+# address are displayed and a lookup for the IP addresses country is made in the
+# GeoIP database. The user can then choose to ban or ignore the address.
+# Addresses chosen to be banned are appended to the $banlist.
+################################################################################
+function process_file() {
+	local file="${1}"
+	local line=''
+	local count=0
+	local addronly=''
+	local addrwithsuffix=''
+	local banaction=''
+	local nline=1
+	local country=
+	# Read the contents from filedescriptor 3 (important: Don's use the
+	# standard filedescriptor because we need to handle user input from
+	# within the loop).
+	while IFS= read -r -u3 line; do
+		line="$(echo "${line}" | tr -s '[:blank:]')"
+		count="$(echo "${line}" | cut -d' ' -f2)"
+		addronly="$(echo "${line}" | cut -d' ' -f3-)${ext}"
+		addrwithsuffix="${addronly}${suffix}"
+		set_highlight_color "${count}"
+		country="$("${curdir}/geoip-lookup.py" -f "${database}" "${addronly}")"
+		if [[ ${autopilot} -eq 0 ]]; then
+			echo "Country: '${yellow}${country}${reset}'"
+		fi
+		echo -n "Address ${bold}$((nline++)) of ${nlines}${reset}: \
+Found '${blue}${addrwithsuffix}${reset}' ${hilite}${count}${reset} times."
+
+		if [[ ${autopilot} -eq 0 ]]; then
+			echo -n " Ban [y/N/s=No, and skip remaining]? "
+			read banaction
+		else
+			if [[ " ${bancountries[@]} " =~ " ${country} " ]]; then
+				if [[ ${count} -ge ${autopilot} ]]; then
+					echo -en "\n${red}Autopilot active. ${reset}"
+					banaction=y
+				else
+					echo -e "\n${yellow}Autopilot active. Ignoring remaining addresses due to limit of ${autopilot}.${reset}"
+					return
+				fi
+			else
+				if [[ ${count} -ge ${autopilot} ]]; then
+					echo -en "\n${green}Autopilot active. ${reset}"
+					banaction=n
+				else
+					echo -e "\n${green}Autopilot active.${reset} ${yellow}Ignoring remaining addresses due to limit of ${autopilot}.${reset}"
+					return
+				fi
+			fi
+		fi
+
+		case "${banaction}" in
+		"s" | "S")
+			echo "Not banning '${blue}${addrwithsuffix}${reset}', \
+skipping remaining addresses."
+			return
+			;;
+		"y" | "Y")
+			echo "Adding '${blue}${addrwithsuffix}${reset}' to \
+banlist (country=${yellow}${country}${reset})."
+			echo "${addrwithsuffix}" >>"${banlist}"
+			;;
+		"n" | "N" | *)
+			echo "Not banning '${blue}${addrwithsuffix}${reset}' (country=${yellow}${country}${reset})."
+			;;
+		esac
+		# Here goes: Pipe the file contents via filedescriptor 3.
+	done 3<"${file}"
+	echo "Processed all entries in ${file}."
+}
+
+# Create a temp directory, chdir into it and create the (initially empty)
+# banlist file.
+tmpdir="$(mktemp -d)"
+
+# Set up all file paths
+curdir="$(dirname "$0")"
+# Define the files that will contain the addresses an subnets.
+fileraw="${tmpdir}/raw-http.txt"
+filtered="${tmpdir}/filtered-http.txt"
+file8="${tmpdir}/sorted-http-8.txt"
+file16="${tmpdir}/sorted-http-16.txt"
+file24="${tmpdir}/sorted-http-24.txt"
+file32="${tmpdir}/sorted-http-32.txt"
+# This file will contain the addresses to be banned.
+banlist="${tmpdir}/banlist.txt"
+
+touch "${banlist}"
+
+# Parse the command line options
+autopilot=
+netmask=
+jail=
+bancountries=
+database=
+port=
+
+parse_command_line_args "$@"
+validate_parameter_values
+
+check_dependencies
+dependencies_ok=$?
+if [[ ${dependencies_ok} -ne 0 ]]; then
+	exit ${dependencies_ok}
+fi
 
 # List already banned addresses in the chosen jail
-banned="$(execAsRoot fail2ban-client get "${jail}" banip)"
+banned="$(exec_as_root fail2ban-client get "${jail}" banip)"
 
 # Determine the current connections to the desired port; store the raw data in
 # $fileraw.
-netstat -nt | grep "${MY_IP}:${MY_PORT}" | tr -s '[:blank:]' | cut -d' ' -f5 \
-  | cut -d: -f1 | sort > "${fileraw}"
+connections=$(ss -HOn state established "( sport = :${port} )" | tr -s '[:blank:]' | cut -d' ' -f5)
+
+# IPv6-mapped-IPv4: [::ffff:192.168.0.1]:443
+echo "${connections}" | grep '^\[::ffff:' - | cut -d: -f4 | cut -d] -f1 | grep -v '^$' >"${fileraw}"
+# Pure IPv4: 192.168.0.1:443
+echo "${connections}" | grep -v '^\[' - | cut -d: -f1 | grep -v '^$' >>"${fileraw}"
 
 # Group and sort the data into the subnet-specific files.
-cp "${fileraw}" "${file32}"
+sort "${fileraw}" >"${file32}"
 cut -d. -f1-3 "${fileraw}" | sort >"${file24}"
 cut -d. -f1-2 "${fileraw}" | sort >"${file16}"
 cut -d. -f1 "${fileraw}" | sort >"${file8}"
 
-function filter() {
-	# list of current connections
-	file="$1"
-	# subnet extension, e.g. ".0.0/16"
-	ext="$2"
-	rm -f "${filtered}"
-
-	# Reject already banned addresses
-	while read -r -u3 address ; do
-		if [[ "${banned}" != *"${address}${ext}"* ]] ; then
-			echo "Considering ${address}."
-			echo "${address}" >> "${filtered}"
-		else
-			echo "IGNORING ${address}, already banned."
-		fi
-	done 3< "${file}"
-
-	mv "${filtered}" "${file}"
-}
-
 # Filter already banned addresses
-filter "${file32}" "${ext32}"
-filter "${file24}" "${ext24}"
-filter "${file16}" "${ext16}"
-filter "${file8}" "${ext8}"
+filter "${file32}" "${ext32}" "${suffix32}"
+filter "${file24}" "${ext24}" "${suffix24}"
+filter "${file16}" "${ext16}" "${suffix16}"
+filter "${file8}" "${ext8}" "${suffix8}"
 
 # Determine the number of connections per address
 uniq -c "${file32}" | sort -rn | sponge "${file32}"
@@ -267,149 +642,16 @@ TEMP="file${netmask}"
 file="${!TEMP}"
 TEMP="ext${netmask}"
 ext="${!TEMP}"
+TEMP="suffix${netmask}"
+suffix="${!TEMP}"
 TEMP="nlines${netmask}"
 nlines="${!TEMP}"
 unset TEMP
 
 echo "Processing ${file}."
 
-################################################################################
-# Set the highlighting color for the address count. The color goes from green
-# (low count) over yellow (intermediate count) to red (high count). Depending
-# on the size of the subnet (/8, /16, /24 or /32), the transitions from one
-# color to the next happen at different values.
-################################################################################
-function setHilite() {
-	local count=$1
-	case "${choice}" in
-		"1" )
-		# /32: 0 <= green < 3 <= yellow < 5 <= red
-			if [ $count -ge 5 ] ; then
-				hilite="${red}"
-			elif [ $count -ge 3 ] ; then
-				hilite="${yellow}"
-			else
-				hilite="${green}"
-			fi
-		;;
-		"2" )
-		# /24: 0 <= green < 7 <= yellow < 13 <= red
-			if [ $count -ge 13 ] ; then
-				hilite="${red}"
-			elif [ $count -ge 7 ] ; then
-				hilite="${yellow}"
-			else
-				hilite="${green}"
-			fi
-		;;
-		"3" )
-		# /16: 0 <= green < 13 <= yellow < 25 <= red
-			if [ $count -ge 25 ] ; then
-				hilite="${red}"
-			elif [ $count -ge 13 ] ; then
-				hilite="${yellow}"
-			else
-				hilite="${green}"
-			fi
-		;;
-		"4" )
-		# /8: 0 <= green < 21 <= yellow < 49 <= red
-			if [ $count -ge 49 ] ; then
-				hilite="${red}"
-			elif [ $count -ge 21 ] ; then
-				hilite="${yellow}"
-			else
-				hilite="${green}"
-			fi
-		;;
-		* )
-		# ???: We should never get here. As a fall-back, just use no
-		# highlighting.
-			hilite=""
-		;;
-	esac
-}
-
-################################################################################
-# Process the file denoted by $1. For each line in the file, the count and the
-# address are displayed and a whois request is made. The user can then choose to
-# ban or ignore the address. Addresses chosen to be banned are appended to the
-# $banlist.
-################################################################################
-function processFile () {
-	local file="${1}"
-	local line=''
-	local count=0
-	local addr=''
-	local banaction=''
-	local nline=1
-	local country_cn=1
-	local source_apnic=1
-	# Read the contents from filedescriptor 3 (important: Don's use the
-	# standard filedescriptor because we need to handle user input from
-	# within the loop).
-	while IFS= read -r -u3 line ; do
-		line="$(echo "${line}" | tr -s '[:blank:]')"
-		count="$(echo "${line}" | cut -d' ' -f2)"
-		addr="$(echo "${line}" | cut -d' ' -f3-)${ext}"
-		setHilite "${count}"
-		if [[ autopilot -eq 0 ]] ; then
-			whois "${addr}" | tee "${whoisoutput}"
-		else
-			whois "${addr}" > "${whoisoutput}"
-		fi
-		grep -iq "^country: *cn$" "${whoisoutput}"
-		country_cn=$?
-		grep -iq "^source: *apnic$" "${whoisoutput}"
-		source_apnic=$?
-		echo -en "Address ${bold}$((nline++)) of ${nlines}${reset}: \
-Found '${blue}${addr}${reset}' ${hilite}${count}${reset} times."
-
-		if [[ ${autopilot} -eq 0 ]] ; then
-			echo -en  "Ban [y/N/s=No, and skip remaining]? "
-			read banaction
-		else
-			if [[ ${country_cn} -eq 0 && ${source_apnic} -eq 0 ]] ; then
-				if [[ $count -ge $autopilot ]] ; then
-					echo -en "\n${red}Autopilot active. ${reset}"
-					banaction=y
-				else
-					echo -e "\n${yellow}Autopilot active. Ignoring remaining addresses due to limit of ${autopilot}.${reset}"
-					return
-				fi
-			else
-				if [[ $count -ge $autopilot ]] ; then
-					echo -en "\n${green}Autopilot active. ${reset}"
-					banaction=n
-				else
-					echo -e "\n${green}Autopilot active.${reset} ${yellow}Ignoring remaining addresses due to limit of ${autopilot}.${reset}"
-					return
-				fi
-			fi
-		fi
-
-		case "${banaction}" in
-			"s" | "S" )
-				echo -e "Not banning '${blue}${addr}${reset}', \
-skipping remaining addresses."
-				return
-			;;
-			"y" | "Y" )
-				echo -e "Adding '${blue}${addr}${reset}' to \
-banlist."
-				echo "${addr}" >> "${banlist}"
-			;;
-			"n" | "N" | * )
-				echo -e "Not banning '${blue}${addr}${reset}'."
-			;;
-		esac
-	# Here goes: Pipe the file contents via filedescriptor 3.
-	done 3< "${file}"
-	echo "Processed all entries in ${file}."
-}
-
 # Invoke the processing function on the chosen file.
-processFile "${file}"
+process_file "${file}"
 
 echo "These are the addresses to be banned:"
 cat "${banlist}"
@@ -420,9 +662,11 @@ sudo -k
 
 # Iterate over all addresses in $banlist and invoke fail2ban-client on each
 # one of them.
-while read -r addr ; do
-	echo "Banning ${addr} ..."
-	execAsRoot fail2ban-client set "${jail}" banip "${addr}"
+while read -r addrwithsuffix; do
+	echo "Banning ${addrwithsuffix} ..."
+	exec_as_root fail2ban-client set "${jail}" banip "${addrwithsuffix}"
 done <"${banlist}"
 
-echo -e "${green}All done!${reset}"
+end="$(date +%s)"
+
+echo "${green}All done in $((end - start)) seconds!${reset}"

geoip-lookup.py

@@ -0,0 +1,107 @@
+#!/usr/bin/python
+
+import argparse
+import sys
+
+try:
+    import geoip2.database
+    import geoip2.errors
+except ImportError:
+    print(
+        "Required modules geoip2.database and geoip2.errors not found. On Gentoo Linux, please install dev-python/geoip2 from the 'fritteli' overlay.",
+        file=sys.stderr)
+    exit(1)
+
+try:
+    import maxminddb.errors
+except ImportError:
+    print(
+        "Required module maxminddb.errors not found. On Gentoo Linux, please install dev-python/maxminddb from the 'fritteli' overlay.",
+        file=sys.stderr)
+    exit(1)
+
+class LookupException(Exception):
+    """
+    General Exception class that is raised if anything goes wrong.
+    """
+    pass
+
+
+def get_county_code(ipaddress, dbfile):
+    """
+    Determine the country code that the given ipaddress comes from.
+    :param ipaddress: The IP address to look up
+    :param dbfile: The path to the GeoIP2/GeoLite2 database file (Country or City database)
+    :return: The ISO country code (2 letters)
+    """
+    if not ipaddress:
+        raise LookupException("No address given")
+    if not dbfile:
+        raise LookupException("No db file given")
+
+    reader = None
+    try:
+        reader = geoip2.database.Reader(dbfile)
+        dbtype = reader.metadata().database_type
+        country = None
+        if dbtype == 'GeoLite2-City' or dbtype == 'GeoIP2-City':
+            country = reader.city(ipaddress).country
+        elif dbfile == 'GeoLite2-Country' or dbtype == 'GeoIP2-Country':
+            country = reader.country(ipaddress).country
+        # ASN is not supported
+        # elif dbfile == 'GeoLite2-ASN' or dbtype == 'GeoIP2-ASN':
+
+        if not country:
+            raise LookupException("Unsupported DB type: " + dbtype)
+
+        return country.iso_code
+    except FileNotFoundError as e:
+        raise LookupException(e.args)
+    except maxminddb.errors.InvalidDatabaseError as e:
+        raise LookupException(e.args)
+    except geoip2.errors.AddressNotFoundError as e:
+        raise LookupException(e.args)
+    except ValueError as e:
+        raise LookupException(e.args)
+    finally:
+        if reader:
+            reader.close()
+
+
+def parse_command_line(argv):
+    """
+    Parse the command line. First, the database file must be specified ("-f /path/to/db/file.mmdb"), then the IP address
+    to look up
+    :param argv:
+    :return:
+    """
+    dbfile = None
+    parser = argparse.ArgumentParser(description='Get the country code from an IP address')
+    parser.add_argument('-f', dest='dbfile', required=True, help="Path to the GeoIP2 database file")
+    parser.add_argument('address', help="The IP address to check")
+    args = parser.parse_args()
+    
+    return args.dbfile, args.address
+
+def main(argv):
+    """
+    Read the database file and the IP address from the command line and print the corresponding ISO country code on
+    stdout.
+    :param argv: Format: "-f /path/to/database.mmdb ip.v4.add.ress"
+    :return:
+    """
+    try:
+        (dbfile, ipaddress) = parse_command_line(argv)
+        code = get_county_code(ipaddress, dbfile)
+        print(code)
+    except LookupException as e:
+        print(e.args, file=sys.stderr)
+        print("Unknown")
+
+
+if __name__ == '__main__':
+    try:
+        main(sys.argv[1:])
+    except BaseException as e:
+        print("Unknown")
+        raise e