Update to use GeoIP2 database.

The legacy databases we were using were discontinued
(https://support.maxmind.com/geolite-legacy-discontinuation-notice/), so
we need to use a different nginx plugin to integrate with the newer
GeoLite2 databases.

#440

Author: GUI

Taskfile.yml

@@ -15,9 +15,10 @@
 #   within the dependency chain. We use these stamp files so that the
 #   checksumming approach properly invalidates the entire chain of dependencies
 #   when build files change (whereas just using "deps" in go-task doesn't fully
-#   work for multiple levels of dependencies). So, for example, if the libgeoip
-#   dependency changes, rebuilding it properly cascades and triggers a rebuild
-#   of openresty (1st level dependency) and luarocks (2nd level dependency).
+#   work for multiple levels of dependencies). So, for example, if the
+#   libmaxminddb dependency changes, rebuilding it properly cascades and
+#   triggers a rebuild of openresty (1st level dependency) and luarocks (2nd
+#   level dependency).
 # - For colorized output we use the "unbuffer" tool to force color output on
 #   some commands, since Task doesn't easily support detecting colorized output
 #   (as of the output changes in 2.0.3):
@@ -91,14 +92,14 @@ tasks:
       - ./build/work/stamp/deps/libfastjson
     method: checksum
 
-  deps:libgeoip:
+  deps:libmaxminddb:
     cmds:
-      - ./tasks/deps/libgeoip
+      - ./tasks/deps/libmaxminddb
     sources:
-      - ./tasks/deps/libgeoip
+      - ./tasks/deps/libmaxminddb
       - ./tasks/helpers.sh
     generates:
-      - ./build/work/stamp/deps/libgeoip
+      - ./build/work/stamp/deps/libmaxminddb
     method: checksum
 
   deps:luarocks:
@@ -136,11 +137,11 @@ tasks:
 
   deps:openresty:
     deps:
-      - deps:libgeoip
+      - deps:libmaxminddb
     cmds:
       - ./tasks/deps/openresty
     sources:
-      - ./build/work/stamp/deps/libgeoip
+      - ./build/work/stamp/deps/libmaxminddb
       - ./tasks/deps/openresty
       - ./tasks/helpers.sh
     generates:
@@ -299,19 +300,6 @@ tasks:
       - ./build/work/stamp/app-deps/lua/cmsgpack
     method: checksum
 
-  app-deps:lua:iconv:
-    deps:
-      - deps:luarocks
-    cmds:
-      - ./tasks/app-deps/lua/iconv
-    sources:
-      - ./build/work/stamp/deps/luarocks
-      - ./tasks/app-deps/lua/iconv
-      - ./tasks/helpers.sh
-    generates:
-      - ./build/work/stamp/app-deps/lua/iconv
-    method: checksum
-
   app-deps:lua:icu-date:
     deps:
       - deps:luarocks
@@ -517,7 +505,6 @@ tasks:
       - task: app-deps:admin-ui:yarn
       - task: app-deps:lua:argparse
       - task: app-deps:lua:cmsgpack
-      - task: app-deps:lua:iconv
       - task: app-deps:lua:icu-date
       - task: app-deps:lua:inspect
       - task: app-deps:lua:libcidr-ffi

scripts/rake/outdated_packages.rb

@@ -36,18 +36,15 @@ class OutdatedPackages
     "libfastjson" => {
       :git => "https://github.com/rsyslog/libfastjson.git",
     },
-    "libgeoip" => {
-      :git => "https://github.com/maxmind/geoip-api-c.git",
+    "libmaxminddb" => {
+      :git => "https://github.com/maxmind/libmaxminddb.git",
     },
     "lua_argparse" => {
       :luarock => "argparse",
     },
     "lua_cmsgpack" => {
       :luarock => "lua-cmsgpack",
     },
-    "lua_iconv" => {
-      :luarock => "lua-iconv",
-    },
     "lua_icu_date" => {
       :git => "https://github.com/GUI/lua-icu-date.git",
       :git_ref => "master",
@@ -114,6 +111,9 @@ class OutdatedPackages
       :git => "https://github.com/emicklei/mora.git",
       :git_ref => "master",
     },
+    "ngx_http_geoip2_module" => {
+      :git => "https://github.com/leev/ngx_http_geoip2_module.git",
+    },
     "nodejs" => {
       :git => "https://github.com/nodejs/node.git",
       :constraint => "~> 10.13",

src/api-umbrella/cli/setup.lua

@@ -130,9 +130,9 @@ local function ensure_geoip_db()
   -- If the city db path doesn't exist, copy it from the package installation
   -- location to the runtime location (this path will then be overwritten by
   -- the auto-updater so we don't touch the original packaged file).
-  local city_db_path = path.join(config["db_dir"], "geoip/city-v6.dat")
+  local city_db_path = path.join(config["db_dir"], "geoip/GeoLite2-City.mmdb")
   if not path.exists(city_db_path) then
-    local default_city_db_path = path.join(config["_embedded_root_dir"], "var/db/geoip/city-v6.dat")
+    local default_city_db_path = path.join(config["_embedded_root_dir"], "var/db/geoip/GeoLite2-City.mmdb")
     dir.makepath(path.dirname(city_db_path))
     file.copy(default_city_db_path, city_db_path)
     chmod(city_db_path, tonumber("0640", 8))

src/api-umbrella/proxy/hooks/init_preload_modules.lua

@@ -56,7 +56,6 @@ require "api-umbrella.utils.round"
 require "api-umbrella.utils.xpcall_error_handler"
 require "cjson"
 require "cmsgpack"
-require "iconv"
 require "icu-date"
 require "libcidr-ffi"
 require "lustache"

src/api-umbrella/proxy/log_utils.lua

@@ -1,6 +1,5 @@
 local config = require "api-umbrella.proxy.models.file_config"
 local escape_uri_non_ascii = require "api-umbrella.utils.escape_uri_non_ascii"
-local iconv = require "iconv"
 local icu_date = require "icu-date"
 local json_encode = require "api-umbrella.utils.json_encode"
 local logger = require "resty.logger.socket"
@@ -232,34 +231,14 @@ function _M.cache_new_city_geocode(data)
 end
 
 function _M.set_request_ip_geo_fields(data, ngx_var)
-  -- The GeoIP module returns ISO-8859-1 encoded city names, but we need UTF-8
-  -- for inserting into ElasticSearch.
-  local geoip_city = ngx_var.geoip_city
-  if geoip_city then
-    local encoding_converter = iconv.new("utf-8//IGNORE", "iso-8859-1")
-    local geoip_city_encoding_err
-    geoip_city, geoip_city_encoding_err  = encoding_converter:iconv(geoip_city)
-    if geoip_city_encoding_err then
-      ngx.log(ngx.ERR, "encoding error for geoip city: ", geoip_city_encoding_err, geoip_city)
-    end
-  end
-
-  -- The geoip database returns "00" for unknown regions sometimes:
-  -- http://maxmind.com/download/geoip/kml/index.html Remove these and treat
-  -- these as nil.
-  local geoip_region = ngx_var.geoip_region
-  if geoip_region == "00" then
-    geoip_region = nil
-  end
-
-  data["request_ip_city"] = geoip_city
-  data["request_ip_country"] = ngx_var.geoip_city_country_code
-  data["request_ip_region"] = geoip_region
+  data["request_ip_city"] = ngx_var.geoip2_data_city_name
+  data["request_ip_country"] = ngx_var.geoip2_data_country_code
+  data["request_ip_region"] = ngx_var.geoip2_data_subdivision_code
 
-  local geoip_latitude = ngx_var.geoip_latitude
+  local geoip_latitude = ngx_var.geoip2_data_latitude
   if geoip_latitude then
     data["request_ip_lat"] = tonumber(geoip_latitude)
-    data["request_ip_lon"] = tonumber(ngx_var.geoip_longitude)
+    data["request_ip_lon"] = tonumber(ngx_var.geoip2_data_longitude)
   end
 end
 

tasks/app-deps/lua/iconv

@@ -7,15 +7,18 @@ set +x
 # If the file exists and was created within the past day, then skip doing
 # anything else. This approach ensures that new files are downloaded, but only
 # once per day.
-if [[ $(find "$STAGE_EMBEDDED_DIR/var/db/geoip/city-v6.dat" -mmin -1440) ]]; then
+if [[ $(find "$STAGE_EMBEDDED_DIR/var/db/geoip/GeoLite2-City.mmdb" -mmin -1440) ]]; then
   exit 0
 fi
 
 set -x
 
 task_working_dir
-curl --location --retry 3 --fail --remote-name https://geolite.maxmind.com/download/geoip/database/GeoLiteCityv6-beta/GeoLiteCityv6.dat.gz
-gunzip -c GeoLiteCityv6.dat.gz > GeoLiteCityv6.dat
-install -D -m 644 ./GeoLiteCityv6.dat "$STAGE_EMBEDDED_DIR/var/db/geoip/city-v6.dat"
+curl --location --retry 3 --fail --remote-name https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz
+dir="GeoLite2-City"
+rm -rf "$dir"
+mkdir -p "$dir"
+tar -xf GeoLite2-City.tar.gz -C "$dir" --strip-components 1
+install -D -m 644 ./GeoLite2-City/GeoLite2-City.mmdb "$STAGE_EMBEDDED_DIR/var/db/geoip/GeoLite2-City.mmdb"
 
 stamp

tasks/deps/geolitecity

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+libmaxminddb_version="1.3.2"
+libmaxminddb_hash="67a861965b30d045bf29a2126bcc05ed"
+
+set -e -u -x
+source ./tasks/helpers.sh
+
+task_working_dir
+download "https://github.com/maxmind/libmaxminddb/releases/download/$libmaxminddb_version/libmaxminddb-$libmaxminddb_version.tar.gz" "md5" "$libmaxminddb_hash"
+extract_download "libmaxminddb-$libmaxminddb_version.tar.gz"
+
+cd "libmaxminddb-$libmaxminddb_version"
+./configure --prefix="$INSTALL_PREFIX_EMBEDDED"
+make -j"$NPROC"
+make install DESTDIR="$STAGE_DIR"
+find "$STAGE_EMBEDDED_DIR/bin/" -name "mmdblookup" -exec chrpath -d {} \;
+
+stamp

tasks/deps/libgeoip

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+ngx_http_geoip2_module_version="3.2"
+ngx_http_geoip2_module_hash="2e60d26460d178068f7a4e9a4e64990a"
 openresty_version="1.13.6.2"
 openresty_hash="d95bc4bbe15e4b045a0593b4ecc0db38"
 openssl_version="1.0.2q"
@@ -21,6 +23,9 @@ extract_download "openssl-$openssl_version.tar.gz"
 download "https://openresty.org/download/openresty-$openresty_version.tar.gz" "md5" "$openresty_hash"
 extract_download "openresty-$openresty_version.tar.gz"
 
+download "https://github.com/leev/ngx_http_geoip2_module/archive/$ngx_http_geoip2_module_version.tar.gz" "md5" "$ngx_http_geoip2_module_hash"
+extract_download "$ngx_http_geoip2_module_version.tar.gz"
+
 cd "openresty-$openresty_version"
 patch -p1 < "$SOURCE_DIR/build/patches/openresty-opm.patch"
 patch -p1 < "$SOURCE_DIR/build/patches/openresty-cli.patch"
@@ -36,12 +41,12 @@ patch -p1 < "$SOURCE_DIR/build/patches/openresty-cli.patch"
   --with-pcre-opt=-g \
   --with-pcre-conf-opt=--enable-unicode-properties \
   --with-pcre-jit \
-  --with-http_geoip_module \
   --with-http_gunzip_module \
   --with-http_gzip_static_module \
   --with-http_realip_module \
   --with-http_ssl_module \
-  --with-http_stub_status_module
+  --with-http_stub_status_module \
+  --add-module="../ngx_http_geoip2_module-$ngx_http_geoip2_module_version"
 make -j"$NPROC"
 make install DESTDIR="$STAGE_DIR"
 chrpath -d "$STAGE_EMBEDDED_DIR/openresty/nginx/sbin/nginx"