Merge pull request #16394 from omoerbeek/rec-pubsuffix-eod

rec: don't use a vector of string for internal pubsuffixlist

Author: omoerbeek

pdns/recursordist/mkpubsuffixcc

@@ -10,9 +10,9 @@ trap "rm -f $temp" 0 1 2 3 15
 set -e
 curl -o $temp -s -S https://publicsuffix.org/list/public_suffix_list.dat
 (echo "#include \"pubsuffix.hh\""
- echo "const std::vector<std::string> g_pubsuffix = {";
+ echo "const std::string g_pubsuffix = ";
 	for a in $(grep -v "//" "$temp" | grep \\. | egrep "^[.0-9a-z-]*$")
 	do
-		echo \"$a\",
+		echo \"$a\\\\n\"
 	done
-echo "};") > "$1"
+echo ";") > "$1"

pdns/recursordist/pubsuffix.hh

@@ -25,7 +25,7 @@
 #include <vector>
 
 extern std::vector<std::vector<std::string>> g_pubs;
-extern const std::vector<std::string> g_pubsuffix;
+extern const std::string g_pubsuffix;
 
 /* initialize the g_pubs variable with the public suffix list,
    using the file passed in parameter if any, or the built-in

pdns/recursordist/pubsuffixloader.cc

@@ -30,44 +30,58 @@
 
 std::vector<std::vector<std::string>> g_pubs;
 
-void initPublicSuffixList(const std::string& file)
+static bool initPublicSuffixList(const std::string& file, std::istream& stream, std::vector<std::vector<std::string>>& pbList)
 {
-  std::vector<std::vector<std::string>> pbList;
 
-  bool loaded = false;
-  if (!file.empty()) {
-    try {
-      Regex reg("^[.0-9a-z-]*$");
-      std::ifstream suffixFile(file);
-      if (!suffixFile.is_open()) {
-        throw std::runtime_error("Error opening the public suffix list file");
-      }
+  try {
+    Regex reg("^[.0-9a-z-]*$");
 
-      std::string line;
-      while (std::getline(suffixFile, line)) {
-        if (line.empty() || (line.rfind("//", 0) == 0)) {
-          /* skip empty and commented lines */
+    std::string line;
+    while (std::getline(stream, line)) {
+      if (line.empty() || (line.rfind("//", 0) == 0)) {
+        /* skip empty and commented lines */
+        continue;
+      }
+      try {
+        line = toLower(line);
+        if (!reg.match(line)) {
           continue;
         }
-        try {
-          line = toLower(line);
-          if (!reg.match(line)) {
-            continue;
-          }
-          DNSName name(toLower(line));
-          if (name.countLabels() < 2) {
-            continue;
-          }
-          pbList.emplace_back(name.labelReverse().getRawLabels());
-        }
-        catch (...) {
-          /* not a DNS name, ignoring */
+        DNSName name(line);
+        if (name.countLabels() < 2) {
           continue;
         }
+        pbList.emplace_back(name.labelReverse().getRawLabels());
+      }
+      catch (...) {
+        /* not a DNS name, ignoring */
+        continue;
       }
+    }
 
+    if (file != "internal") {
       g_slog->withName("runtime")->info(Logr::Info, "Loaded the Public Suffix List", "file", Logging::Loggable(file));
-      loaded = true;
+    }
+    return true;
+  }
+  catch (const std::exception& e) {
+    g_slog->withName("runtime")->error(Logr::Error, e.what(), "Error while loading the Public Suffix List", "file", Logging::Loggable(file));
+  }
+  return false;
+}
+
+void initPublicSuffixList(const std::string& file)
+{
+  bool loaded = false;
+  std::vector<std::vector<std::string>> pbList;
+
+  if (!file.empty()) {
+    try {
+      std::ifstream suffixFile(file);
+      if (!suffixFile.is_open()) {
+        throw std::runtime_error("Error opening the public suffix list file");
+      }
+      loaded = initPublicSuffixList(file, suffixFile, pbList);
     }
     catch (const std::exception& e) {
       g_slog->withName("runtime")->error(Logr::Error, e.what(), "Error while loading the Public Suffix List", "file", Logging::Loggable(file));
@@ -76,17 +90,9 @@ void initPublicSuffixList(const std::string& file)
 
   if (!loaded) {
     pbList.clear();
-
-    for (const auto& entry : g_pubsuffix) {
-      const auto low = toLower(entry);
-      std::vector<std::string> parts;
-      stringtok(parts, low, ".");
-      std::reverse(parts.begin(), parts.end());
-      parts.shrink_to_fit();
-      pbList.emplace_back(std::move(parts));
-    }
+    std::istringstream stream(g_pubsuffix);
+    initPublicSuffixList("internal", stream, pbList);
   }
-
   std::sort(pbList.begin(), pbList.end());
   g_pubs = std::move(pbList);
 }