[Kubernetes]: The kube server could be used as http-proxy for docker

build_debian.sh

@@ -245,7 +245,7 @@ fi
 ## Add docker config drop-in to specify dockerd command line
 sudo mkdir -p $FILESYSTEM_ROOT/etc/systemd/system/docker.service.d/
 ## Note: $_ means last argument of last command
-sudo cp files/docker/docker.service.conf $_
+sudo cp files/docker/*.conf $_
 ## Fix systemd race between docker and containerd
 sudo sed -i '/After=/s/$/ containerd.service/' $FILESYSTEM_ROOT/lib/systemd/system/docker.service
 

src/sonic-ctrmgrd/ctrmgr/ctrmgr_iptables.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+
+import ipaddress
+import os
+import re
+import socket
+import subprocess
+import syslog
+
+UNIT_TESTING = 0
+
+# NOTE:
+# Unable to use python-iptables as that does not create rules per ip-tables default
+# which is nf_tables. So rules added via iptc package will not be listed under
+# "sudo iptables -t nat -L -n". But available in kernel. To list, we need to 
+# use legacy mode as "sudo iptables-legacy -t nat -L -n".
+# As we can't use two modes and using non-default could make any debugging effort
+# very tough.
+
+
+from urllib.parse import urlparse
+
+DST_FILE = "/etc/systemd/system/docker.service.d/http_proxy.conf"
+DST_IP = None
+DST_PORT = None
+SQUID_PORT = "3128"
+
+def _get_ip(ip_str):
+    ret = ""
+    if ip_str:
+        try:
+            ipaddress.ip_address(ip_str)
+            ret = ip_str
+        except ValueError:
+            pass
+
+        if not ret:
+            try:
+                ret = socket.gethostbyname(ip_str)
+            except (OSError, socket.error):
+                pass
+        if not ret:
+            syslog.syslog(syslog.LOG_ERR, "{} is neither IP nor resolves to IP".
+                    format(ip_str))
+    return ret
+
+
+def _get_dst_info():
+    global DST_IP, DST_PORT
+    DST_IP = None
+    DST_PORT = None
+    print("DST_FILE={}".format(DST_FILE))
+    if os.path.exists(DST_FILE):
+        with open(DST_FILE, "r") as s:
+            for line in s.readlines():
+                url_match = re.search('^Environment=.HTTP_PROXY=(.+?)"', line)
+                if url_match:
+                    url = urlparse(url_match.group(1))
+                    DST_IP = _get_ip(url.hostname)
+                    DST_PORT = url.port
+                    break
+    else:
+        print("{} not available".format(DST_FILE))
+    print("DST_IP={}".format(DST_IP))
+
+
+def _is_rule_match(rule):
+    expect = "DNAT tcp -- 0.0.0.0/0 {} tcp dpt:{} to:".format(
+            DST_IP, DST_PORT)
+
+    # Remove duplicate spaces
+    rule = " ".join(rule.split()).strip()
+
+    if rule.startswith(expect):
+        return rule[len(expect):]
+    else:
+        return ""
+
+
+def check_proc(proc):
+    if proc.returncode:
+        syslog.syslog(syslog.LOG_ERR, "Failed to run: cmd: {}".format(proc.args))
+        syslog.syslog(syslog.LOG_ERR, "Failed to run: stdout: {}".format(proc.stdout))
+        syslog.syslog(syslog.LOG_ERR, "Failed to run: stderr: {}".format(proc.stderr))
+        if not UNIT_TESTING:
+            assert False
+
+
+def iptable_proxy_rule_upd(ip_str, port = SQUID_PORT):
+    _get_dst_info()
+    if not DST_IP:
+        # There is no proxy in use. Bail out.
+        return ""
+
+    destination = ""
+    if ip_str:
+        upd_ip = _get_ip(ip_str)
+        if not upd_ip:
+            return ""
+        destination = "{}:{}".format(upd_ip, port)
+
+    found = False
+    num = 0
+
+    while True:
+        num += 1
+
+        cmd = "sudo iptables -t nat -n -L OUTPUT {}".format(num)
+        proc = subprocess.run(cmd, shell=True, capture_output=True)
+        check_proc(proc)
+
+        if not proc.stdout:
+            # No more rule
+            break
+
+        rule_dest = _is_rule_match(proc.stdout.decode("utf-8").strip())
+        if rule_dest:
+            if not found and destination and (rule_dest == destination):
+                found = True
+            else:
+                # Duplicate or different IP - delete it
+                cmd = "sudo iptables -t nat -D OUTPUT {}".format(num)
+                proc = subprocess.run(cmd, shell=True, capture_output=True)
+                check_proc(proc)
+                # Decrement number to accommodate deleted rule
+                num -= 1
+
+    if destination and not found:
+        cmd = "sudo iptables -t nat -A OUTPUT -p tcp -d {} --dport {} -j DNAT --to-destination {}".format(
+                DST_IP, DST_PORT, destination)
+        proc = subprocess.run(cmd, shell=True, capture_output=True)
+
+        check_proc(proc)
+
+    return destination

src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import argparse
 import datetime
 import inspect
 import json
@@ -8,6 +9,7 @@
 import syslog
 
 from collections import defaultdict
+from ctrmgr.ctrmgr_iptables import iptable_proxy_rule_upd
 
 from swsscommon import swsscommon
 from sonic_py_common import device_info
@@ -54,6 +56,7 @@
 KUBE_LABEL_SET_KEY = "SET"
 
 remote_connected = False
+use_k8s_master_as_docker_proxy = False
 
 dflt_cfg_ser = {
         CFG_SER_IP: "",
@@ -309,6 +312,9 @@ def __init__(self, server):
 
         self.start_time = datetime.datetime.now()
 
+        if use_k8s_master_as_docker_proxy:
+            iptable_proxy_rule_upd(self.cfg_server[CFG_SER_IP])
+
         if not self.st_server[ST_FEAT_UPDATE_TS]:
             # This is upon system start. Sleep 10m before join
             self.start_time += datetime.timedelta(
@@ -336,6 +342,9 @@ def on_config_update(self, key, op, data):
         log_debug("Received config update: {}".format(str(data)))
         self.cfg_server = cfg_data
 
+        if use_k8s_master_as_docker_proxy:
+            iptable_proxy_rule_upd(self.cfg_server[CFG_SER_IP])
+
         if self.pending:
             tnow = datetime.datetime.now()
             if tnow < self.start_time:
@@ -359,7 +368,7 @@ def handle_update(self):
 
         ip = self.cfg_server[CFG_SER_IP]
         disable = self.cfg_server[CFG_SER_DISABLE] != "false"
-        
+
         pre_state = dict(self.st_server)
         log_debug("server: handle_update: disable={} ip={}".format(disable, ip))
         if disable or not ip:
@@ -582,6 +591,14 @@ def update_node_labels(self):
 
 
 def main():
+    global use_k8s_master_as_docker_proxy
+
+    parser = argparse.ArgumentParser(description="ctrmgrd service")
+    parser.add_argument("-p", "--proxy", action='store_true',
+            help="Act as docker http-proxy", default=False)
+    args = parser.parse_args()
+    use_k8s_master_as_docker_proxy = args.proxy
+
     init()
     server = MainServer()
     RemoteServerHandler(server)

src/sonic-ctrmgrd/tests/ctrmgr_iptables_test.py

@@ -0,0 +1,178 @@
+import os
+import re
+import sys
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from . import common_test
+
+sys.path.append("ctrmgr")
+import ctrmgr_iptables
+
+
+PROXY_FILE="http_proxy.conf"
+
+test_data = {
+    "1": {
+        "ip": "10.10.20.20",
+        "port": "3128",
+        "pre_rules": [
+            "DNAT tcp -- 20.20.0.0/0 172.16.1.1 tcp dpt:8080 to:100.127.20.21:8080",
+            "DNAT tcp -- 0.0.0.0/0 172.16.1.1 tcp dpt:3128 to:11.11.11.11:8080",
+            "DNAT tcp -- 0.0.0.0/0 172.16.1.1 tcp dpt:3128 to:11.11.11.11:8080",
+            "DNAT tcp -- 0.0.0.0/0 172.16.1.1 tcp dpt:3128 to:11.11.11.11:8088",
+            "DNAT tcp -- 0.0.0.0/0 172.16.1.1 tcp dpt:3129 to:11.11.11.11:8088"
+        ],
+        "post_rules": [
+            "DNAT tcp -- 20.20.0.0/0 172.16.1.1 tcp dpt:8080 to:100.127.20.21:8080",
+            "DNAT tcp -- 0.0.0.0/0 172.16.1.1 tcp dpt:3129 to:11.11.11.11:8088",
+            "DNAT tcp -- 0.0.0.0/0 172.16.1.1 tcp dpt:3128 to:10.10.20.20:3128"
+            ],
+        "ret": "10.10.20.20:3128"
+    },
+    "2": {
+        "ip": "",
+        "port": "",
+        "pre_rules": [
+            "DNAT tcp -- 20.20.0.0/0 172.16.1.1 tcp dpt:8080 to:100.127.20.21:8080",
+            "DNAT tcp -- 0.0.0.0/0 172.16.1.1 tcp dpt:3128 to:11.11.11.11:8080",
+            "DNAT tcp -- 0.0.0.0/0 172.16.1.1 tcp dpt:3128 to:11.11.11.11:8080",
+            "DNAT tcp -- 0.0.0.0/0 172.16.1.1 tcp dpt:3128 to:11.11.11.11:8088"
+        ],
+        "post_rules": [
+            "DNAT tcp -- 20.20.0.0/0 172.16.1.1 tcp dpt:8080 to:100.127.20.21:8080"
+        ],
+        "ret": ""
+    },
+    "3": {
+        "ip": "www.google.com",
+        "port": "3128",
+        "pre_rules": [
+            "DNAT tcp -- 20.20.0.0/0 172.16.1.1 tcp dpt:8080 to:100.127.20.21:8080",
+            "DNAT tcp -- 0.0.0.0/0 172.16.1.1 tcp dpt:3128 to:11.11.11.11:8080"
+        ],
+        "post_rules": [
+            "DNAT tcp -- 20.20.0.0/0 172.16.1.1 tcp dpt:8080 to:100.127.20.21:8080",
+            "DNAT tcp -- 0.0.0.0/0 172.16.1.1 tcp dpt:3128 to:.*3128"
+        ]
+    },
+    "4": {
+        "ip": "www.google.comx",
+        "port": "3128",
+        "pre_rules": [
+            "DNAT tcp -- 20.20.0.0/0 172.16.1.1 tcp dpt:8080 to:100.127.20.21:8080",
+            "DNAT tcp -- 0.0.0.0/0 172.16.1.1 tcp dpt:3128 to:11.11.11.11:8080"
+        ],
+        "post_rules": [
+            "DNAT tcp -- 20.20.0.0/0 172.16.1.1 tcp dpt:8080 to:100.127.20.21:8080",
+            "DNAT tcp -- 0.0.0.0/0 172.16.1.1 tcp dpt:3128 to:11.11.11.11:8080"
+        ],
+        "ret": ""
+    },
+    "5": {
+        "ip": "www.google.comx",
+        "port": "3128",
+        "conf_file": "no_proxy.conf",
+        "pre_rules": [
+            "DNAT tcp -- 20.20.0.0/0 172.16.1.1 tcp dpt:8080 to:100.127.20.21:8080",
+            "DNAT tcp -- 0.0.0.0/0 172.16.1.1 tcp dpt:3128 to:11.11.11.11:8080"
+        ],
+        "post_rules": [
+            "DNAT tcp -- 20.20.0.0/0 172.16.1.1 tcp dpt:8080 to:100.127.20.21:8080",
+            "DNAT tcp -- 0.0.0.0/0 172.16.1.1 tcp dpt:3128 to:11.11.11.11:8080"
+        ],
+        "ret": ""
+    }
+}
+
+
+current_tc = None
+current_rules = None
+
+class proc:
+    returncode = 0
+    stdout = None
+    stderr = None
+
+    def __init__(self, ret, stdout, stderr):
+        self.returncode = ret
+        self.stdout = bytearray(stdout, 'utf-8')
+        self.stderr = bytearray(stderr, 'utf-8')
+        print("out={} err={}".format(stdout, stderr))
+
+
+def mock_subproc_run(cmd, shell, capture_output):
+    cmd_prefix = "sudo iptables -t nat "
+    list_cmd = "{}-n -L OUTPUT ".format(cmd_prefix)
+    del_cmd = "{}-D OUTPUT ".format(cmd_prefix)
+    ins_cmd = "{}-A OUTPUT -p tcp -d ".format(cmd_prefix)
+
+    assert shell
+
+    print("cmd={}".format(cmd))
+    if cmd.startswith(list_cmd):
+        num = int(cmd[len(list_cmd):])
+        out = current_rules[num] if len(current_rules) > num else ""
+        return proc(0, out, "")
+
+    if cmd.startswith(del_cmd):
+        num = int(cmd[len(del_cmd):])
+        if num >= len(current_rules):
+            print("delete num={} is greater than len={}".format(num, len(current_rules)))
+            print("current_rules = {}".format(current_rules))
+            assert False
+        del current_rules[num]
+        return proc(0, "", "")
+
+    if cmd.startswith(ins_cmd):
+        l = cmd.split()
+        assert len(l) == 16
+        rule = "DNAT tcp -- 0.0.0.0/0 {} tcp dpt:{} to:{}".format(l[9], l[11], l[-1])
+        current_rules.append(rule)
+        return proc(0, "", "")
+
+    print("unknown cmd: {}".format(cmd))
+    return None
+
+
+def match_rules(pattern_list, str_list):
+    if len(pattern_list) != len(str_list):
+        print("pattern len {} != given {}".format(
+            len(pattern_list), len(str_list)))
+        return False
+
+    for i in range(len(pattern_list)):
+        if not re.match(pattern_list[i], str_list[i]):
+            print("{}: {} != {}".format(i, pattern_list[i], str_list[i]))
+            return False
+    return True
+
+
+class TestIPTableUpdate(object):
+
+    @patch("ctrmgr_iptables.subprocess.run")
+    def test_table(self, mock_proc):
+        global current_rules, current_tc
+
+        mock_proc.side_effect = mock_subproc_run
+        for i, tc in test_data.items():
+            print("----- Test: {} Start ------------------".format(i))
+            current_tc = tc
+            current_rules = tc["pre_rules"].copy()
+
+            ctrmgr_iptables.DST_IP = ""
+            ctrmgr_iptables.DST_PORT = ""
+            ctrmgr_iptables.DST_FILE = os.path.join(
+                    os.path.dirname(os.path.realpath(__file__)),
+                    tc.get("conf_file", PROXY_FILE))
+            ret = ctrmgr_iptables.iptable_proxy_rule_upd(tc["ip"], tc["port"])
+            if "ret" in tc:
+                assert ret == tc["ret"]
+            if not match_rules(tc["post_rules"], current_rules):
+                print("current_rules={}".format(current_rules))
+                print("post_rules={}".format(tc["post_rules"]))
+                assert False
+            print("----- Test: {} End   ------------------".format(i))
+
+