From ca79af77e174677f7dda9e416099bc476fc55f37 Mon Sep 17 00:00:00 2001 From: isabelmsft Date: Mon, 26 Oct 2020 23:59:15 +0000 Subject: [PATCH 1/6] Initial commit on cleaned branch 2 from noha --- ansible/.k8s_ubuntu_vtb.swp | Bin 0 -> 12288 bytes ansible/doc/README.testbed.k8s.Setup.md | 45 +++++++++--- ansible/files/br1.xml | 15 ++++ ansible/host_vars/STR-ACS-VSERV-21.yml | 4 ++ ansible/{k8s-ubuntu => k8s_ubuntu} | 29 ++++---- ansible/k8s_ubuntu_vtb | 66 ++++++++++++++++++ .../k8s_master/tasks/init_master_leader.yml | 1 - ansible/roles/k8s_master/tasks/main.yml | 1 + ansible/roles/k8s_master/vars/main.yml | 2 +- ansible/roles/vm_set/tasks/start_k8s.yml | 2 + ansible/roles/vm_set/tasks/start_k8s_vm.yml | 10 +-- ansible/roles/vm_set/tasks/stop_k8s_vm.yml | 1 + .../vm_set/templates/network-config-v2.yml.j2 | 6 +- ansible/setup-k8s-management-network.sh | 57 +++++++++++++++ ansible/testbed-cli.sh | 6 +- 15 files changed, 209 insertions(+), 36 deletions(-) create mode 100644 ansible/.k8s_ubuntu_vtb.swp create mode 100644 ansible/files/br1.xml create mode 100644 ansible/host_vars/STR-ACS-VSERV-21.yml rename ansible/{k8s-ubuntu => k8s_ubuntu} (96%) create mode 100644 ansible/k8s_ubuntu_vtb create mode 100755 ansible/setup-k8s-management-network.sh diff --git a/ansible/.k8s_ubuntu_vtb.swp b/ansible/.k8s_ubuntu_vtb.swp new file mode 100644 index 0000000000000000000000000000000000000000..d60d8821bab233e164d303e216ace069e91be132 GIT binary patch literal 12288 zcmeI2zl+mg9EacD=vnX3!Nnn`;vh7?YHb_1LFEJo5w3R~4N053)+TL9-Ue@SIK)Bl z4>%MYTyzo~RNNhI5X4>T>f$D%i=Q`1drjz_RJt5|Uw8<8-#4H4eV_NuP!e0M-?}YT zWR;;TGge%DV%kej7k(`=)(%`z`_EFURvP6-v!QD0)={06ro}BKfDDiUGC&5%02v?yWPl8i z0Wxq54OmUa)=uDEKi>C?_y42+|9_Sk`vJa!Pv9+h1@^#0&<7^C11g{hE`oF5`zhQ9 z_Q8Ab4!i`Ii58MT-;4)YSIWWXpKY};l1=s~$-~ktOz&#+GYy&D}fDDiUGC&5%02v?yWZ?e{ zNKt87eZO~q$B?)`a3mI2Cf~rCtzN(lNmFILq{^zSsnw`MC-ii!ptmd|>F-$qcl#|lB z2R&-8ZLAlr-KZ66YwPub*uTuKMvupMUz44VhZW&*Mbs&piX=5!ZMW$=-T3f{G0o8O z&8@cU7}0?dKLcenZao=T-tokIMO%EJu!zv{lTc~&(8eQX+OmU+NyS}|C1xo~>X61& f ~/.password` - - `k8s-server-name` corresponds to the group name used to describe the testbed server in the [`ansible/k8s-ubuntu`](../k8s-ubuntu) inventory file, of the form `k8s_server_{unit}`. +8. From `docker-sonic-mgmt` container, `cd` into `sonic-mgmt/ansible` directory and run `./testbed-cli.sh -m k8s_ubuntu [additional OPTIONS] create-master ~/.password` + - `k8s-server-name` corresponds to the group name used to describe the testbed server in the [`ansible/k8s_ubuntu`](../k8s_ubuntu) inventory file, of the form `k8s_server_{unit}`. - Please note: `~/.password` is the ansible vault password file name/path. Ansible allows users to use ansible-vault to encrypt password files. By default, this shell script requires a password file. If you are not using ansible-vault, just create an empty file and pass the file name to the command line. The file name and location are created and maintained by the user. - For HA Kubernetes master set 1 running on server 19 shown above, the proper command would be: -`./testbed-cli.sh -m k8s-ubuntu create-master k8s_server_19 ~/.password` +`./testbed-cli.sh -m k8s_ubuntu create-master k8s_server_19 ~/.password` - OPTIONAL: We offer the functionality to run multiple master sets on one server. - Each master set is one HA Kubernetes master composed of 4 Linux KVMs. - - Should an additional HA master set be necessary on an occupied server, add the option `-s `, where `msetnumber` would be 2 if this is the 2nd master set running on ``. Make sure that [`ansible/k8s-ubuntu`](../k8s-ubuntu) is updated accordingly. `msetnumber` is 1 by default. + - Should an additional HA master set be necessary on an occupied server, add the option `-s `, where `msetnumber` would be 2 if this is the 2nd master set running on ``. Make sure that [`ansible/k8s_ubuntu`](../k8s_ubuntu) is updated accordingly. Specifically, make sure that the IPS are set in the correct group `k8s_vms{msetnumber}_{servernumber}` and the `children` are properly updated for `k8s_server_{servernumber}` at the bottom of the inventory file. `msetnumber` is 1 by default. 9. Join Kubernetes-enabled SONiC DUT to cluster (kube_join function to be written). -#### To remove a HA Kubernetes master: -- Run `./testbed-cli.sh -m k8s-ubuntu [additional OPTIONS] destroy-master ~/.password` +#### To remove a HA Kubernetes master for Physical DUT: +- Run `./testbed-cli.sh -m k8s_ubuntu [additional OPTIONS] destroy-master ~/.password` - For HA Kubernetes master set 1 running on server 19 shown above, the proper command would be: -`./testbed-cli.sh -m k8s-ubuntu destroy-master k8s_server_19 ~/.password` +`./testbed-cli.sh -m k8s_ubuntu destroy-master k8s_server_19 ~/.password` + +## How to Setup High Availability Kubernetes Master for Virtual DUT + +#### To create a HA Kubernetes master for Virtual DUT: +1. Setup internal management network: +``` +$ git clone https://github.com/Azure/sonic-mgmt +$ cd sonic-mgmt/ansible +$ sudo ./setup-k8s-management-network.sh +``` +2. Setup virtual switch testbed as described [here](https://github.com/Azure/sonic-mgmt/blob/master/ansible/doc/README.testbed.VsSetup.md). Note that k8s internal management network is set up in step 1 above; do NOT use `setup-management-network.sh`. +3. In [`ansible/k8s_ubuntu_vtb`](../k8s_ubuntu_vtb), replace `use_own_value` with the username for the server, corresponds to the username used while setting up [`ansible/veos_vtb`](../veos_vtb) for the virtual switch testbed. +4. From inside the `sonic-mgmt` docker set up in step 2, run: +``` +$ cd /data/sonic-mgmt/ansible +$ ./testbed-cli.sh k8s_ubuntu_vtb create-master k8s_server_21 password.txt +``` +#### To remove a HA Kubernetes master for Virtual DUT: +``` +$ cd /data/sonic-mgmt/ansible +$ ./testbed-cli.sh k8s_ubuntu_vtb destroy-master k8s_server_21 password.txt +``` ## Testing Scope diff --git a/ansible/files/br1.xml b/ansible/files/br1.xml new file mode 100644 index 00000000000..250a39e8849 --- /dev/null +++ b/ansible/files/br1.xml @@ -0,0 +1,15 @@ + + br1 + + + + + + + + + + + + + diff --git a/ansible/host_vars/STR-ACS-VSERV-21.yml b/ansible/host_vars/STR-ACS-VSERV-21.yml new file mode 100644 index 00000000000..91f32497403 --- /dev/null +++ b/ansible/host_vars/STR-ACS-VSERV-21.yml @@ -0,0 +1,4 @@ +mgmt_bridge_k8s: br1 +mgmt_prefixlen_k8s: 24 +mgmt_gw_k8s: 10.250.0.1 + diff --git a/ansible/k8s-ubuntu b/ansible/k8s_ubuntu similarity index 96% rename from ansible/k8s-ubuntu rename to ansible/k8s_ubuntu index afdbb48aaba..d221b27d234 100644 --- a/ansible/k8s-ubuntu +++ b/ansible/k8s_ubuntu @@ -14,7 +14,20 @@ all: children: k8s_server_19: k8s_server_20: - + +k8s_server_19: + vars: + host_var_file: host_vars/STR-ACS-SERV-19.yml + children: + k8s_vm_host19: + k8s_vms1_19: + +k8s_server_20: + vars: + host_var_file: host_vars/STR-ACS-SERV-20.yml + children: + k8s_vm_host20: + k8s_vms1_20: k8s_vm_host19: hosts: @@ -98,18 +111,4 @@ k8s_vms2_20: ansible_host: 10.251.0.118 haproxy: true -# The groups below are helper to limit running playbooks to specific server(s) only -k8s_server_19: - vars: - host_var_file: host_vars/STR-ACS-SERV-19.yml - children: - k8s_vm_host19: - k8s_vms1_19: - -k8s_server_20: - vars: - host_var_file: host_vars/STR-ACS-SERV-20.yml - children: - k8s_vm_host20: - k8s_vms1_20: diff --git a/ansible/k8s_ubuntu_vtb b/ansible/k8s_ubuntu_vtb new file mode 100644 index 00000000000..6cd8bcf3eca --- /dev/null +++ b/ansible/k8s_ubuntu_vtb @@ -0,0 +1,66 @@ +all: + children: + k8s_vm_host: + children: + k8s_vm_host21: + k8s_ubu: + children: + k8s_vms1_21: + k8s_vms2_21: + k8s_servers: + children: + k8s_server_21: + +k8s_server_21: + vars: + host_var_file: host_vars/STR-ACS-VSERV-21.yml + children: + k8s_vm_host21: + k8s_vms1_21: + +k8s_vm_host21: + hosts: + STR-ACS-VSERV-21: + ansible_host: 172.17.0.1 + ansible_user: use_own_value + +k8s_vms1_21: + hosts: + kvm21-1m1: + ansible_host: 10.250.0.106 + master: true + master_leader: true + kvm21-1m2: + ansible_host: 10.250.0.107 + master: true + master_member: true + kvm21-1m3: + ansible_host: 10.250.0.108 + master_member: true + master: true + kvm21-1ha: + ansible_host: 10.250.0.109 + haproxy: true + +k8s_vms2_21: + hosts: + kvm21-2m1: + ansible_host: 10.250.0.110 + master: true + master_leader: true + kvm21-2m2: + ansible_host: 10.250.0.111 + master: true + master_member: true + kvm21-2m3: + ansible_host: 10.250.0.112 + master_member: true + master: true + kvm21-2ha: + ansible_host: 10.250.0.113 + haproxy: true + + + + + diff --git a/ansible/roles/k8s_master/tasks/init_master_leader.yml b/ansible/roles/k8s_master/tasks/init_master_leader.yml index ac1f83c6b3d..d5312bb7123 100644 --- a/ansible/roles/k8s_master/tasks/init_master_leader.yml +++ b/ansible/roles/k8s_master/tasks/init_master_leader.yml @@ -20,7 +20,6 @@ - name: Store logs and generated token for future reference local_action: copy content={{ output.stdout }} dest="./token" - - name: Get join command for other master nodes shell: kubeadm token create --print-join-command register: join_command diff --git a/ansible/roles/k8s_master/tasks/main.yml b/ansible/roles/k8s_master/tasks/main.yml index 5c880785839..dbf9684e905 100644 --- a/ansible/roles/k8s_master/tasks/main.yml +++ b/ansible/roles/k8s_master/tasks/main.yml @@ -51,6 +51,7 @@ template: src: 'docker-proxy.j2' dest: '/etc/systemd/system/docker.service.d/http-proxy.conf' + when: proxy_env['https_proxy'] is defined - name: Enable docker systemd service service: diff --git a/ansible/roles/k8s_master/vars/main.yml b/ansible/roles/k8s_master/vars/main.yml index 98f31cd5573..e51ca4fb1a9 100644 --- a/ansible/roles/k8s_master/vars/main.yml +++ b/ansible/roles/k8s_master/vars/main.yml @@ -1,2 +1,2 @@ --- -KUBERNETES_VERSION: 1.19.2-00 +KUBERNETES_VERSION: 1.19.2-00 \ No newline at end of file diff --git a/ansible/roles/vm_set/tasks/start_k8s.yml b/ansible/roles/vm_set/tasks/start_k8s.yml index fd87bd5b1b3..32164d7bd51 100644 --- a/ansible/roles/vm_set/tasks/start_k8s.yml +++ b/ansible/roles/vm_set/tasks/start_k8s.yml @@ -6,6 +6,7 @@ - name: Ensure {{ k8s_root_path }} exists file: path={{ k8s_root_path }} state=directory + become: yes - name: Find current server group set_fact: current_server={{ group_names | extract_by_prefix('k8s_server_') }} @@ -15,6 +16,7 @@ - name: Create directory for Ubuntu VM image file: path={{ k8s_root_path }}/images state=directory mode=0755 + become: yes - name: Check local cloud image stat: diff --git a/ansible/roles/vm_set/tasks/start_k8s_vm.yml b/ansible/roles/vm_set/tasks/start_k8s_vm.yml index 1594f1c5dd0..97c96010af1 100644 --- a/ansible/roles/vm_set/tasks/start_k8s_vm.yml +++ b/ansible/roles/vm_set/tasks/start_k8s_vm.yml @@ -1,5 +1,6 @@ - name: Create directory for Ubuntu VM disks - file: path={{ k8s_root_path }}/{{ vm_name }} state=directory mode=0755 + file: path={{ k8s_root_path }}/{{ vm_name }} state=directory mode=0777 + become: yes - name: Check destination file existence stat: path={{ disk_image }} get_checksum="no" @@ -41,14 +42,14 @@ --noautoconsole --name {{ vm_name }} --memory 4098 - --vcpus 2 + --vcpus 2 --disk {{ disk_image }},device=disk,bus=virtio --disk {{ k8s_root_path }}/{{ vm_name }}/{{ vm_name }}_cloud-config.img,device=cdrom --os-type linux --os-variant ubuntu18.04 --virt-type kvm --graphics none - --network bridge={{ mgmt_bridge_k8s }},model=virtio + --network bridge={{ mgmt_bridge_k8s }},model=virtio --import become: yes when: vm_name not in vm_list_defined.list_vms @@ -56,7 +57,8 @@ - name: "{{ vm_name }} start with host startup" virt: name: "{{ vm_name }}" - autostart: true + autostart: true + become: yes - name: "Wait for K8s Ubuntu VM to be accessible via SSH" wait_for: diff --git a/ansible/roles/vm_set/tasks/stop_k8s_vm.yml b/ansible/roles/vm_set/tasks/stop_k8s_vm.yml index 2fd4820f660..8d05aa4eded 100644 --- a/ansible/roles/vm_set/tasks/stop_k8s_vm.yml +++ b/ansible/roles/vm_set/tasks/stop_k8s_vm.yml @@ -14,4 +14,5 @@ - name: Remove VM disk images and config files for {{ vm_name }} file: path={{ vm_folder }} state=absent + become: yes diff --git a/ansible/roles/vm_set/templates/network-config-v2.yml.j2 b/ansible/roles/vm_set/templates/network-config-v2.yml.j2 index e0f3f6017b3..2e390bbfb9a 100644 --- a/ansible/roles/vm_set/templates/network-config-v2.yml.j2 +++ b/ansible/roles/vm_set/templates/network-config-v2.yml.j2 @@ -1,7 +1,11 @@ version: 2 ethernets: ens2: +{% if 'vtb' in inventory_file %} + dhcp4: true +{% else %} dhcp4: false +{% endif %} + dhcp4: true addresses: [{{ mgmt_ip_address }}/{{ mgmt_prefixlen_k8s }}, ] gateway4: {{ mgmt_gw }} - diff --git a/ansible/setup-k8s-management-network.sh b/ansible/setup-k8s-management-network.sh new file mode 100755 index 00000000000..579cd402e0e --- /dev/null +++ b/ansible/setup-k8s-management-network.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +if [[ $(id -u) -ne 0 ]]; then + echo "Root privelege required" + exit +fi + +echo "Checking for libvirt package..." +if ! command -v virsh; then + echo "libvirt not found, installing libvirt" + apt-get install -y libvirt-bin +fi +echo + +echo "Checking for bridge-utils package..." +if ! command -v brctl; then + echo "brctl not found, installing bridge-utils" + apt-get install -y bridge-utils +fi +echo + +echo "Checking for net-tools package..." +if ! command -v ifconfig; then + echo "ifconfig no t found, install net-tools" + apt-get install -y net-tools +fi +echo + +echo "Checking if br1 already exists..." +if ifconfig br1; then + echo "br1 already exists, delete and then rerun this script" + exit +fi +echo + +echo "Creating management bridge br1..." +virsh net-define files/br1.xml +virsh net-start br1 +virsh net-autostart br1 +echo + +echo "Enabling br1 forwarding..." +l=$(iptables -v -L | grep -n 'Chain FORWARD') +lowerbound="${l%%:*}" +sync=0 +iptables -v -L | grep -n 'REJECT.*br1' | while read -r line; do + upperbound=${line%%:*} + let index=$upperbound-$lowerbound-$sync-1 + iptables -D FORWARD $index + ((sync=sync+1)) +done + +echo "COMPLETE. Bridge info:" +echo +brctl show br1 +echo +ifconfig br1 diff --git a/ansible/testbed-cli.sh b/ansible/testbed-cli.sh index d7ac633a7ea..f6f2d64cedd 100755 --- a/ansible/testbed-cli.sh +++ b/ansible/testbed-cli.sh @@ -27,7 +27,7 @@ function usage echo " : Path to file containing Ansible Vault password" echo " : Name of the target topology" echo " : Name of the Ansible inventory containing the DUT" - echo " : Server identifier in form k8s_server_{id}, corresponds to k8s-ubuntu inventory group name" + echo " : Server identifier in form k8s_server_{id}, corresponds to k8s_ubuntu inventory group name" echo echo "To start all VMs on a server: $0 start-vms 'server-name' ~/.password" echo "To restart a subset of VMs:" @@ -55,8 +55,8 @@ function usage echo " -e enable_data_plane_acl=true" echo " -e enable_data_plane_acl=false" echo " by default, data acl is enabled" - echo "To create Kubernetes master on a server: $0 -m k8s-ubuntu create-master 'k8s-server-name' ~/.password" - echo "To destroy Kubernetes master on a server: $0 -m k8s-ubuntu destroy-master 'k8s-server-name' ~/.password" + echo "To create Kubernetes master on a server: $0 -m k8s_ubuntu create-master 'k8s-server-name' ~/.password" + echo "To destroy Kubernetes master on a server: $0 -m k8s_ubuntu destroy-master 'k8s-server-name' ~/.password" echo echo "You should define your topology in testbed CSV file" echo From dea5d900ddbbadfe50e6a357fd632a90abd06cc6 Mon Sep 17 00:00:00 2001 From: isabelmsft <67024108+isabelmsft@users.noreply.github.com> Date: Mon, 26 Oct 2020 23:30:17 -0500 Subject: [PATCH 2/6] Update README.testbed.k8s.Setup.md --- ansible/doc/README.testbed.k8s.Setup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/doc/README.testbed.k8s.Setup.md b/ansible/doc/README.testbed.k8s.Setup.md index 1854791957d..ed93e0d6692 100644 --- a/ansible/doc/README.testbed.k8s.Setup.md +++ b/ansible/doc/README.testbed.k8s.Setup.md @@ -143,7 +143,7 @@ $ sudo ./setup-k8s-management-network.sh 4. From inside the `sonic-mgmt` docker set up in step 2, run: ``` $ cd /data/sonic-mgmt/ansible -$ ./testbed-cli.sh k8s_ubuntu_vtb create-master k8s_server_21 password.txt +$ ./testbed-cli.sh -m k8s_ubuntu_vtb create-master k8s_server_21 password.txt ``` #### To remove a HA Kubernetes master for Virtual DUT: ``` From 86086b2b0f10d64d8a680f26d71d6539b1b351f6 Mon Sep 17 00:00:00 2001 From: isabelmsft Date: Tue, 27 Oct 2020 21:32:38 +0000 Subject: [PATCH 3/6] Remove hidden swp file --- ansible/.k8s_ubuntu_vtb.swp | Bin 12288 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 ansible/.k8s_ubuntu_vtb.swp diff --git a/ansible/.k8s_ubuntu_vtb.swp b/ansible/.k8s_ubuntu_vtb.swp deleted file mode 100644 index d60d8821bab233e164d303e216ace069e91be132..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2zl+mg9EacD=vnX3!Nnn`;vh7?YHb_1LFEJo5w3R~4N053)+TL9-Ue@SIK)Bl z4>%MYTyzo~RNNhI5X4>T>f$D%i=Q`1drjz_RJt5|Uw8<8-#4H4eV_NuP!e0M-?}YT zWR;;TGge%DV%kej7k(`=)(%`z`_EFURvP6-v!QD0)={06ro}BKfDDiUGC&5%02v?yWPl8i z0Wxq54OmUa)=uDEKi>C?_y42+|9_Sk`vJa!Pv9+h1@^#0&<7^C11g{hE`oF5`zhQ9 z_Q8Ab4!i`Ii58MT-;4)YSIWWXpKY};l1=s~$-~ktOz&#+GYy&D}fDDiUGC&5%02v?yWZ?e{ zNKt87eZO~q$B?)`a3mI2Cf~rCtzN(lNmFILq{^zSsnw`MC-ii!ptmd|>F-$qcl#|lB z2R&-8ZLAlr-KZ66YwPub*uTuKMvupMUz44VhZW&*Mbs&piX=5!ZMW$=-T3f{G0o8O z&8@cU7}0?dKLcenZao=T-tokIMO%EJu!zv{lTc~&(8eQX+OmU+NyS}|C1xo~>X61& f Date: Wed, 28 Oct 2020 05:08:07 +0000 Subject: [PATCH 4/6] Encompass row header offset into sync variable --- ansible/setup-k8s-management-network.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/setup-k8s-management-network.sh b/ansible/setup-k8s-management-network.sh index 579cd402e0e..23583fa0c5e 100755 --- a/ansible/setup-k8s-management-network.sh +++ b/ansible/setup-k8s-management-network.sh @@ -42,10 +42,10 @@ echo echo "Enabling br1 forwarding..." l=$(iptables -v -L | grep -n 'Chain FORWARD') lowerbound="${l%%:*}" -sync=0 +sync=1 iptables -v -L | grep -n 'REJECT.*br1' | while read -r line; do upperbound=${line%%:*} - let index=$upperbound-$lowerbound-$sync-1 + let index=$upperbound-$lowerbound-$sync iptables -D FORWARD $index ((sync=sync+1)) done From 9ec4c85b19ff38901bd9c8cd47941e316165c2ce Mon Sep 17 00:00:00 2001 From: isabelmsft Date: Wed, 28 Oct 2020 23:26:48 +0000 Subject: [PATCH 5/6] Simplify NAT and DNS setup --- ansible/doc/README.testbed.k8s.Setup.md | 8 ++- ansible/host_vars/STR-ACS-VSERV-21.yml | 1 + .../vm_set/templates/network-config-v2.yml.j2 | 9 ++- ansible/setup-br1-nat.sh | 18 ++++++ ansible/setup-k8s-management-network.sh | 57 ------------------- 5 files changed, 28 insertions(+), 65 deletions(-) create mode 100755 ansible/setup-br1-nat.sh delete mode 100755 ansible/setup-k8s-management-network.sh diff --git a/ansible/doc/README.testbed.k8s.Setup.md b/ansible/doc/README.testbed.k8s.Setup.md index ed93e0d6692..be6390ad4d3 100644 --- a/ansible/doc/README.testbed.k8s.Setup.md +++ b/ansible/doc/README.testbed.k8s.Setup.md @@ -136,11 +136,13 @@ k8s_server_19: ``` $ git clone https://github.com/Azure/sonic-mgmt $ cd sonic-mgmt/ansible -$ sudo ./setup-k8s-management-network.sh +$ sudo ./setup-management-network.sh +$ sudo ./setup-br1-nat.sh ``` -2. Setup virtual switch testbed as described [here](https://github.com/Azure/sonic-mgmt/blob/master/ansible/doc/README.testbed.VsSetup.md). Note that k8s internal management network is set up in step 1 above; do NOT use `setup-management-network.sh`. +2. Setup virtual switch testbed as described [here](https://github.com/Azure/sonic-mgmt/blob/master/ansible/doc/README.testbed.VsSetup.md). 3. In [`ansible/k8s_ubuntu_vtb`](../k8s_ubuntu_vtb), replace `use_own_value` with the username for the server, corresponds to the username used while setting up [`ansible/veos_vtb`](../veos_vtb) for the virtual switch testbed. -4. From inside the `sonic-mgmt` docker set up in step 2, run: +4. Specify DNS server IP to be used by Ubuntu KVMs in [`ansible/host_vars/STR-ACS-VSERV-21.yml`](../host_vars/STR-ACS-VSERV-21.yml); this should be the same DNS server IP as used by the testbed server. +5. From inside the `sonic-mgmt` docker set up in step 2, run: ``` $ cd /data/sonic-mgmt/ansible $ ./testbed-cli.sh -m k8s_ubuntu_vtb create-master k8s_server_21 password.txt diff --git a/ansible/host_vars/STR-ACS-VSERV-21.yml b/ansible/host_vars/STR-ACS-VSERV-21.yml index 91f32497403..13a190ea7d5 100644 --- a/ansible/host_vars/STR-ACS-VSERV-21.yml +++ b/ansible/host_vars/STR-ACS-VSERV-21.yml @@ -1,4 +1,5 @@ mgmt_bridge_k8s: br1 mgmt_prefixlen_k8s: 24 mgmt_gw_k8s: 10.250.0.1 +dns_server_ip: use_own_value diff --git a/ansible/roles/vm_set/templates/network-config-v2.yml.j2 b/ansible/roles/vm_set/templates/network-config-v2.yml.j2 index 2e390bbfb9a..4c3d6c73f95 100644 --- a/ansible/roles/vm_set/templates/network-config-v2.yml.j2 +++ b/ansible/roles/vm_set/templates/network-config-v2.yml.j2 @@ -1,11 +1,10 @@ version: 2 ethernets: ens2: -{% if 'vtb' in inventory_file %} - dhcp4: true -{% else %} dhcp4: false -{% endif %} - dhcp4: true addresses: [{{ mgmt_ip_address }}/{{ mgmt_prefixlen_k8s }}, ] gateway4: {{ mgmt_gw }} +{% if 'vtb' in inventory_file %} + nameservers: + addresses: [{{ dns_server_ip }}] +{% endif %} diff --git a/ansible/setup-br1-nat.sh b/ansible/setup-br1-nat.sh new file mode 100755 index 00000000000..9c52240934b --- /dev/null +++ b/ansible/setup-br1-nat.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +if [[ $(id -u) -ne 0 ]]; then + echo "Root privelege required" + exit +fi + +if [ $# -eq 0 ]; then + echo "Please specify server's external facing port name" + exit +fi + +echo "Setting up NAT..." +iptables -t nat -A POSTROUTING -s 10.250.0.0/24 -o $1 -j MASQUERADE +iptables -A FORWARD -i $1 -j ACCEPT +iptables -A FORWARD -i br1 -j ACCEPT +echo "Updated IP tables rules: " +iptables -v -L diff --git a/ansible/setup-k8s-management-network.sh b/ansible/setup-k8s-management-network.sh deleted file mode 100755 index 23583fa0c5e..00000000000 --- a/ansible/setup-k8s-management-network.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -if [[ $(id -u) -ne 0 ]]; then - echo "Root privelege required" - exit -fi - -echo "Checking for libvirt package..." -if ! command -v virsh; then - echo "libvirt not found, installing libvirt" - apt-get install -y libvirt-bin -fi -echo - -echo "Checking for bridge-utils package..." -if ! command -v brctl; then - echo "brctl not found, installing bridge-utils" - apt-get install -y bridge-utils -fi -echo - -echo "Checking for net-tools package..." -if ! command -v ifconfig; then - echo "ifconfig no t found, install net-tools" - apt-get install -y net-tools -fi -echo - -echo "Checking if br1 already exists..." -if ifconfig br1; then - echo "br1 already exists, delete and then rerun this script" - exit -fi -echo - -echo "Creating management bridge br1..." -virsh net-define files/br1.xml -virsh net-start br1 -virsh net-autostart br1 -echo - -echo "Enabling br1 forwarding..." -l=$(iptables -v -L | grep -n 'Chain FORWARD') -lowerbound="${l%%:*}" -sync=1 -iptables -v -L | grep -n 'REJECT.*br1' | while read -r line; do - upperbound=${line%%:*} - let index=$upperbound-$lowerbound-$sync - iptables -D FORWARD $index - ((sync=sync+1)) -done - -echo "COMPLETE. Bridge info:" -echo -brctl show br1 -echo -ifconfig br1 From 9b7ba4189f9e8ec037913268655b616b34efd0b7 Mon Sep 17 00:00:00 2001 From: isabelmsft Date: Wed, 28 Oct 2020 23:29:55 +0000 Subject: [PATCH 6/6] Remove now unnecessary libvirt br1 xml file --- ansible/files/br1.xml | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 ansible/files/br1.xml diff --git a/ansible/files/br1.xml b/ansible/files/br1.xml deleted file mode 100644 index 250a39e8849..00000000000 --- a/ansible/files/br1.xml +++ /dev/null @@ -1,15 +0,0 @@ - - br1 - - - - - - - - - - - - -