diff --git a/.azure-pipelines/meta_validator.yml b/.azure-pipelines/meta_validator.yml index 4039d153375..df5e83fe1ab 100644 --- a/.azure-pipelines/meta_validator.yml +++ b/.azure-pipelines/meta_validator.yml @@ -58,10 +58,12 @@ validators: exclude_devices: - sonic-s6100-dut1 # E3002 Device has no console connection - sonic-s6100-dut2 # E3002 Device has no console connection + - vnut-.* # E3002 Virtual devices have no console connection - name: pdu enabled: true config: exclude_devices: - sonic-s6100-dut1 # E4002 Device has no PDU connection - sonic-s6100-dut2 # E4002 Device has no PDU connection + - vnut-.* # E4002 Virtual devices have no PDU connection issue_severities: {} diff --git a/ansible/files/sonic_lab_devices.csv b/ansible/files/sonic_lab_devices.csv index 7315ab286a2..7a6eb511d81 100644 --- a/ansible/files/sonic_lab_devices.csv +++ b/ansible/files/sonic_lab_devices.csv @@ -11,3 +11,7 @@ management-1,192.168.10.3/23,Sonic,MgmtTsToRRouter,,sonic switch01,192.168.0.100/24,SKU,DevSonic,,sonic switch01-bmc,192.168.0.101/24,bmc,DevSonic,,sonic switch01-bmc-con,192.168.0.101/24,bmc,ConsoleServer,ssh,sonic +vnut-t0-01,10.250.0.210/24,Force10-S6000,DevSonic,,sonic, +vnut-t0-02,10.250.0.211/24,Force10-S6000,DevSonic,,sonic, +vnut-t1-01,10.250.0.212/24,Force10-S6000,DevSonic,,sonic, +vnut-tg-01,10.250.0.220/24,IxiaChassis,DevIxiaChassis,,ixia, diff --git a/ansible/files/sonic_lab_links.csv b/ansible/files/sonic_lab_links.csv index bdea251f811..3af14f42092 100644 --- a/ansible/files/sonic_lab_links.csv +++ b/ansible/files/sonic_lab_links.csv @@ -33,3 +33,7 @@ str-msn2700-01,Ethernet120,str-7260-10,Ethernet31,40000,1711,Access,on str-msn2700-01,Ethernet124,str-7260-10,Ethernet32,40000,1712,Access,on str-7260-11,Ethernet19,str-acs-serv-01,p4p1,40000,,Trunk,on str-7260-11,Ethernet30,str-7260-10,Ethernet64,40000,1681-1712,Trunk,on +vnut-t0-01,Ethernet0,vnut-tg-01,Ethernet0,10000,,, +vnut-t0-01,Ethernet4,vnut-t1-01,Ethernet0,10000,,, +vnut-t0-02,Ethernet0,vnut-tg-01,Ethernet4,10000,,, +vnut-t0-02,Ethernet4,vnut-t1-01,Ethernet4,10000,,, diff --git a/ansible/lab b/ansible/lab index 35428f864be..059a0f38101 100644 --- a/ansible/lab +++ b/ansible/lab @@ -16,6 +16,7 @@ all: sonic_msft_sup: sonic_msft_lc_100G: sonic_cisco_vs: + sonic_vnut: fanout: hosts: str-7260-10: @@ -53,6 +54,8 @@ all: ansible_ssh_user: root ansible_ssh_pass: root hosts: + vnut-tg-01: + ansible_host: 10.250.0.220 ptf_ptf1: ansible_host: 10.255.0.188 ansible_hostv6: 2001:db8:1::1/64 @@ -284,3 +287,27 @@ sonic_cisco_vs: ansible_user: admin ansible_ssh_user: admin ansible_altpassword: admin + +sonic_vnut: + vars: + hwsku: Force10-S6000 + iface_speed: 10000 + hosts: + vnut-t0-01: + ansible_host: 10.250.0.210 + type: kvm + hwsku: Force10-S6000 + serial_port: 9100 + num_asics: 1 + vnut-t0-02: + ansible_host: 10.250.0.211 + type: kvm + hwsku: Force10-S6000 + serial_port: 9101 + num_asics: 1 + vnut-t1-01: + ansible_host: 10.250.0.212 + type: kvm + hwsku: Force10-S6000 + serial_port: 9102 + num_asics: 1 diff --git a/ansible/library/vnut_network.py b/ansible/library/vnut_network.py new file mode 100644 index 00000000000..2ae9e9e823e --- /dev/null +++ b/ansible/library/vnut_network.py @@ -0,0 +1,204 @@ +#!/usr/bin/python3 +""" +Ansible module to manage virtual network links (veth pairs) for NUT virtual testbed. + +Usage in Ansible: + - name: Connect container to management bridge + vnut_network: + action: connect_mgmt + device: "switch-t0-1" + mgmt_ip: "10.0.0.100/24" + mgmt_gateway: "10.0.0.1" + mgmt_bridge: "br-mgmt" + testbed_name: "nut-ci-1" + container_prefix: "net" + + - name: Create management bridge + vnut_network: + action: create_bridge + bridge_name: "br-mgmt" + bridge_ip: "10.0.0.1/24" +""" + +import hashlib +import subprocess + +from ansible.module_utils.basic import AnsibleModule + + +def run_cmd(cmd_args, check=True): + """Run a command and return (rc, stdout, stderr). + + Args: + cmd_args: List of command arguments (e.g. ["ip", "link", "show", "eth0"]). + check: If True, raise RuntimeError on non-zero exit code. + """ + result = subprocess.run(cmd_args, capture_output=True, text=True, timeout=60) + if check and result.returncode != 0: + raise RuntimeError( + "Command failed: {}\nstdout: {}\nstderr: {}".format( + " ".join(cmd_args), result.stdout.strip(), result.stderr.strip() + ) + ) + return result.returncode, result.stdout.strip(), result.stderr.strip() + + +def link_exists_on_host(link_name): + """Check if a network link exists on the host.""" + rc, _, _ = run_cmd(["ip", "link", "show", link_name], check=False) + return rc == 0 + + +def bridge_exists(bridge_name): + """Check if a bridge exists on the host.""" + rc, _, _ = run_cmd( + ["ip", "link", "show", "type", "bridge", "dev", bridge_name], check=False + ) + return rc == 0 + + +def get_container_pid(container_name): + """Get the PID of a running Docker container.""" + rc, stdout, stderr = run_cmd( + ["docker", "inspect", "-f", "{{.State.Pid}}", container_name], check=False + ) + if rc != 0: + raise RuntimeError( + "Container '{}' not found or not running: {}".format(container_name, stderr) + ) + pid = stdout.strip() + if pid == "0": + raise RuntimeError("Container '{}' is not running (PID=0)".format(container_name)) + return pid + + +def container_name(prefix, testbed, device): + """Build the Docker container name from components.""" + return "{}_{}_{}".format(prefix, testbed, device) + + +def interface_exists_in_ns(pid, iface_name): + """Check if an interface exists inside a container network namespace.""" + rc, _, _ = run_cmd( + ["nsenter", "-t", pid, "-n", "ip", "link", "show", iface_name], check=False + ) + return rc == 0 + + +def action_connect_mgmt(module): + """Connect a container to a management bridge.""" + p = module.params + device = p["device"] + mgmt_ip = p["mgmt_ip"] + mgmt_gw = p["mgmt_gateway"] + bridge = p["mgmt_bridge"] + testbed_name = p["testbed_name"] + prefix = p["container_prefix"] + + cname = container_name(prefix, testbed_name, device) + pid = get_container_pid(cname) + + # Idempotency: if eth0 already exists inside container, skip + if interface_exists_in_ns(pid, "eth0"): + module.exit_json( + changed=False, + msg="Management interface eth0 already exists in {}".format(cname), + ) + + short_id = hashlib.md5(cname.encode()).hexdigest()[:8] + veth_a = "vm{}a".format(short_id) # 12 chars, well under 15 + veth_b = "vm{}b".format(short_id) # 12 chars, well under 15 + + # Clean up if host-side veth exists + if link_exists_on_host(veth_a): + run_cmd(["ip", "link", "delete", veth_a]) + + # Create veth pair and move into container; clean up on failure + run_cmd(["ip", "link", "add", veth_a, "type", "veth", "peer", "name", veth_b]) + try: + # Move one end into container as eth0 + run_cmd(["ip", "link", "set", veth_a, "netns", pid]) + run_cmd(["nsenter", "-t", pid, "-n", "ip", "link", "set", veth_a, "name", "eth0"]) + run_cmd(["nsenter", "-t", pid, "-n", "ip", "addr", "add", mgmt_ip, "dev", "eth0"]) + run_cmd(["nsenter", "-t", pid, "-n", "ip", "link", "set", "eth0", "up"]) + run_cmd(["nsenter", "-t", pid, "-n", "ip", "route", "add", "default", "via", mgmt_gw]) + + # Attach host end to bridge + run_cmd(["ip", "link", "set", veth_b, "master", bridge]) + run_cmd(["ip", "link", "set", veth_b, "up"]) + except Exception: + # Clean up the veth pair to avoid dangling interfaces + for iface in (veth_a, veth_b): + try: + run_cmd(["ip", "link", "delete", iface]) + except Exception: + pass # Best-effort cleanup; re-raise original exception below + raise + + module.exit_json( + changed=True, + msg="Connected {} to bridge {} with IP {}".format(cname, bridge, mgmt_ip), + ) + + +def action_create_bridge(module): + """Create a Linux bridge with an IP address.""" + p = module.params + bridge_name = p["bridge_name"] + bridge_ip = p["bridge_ip"] + + if bridge_exists(bridge_name): + module.exit_json(changed=False, msg="Bridge {} already exists".format(bridge_name)) + + run_cmd(["ip", "link", "add", bridge_name, "type", "bridge"]) + run_cmd(["ip", "addr", "add", bridge_ip, "dev", bridge_name]) + run_cmd(["ip", "link", "set", bridge_name, "up"]) + + module.exit_json( + changed=True, + msg="Created bridge {} with IP {}".format(bridge_name, bridge_ip), + ) + + +def main(): + module = AnsibleModule( + argument_spec=dict( + action=dict( + type="str", + required=True, + choices=["connect_mgmt", "create_bridge"], + ), + # connect_mgmt params + device=dict(type="str"), + mgmt_ip=dict(type="str"), + mgmt_gateway=dict(type="str"), + mgmt_bridge=dict(type="str", default="br-mgmt"), + # create_bridge params + bridge_name=dict(type="str"), + bridge_ip=dict(type="str"), + # common params + testbed_name=dict(type="str"), + container_prefix=dict(type="str", default="net"), + ), + required_if=[ + ("action", "connect_mgmt", ["device", "mgmt_ip", "mgmt_gateway", "testbed_name"]), + ("action", "create_bridge", ["bridge_name", "bridge_ip"]), + ], + supports_check_mode=False, + ) + + action = module.params["action"] + + try: + if action == "connect_mgmt": + action_connect_mgmt(module) + elif action == "create_bridge": + action_create_bridge(module) + except RuntimeError as e: + module.fail_json(msg=str(e)) + except Exception as e: + module.fail_json(msg="Unexpected error: {}".format(e)) + + +if __name__ == "__main__": + main() diff --git a/ansible/roles/testbed/nut-vtopo-create/defaults/main.yml b/ansible/roles/testbed/nut-vtopo-create/defaults/main.yml new file mode 100644 index 00000000000..0f1a6d31ce2 --- /dev/null +++ b/ansible/roles/testbed/nut-vtopo-create/defaults/main.yml @@ -0,0 +1,22 @@ +--- +# Default Docker images (TG container only) +vnut_ptf_image: "docker-ptf:latest" + +# Naming prefix for TG Docker containers +vnut_container_prefix: "net" + +# KVM VM settings +vnut_vm_memory_gb: 4 +vnut_vm_vcpus: 2 +vnut_sonic_vm_storage: "{{ ansible_env.HOME }}/sonic-vm" +vnut_sonic_vs_image: "{{ vnut_sonic_vm_storage }}/images/sonic-vs.img" +vnut_vm_disk_dir: "{{ vnut_sonic_vm_storage }}/disks" +vnut_vm_serial_port_base: 9100 + +# Front-panel MTU +vnut_fp_mtu: 9100 + +# Management network +mgmt_bridge: "vnut_mgmt" +mgmt_gw: "10.250.0.1" +mgmt_prefixlen: "24" diff --git a/ansible/roles/testbed/nut-vtopo-create/library/kickstart.py b/ansible/roles/testbed/nut-vtopo-create/library/kickstart.py new file mode 120000 index 00000000000..eb69c9c9c84 --- /dev/null +++ b/ansible/roles/testbed/nut-vtopo-create/library/kickstart.py @@ -0,0 +1 @@ +../../../vm_set/library/kickstart.py \ No newline at end of file diff --git a/ansible/roles/testbed/nut-vtopo-create/library/sonic_kickstart.py b/ansible/roles/testbed/nut-vtopo-create/library/sonic_kickstart.py new file mode 120000 index 00000000000..6e46b6c065b --- /dev/null +++ b/ansible/roles/testbed/nut-vtopo-create/library/sonic_kickstart.py @@ -0,0 +1 @@ +../../../vm_set/library/sonic_kickstart.py \ No newline at end of file diff --git a/ansible/roles/testbed/nut-vtopo-create/tasks/connect_tg_links.yml b/ansible/roles/testbed/nut-vtopo-create/tasks/connect_tg_links.yml new file mode 100644 index 00000000000..ba0ed95b548 --- /dev/null +++ b/ansible/roles/testbed/nut-vtopo-create/tasks/connect_tg_links.yml @@ -0,0 +1,64 @@ +--- +# connect_tg_links.yml — Connect TG container interfaces to front-panel link bridges +# For each link involving a TG device, create a veth pair with one end in the +# TG container namespace and the other end attached to the link bridge. + +- name: Identify TG links + set_fact: + vnut_tg_links: >- + {{ (vnut_links | selectattr('StartDevice', 'in', testbed_tg_names) | list + + vnut_links | selectattr('EndDevice', 'in', testbed_tg_names) | list) | unique }} + +- name: Connect TG containers to link bridges + shell: | + {% for link in vnut_links %} + {% if link.StartDevice in testbed_tg_names %} + {# TG is the start device #} + BRIDGE="vbr_{{ testbed_name[:8] }}_{{ loop.index0 }}" + CONTAINER="{{ vnut_container_prefix }}_{{ testbed_name }}_{{ link.StartDevice }}" + PORT="{{ link.StartPort }}" + VETH_A="vtg{{ testbed_name[:6] }}_{{ loop.index0 }}a" + VETH_B="vtg{{ testbed_name[:6] }}_{{ loop.index0 }}b" + PID=$(docker inspect -f '{% raw %}{{.State.Pid}}{% endraw %}' "$CONTAINER") + + # Check if port already exists in container + if nsenter -t "$PID" -n ip link show "$PORT" >/dev/null 2>&1; then + echo "Port $PORT already exists in $CONTAINER" + else + ip link show "$VETH_A" 2>/dev/null || ip link add "$VETH_A" type veth peer name "$VETH_B" + ip link set "$VETH_A" master "$BRIDGE" + ip link set "$VETH_A" up + ip link set "$VETH_B" netns "$PID" + nsenter -t "$PID" -n ip link set "$VETH_B" name "$PORT" + nsenter -t "$PID" -n ip link set "$PORT" up + echo "Connected $CONTAINER:$PORT to $BRIDGE" + fi + + {% elif link.EndDevice in testbed_tg_names %} + {# TG is the end device #} + BRIDGE="vbr_{{ testbed_name[:8] }}_{{ loop.index0 }}" + CONTAINER="{{ vnut_container_prefix }}_{{ testbed_name }}_{{ link.EndDevice }}" + PORT="{{ link.EndPort }}" + VETH_A="vtg{{ testbed_name[:6] }}_{{ loop.index0 }}a" + VETH_B="vtg{{ testbed_name[:6] }}_{{ loop.index0 }}b" + PID=$(docker inspect -f '{% raw %}{{.State.Pid}}{% endraw %}' "$CONTAINER") + + if nsenter -t "$PID" -n ip link show "$PORT" >/dev/null 2>&1; then + echo "Port $PORT already exists in $CONTAINER" + else + ip link show "$VETH_A" 2>/dev/null || ip link add "$VETH_A" type veth peer name "$VETH_B" + ip link set "$VETH_A" master "$BRIDGE" + ip link set "$VETH_A" up + ip link set "$VETH_B" netns "$PID" + nsenter -t "$PID" -n ip link set "$VETH_B" name "$PORT" + nsenter -t "$PID" -n ip link set "$PORT" up + echo "Connected $CONTAINER:$PORT to $BRIDGE" + fi + + {% endif %} + {% endfor %} + args: + executable: /bin/bash + register: connect_tg_result + changed_when: "'Connected' in connect_tg_result.stdout" + when: vnut_tg_links | length > 0 diff --git a/ansible/roles/testbed/nut-vtopo-create/tasks/create_links.yml b/ansible/roles/testbed/nut-vtopo-create/tasks/create_links.yml new file mode 100644 index 00000000000..81353079a07 --- /dev/null +++ b/ansible/roles/testbed/nut-vtopo-create/tasks/create_links.yml @@ -0,0 +1,25 @@ +--- +# create_links.yml — Create Linux bridges for each front-panel link. +# KVM VM interfaces and TG container interfaces will attach to these bridges. +# Bridge naming: vbr__ (max 15 chars for Linux interface name) + +- name: Create front-panel link bridges + shell: | + BRIDGE="vbr_{{ testbed_name[:8] }}_{{ idx }}" + if ip link show "$BRIDGE" >/dev/null 2>&1; then + echo "Bridge $BRIDGE already exists" + else + ip link add "$BRIDGE" type bridge + ip link set "$BRIDGE" up + echo "Created bridge $BRIDGE" + fi + loop: "{{ vnut_links }}" + loop_control: + index_var: idx + label: "{{ item.StartDevice }}:{{ item.StartPort }} <-> {{ item.EndDevice }}:{{ item.EndPort }}" + register: bridge_results + changed_when: "'Created' in bridge_results.stdout" + +- name: Display link bridge creation summary + debug: + msg: "Created {{ vnut_links | length }} front-panel link bridges" diff --git a/ansible/roles/testbed/nut-vtopo-create/tasks/create_mgmt_network.yml b/ansible/roles/testbed/nut-vtopo-create/tasks/create_mgmt_network.yml new file mode 100644 index 00000000000..3def875e918 --- /dev/null +++ b/ansible/roles/testbed/nut-vtopo-create/tasks/create_mgmt_network.yml @@ -0,0 +1,64 @@ +--- +- name: Derive management subnet CIDR + set_fact: + mgmt_subnet_cidr: "{{ (mgmt_gw + '/' + mgmt_prefixlen | string) | ansible.utils.ipaddr('network/prefix') }}" + +- name: Find other bridges with conflicting IP on management subnet + shell: | + for br in $(ip -o addr show | grep ' {{ mgmt_gw }}/' | awk '{print $2}'); do + if [ "$br" != "{{ mgmt_bridge }}" ]; then + echo "$br" + fi + done + register: conflicting_bridges + changed_when: false + +- name: Remove conflicting bridges to avoid routing conflicts + shell: | + ip addr flush dev {{ item }} + ip link set {{ item }} down + ip link delete {{ item }} 2>/dev/null || true + loop: "{{ conflicting_bridges.stdout_lines }}" + when: conflicting_bridges.stdout_lines | length > 0 + +- name: Check if management bridge exists + command: "ip link show {{ mgmt_bridge }}" + register: bridge_check + failed_when: false + changed_when: false + +- name: Create management bridge (if not already present) + vnut_network: + action: create_bridge + bridge_name: "{{ mgmt_bridge }}" + bridge_ip: "{{ mgmt_gw }}/{{ mgmt_prefixlen }}" + when: bridge_check.rc != 0 + +- name: Enable IP forwarding + ansible.posix.sysctl: + name: net.ipv4.ip_forward + value: '1' + sysctl_set: yes + state: present + +- name: Check if NAT masquerade rule exists + shell: > + nsenter -t 1 -m -u -n -i -p -- iptables -t nat -C POSTROUTING -s {{ mgmt_subnet_cidr }} ! -d {{ mgmt_subnet_cidr }} -j MASQUERADE 2>/dev/null + register: nat_check + failed_when: false + changed_when: false + +- name: Set up NAT masquerade for management network + shell: > + nsenter -t 1 -m -u -n -i -p -- iptables -t nat -A POSTROUTING -s {{ mgmt_subnet_cidr }} ! -d {{ mgmt_subnet_cidr }} -j MASQUERADE + when: nat_check.rc != 0 + changed_when: nat_check.rc != 0 + +- name: Allow forwarding from management bridge + shell: | + changed=false + nsenter -t 1 -m -u -n -i -p -- iptables -C FORWARD -i {{ mgmt_bridge }} -j ACCEPT 2>/dev/null || { nsenter -t 1 -m -u -n -i -p -- iptables -I FORWARD -i {{ mgmt_bridge }} -j ACCEPT; changed=true; } + nsenter -t 1 -m -u -n -i -p -- iptables -C FORWARD -o {{ mgmt_bridge }} -m state --state RELATED,ESTABLISHED -j ACCEPT 2>/dev/null || { nsenter -t 1 -m -u -n -i -p -- iptables -I FORWARD -o {{ mgmt_bridge }} -m state --state RELATED,ESTABLISHED -j ACCEPT; changed=true; } + echo "changed=$changed" + register: fwd_result + changed_when: "'changed=true' in fwd_result.stdout" diff --git a/ansible/roles/testbed/nut-vtopo-create/tasks/launch_one_dut.yml b/ansible/roles/testbed/nut-vtopo-create/tasks/launch_one_dut.yml new file mode 100644 index 00000000000..81f5a39ff0f --- /dev/null +++ b/ansible/roles/testbed/nut-vtopo-create/tasks/launch_one_dut.yml @@ -0,0 +1,97 @@ +--- +# launch_one_dut.yml — Create and start a SONiC KVM VM for a DUT +# Called with: device (dict with Hostname, ManagementIp, HwSku, etc.), device_idx + +- name: "Set VM facts for {{ device.Hostname }}" + set_fact: + vm_name: "{{ device.Hostname }}" + disk_image: "{{ vnut_vm_disk_dir }}/sonic_{{ device.Hostname }}.img" + serial_port: "{{ vnut_vm_serial_port_base + device_idx }}" + +- name: "Map interfaces to bridges for {{ device.Hostname }}" + set_fact: + vm_interfaces: >- + {%- set ns = namespace(items=[]) -%} + {%- for link in vnut_links -%} + {%- if link.StartDevice == device.Hostname -%} + {%- set ns.items = ns.items + [{"name": link.StartPort, "bridge": "vbr_" ~ testbed_name[:8] ~ "_" ~ loop.index0}] -%} + {%- elif link.EndDevice == device.Hostname -%} + {%- set ns.items = ns.items + [{"name": link.EndPort, "bridge": "vbr_" ~ testbed_name[:8] ~ "_" ~ loop.index0}] -%} + {%- endif -%} + {%- endfor -%} + {{ ns.items }} + +- name: "Create VM disk directories" + file: + path: "{{ item }}" + state: directory + mode: '0755' + loop: + - "{{ vnut_sonic_vm_storage }}/images" + - "{{ vnut_vm_disk_dir }}" + +- name: "Check if disk image exists for {{ device.Hostname }}" + stat: + path: "{{ disk_image }}" + register: disk_stat + +- name: "Check source VS image exists" + stat: + path: "{{ vnut_sonic_vs_image }}" + register: src_image_stat + when: not disk_stat.stat.exists + +- name: "Fail if source VS image not found" + fail: + msg: >- + SONiC VS image not found at {{ vnut_sonic_vs_image }}. + Please download sonic-vs.img and place it there. + when: + - not disk_stat.stat.exists + - not (src_image_stat.stat.exists | default(false)) + +- name: "Copy sonic-vs.img for {{ device.Hostname }}" + copy: + src: "{{ vnut_sonic_vs_image }}" + dest: "{{ disk_image }}" + remote_src: yes + when: not disk_stat.stat.exists + +- name: "Get list of defined VMs" + virt: + command: list_vms + uri: qemu:///system + register: vm_list_defined + become: yes + +- name: "Define SONiC VM {{ device.Hostname }}" + virt: + name: "{{ vm_name }}" + command: define + xml: "{{ lookup('template', 'templates/vnut-sonic.xml.j2') }}" + uri: qemu:///system + when: vm_name not in vm_list_defined.list_vms + become: yes + +- name: "Get list of running VMs" + virt: + command: list_vms + state: running + uri: qemu:///system + register: vm_list_running + become: yes + +- name: "Start SONiC VM {{ device.Hostname }}" + virt: + name: "{{ vm_name }}" + state: running + uri: qemu:///system + when: vm_name not in (vm_list_running.list_vms | default([])) + become: yes + +- name: "Debug VM info for {{ device.Hostname }}" + debug: + msg: >- + VM {{ vm_name }}: serial_port={{ serial_port }}, + mgmt_ip={{ device.ManagementIp }}, disk={{ disk_image }}, + interfaces={{ vm_interfaces | length }} front-panel ports diff --git a/ansible/roles/testbed/nut-vtopo-create/tasks/launch_one_tg.yml b/ansible/roles/testbed/nut-vtopo-create/tasks/launch_one_tg.yml new file mode 100644 index 00000000000..1fe95b83192 --- /dev/null +++ b/ansible/roles/testbed/nut-vtopo-create/tasks/launch_one_tg.yml @@ -0,0 +1,18 @@ +--- +- name: "Create TG container for {{ device.Hostname }}" + community.docker.docker_container: + name: "{{ vnut_container_prefix }}_{{ testbed_name }}_{{ device.Hostname }}" + image: "{{ vnut_ptf_image }}" + privileged: yes + network_mode: "none" + state: started + +- name: "Connect {{ device.Hostname }} to management network" + vnut_network: + action: connect_mgmt + device: "{{ device.Hostname }}" + mgmt_ip: "{{ device.ManagementIp }}" + mgmt_gateway: "{{ mgmt_gw }}" + mgmt_bridge: "{{ mgmt_bridge }}" + testbed_name: "{{ testbed_name }}" + container_prefix: "{{ vnut_container_prefix }}" diff --git a/ansible/roles/testbed/nut-vtopo-create/tasks/main.yml b/ansible/roles/testbed/nut-vtopo-create/tasks/main.yml new file mode 100644 index 00000000000..a12371c3214 --- /dev/null +++ b/ansible/roles/testbed/nut-vtopo-create/tasks/main.yml @@ -0,0 +1,35 @@ +--- +- name: Read testbed definition and CSV files + include_tasks: read_testbed.yml + +- name: Create management network + include_tasks: create_mgmt_network.yml + +- name: Create front-panel link bridges (before VMs, so XML can reference them) + include_tasks: create_links.yml + +- name: Launch DUT KVM VMs + include_tasks: launch_one_dut.yml + loop: "{{ vnut_devices | selectattr('Type', 'equalto', 'DevSonic') | list }}" + loop_control: + loop_var: device + index_var: device_idx + +- name: Kickstart SONiC on DUT VMs + include_tasks: start_sonic.yml + loop: "{{ vnut_devices | selectattr('Type', 'equalto', 'DevSonic') | list }}" + loop_control: + loop_var: device + index_var: device_idx + +- name: Launch TG containers + include_tasks: launch_one_tg.yml + loop: "{{ vnut_devices | selectattr('Type', 'equalto', 'DevIxiaChassis') | list }}" + loop_control: + loop_var: device + +- name: Connect TG containers to front-panel link bridges + include_tasks: connect_tg_links.yml + +- name: Wait for testbed to be ready + include_tasks: wait_testbed_ready.yml diff --git a/ansible/roles/testbed/nut-vtopo-create/tasks/read_testbed.yml b/ansible/roles/testbed/nut-vtopo-create/tasks/read_testbed.yml new file mode 100644 index 00000000000..518a5bba2de --- /dev/null +++ b/ansible/roles/testbed/nut-vtopo-create/tasks/read_testbed.yml @@ -0,0 +1,72 @@ +--- +# Read testbed.nut.yaml and extract device/link info from CSVs +# Inputs: testbed_name, testbed_file +# Outputs (set_fact): vnut_devices (list of dicts), vnut_links (list of dicts) + +- name: Validate testbed_name is provided + fail: + msg: "testbed_name must be specified" + when: testbed_name is not defined or testbed_name | length == 0 + +- name: Resolve ansible base directory + # Use playbook_dir instead of role_path so we can locate testbed files and lab data + # relative to the top-level ansible directory regardless of role nesting depth. + set_fact: + vnut_ansible_dir: "{{ playbook_dir }}" + vnut_lab_files_dir: "{{ playbook_dir }}/files" + +- name: Read testbed YAML file + set_fact: + testbed_data: "{{ lookup('file', vnut_ansible_dir + '/' + testbed_file) | from_yaml }}" + +- name: Extract testbed entry + set_fact: + testbed_entry: "{{ testbed_data | selectattr('name', 'equalto', testbed_name) | list }}" + +- name: Validate testbed entry exists + assert: + that: + - testbed_entry | length > 0 + fail_msg: >- + Testbed '{{ testbed_name }}' not found in {{ testbed_file }}. + Available testbeds: {{ testbed_data | map(attribute='name') | list }} + +- name: Set testbed entry fact + set_fact: + testbed_entry: "{{ testbed_entry | first }}" + +- name: Get DUT and TG names from testbed + set_fact: + testbed_dut_names: "{{ testbed_entry.duts | default([]) }}" + testbed_tg_names: "{{ testbed_entry.tgs | default([]) }}" + testbed_all_devices: "{{ (testbed_entry.duts | default([])) + (testbed_entry.tgs | default([])) }}" + +- name: Read sonic_lab_devices.csv + read_csv: + path: "{{ vnut_lab_files_dir }}/sonic_lab_devices.csv" + delegate_to: localhost + register: lab_devices_csv + +- name: Filter devices for this testbed + set_fact: + vnut_devices: "{{ lab_devices_csv.list | selectattr('Hostname', 'in', testbed_all_devices) | list }}" + +- name: Read sonic_lab_links.csv + read_csv: + path: "{{ vnut_lab_files_dir }}/sonic_lab_links.csv" + delegate_to: localhost + register: lab_links_csv + +- name: Filter links for this testbed's devices + set_fact: + vnut_links: >- + {{ (lab_links_csv.list | selectattr('StartDevice', 'in', testbed_all_devices) | list + + lab_links_csv.list | selectattr('EndDevice', 'in', testbed_all_devices) | list) + | unique }} + +- name: Display parsed testbed info + debug: + msg: | + Testbed: {{ testbed_name }} + Devices: {{ vnut_devices | map(attribute='Hostname') | list }} + Links: {{ vnut_links | length }} links diff --git a/ansible/roles/testbed/nut-vtopo-create/tasks/start_sonic.yml b/ansible/roles/testbed/nut-vtopo-create/tasks/start_sonic.yml new file mode 100644 index 00000000000..f2a73c6f1eb --- /dev/null +++ b/ansible/roles/testbed/nut-vtopo-create/tasks/start_sonic.yml @@ -0,0 +1,25 @@ +--- +# start_sonic.yml — Kickstart SONiC VM via serial console +# Called with: device (dict), device_idx + +- name: "Set serial port for {{ device.Hostname }}" + set_fact: + serial_port: "{{ vnut_vm_serial_port_base + device_idx }}" + +- name: "Kickstart SONiC VM {{ device.Hostname }}" + sonic_kickstart: + telnet_port: "{{ serial_port }}" + login: "{{ sonicadmin_user }}" + passwords: "{{ sonic_default_passwords }}" + hostname: "{{ device.Hostname }}" + mgmt_ip: "{{ device.ManagementIp }}" + mgmt_gw: "{{ mgmt_gw }}" + new_password: "{{ sonicadmin_password }}" + num_asic: 1 + no_log: true + register: kickstart_output + +- name: "Fail if kickstart failed for {{ device.Hostname }}" + fail: + msg: "Kickstart failed for {{ device.Hostname }}: {{ kickstart_output }}" + when: kickstart_output.kickstart_code != 0 diff --git a/ansible/roles/testbed/nut-vtopo-create/tasks/wait_testbed_ready.yml b/ansible/roles/testbed/nut-vtopo-create/tasks/wait_testbed_ready.yml new file mode 100644 index 00000000000..6ba4ba650b3 --- /dev/null +++ b/ansible/roles/testbed/nut-vtopo-create/tasks/wait_testbed_ready.yml @@ -0,0 +1,31 @@ +--- +- name: Wait for DUT devices to become reachable via SSH + become: false + local_action: wait_for + args: + host: "{{ item.ManagementIp.split('/')[0] }}" + port: 22 + state: started + search_regex: "OpenSSH_[\\w\\.]+ Debian" + delay: 10 + timeout: 600 + changed_when: false + loop: "{{ vnut_devices | selectattr('Type', 'equalto', 'DevSonic') | list }}" + loop_control: + label: "{{ item.Hostname }}" + +- name: Wait for all monit services to be healthy on DUT devices + ansible.builtin.shell: > + sshpass -p {{ sonicadmin_password }} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + {{ sonicadmin_user }}@{{ item.ManagementIp.split('/')[0] }} + "sudo monit summary | grep -v 'OK\|Running\|Monit\|^$\|Status\|---' | wc -l" + no_log: true + delegate_to: localhost + register: monit_result + until: monit_result.rc == 0 and monit_result.stdout | trim == '0' + retries: 60 + delay: 10 + changed_when: false + loop: "{{ vnut_devices | selectattr('Type', 'equalto', 'DevSonic') | list }}" + loop_control: + label: "{{ item.Hostname }}" diff --git a/ansible/roles/testbed/nut-vtopo-create/templates/vnut-sonic.xml.j2 b/ansible/roles/testbed/nut-vtopo-create/templates/vnut-sonic.xml.j2 new file mode 100644 index 00000000000..0b567adce34 --- /dev/null +++ b/ansible/roles/testbed/nut-vtopo-create/templates/vnut-sonic.xml.j2 @@ -0,0 +1,57 @@ + + {{ vm_name }} + {{ vnut_vm_memory_gb }} + {{ vnut_vm_memory_gb }} + {{ vnut_vm_vcpus }} + + + + + + hvm + + + + + + + + destroy + restart + restart + + /usr/bin/qemu-system-x86_64 + + + + + + + + + + + {# Management interface — connected to mgmt bridge #} + {%- set mac_hash = (vm_name | hash('md5'))[:6] -%} + + + + + +{% for iface in vm_interfaces %} + {# Front-panel interface {{ iface.name }} — connected to bridge {{ iface.bridge }} #} + + + + + +{% endfor %} + + + +
+ + + + + diff --git a/ansible/roles/testbed/nut-vtopo-remove/defaults/main.yml b/ansible/roles/testbed/nut-vtopo-remove/defaults/main.yml new file mode 100644 index 00000000000..a481954ae78 --- /dev/null +++ b/ansible/roles/testbed/nut-vtopo-remove/defaults/main.yml @@ -0,0 +1,7 @@ +--- +# Naming prefix for TG Docker containers +vnut_container_prefix: "net" + +# VM storage paths +vnut_sonic_vm_storage: "{{ ansible_env.HOME }}/sonic-vm" +vnut_vm_disk_dir: "{{ vnut_sonic_vm_storage }}/disks" diff --git a/ansible/roles/testbed/nut-vtopo-remove/tasks/main.yml b/ansible/roles/testbed/nut-vtopo-remove/tasks/main.yml new file mode 100644 index 00000000000..53b217da304 --- /dev/null +++ b/ansible/roles/testbed/nut-vtopo-remove/tasks/main.yml @@ -0,0 +1,100 @@ +--- +- name: Read testbed definition + include_tasks: read_testbed.yml + +# --- Destroy KVM VMs --- +- name: Get list of running VMs + virt: + command: list_vms + state: running + uri: qemu:///system + register: vm_list_running + become: yes + ignore_errors: yes + +- name: Destroy DUT VMs + virt: + name: "{{ item.Hostname }}" + state: destroyed + uri: qemu:///system + loop: "{{ vnut_devices | selectattr('Type', 'equalto', 'DevSonic') | list }}" + loop_control: + label: "{{ item.Hostname }}" + when: item.Hostname in (vm_list_running.list_vms | default([])) + become: yes + ignore_errors: yes + +- name: Get list of defined VMs + virt: + command: list_vms + uri: qemu:///system + register: vm_list_defined + become: yes + ignore_errors: yes + +- name: Undefine DUT VMs + virt: + name: "{{ item.Hostname }}" + command: undefine + uri: qemu:///system + loop: "{{ vnut_devices | selectattr('Type', 'equalto', 'DevSonic') | list }}" + loop_control: + label: "{{ item.Hostname }}" + when: item.Hostname in (vm_list_defined.list_vms | default([])) + become: yes + ignore_errors: yes + +- name: Remove DUT disk images + file: + path: "{{ vnut_vm_disk_dir }}/sonic_{{ item.Hostname }}.img" + state: absent + loop: "{{ vnut_devices | selectattr('Type', 'equalto', 'DevSonic') | list }}" + loop_control: + label: "{{ item.Hostname }}" + ignore_errors: yes + +# --- Remove TG containers --- +- name: Stop and remove TG containers + community.docker.docker_container: + name: "{{ vnut_container_prefix }}_{{ testbed_name }}_{{ item.Hostname }}" + state: absent + force_kill: yes + loop: "{{ vnut_devices | selectattr('Type', 'equalto', 'DevIxiaChassis') | list }}" + loop_control: + label: "{{ item.Hostname }}" + ignore_errors: yes + +# --- Clean up link bridges --- +- name: Remove front-panel link bridges + shell: | + {% for idx in range(vnut_links | length) %} + BRIDGE="vbr_{{ testbed_name[:8] }}_{{ idx }}" + ip link delete "$BRIDGE" 2>/dev/null || true + {% endfor %} + echo "Cleaned up link bridges" + args: + executable: /bin/bash + changed_when: false + ignore_errors: yes + +# --- Clean up TG veth pairs --- +- name: Clean up TG veth pairs + shell: | + {% for idx in range(vnut_links | length) %} + {% if vnut_links[idx].StartDevice in (vnut_devices | selectattr('Type', 'equalto', 'DevIxiaChassis') | map(attribute='Hostname') | list) or vnut_links[idx].EndDevice in (vnut_devices | selectattr('Type', 'equalto', 'DevIxiaChassis') | map(attribute='Hostname') | list) %} + ip link delete "vtg{{ testbed_name[:6] }}_{{ idx }}a" 2>/dev/null || true + {% endif %} + {% endfor %} + echo "Cleaned up TG veth pairs" + args: + executable: /bin/bash + changed_when: false + ignore_errors: yes + +# --- Management network --- +# The management bridge and iptables rules are shared with other testbeds, +# so we do NOT remove them during vnut teardown. + +- name: Display teardown summary + debug: + msg: "Removed all VMs, containers, and front-panel links for testbed {{ testbed_name }}" diff --git a/ansible/roles/testbed/nut-vtopo-remove/tasks/read_testbed.yml b/ansible/roles/testbed/nut-vtopo-remove/tasks/read_testbed.yml new file mode 100644 index 00000000000..518a5bba2de --- /dev/null +++ b/ansible/roles/testbed/nut-vtopo-remove/tasks/read_testbed.yml @@ -0,0 +1,72 @@ +--- +# Read testbed.nut.yaml and extract device/link info from CSVs +# Inputs: testbed_name, testbed_file +# Outputs (set_fact): vnut_devices (list of dicts), vnut_links (list of dicts) + +- name: Validate testbed_name is provided + fail: + msg: "testbed_name must be specified" + when: testbed_name is not defined or testbed_name | length == 0 + +- name: Resolve ansible base directory + # Use playbook_dir instead of role_path so we can locate testbed files and lab data + # relative to the top-level ansible directory regardless of role nesting depth. + set_fact: + vnut_ansible_dir: "{{ playbook_dir }}" + vnut_lab_files_dir: "{{ playbook_dir }}/files" + +- name: Read testbed YAML file + set_fact: + testbed_data: "{{ lookup('file', vnut_ansible_dir + '/' + testbed_file) | from_yaml }}" + +- name: Extract testbed entry + set_fact: + testbed_entry: "{{ testbed_data | selectattr('name', 'equalto', testbed_name) | list }}" + +- name: Validate testbed entry exists + assert: + that: + - testbed_entry | length > 0 + fail_msg: >- + Testbed '{{ testbed_name }}' not found in {{ testbed_file }}. + Available testbeds: {{ testbed_data | map(attribute='name') | list }} + +- name: Set testbed entry fact + set_fact: + testbed_entry: "{{ testbed_entry | first }}" + +- name: Get DUT and TG names from testbed + set_fact: + testbed_dut_names: "{{ testbed_entry.duts | default([]) }}" + testbed_tg_names: "{{ testbed_entry.tgs | default([]) }}" + testbed_all_devices: "{{ (testbed_entry.duts | default([])) + (testbed_entry.tgs | default([])) }}" + +- name: Read sonic_lab_devices.csv + read_csv: + path: "{{ vnut_lab_files_dir }}/sonic_lab_devices.csv" + delegate_to: localhost + register: lab_devices_csv + +- name: Filter devices for this testbed + set_fact: + vnut_devices: "{{ lab_devices_csv.list | selectattr('Hostname', 'in', testbed_all_devices) | list }}" + +- name: Read sonic_lab_links.csv + read_csv: + path: "{{ vnut_lab_files_dir }}/sonic_lab_links.csv" + delegate_to: localhost + register: lab_links_csv + +- name: Filter links for this testbed's devices + set_fact: + vnut_links: >- + {{ (lab_links_csv.list | selectattr('StartDevice', 'in', testbed_all_devices) | list + + lab_links_csv.list | selectattr('EndDevice', 'in', testbed_all_devices) | list) + | unique }} + +- name: Display parsed testbed info + debug: + msg: | + Testbed: {{ testbed_name }} + Devices: {{ vnut_devices | map(attribute='Hostname') | list }} + Links: {{ vnut_links | length }} links diff --git a/ansible/roles/testbed/nut/tasks/testbed_facts.yml b/ansible/roles/testbed/nut/tasks/testbed_facts.yml index 236159be7ea..12056e2a68c 100644 --- a/ansible/roles/testbed/nut/tasks/testbed_facts.yml +++ b/ansible/roles/testbed/nut/tasks/testbed_facts.yml @@ -11,7 +11,7 @@ delegate_to: localhost - fail: msg="The DUT you are trying to run test does not belongs to this testbed" - when: (inventory_hostname not in testbed_facts['duts'] and inventory_hostname not in testbed_facts['l1s']) + when: (inventory_hostname not in (testbed_facts.duts | default([])) and inventory_hostname not in (testbed_facts.l1s | default([]))) - name: output testbed facts debug: @@ -19,6 +19,6 @@ - name: get connection graph if defined for dut conn_graph_facts: - hosts: "{{ testbed_facts['duts'] + testbed_facts['tgs'] + testbed_facts['l1s'] }}" + hosts: "{{ (testbed_facts.duts | default([])) + (testbed_facts.tgs | default([])) + (testbed_facts.l1s | default([])) }}" forced_mgmt_routes: "{{ forced_mgmt_routes | default([]) }}" delegate_to: localhost diff --git a/ansible/testbed-cli.sh b/ansible/testbed-cli.sh index 8b4ba221a34..e1116e22eab 100755 --- a/ansible/testbed-cli.sh +++ b/ansible/testbed-cli.sh @@ -10,6 +10,7 @@ function usage echo " $0 [options] (start-topo-vms | stop-topo-vms) " echo " $0 [options] (deploy-topo-with-cache) " echo " $0 [options] (add-topo | remove-topo | redeploy-topo | renumber-topo | connect-topo) " + echo " $0 [options] (add-vnut-topo | remove-vnut-topo) " echo " $0 [options] refresh-dut " echo " $0 [options] (connect-vms | disconnect-vms) " echo " $0 [options] config-vm " @@ -230,7 +231,7 @@ function read_nut_file content=$(python -c "from __future__ import print_function; import yaml; print('+'.join(str(tb) for tb in yaml.safe_load(open('$tbfile')) if '$1'==tb['$keyName']))") echo "" - IFS=$'+' read -r -a tb_lines <<< $content + IFS=$'+' read -r -a tb_lines <<< "$content" linecount=${#tb_lines[@]} if [ $linecount == 0 ] @@ -847,7 +848,7 @@ function deploy_config echo "Deploying config to testbed '$testbed_name'" - read_nut_file $testbed_name + read_nut_file "$testbed_name" devices=$duts if [ ! -z "$l1s" ]; then @@ -873,7 +874,7 @@ function deploy_l1 echo "Deploying L1 config to testbed '$testbed_name'" - read_nut_file $testbed_name + read_nut_file "$testbed_name" devices=$duts if [ ! -z "$l1s" ]; then @@ -898,7 +899,7 @@ function generate_config echo "Generate config for testbed '$testbed_name' for testing" - read_nut_file $testbed_name + read_nut_file "$testbed_name" devices=$duts if [ ! -z "$l1s" ]; then @@ -1197,6 +1198,48 @@ then usage fi +function add_vnut_topo +{ + testbed_name="$1" + inventory="$2" + passwd="$3" + shift; shift; shift + echo "Deploying virtual NUT topology for testbed '${testbed_name}'" + + read_nut_file "${testbed_name}" + + ANSIBLE_SCP_IF_SSH=y ansible-playbook -i "${inventory}" \ + testbed_add_nut_topo.yml \ + --vault-password-file="${passwd}" \ + -e testbed_name="${testbed_name}" \ + -e testbed_file="${tbfile}" \ + -e duts_name="${duts}" \ + "$@" + + echo Done +} + +function remove_vnut_topo +{ + testbed_name="$1" + inventory="$2" + passwd="$3" + shift; shift; shift + echo "Removing virtual NUT topology for testbed '${testbed_name}'" + + read_nut_file "${testbed_name}" + + ANSIBLE_SCP_IF_SSH=y ansible-playbook -i "${inventory}" \ + testbed_remove_nut_topo.yml \ + --vault-password-file="${passwd}" \ + -e testbed_name="${testbed_name}" \ + -e testbed_file="${tbfile}" \ + -e duts_name="${duts}" \ + "$@" + + echo Done +} + subcmd=$1 shift case "${subcmd}" in @@ -1261,6 +1304,10 @@ case "${subcmd}" in ;; config-vs-chassis) config_vs_chassis $@ ;; + add-vnut-topo) add_vnut_topo "$@" + ;; + remove-vnut-topo) remove_vnut_topo "$@" + ;; *) usage ;; esac diff --git a/ansible/testbed.vnut.yaml b/ansible/testbed.vnut.yaml new file mode 100644 index 00000000000..57311d90705 --- /dev/null +++ b/ansible/testbed.vnut.yaml @@ -0,0 +1,13 @@ +--- +- name: vnut-2tier-test + comment: "Virtual NUT 2-tier testbed for local testing" + inv_name: lab + topo: nut-2tiers + test_tags: [] + duts: + - vnut-t0-01 + - vnut-t0-02 + - vnut-t1-01 + tgs: + - vnut-tg-01 + tg_api_server: "10.250.0.220:443" diff --git a/ansible/testbed_add_nut_topo.yml b/ansible/testbed_add_nut_topo.yml new file mode 100644 index 00000000000..58fbe7fefed --- /dev/null +++ b/ansible/testbed_add_nut_topo.yml @@ -0,0 +1,11 @@ +--- +- name: Deploy virtual NUT topology + hosts: localhost + connection: local + gather_facts: yes + become: yes + vars_files: + - group_vars/all/creds.yml + - group_vars/lab/secrets.yml + roles: + - role: testbed/nut-vtopo-create diff --git a/ansible/testbed_remove_nut_topo.yml b/ansible/testbed_remove_nut_topo.yml new file mode 100644 index 00000000000..384d6c2dc26 --- /dev/null +++ b/ansible/testbed_remove_nut_topo.yml @@ -0,0 +1,11 @@ +--- +- name: Remove virtual NUT topology + hosts: localhost + connection: local + gather_facts: yes + become: yes + vars_files: + - group_vars/all/creds.yml + - group_vars/lab/secrets.yml + roles: + - role: testbed/nut-vtopo-remove