From e681c3fea57f29fa0fff4ba91533bef78c1f3b8f Mon Sep 17 00:00:00 2001 From: Rich Megginson Date: Wed, 21 Feb 2024 16:49:10 -0700 Subject: [PATCH] test: improve bond test failure debugging Improve bond test failure debugging * put test setup into block/always so that cleanup happens for setup failures * trace shell commands so that we can determine exactly which command failed * add error reporting so that when certain commands fail, we can determine the error code Signed-off-by: Rich Megginson --- tests/playbooks/tests_bond.yml | 36 +++++++++---------- tests/playbooks/tests_bond_cloned_mac.yml | 19 +++++----- tests/playbooks/tests_bond_deprecated.yml | 36 +++++++++---------- tests/playbooks/tests_bond_removal.yml | 36 +++++++++---------- tests/tasks/cleanup_802_1x_server.yml | 2 ++ .../create_test_interfaces_with_dhcp.yml | 22 +++++++++--- .../remove_test_interfaces_with_dhcp.yml | 21 ++++++++--- 7 files changed, 100 insertions(+), 72 deletions(-) diff --git a/tests/playbooks/tests_bond.yml b/tests/playbooks/tests_bond.yml index 6a49c46..3482077 100644 --- a/tests/playbooks/tests_bond.yml +++ b/tests/playbooks/tests_bond.yml @@ -10,26 +10,26 @@ port2_profile: bond0.1 dhcp_interface2: test2 tasks: - - name: "INIT Prepare setup" - debug: - msg: "##################################################" - - name: Import the task 'create_test_interfaces_with_dhcp.yml' - import_tasks: tasks/create_test_interfaces_with_dhcp.yml - - name: Import the task 'assert_device_present.yml' - import_tasks: tasks/assert_device_present.yml - vars: - interface: "{{ dhcp_interface1 }}" - - name: Import the task 'assert_device_present.yml' - import_tasks: tasks/assert_device_present.yml - vars: - interface: "{{ dhcp_interface2 }}" - - name: "Backup the /etc/resolv.conf for initscript" - command: cp -vf /etc/resolv.conf /etc/resolv.conf.bak - when: - - network_provider == "initscripts" - changed_when: false - name: Test bond device block: + - name: "INIT Prepare setup" + debug: + msg: "##################################################" + - name: Import the task 'create_test_interfaces_with_dhcp.yml' + import_tasks: tasks/create_test_interfaces_with_dhcp.yml + - name: Import the task 'assert_device_present.yml' + import_tasks: tasks/assert_device_present.yml + vars: + interface: "{{ dhcp_interface1 }}" + - name: Import the task 'assert_device_present.yml' + import_tasks: tasks/assert_device_present.yml + vars: + interface: "{{ dhcp_interface2 }}" + - name: "Backup the /etc/resolv.conf for initscript" + command: cp -vf /etc/resolv.conf /etc/resolv.conf.bak + when: + - network_provider == "initscripts" + changed_when: false - name: "TEST Add Bond with 2 ports" debug: msg: "##################################################" diff --git a/tests/playbooks/tests_bond_cloned_mac.yml b/tests/playbooks/tests_bond_cloned_mac.yml index 0c1fe4c..511ff0e 100644 --- a/tests/playbooks/tests_bond_cloned_mac.yml +++ b/tests/playbooks/tests_bond_cloned_mac.yml @@ -10,18 +10,17 @@ port2_profile: bond0.1 dhcp_interface2: test2 tasks: - - name: INIT Prepare setup - debug: - msg: "##################################################" - - name: Import the task 'create_test_interfaces_with_dhcp.yml' - import_tasks: tasks/create_test_interfaces_with_dhcp.yml - - name: Backup the /etc/resolv.conf for initscript - command: cp -vf /etc/resolv.conf /etc/resolv.conf.bak - when: - - network_provider == "initscripts" - changed_when: false - name: Test bond cloned MAC address block: + - name: INIT Prepare setup + debug: + msg: "##################################################" + - name: Import the task 'create_test_interfaces_with_dhcp.yml' + import_tasks: tasks/create_test_interfaces_with_dhcp.yml + - name: Backup the /etc/resolv.conf for initscript + command: cp -vf /etc/resolv.conf /etc/resolv.conf.bak + when: network_provider == "initscripts" + changed_when: false - name: TEST Add Bond with 2 ports debug: msg: "##################################################" diff --git a/tests/playbooks/tests_bond_deprecated.yml b/tests/playbooks/tests_bond_deprecated.yml index 3aca8c8..e90952c 100644 --- a/tests/playbooks/tests_bond_deprecated.yml +++ b/tests/playbooks/tests_bond_deprecated.yml @@ -10,26 +10,26 @@ port2_profile: bond0.1 dhcp_interface2: test2 tasks: - - name: "INIT Prepare setup" - debug: - msg: "##################################################" - - name: Import the task 'create_test_interfaces_with_dhcp.yml' - import_tasks: tasks/create_test_interfaces_with_dhcp.yml - - name: Import the task 'assert_device_present.yml' - import_tasks: tasks/assert_device_present.yml - vars: - interface: "{{ dhcp_interface1 }}" - - name: Import the task 'assert_device_present.yml' - import_tasks: tasks/assert_device_present.yml - vars: - interface: "{{ dhcp_interface2 }}" - - name: "Backup the /etc/resolv.conf for initscript" - command: cp -vf /etc/resolv.conf /etc/resolv.conf.bak - when: - - network_provider == "initscripts" - changed_when: false - name: Test bond device using deprecated 'master' argument block: + - name: "INIT Prepare setup" + debug: + msg: "##################################################" + - name: Import the task 'create_test_interfaces_with_dhcp.yml' + import_tasks: tasks/create_test_interfaces_with_dhcp.yml + - name: Import the task 'assert_device_present.yml' + import_tasks: tasks/assert_device_present.yml + vars: + interface: "{{ dhcp_interface1 }}" + - name: Import the task 'assert_device_present.yml' + import_tasks: tasks/assert_device_present.yml + vars: + interface: "{{ dhcp_interface2 }}" + - name: "Backup the /etc/resolv.conf for initscript" + command: cp -vf /etc/resolv.conf /etc/resolv.conf.bak + when: + - network_provider == "initscripts" + changed_when: false - name: "TEST Add Bond with 2 ports using deprecated 'master' argument" debug: msg: "##################################################" diff --git a/tests/playbooks/tests_bond_removal.yml b/tests/playbooks/tests_bond_removal.yml index 96fd4aa..3cf16c5 100644 --- a/tests/playbooks/tests_bond_removal.yml +++ b/tests/playbooks/tests_bond_removal.yml @@ -10,26 +10,26 @@ port2_profile: bond0.1 dhcp_interface2: test2 tasks: - - name: "INIT Prepare setup" - debug: - msg: "##################################################" - - name: Import the task 'create_test_interfaces_with_dhcp.yml' - import_tasks: tasks/create_test_interfaces_with_dhcp.yml - - name: Import the task 'assert_device_present.yml' - import_tasks: tasks/assert_device_present.yml - vars: - interface: "{{ dhcp_interface1 }}" - - name: Import the task 'assert_device_present.yml' - import_tasks: tasks/assert_device_present.yml - vars: - interface: "{{ dhcp_interface2 }}" - - name: "Backup the /etc/resolv.conf for initscript" - command: cp -vf /etc/resolv.conf /etc/resolv.conf.bak - when: - - network_provider == "initscripts" - changed_when: false - name: Test removing bond device block: + - name: "INIT Prepare setup" + debug: + msg: "##################################################" + - name: Import the task 'create_test_interfaces_with_dhcp.yml' + import_tasks: tasks/create_test_interfaces_with_dhcp.yml + - name: Import the task 'assert_device_present.yml' + import_tasks: tasks/assert_device_present.yml + vars: + interface: "{{ dhcp_interface1 }}" + - name: Import the task 'assert_device_present.yml' + import_tasks: tasks/assert_device_present.yml + vars: + interface: "{{ dhcp_interface2 }}" + - name: "Backup the /etc/resolv.conf for initscript" + command: cp -vf /etc/resolv.conf /etc/resolv.conf.bak + when: + - network_provider == "initscripts" + changed_when: false - name: "TEST Add Bond with 2 ports" debug: msg: "##################################################" diff --git a/tests/tasks/cleanup_802_1x_server.yml b/tests/tasks/cleanup_802_1x_server.yml index 058a6e6..a444e9d 100644 --- a/tests/tasks/cleanup_802_1x_server.yml +++ b/tests/tasks/cleanup_802_1x_server.yml @@ -2,6 +2,8 @@ --- - name: Remove test interfaces shell: | + set -uxo pipefail + exec 1>&2 ip netns delete ns1 ip link delete veth1-br ip link delete veth2-br diff --git a/tests/tasks/create_test_interfaces_with_dhcp.yml b/tests/tasks/create_test_interfaces_with_dhcp.yml index 1c42c34..0ce13b0 100644 --- a/tests/tasks/create_test_interfaces_with_dhcp.yml +++ b/tests/tasks/create_test_interfaces_with_dhcp.yml @@ -37,7 +37,8 @@ - name: Create test interfaces shell: | - set -euo pipefail + set -euxo pipefail + exec 1>&2 ip link add {{ dhcp_interface1 }} type veth peer name {{ dhcp_interface1 }}p ip link add {{ dhcp_interface2 }} type veth peer name {{ dhcp_interface2 }}p if [ -n "$(pgrep NetworkManager)" ];then @@ -63,10 +64,23 @@ while ! ip addr show testbr | grep -q 'inet [1-9]' do let "timer+=1" - if [ $timer -eq 30 ]; then break; fi + if [ $timer -eq 30 ]; then + echo ERROR - could not add testbr + ip addr + exit 1 + fi sleep 1 - ip addr add 192.0.2.1/24 dev testbr - ip -6 addr add 2001:DB8::1/32 dev testbr + rc=0 + ip addr add 192.0.2.1/24 dev testbr || rc="$?" + if [ "$rc" != 0 ]; then + echo NOTICE - could not add testbr - error code "$rc" + continue + fi + ip -6 addr add 2001:DB8::1/32 dev testbr || rc="$?" + if [ "$rc" != 0 ]; then + echo NOTICE - could not add testbr - error code "$rc" + continue + fi done if grep 'release 6' /etc/redhat-release; then diff --git a/tests/tasks/remove_test_interfaces_with_dhcp.yml b/tests/tasks/remove_test_interfaces_with_dhcp.yml index d3905a2..6e37a30 100644 --- a/tests/tasks/remove_test_interfaces_with_dhcp.yml +++ b/tests/tasks/remove_test_interfaces_with_dhcp.yml @@ -2,14 +2,28 @@ --- - name: Remove test interfaces shell: | - ip link delete {{ dhcp_interface1 }} - ip link delete {{ dhcp_interface2 }} - ip link delete testbr + set -euxo pipefail + exec 1>&2 + rc=0 + ip link delete {{ dhcp_interface1 }} || rc="$?" + if [ "$rc" != 0 ]; then + echo ERROR - could not delete link {{ dhcp_interface1 }} - error "$rc" + fi + ip link delete {{ dhcp_interface2 }} || rc="$?" + if [ "$rc" != 0 ]; then + echo ERROR - could not delete link {{ dhcp_interface2 }} - error "$rc" + fi + ip link delete testbr || rc="$?" + if [ "$rc" != 0 ]; then + echo ERROR - could not delete link testbr - error "$rc" + fi changed_when: false - name: Stop dnsmasq/radvd services shell: | + set -uxo pipefail + exec 1>&2 pkill -F /run/dhcp_testbr.pid rm -rf /run/dhcp_testbr.pid rm -rf /run/dhcp_testbr.lease @@ -17,6 +31,5 @@ # Stop radvd server service radvd stop iptables -D INPUT -i testbr -p udp --dport 67:68 --sport 67:68 -j ACCEPT - fi changed_when: false