test: improve bond test failure debugging

Improve bond test failure debugging
* put test setup into block/always so that cleanup happens for setup failures
* trace shell commands so that we can determine exactly which command failed
* add error reporting so that when certain commands fail, we can determine the error code

Signed-off-by: Rich Megginson <rmeggins@redhat.com>

tests/playbooks/tests_bond.yml

@@ -10,26 +10,26 @@
     port2_profile: bond0.1
     dhcp_interface2: test2
   tasks:
-    - name: "INIT Prepare setup"
-      debug:
-        msg: "##################################################"
-    - name: Import the task 'create_test_interfaces_with_dhcp.yml'
-      import_tasks: tasks/create_test_interfaces_with_dhcp.yml
-    - name: Import the task 'assert_device_present.yml'
-      import_tasks: tasks/assert_device_present.yml
-      vars:
-        interface: "{{ dhcp_interface1 }}"
-    - name: Import the task 'assert_device_present.yml'
-      import_tasks: tasks/assert_device_present.yml
-      vars:
-        interface: "{{ dhcp_interface2 }}"
-    - name: "Backup the /etc/resolv.conf for initscript"
-      command: cp -vf /etc/resolv.conf /etc/resolv.conf.bak
-      when:
-        - network_provider == "initscripts"
-      changed_when: false
     - name: Test bond device
       block:
+        - name: "INIT Prepare setup"
+          debug:
+            msg: "##################################################"
+        - name: Import the task 'create_test_interfaces_with_dhcp.yml'
+          import_tasks: tasks/create_test_interfaces_with_dhcp.yml
+        - name: Import the task 'assert_device_present.yml'
+          import_tasks: tasks/assert_device_present.yml
+          vars:
+            interface: "{{ dhcp_interface1 }}"
+        - name: Import the task 'assert_device_present.yml'
+          import_tasks: tasks/assert_device_present.yml
+          vars:
+            interface: "{{ dhcp_interface2 }}"
+        - name: "Backup the /etc/resolv.conf for initscript"
+          command: cp -vf /etc/resolv.conf /etc/resolv.conf.bak
+          when:
+            - network_provider == "initscripts"
+          changed_when: false
         - name: "TEST Add Bond with 2 ports"
           debug:
             msg: "##################################################"

tests/playbooks/tests_bond_cloned_mac.yml

@@ -10,18 +10,17 @@
     port2_profile: bond0.1
     dhcp_interface2: test2
   tasks:
-    - name: INIT Prepare setup
-      debug:
-        msg: "##################################################"
-    - name: Import the task 'create_test_interfaces_with_dhcp.yml'
-      import_tasks: tasks/create_test_interfaces_with_dhcp.yml
-    - name: Backup the /etc/resolv.conf for initscript
-      command: cp -vf /etc/resolv.conf /etc/resolv.conf.bak
-      when:
-        - network_provider == "initscripts"
-      changed_when: false
     - name: Test bond cloned MAC address
       block:
+        - name: INIT Prepare setup
+          debug:
+            msg: "##################################################"
+        - name: Import the task 'create_test_interfaces_with_dhcp.yml'
+          import_tasks: tasks/create_test_interfaces_with_dhcp.yml
+        - name: Backup the /etc/resolv.conf for initscript
+          command: cp -vf /etc/resolv.conf /etc/resolv.conf.bak
+          when: network_provider == "initscripts"
+          changed_when: false
         - name: TEST Add Bond with 2 ports
           debug:
             msg: "##################################################"

tests/playbooks/tests_bond_deprecated.yml

@@ -10,26 +10,26 @@
     port2_profile: bond0.1
     dhcp_interface2: test2
   tasks:
-    - name: "INIT Prepare setup"
-      debug:
-        msg: "##################################################"
-    - name: Import the task 'create_test_interfaces_with_dhcp.yml'
-      import_tasks: tasks/create_test_interfaces_with_dhcp.yml
-    - name: Import the task 'assert_device_present.yml'
-      import_tasks: tasks/assert_device_present.yml
-      vars:
-        interface: "{{ dhcp_interface1 }}"
-    - name: Import the task 'assert_device_present.yml'
-      import_tasks: tasks/assert_device_present.yml
-      vars:
-        interface: "{{ dhcp_interface2 }}"
-    - name: "Backup the /etc/resolv.conf for initscript"
-      command: cp -vf /etc/resolv.conf /etc/resolv.conf.bak
-      when:
-        - network_provider == "initscripts"
-      changed_when: false
     - name: Test bond device using deprecated 'master' argument
       block:
+        - name: "INIT Prepare setup"
+          debug:
+            msg: "##################################################"
+        - name: Import the task 'create_test_interfaces_with_dhcp.yml'
+          import_tasks: tasks/create_test_interfaces_with_dhcp.yml
+        - name: Import the task 'assert_device_present.yml'
+          import_tasks: tasks/assert_device_present.yml
+          vars:
+            interface: "{{ dhcp_interface1 }}"
+        - name: Import the task 'assert_device_present.yml'
+          import_tasks: tasks/assert_device_present.yml
+          vars:
+            interface: "{{ dhcp_interface2 }}"
+        - name: "Backup the /etc/resolv.conf for initscript"
+          command: cp -vf /etc/resolv.conf /etc/resolv.conf.bak
+          when:
+            - network_provider == "initscripts"
+          changed_when: false
         - name: "TEST Add Bond with 2 ports using deprecated 'master' argument"
           debug:
             msg: "##################################################"

tests/playbooks/tests_bond_removal.yml

@@ -10,26 +10,26 @@
     port2_profile: bond0.1
     dhcp_interface2: test2
   tasks:
-    - name: "INIT Prepare setup"
-      debug:
-        msg: "##################################################"
-    - name: Import the task 'create_test_interfaces_with_dhcp.yml'
-      import_tasks: tasks/create_test_interfaces_with_dhcp.yml
-    - name: Import the task 'assert_device_present.yml'
-      import_tasks: tasks/assert_device_present.yml
-      vars:
-        interface: "{{ dhcp_interface1 }}"
-    - name: Import the task 'assert_device_present.yml'
-      import_tasks: tasks/assert_device_present.yml
-      vars:
-        interface: "{{ dhcp_interface2 }}"
-    - name: "Backup the /etc/resolv.conf for initscript"
-      command: cp -vf /etc/resolv.conf /etc/resolv.conf.bak
-      when:
-        - network_provider == "initscripts"
-      changed_when: false
     - name: Test removing bond device
       block:
+        - name: "INIT Prepare setup"
+          debug:
+            msg: "##################################################"
+        - name: Import the task 'create_test_interfaces_with_dhcp.yml'
+          import_tasks: tasks/create_test_interfaces_with_dhcp.yml
+        - name: Import the task 'assert_device_present.yml'
+          import_tasks: tasks/assert_device_present.yml
+          vars:
+            interface: "{{ dhcp_interface1 }}"
+        - name: Import the task 'assert_device_present.yml'
+          import_tasks: tasks/assert_device_present.yml
+          vars:
+            interface: "{{ dhcp_interface2 }}"
+        - name: "Backup the /etc/resolv.conf for initscript"
+          command: cp -vf /etc/resolv.conf /etc/resolv.conf.bak
+          when:
+            - network_provider == "initscripts"
+          changed_when: false
         - name: "TEST Add Bond with 2 ports"
           debug:
             msg: "##################################################"

tests/tasks/cleanup_802_1x_server.yml

@@ -2,6 +2,8 @@
 ---
 - name: Remove test interfaces
   shell: |
+    set -uxo pipefail
+    exec 1>&2
     ip netns delete ns1
     ip link delete veth1-br
     ip link delete veth2-br

tests/tasks/create_test_interfaces_with_dhcp.yml

@@ -37,7 +37,8 @@
 
 - name: Create test interfaces
   shell: |
-    set -euo pipefail
+    set -euxo pipefail
+    exec 1>&2
     ip link add {{ dhcp_interface1 }} type veth peer name {{ dhcp_interface1 }}p
     ip link add {{ dhcp_interface2 }} type veth peer name {{ dhcp_interface2 }}p
     if [ -n "$(pgrep NetworkManager)" ];then
@@ -63,10 +64,23 @@
     while ! ip addr show testbr | grep -q 'inet [1-9]'
     do
         let "timer+=1"
-        if [ $timer -eq 30 ]; then break; fi
+        if [ $timer -eq 30 ]; then
+          echo ERROR - could not add testbr
+          ip addr
+          exit 1
+        fi
         sleep 1
-        ip addr add 192.0.2.1/24 dev testbr
-        ip -6 addr add 2001:DB8::1/32 dev testbr
+        rc=0
+        ip addr add 192.0.2.1/24 dev testbr || rc="$?"
+        if [ "$rc" != 0 ]; then
+          echo NOTICE - could not add testbr - error code "$rc"
+          continue
+        fi
+        ip -6 addr add 2001:DB8::1/32 dev testbr || rc="$?"
+        if [ "$rc" != 0 ]; then
+          echo NOTICE - could not add testbr - error code "$rc"
+          continue
+        fi
     done
 
     if grep 'release 6' /etc/redhat-release; then

tests/tasks/remove_test_interfaces_with_dhcp.yml

@@ -2,14 +2,28 @@
 ---
 - name: Remove test interfaces
   shell: |
-    ip link delete {{ dhcp_interface1 }}
-    ip link delete {{ dhcp_interface2 }}
-    ip link delete testbr
+    set -euxo pipefail
+    exec 1>&2
+    rc=0
+    ip link delete {{ dhcp_interface1 }} || rc="$?"
+    if [ "$rc" != 0 ]; then
+      echo ERROR - could not delete link {{ dhcp_interface1 }} - error "$rc"
+    fi
+    ip link delete {{ dhcp_interface2 }} || rc="$?"
+    if [ "$rc" != 0 ]; then
+      echo ERROR - could not delete link {{ dhcp_interface2 }} - error "$rc"
+    fi
+    ip link delete testbr || rc="$?"
+    if [ "$rc" != 0 ]; then
+      echo ERROR - could not delete link testbr - error "$rc"
+    fi
   changed_when: false
 
 
 - name: Stop dnsmasq/radvd services
   shell: |
+    set -uxo pipefail
+    exec 1>&2
     pkill -F /run/dhcp_testbr.pid
     rm -rf /run/dhcp_testbr.pid
     rm -rf /run/dhcp_testbr.lease
@@ -17,6 +31,5 @@
         # Stop radvd server
         service radvd stop
         iptables -D INPUT -i testbr -p udp --dport 67:68 --sport 67:68 -j ACCEPT
-
     fi
   changed_when: false