Integration tests: Explicitly select provider

Instead of determining the non-default provider, run all integration
tests explicitly for both providers or only the providers that are
required. Since Ansible does not allow to pass variables to playbooks
imported with `import_playbook` to select another playbook to import,
generate the files instead of using a shared playbook like
`run_with_nm.yml` or `run_with_initscripts.yml`.

.travis/config.sh

@@ -39,6 +39,13 @@
 #
 #       - RUN_FLAKE8_DISABLED
 #       - RUN_FLAKE8_EXTRA_ARGS
+if [[ "$(python2 -c "import sys; print(sys.version_info.major)")" == "2" ]]
+then
+    PYTHON2_EXCLUDES="tests/ensure_provider_tests.py"
+    FLAKE8_DEFAULT_EXCLUDES=".svn,CVS,.bzr,.hg,.git,__pycache__,.tox,.eggs,*.egg"
+    RUN_PYLINT_EXCLUDE='^(\..*|ensure_provider_tests\.py)$'
+    RUN_FLAKE8_EXTRA_ARGS="--exclude ${FLAKE8_DEFAULT_EXCLUDES},${PYTHON2_EXCLUDES}"
+fi
 #
 #   * .travis/runsyspycmd.sh:
 #

.travis/custom.sh

@@ -11,5 +11,5 @@ SCRIPTDIR=$(readlink -f $(dirname $0))
 
 # Write your custom commands here that should be run when `tox -e custom`:
 if [[ -z "${TRAVIS}" ]] || lsr_check_python_version python -eq '3.6'; then
-  (set -x; cd ${TOPDIR}/tests; ${ENVPYTHON} ./ensure_non_running_provider.py)
+    (set -x; cd ${TOPDIR}/tests; ${ENVPYTHON} ./ensure_provider_tests.py)
 fi

examples/down-profile.yml

@@ -1 +0,0 @@
-../tests/down-profile.yml

examples/down_profile.yml

@@ -0,0 +1 @@
+../tests/playbooks/down_profile.yml

examples/remove-profile.yml

@@ -1 +0,0 @@
-../tests/remove-profile.yml

examples/remove_profile.yml

@@ -0,0 +1 @@
+../tests/playbooks/remove_profile.yml

tests/ensure_non_running_provider.py

@@ -1,118 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: BSD-3-Clause
-""" Check that there is a playbook to run all role tests with the non-default
-provider as well """
-# vim: fileencoding=utf8
-
-import glob
-import os
-import sys
-
-
-import yaml
-
-OTHER_PROVIDER_SUFFIX = "_other_provider.yml"
-
-IGNORE = [
-    "tests_802_1x_nm.yml",
-    "tests_default_initscripts.yml",
-    "tests_default_nm.yml",
-    "tests_default.yml",
-    "tests_ethtool_features_initscripts.yml",
-    "tests_ethtool_features_nm.yml",
-    "tests_helpers-and-asserts.yml",
-    "tests_reapply_nm.yml",
-    "tests_regression_nm.yml",
-    "tests_states.yml",
-    "tests_unit.yml",
-    "tests_vlan_mtu_initscripts.yml",
-    "tests_vlan_mtu_nm.yml",
-]
-
-OTHER_PLAYBOOK = """
-# SPDX-License-Identifier: BSD-3-Clause
----
-- name: Run playbook '{tests_playbook}' with non-default provider
-  hosts: all
-  vars:
-      network_provider_current:
-  tasks:
-  # required for the code to set network_provider_current
-  - name: Get service facts
-    service_facts:
-  - name: Set network provider
-    set_fact:
-      network_provider: '{{{{ "initscripts" if network_provider_current == "nm"
-                                  else "nm" }}}}'
-
-- import_playbook: "{tests_playbook}"
-  when:
-    - ansible_distribution_major_version != '6'
-"""  # noqa: E501 # ignore that the line is too long
-
-
-def get_current_provider_code():
-    with open("../defaults/main.yml") as defaults:
-        yaml_defaults = yaml.safe_load(defaults)
-    current_provider = yaml_defaults["network_provider_current"]
-    return current_provider
-
-
-def generate_nominal_other_playbook(tests_playbook):
-    nominal_other_testfile_data = OTHER_PLAYBOOK.format(tests_playbook=tests_playbook)
-    nominal = yaml.safe_load(nominal_other_testfile_data)
-    nominal[0]["vars"]["network_provider_current"] = get_current_provider_code()
-    yaml_dump = yaml.dump(
-        nominal, default_flow_style=False, explicit_start=True, width=80
-    )
-    return "# yamllint disable\n" + yaml_dump
-
-
-def main():
-    testsfiles = glob.glob("tests_*.yml")
-    missing = []
-    returncode = 0
-
-    # Generate files when specified
-    generate = bool(len(sys.argv) > 1 and sys.argv[1] == "generate")
-
-    if not testsfiles:
-        print("ERROR: No tests found")
-        returncode = 1
-
-    for filename in testsfiles:
-        if filename.endswith(OTHER_PROVIDER_SUFFIX):
-            continue
-
-        if filename in IGNORE:
-            continue
-
-        fileroot = os.path.splitext(filename)[0]
-        other_testfile = fileroot + OTHER_PROVIDER_SUFFIX
-        nominal_other_testfile_data = generate_nominal_other_playbook(filename)
-
-        if generate:
-            with open(other_testfile, "w") as ofile:
-                ofile.write(nominal_other_testfile_data)
-
-        if other_testfile not in testsfiles and not generate:
-            missing.append(filename)
-        else:
-            with open(other_testfile) as ifile:
-                testdata = ifile.read()
-                if testdata != nominal_other_testfile_data:
-                    print(
-                        "ERROR: Playbook does not match nominal value " + other_testfile
-                    )
-                    returncode = 1
-
-    if missing:
-        print("ERROR: No tests for other provider found for:\n" + ", \n".join(missing))
-        print("Try to generate them with '{} generate'".format(sys.argv[0]))
-        returncode = 1
-
-    return returncode
-
-
-if __name__ == "__main__":
-    sys.exit(main())

tests/ensure_provider_tests.py

@@ -0,0 +1,187 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+""" Check that there is a playbook to run all role tests with both providers
+"""
+# vim: fileencoding=utf8
+
+import glob
+import os
+import sys
+
+
+GET_NM_VERSION = """
+    - block:
+        - name: Install NetworkManager
+          package:
+            name: NetworkManager
+            state: present
+        - name: Get NetworkManager version
+          command: rpm -q --qf "%{version}" NetworkManager
+          args:
+            warn: false
+          register: NetworkManager_version
+          when: true
+      when:
+        - ansible_distribution_major_version != '6'
+"""
+
+MINIMUM_NM_VERSION_CHECK = """
+    - NetworkManager_version.stdout is version({minimum_nm_version}, '>=')
+"""
+
+RUN_PLAYBOOK_WITH_NM = """# SPDX-License-Identifier: BSD-3-Clause
+# This file was generated by ensure_provider_tests.py
+---
+# set network provider and gather facts
+- hosts: all
+  name: Run playbook '{test_playbook}' with nm as provider
+  tasks:
+    - name: Set network provider to 'nm'
+      set_fact:
+        network_provider: nm
+{get_nm_version}
+
+# workaround for: https://github.com/ansible/ansible/issues/27973
+# There is no way in Ansible to abort a playbook hosts with specific OS
+# releases Therefore we include the playbook with the tests only if the hosts
+# would support it.
+# The test requires or should run with NetworkManager, therefore it cannot run
+# on RHEL/CentOS 6
+- import_playbook: {test_playbook}
+  when:
+    - ansible_distribution_major_version != '6'
+{minimum_nm_version_check}"""
+
+MINIMUM_VERSION = "minimum_version"
+NM_ONLY_TESTS = {
+    "playbooks/tests_ethtool_features.yml": {
+        MINIMUM_VERSION: "'1.20.0'",
+        "comment": "# NetworkManager 1.20.0 introduced ethtool settings support",
+    },
+    "playbooks/tests_reapply.yml": {},
+    "playbooks/tests_states.yml": {},
+    "playbooks/tests_802_1x.yml": {},
+}
+
+IGNORE = [
+    # checked by tests_regression_nm.yml
+    "playbooks/tests_checkpoint_cleanup.yml",
+]
+
+RUN_PLAYBOOK_WITH_INITSCRIPTS = """# SPDX-License-Identifier: BSD-3-Clause
+# This file was generated by ensure_provider_tests.py
+---
+- hosts: all
+  name: Run playbook '{test_playbook}' with initscripts as provider
+  tasks:
+    - name: Set network provider to 'initscripts'
+      set_fact:
+        network_provider: initscripts
+
+- import_playbook: {test_playbook}
+"""
+
+
+def create_nm_playbook(test_playbook):
+    fileroot = os.path.splitext(os.path.basename(test_playbook))[0]
+    nm_testfile = fileroot + "_nm.yml"
+
+    minimum_nm_version = NM_ONLY_TESTS.get(test_playbook, {}).get(MINIMUM_VERSION)
+    nm_version_check = ""
+    if minimum_nm_version:
+        nm_version_check = MINIMUM_NM_VERSION_CHECK.format(
+            minimum_nm_version=minimum_nm_version
+        )
+
+    nominal_nm_testfile_data = RUN_PLAYBOOK_WITH_NM.format(
+        test_playbook=test_playbook,
+        get_nm_version=minimum_nm_version and GET_NM_VERSION or "",
+        minimum_nm_version_check=nm_version_check,
+    )
+
+    return nm_testfile, nominal_nm_testfile_data
+
+
+def create_initscripts_playbook(test_playbook):
+    fileroot = os.path.splitext(os.path.basename(test_playbook))[0]
+    init_testfile = fileroot + "_initscripts.yml"
+
+    nominal_data = RUN_PLAYBOOK_WITH_INITSCRIPTS.format(test_playbook=test_playbook)
+
+    return init_testfile, nominal_data
+
+
+def check_playbook(generate, testfile, test_playbook, nominal_data):
+    is_missing = False
+    returncode = None
+    if generate:
+        print(testfile)
+        with open(testfile, "w") as ofile:
+            ofile.write(nominal_data)
+
+    if not os.path.isfile(testfile) and not generate:
+        is_missing = True
+    else:
+        with open(testfile) as ifile:
+            testdata = ifile.read()
+            if testdata != nominal_data:
+                print(f"ERROR: Playbook does not match nominal value: {testfile}")
+                returncode = 1
+
+    return is_missing, returncode
+
+
+def main():
+    testsfiles = glob.glob("playbooks/tests_*.yml")
+    missing = []
+    returncode = 0
+
+    # Generate files when specified
+    generate = bool(len(sys.argv) > 1 and sys.argv[1] == "generate")
+
+    if not testsfiles:
+        print("ERROR: No tests found")
+        returncode = 1
+
+    for test_playbook in testsfiles:
+        if test_playbook in IGNORE:
+            continue
+
+        nm_testfile, nominal_nm_testfile_data = create_nm_playbook(test_playbook)
+
+        is_missing, new_returncode = check_playbook(
+            generate=generate,
+            testfile=nm_testfile,
+            test_playbook=test_playbook,
+            nominal_data=nominal_nm_testfile_data,
+        )
+        if is_missing:
+            missing.append(test_playbook)
+        if new_returncode:
+            returncode = new_returncode
+
+        if test_playbook not in NM_ONLY_TESTS:
+            init_testfile, nominal_init_testfile_data = create_initscripts_playbook(
+                test_playbook
+            )
+            is_missing, new_returncode = check_playbook(
+                generate=generate,
+                testfile=init_testfile,
+                test_playbook=test_playbook,
+                nominal_data=nominal_init_testfile_data,
+            )
+            if is_missing:
+                missing.append(test_playbook)
+            if new_returncode:
+                returncode = new_returncode
+
+    if missing:
+        print("ERROR: No NM or initscripts tests found for:\n" + ", \n".join(missing))
+        print("Try to generate them with '{} generate'".format(sys.argv[0]))
+        returncode = 1
+
+    return returncode
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/playbooks/tests_bridge.yml

@@ -26,30 +26,30 @@
   roles:
     - linux-system-roles.network
 
-- import_playbook: run-tasks.yml
+- import_playbook: run_tasks.yml
   vars:
     task: tasks/assert-device_present.yml
 
-- import_playbook: run-tasks.yml
+- import_playbook: run_tasks.yml
   vars:
     profile: "{{ interface }}"
     task: tasks/assert-profile_present.yml
 
-- import_playbook: down-profile.yml
+- import_playbook: down_profile.yml
   vars:
     profile: "{{ interface }}"
 # FIXME: assert profile/device down
 
-- import_playbook: remove-profile.yml
+- import_playbook: remove_profile.yml
   vars:
     profile: "{{ interface }}"
 
-- import_playbook: run-tasks.yml
+- import_playbook: run_tasks.yml
   vars:
     profile: "{{ interface }}"
     task: tasks/assert-profile_absent.yml
 
 # FIXME: Devices might still be left when profile is absent
-# - import_playbook: run-tasks.yml
+# - import_playbook: run_tasks.yml
 #   vars:
 #     task: tasks/assert-device_absent.yml

tests/playbooks/tests_ethernet.yml

@@ -45,11 +45,11 @@
 
 # FIXME: assert profile present
 # FIXME: assert profile/device up + IP address
-- import_playbook: down-profile.yml
+- import_playbook: down_profile.yml
   vars:
     profile: "{{ interface }}"
 # FIXME: assert profile/device down
-- import_playbook: remove-profile.yml
+- import_playbook: remove_profile.yml
   vars:
     profile: "{{ interface }}"
 # FIXME: assert profile away

tests/tests_802_1x_nm.yml

@@ -1,15 +1,21 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# This file was generated by ensure_provider_tests.py
 ---
 # set network provider and gather facts
 - hosts: all
+  name: Run playbook 'playbooks/tests_802_1x.yml' with nm as provider
   tasks:
     - name: Set network provider to 'nm'
       set_fact:
         network_provider: nm
 
+
 # workaround for: https://github.com/ansible/ansible/issues/27973
 # There is no way in Ansible to abort a playbook hosts with specific OS
 # releases Therefore we include the playbook with the tests only if the hosts
 # would support it.
-# The test requires NetworkManager, therefore it cannot run on RHEL/CentOS 6
+# The test requires or should run with NetworkManager, therefore it cannot run
+# on RHEL/CentOS 6
 - import_playbook: playbooks/tests_802_1x.yml
-  when: ansible_distribution_major_version != '6'
+  when:
+    - ansible_distribution_major_version != '6'

tests/tests_bridge_initscripts.yml

@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# This file was generated by ensure_provider_tests.py
+---
+- hosts: all
+  name: Run playbook 'playbooks/tests_bridge.yml' with initscripts as provider
+  tasks:
+    - name: Set network provider to 'initscripts'
+      set_fact:
+        network_provider: initscripts
+
+- import_playbook: playbooks/tests_bridge.yml

tests/tests_bridge_nm.yml

@@ -0,0 +1,21 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# This file was generated by ensure_provider_tests.py
+---
+# set network provider and gather facts
+- hosts: all
+  name: Run playbook 'playbooks/tests_bridge.yml' with nm as provider
+  tasks:
+    - name: Set network provider to 'nm'
+      set_fact:
+        network_provider: nm
+
+
+# workaround for: https://github.com/ansible/ansible/issues/27973
+# There is no way in Ansible to abort a playbook hosts with specific OS
+# releases Therefore we include the playbook with the tests only if the hosts
+# would support it.
+# The test requires or should run with NetworkManager, therefore it cannot run
+# on RHEL/CentOS 6
+- import_playbook: playbooks/tests_bridge.yml
+  when:
+    - ansible_distribution_major_version != '6'

tests/tests_bridge_other_provider.yml

@@ -1,18 +0,0 @@
-# yamllint disable
----
-- hosts: all
-  name: Run playbook 'tests_bridge.yml' with non-default provider
-  tasks:
-  - name: Get service facts
-    service_facts: null
-  - name: Set network provider
-    set_fact:
-      network_provider: '{{ "initscripts" if network_provider_current == "nm" else
-        "nm" }}'
-  vars:
-    network_provider_current: '{{ ''nm'' if ''NetworkManager.service'' in ansible_facts.services
-      and ansible_facts.services[''NetworkManager.service''][''state''] == ''running''
-      else ''initscripts'' }}'
-- import_playbook: tests_bridge.yml
-  when:
-  - ansible_distribution_major_version != '6'

tests/tests_ethernet_initscripts.yml

@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# This file was generated by ensure_provider_tests.py
+---
+- hosts: all
+  name: Run playbook 'playbooks/tests_ethernet.yml' with initscripts as provider
+  tasks:
+    - name: Set network provider to 'initscripts'
+      set_fact:
+        network_provider: initscripts
+
+- import_playbook: playbooks/tests_ethernet.yml

tests/tests_ethernet_nm.yml

@@ -0,0 +1,21 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# This file was generated by ensure_provider_tests.py
+---
+# set network provider and gather facts
+- hosts: all
+  name: Run playbook 'playbooks/tests_ethernet.yml' with nm as provider
+  tasks:
+    - name: Set network provider to 'nm'
+      set_fact:
+        network_provider: nm
+
+
+# workaround for: https://github.com/ansible/ansible/issues/27973
+# There is no way in Ansible to abort a playbook hosts with specific OS
+# releases Therefore we include the playbook with the tests only if the hosts
+# would support it.
+# The test requires or should run with NetworkManager, therefore it cannot run
+# on RHEL/CentOS 6
+- import_playbook: playbooks/tests_ethernet.yml
+  when:
+    - ansible_distribution_major_version != '6'

tests/tests_ethernet_other_provider.yml

@@ -1,18 +0,0 @@
-# yamllint disable
----
-- hosts: all
-  name: Run playbook 'tests_ethernet.yml' with non-default provider
-  tasks:
-  - name: Get service facts
-    service_facts: null
-  - name: Set network provider
-    set_fact:
-      network_provider: '{{ "initscripts" if network_provider_current == "nm" else
-        "nm" }}'
-  vars:
-    network_provider_current: '{{ ''nm'' if ''NetworkManager.service'' in ansible_facts.services
-      and ansible_facts.services[''NetworkManager.service''][''state''] == ''running''
-      else ''initscripts'' }}'
-- import_playbook: tests_ethernet.yml
-  when:
-  - ansible_distribution_major_version != '6'

tests/tests_ethtool_features_nm.yml

@@ -1,29 +1,37 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# This file was generated by ensure_provider_tests.py
 ---
 # set network provider and gather facts
 - hosts: all
+  name: Run playbook 'playbooks/tests_ethtool_features.yml' with nm as provider
   tasks:
     - name: Set network provider to 'nm'
       set_fact:
         network_provider: nm
-    - name: Install NetworkManager
-      package:
-        name: NetworkManager
-        state: present
-    - name: Get NetworkManager version
-      command: rpm -q --qf "%{version}" NetworkManager
-      args:
-        warn: "no"
-      when: true
-      register: NetworkManager_version
+
+    - block:
+        - name: Install NetworkManager
+          package:
+            name: NetworkManager
+            state: present
+        - name: Get NetworkManager version
+          command: rpm -q --qf "%{version}" NetworkManager
+          args:
+            warn: false
+          register: NetworkManager_version
+          when: true
+      when:
+        - ansible_distribution_major_version != '6'
+
 
 # workaround for: https://github.com/ansible/ansible/issues/27973
 # There is no way in Ansible to abort a playbook hosts with specific OS
 # releases Therefore we include the playbook with the tests only if the hosts
 # would support it.
-# The test should run with NetworkManager, therefore it cannot run on
-# RHEL/CentOS 6
+# The test requires or should run with NetworkManager, therefore it cannot run
+# on RHEL/CentOS 6
 - import_playbook: playbooks/tests_ethtool_features.yml
   when:
     - ansible_distribution_major_version != '6'
-    # NetworkManager 1.20.0 introduced ethtool settings support
+
     - NetworkManager_version.stdout is version('1.20.0', '>=')

tests/tests_reapply_nm.yml

@@ -1,17 +1,21 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# This file was generated by ensure_provider_tests.py
 ---
 # set network provider and gather facts
 - hosts: all
+  name: Run playbook 'playbooks/tests_reapply.yml' with nm as provider
   tasks:
     - name: Set network provider to 'nm'
       set_fact:
         network_provider: nm
 
+
 # workaround for: https://github.com/ansible/ansible/issues/27973
 # There is no way in Ansible to abort a playbook hosts with specific OS
 # releases Therefore we include the playbook with the tests only if the hosts
 # would support it.
-# The test should run with NetworkManager, therefore it cannot run on
-# RHEL/CentOS 6
+# The test requires or should run with NetworkManager, therefore it cannot run
+# on RHEL/CentOS 6
 - import_playbook: playbooks/tests_reapply.yml
   when:
     - ansible_distribution_major_version != '6'

tests/tests_regression_nm.yml

@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: BSD-3-Clause
 ---
 # set network provider and gather facts
 - hosts: all

tests/tests_states.yml

@@ -1,11 +0,0 @@
----
-# empty playbook to gather facts for import_playbook when clause
-- hosts: all
-
-# workaround for: https://github.com/ansible/ansible/issues/27973
-# There is no way in Ansible to abort a playbook hosts with specific OS
-# releases Therefore we include the playbook with the tests only if the hosts
-# would support it.
-# The test requires NetworkManager, therefore it cannot run on RHEL/CentOS 6
-- import_playbook: playbooks/tests_states.yml
-  when: ansible_distribution_major_version != '6'

tests/tests_states_nm.yml

@@ -0,0 +1,21 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# This file was generated by ensure_provider_tests.py
+---
+# set network provider and gather facts
+- hosts: all
+  name: Run playbook 'playbooks/tests_states.yml' with nm as provider
+  tasks:
+    - name: Set network provider to 'nm'
+      set_fact:
+        network_provider: nm
+
+
+# workaround for: https://github.com/ansible/ansible/issues/27973
+# There is no way in Ansible to abort a playbook hosts with specific OS
+# releases Therefore we include the playbook with the tests only if the hosts
+# would support it.
+# The test requires or should run with NetworkManager, therefore it cannot run
+# on RHEL/CentOS 6
+- import_playbook: playbooks/tests_states.yml
+  when:
+    - ansible_distribution_major_version != '6'

tests/tests_vlan_mtu_initscripts.yml

@@ -1,13 +1,11 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# This file was generated by ensure_provider_tests.py
 ---
-# set network provider and gather facts
 - hosts: all
+  name: Run playbook 'playbooks/tests_vlan_mtu.yml' with initscripts as provider
   tasks:
     - name: Set network provider to 'initscripts'
       set_fact:
         network_provider: initscripts
 
-# workaround for: https://github.com/ansible/ansible/issues/27973
-# There is no way in Ansible to abort a playbook hosts with specific OS
-# releases Therefore we include the playbook with the tests only if the hosts
-# would support it.
 - import_playbook: playbooks/tests_vlan_mtu.yml

tests/tests_vlan_mtu_nm.yml

@@ -1,15 +1,21 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# This file was generated by ensure_provider_tests.py
 ---
 # set network provider and gather facts
 - hosts: all
+  name: Run playbook 'playbooks/tests_vlan_mtu.yml' with nm as provider
   tasks:
     - name: Set network provider to 'nm'
       set_fact:
         network_provider: nm
 
+
 # workaround for: https://github.com/ansible/ansible/issues/27973
 # There is no way in Ansible to abort a playbook hosts with specific OS
 # releases Therefore we include the playbook with the tests only if the hosts
 # would support it.
-# The test requires NetworkManager, therefore it cannot run on RHEL/CentOS 6
+# The test requires or should run with NetworkManager, therefore it cannot run
+# on RHEL/CentOS 6
 - import_playbook: playbooks/tests_vlan_mtu.yml
-  when: ansible_distribution_major_version != '6'
+  when:
+    - ansible_distribution_major_version != '6'