diff --git a/tests/playbooks/integration_pytest_python3.yml b/tests/playbooks/integration_pytest_python3.yml index 267dd7a..4c8c61e 100644 --- a/tests/playbooks/integration_pytest_python3.yml +++ b/tests/playbooks/integration_pytest_python3.yml @@ -30,110 +30,16 @@ tasks: - name: Run Pytest tests block: - - name: Create tempdir for code to test - tempfile: - state: directory - prefix: lsrtest_ - register: _rundir - - - name: Get tempfile for tar - tempfile: - prefix: lsrtest_ - suffix: ".tar" - register: temptar - delegate_to: localhost - - - name: Include the task '../tasks/get_modules_and_utils_paths.yml' - include_tasks: ../tasks/get_modules_and_utils_paths.yml - - - name: Get tests directory - set_fact: - tests_directory: "{{ lookup('first_found', params) }}" - vars: - params: - files: - - tests - - network - paths: - - "../.." - - # TODO: using tar and copying the file is a workaround for the - # synchronize module that does not work in test-harness. Related issue: - # https://github.com/linux-system-roles/test-harness/issues/102 - # - - name: Create Tar file - command: > - tar -cvf {{ temptar.path }} --exclude "*.pyc" - --exclude "__pycache__" - -C {{ tests_directory | realpath | dirname }} - {{ tests_directory | basename }} - -C {{ modules_parent_and_dir.stdout_lines[0] }} - {{ modules_parent_and_dir.stdout_lines[1] }} - -C {{ module_utils_parent_and_dir.stdout_lines[0] }} - {{ module_utils_parent_and_dir.stdout_lines[1] }} - # noqa command-instead-of-module - delegate_to: localhost - changed_when: false - - - name: Copy testrepo.tar to the remote system - copy: - src: "{{ temptar.path }}" - dest: "{{ _rundir.path }}" - mode: preserve - - - name: Untar testrepo.tar - unarchive: - src: "{{ _rundir.path }}/{{ temptar.path | basename }}" - dest: "{{ _rundir.path }}" - remote_src: true - - - name: "Create subdirectory './ansible' under '{{ _rundir.path }}'" - file: - state: directory - path: "{{ _rundir.path }}/ansible" - mode: "0755" - - - name: Move module_utils to ansible directory - shell: | - if [ -d {{ _rundir.path }}/module_utils ]; then - mv {{ _rundir.path }}/module_utils {{ _rundir.path }}/ansible - fi - changed_when: false - - - name: Fake out python module directories, primarily for python2 - shell: | - for dir in $(find {{ _rundir.path }} -type d -print); do - if [ ! -f "$dir/__init__.py" ]; then - touch "$dir/__init__.py" - fi - done - changed_when: false - - - name: Set _lsr_python_path - set_fact: - _lsr_python_path: "{{ - _rundir.path ~ '/' ~ - modules_parent_and_dir.stdout_lines[1] ~ ':' ~ _rundir.path - }}" - - - name: Show _lsr_python_path - debug: - msg: path {{ _lsr_python_path }} - - - name: "List the files in {{ _rundir.path }}" - command: ls -alrtFR {{ _rundir.path }} - changed_when: false + - name: Copy python files to the remote system + include_tasks: ../tasks/setup_remote_pytest.yml - name: Run pytest with nm provider block: - name: Run pytest with nm - command: > - pytest-3 - {{ _rundir.path }}/{{ tests_directory | basename }}/integration/ - --provider=nm - register: playbook_run + command: pytest-3 {{ _rundir.path }}/integration/ --provider=nm environment: PYTHONPATH: "{{ _lsr_python_path }}" + register: playbook_run changed_when: false always: - name: Debug stdout_lines of the running playbook with nm @@ -149,13 +55,10 @@ use: "{{ (__network_is_ostree | d(false)) | ternary('ansible.posix.rhel_rpm_ostree', omit) }}" - name: Run pytest with initscripts - command: > - pytest-3 - {{ _rundir.path }}/{{ tests_directory | basename }}/integration/ - --provider=initscripts - register: playbook_run + command: pytest-3 {{ _rundir.path }}/integration/ --provider=initscripts environment: PYTHONPATH: "{{ _lsr_python_path }}" + register: playbook_run changed_when: false always: - name: Debug stdout_lines of the running playbook with initscripts diff --git a/tests/tasks/get_modules_and_utils_paths.yml b/tests/tasks/get_modules_and_utils_paths.yml deleted file mode 100644 index e9bab93..0000000 --- a/tests/tasks/get_modules_and_utils_paths.yml +++ /dev/null @@ -1,98 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause ---- -- name: Set collection paths - set_fact: - collection_paths: | - {{ - (lookup("env","ANSIBLE_COLLECTIONS_PATH").split(":") + - lookup("env","ANSIBLE_COLLECTIONS_PATHS").split(":") + - lookup("config", "COLLECTIONS_PATHS")) | - select | list - }} - -- name: Set search paths - set_fact: - modules_search_path: | - {{ - (lookup("env", "ANSIBLE_LIBRARY").split(":") + - ["../../library", "../library"] + - lookup("config", "DEFAULT_MODULE_PATH")) | - select | list - }} - module_utils_search_path: | - {{ - (lookup("env", "ANSIBLE_MODULE_UTILS").split(":") + - ["../../module_utils", "../module_utils"] + - lookup("config", "DEFAULT_MODULE_UTILS_PATH")) | - select | list - }} - -# the output should be something like -# - path to parent directory to chdir to in order to use tar -# - relative path under parent directory to tar -# e.g. for the local role case -# - ../.. -# - library -# would translate to tar -C ../.. library -# for the collection case -# - /home/user/.ansible/collections -# - ansible_collections/fedora/linux_system_roles/plugins/modules -# would translate to tar -C /home/user/.ansible/collections \ -# ansible_collections/fedora/linux_system_roles/plugins/modules -- name: Find parent directory and path of modules - shell: - executable: /bin/bash # ensure pipefail setting - cmd: | - set -euxo pipefail - for dir in {{ modules_search_path | join(" ") }}; do - if [ -f "$dir/network_connections.py" ]; then - readlink -f "$(dirname "$dir")" - basename "$dir" - exit 0 - fi - done - for dir in {{ collection_paths | join(" ") }}; do - if [ ! -d "$dir" ]; then continue; fi - cd "$dir" - for subdir in ansible_collections/*/*/plugins/modules; do - if [ -f "$subdir/network_connections.py" ]; then - echo "$dir" - echo "$subdir" - exit 0 - fi - done - done - echo network_connections.py not found - exit 1 - delegate_to: localhost - register: modules_parent_and_dir - changed_when: false - -- name: Find parent directory and path of module_utils - shell: - executable: /bin/bash # ensure pipefail setting - cmd: | - set -euxo pipefail - for dir in {{ module_utils_search_path | join(" ") }}; do - if [ -d "$dir/network_lsr" ]; then - readlink -f "$(dirname "$dir")" - basename "$dir" - exit 0 - fi - done - for dir in {{ collection_paths | join(" ") }}; do - if [ ! -d "$dir" ]; then continue; fi - cd "$dir" - for subdir in ansible_collections/*/*/plugins/module_utils; do - if [ -d "$subdir/network_lsr" ]; then - echo "$dir" - echo "$subdir" - exit 0 - fi - done - done - echo network_lsr not found - exit 1 - delegate_to: localhost - register: module_utils_parent_and_dir - changed_when: false diff --git a/tests/tasks/setup_remote_pytest.yml b/tests/tasks/setup_remote_pytest.yml new file mode 100644 index 0000000..c326401 --- /dev/null +++ b/tests/tasks/setup_remote_pytest.yml @@ -0,0 +1,221 @@ +# SPDX-License-Identifier: BSD-3-Clause +# outputs: +# - _rundir - directory on remote machine +# - temptar - tempfile for tar +# - _lsr_python_path - python path to use for pytest +--- +- name: Create tempdir for code to test + tempfile: + state: directory + prefix: lsrtest_ + register: _rundir + +- name: Get tempfile for tar + tempfile: + prefix: lsrtest_ + suffix: ".tar" + register: temptar + delegate_to: localhost + +# The starting assumption is that we are running this from the control node +# in the directory containing the test playbook. +# This is complicated because we have to handle so many different cases: +# - local role from git clone which may not be in a "normal" directory structure +# - git clone in a pull request in legacy format +# - collection format converted by tox -e collection or similar +# - legacy and collection format as built by RPM or from collection build +# - test harness that copies the tests outside of the role/collection to a tempdir +# The output is a list of paths to tar, each path is a list of: +# - path to parent directory to chdir to in order to use tar +# - relative path or filenames under parent directory to tar +# e.g. for the local role case +# - /some/path/to/library +# - network_*.py +# - /some/path/to/module_utils +# - network_lsr +# would translate to tar -C /some/path/to/library network_*.py -C /some/path/to/module_utils network_lsr +# for the collection case +# - /home/user/.tox/ansible_collections/fedora/linux_system_roles/tests/network/plugins/modules +# - network_*.py +# - /home/user/.tox +# - ansible_collections/fedora/linux_system_roles/tests/network/plugins/module_utils/network_lsr/network_lsr +# would translate to tar -C /home/user/.tox/ansible_collections/fedora/linux_system_roles/tests/network/plugins/modules network_*.py \ +# -C /home/user/.tox ansible_collections/fedora/linux_system_roles/tests/network/plugins/module_utils/network_lsr +# because we have to preserve the entire directory structure for the module_utils for the collection case +- name: Get paths to tar + delegate_to: localhost + changed_when: false + shell: + executable: /bin/bash # ensure pipefail setting + cmd: | + set -euo pipefail + # This assumes we are in the directory where the test playbook is located + test_playbook_dir="$(pwd)" + if [[ "$test_playbook_dir" =~ /playbooks$ ]]; then + test_playbook_dir="$(dirname "$test_playbook_dir")" + fi + # full path to parent and subdir for unit tests + echo "$test_playbook_dir" + echo "unit" + # full path to parent and subdir for integration tests + echo "$test_playbook_dir" + echo "integration" + if [[ "$test_playbook_dir" =~ /tests/network$ ]]; then + is_coll=true + else + is_coll=false + fi + # right now we only test network_connections.py, but we may want to test other modules in the future + modules_to_test=(network_connections.py) + if [[ "$test_playbook_dir" =~ (/.*)/(ansible_collections/[a-z0-9_]+/[a-z0-9_]+)/tests/network ]] && \ + [ -d "${BASH_REMATCH[1]}/${BASH_REMATCH[2]}/plugins/modules" ] && \ + [ -d "${BASH_REMATCH[1]}/${BASH_REMATCH[2]}/plugins/module_utils/network_lsr" ]; then + # we don't need the directory for the modules, we just need the filenames + # otherwise we have to have logic in the destination directory to handle library vs plugins/modules + echo "${BASH_REMATCH[1]}/${BASH_REMATCH[2]}/plugins/modules" + echo "${modules_to_test[@]}" + # we do need the entire directory structure for the module_utils due to + # the way Ansible loads module_utils + echo "${BASH_REMATCH[1]}" + echo "${BASH_REMATCH[2]}/plugins/module_utils/network_lsr" + elif [[ "$test_playbook_dir" =~ (/.*)/([a-z]+-system-roles[.]network)/tests ]] && \ + [ -d "${BASH_REMATCH[1]}/${BASH_REMATCH[2]}/library" ] && \ + [ -d "${BASH_REMATCH[1]}/${BASH_REMATCH[2]}/module_utils/network_lsr" ]; then + echo "${BASH_REMATCH[1]}/${BASH_REMATCH[2]}/library" + echo "${modules_to_test[@]}" + echo "${BASH_REMATCH[1]}/${BASH_REMATCH[2]}" + echo module_utils/network_lsr + elif [ "$is_coll" = true ]; then + # search collection paths for modules and module_utils + for dir in {{ collection_paths | join(" ") }}; do + for collections_dir in "$dir/ansible_collections/*/*"; do + if [ -d "$collections_dir/plugins/modules" ] && [ -d "$collections_dir/plugins/module_utils/network_lsr" ]; then + readlink -f "$collections_dir/plugins/modules" + echo "${modules_to_test[@]}" + readlink -f "$dir" + echo "${collections_dir#$dir/}/plugins/module_utils/network_lsr" + break 2 + fi + done + done + else + # look for modules in modules search path + for dir in {{ modules_search_path | join(" ") }}; do + if [ -f "$dir/${modules_to_test[0]}" ] && [ ! -L "$dir/${modules_to_test[0]}" ]; then + readlink -f "$dir" + echo "${modules_to_test[@]}" + break + fi + done + # look for module_utils in module_utils search path + for dir in {{ module_utils_search_path | join(" ") }}; do + if [ -d "$dir/network_lsr" ]; then + echo "$(dirname "$(readlink -f "$dir")")" + echo module_utils/network_lsr + break + fi + done + fi + register: __network_tar_paths + vars: + collection_paths: | + {{ + (lookup("env","ANSIBLE_COLLECTIONS_PATH").split(":") + + lookup("env","ANSIBLE_COLLECTIONS_PATHS").split(":") + + lookup("config", "COLLECTIONS_PATHS")) | + select | unique | list + }} + modules_search_path: | + {{ + (lookup("env", "ANSIBLE_LIBRARY").split(":") + + ["../../library", "../library"] + + lookup("config", "DEFAULT_MODULE_PATH")) | + select | unique | list + }} + module_utils_search_path: | + {{ + (lookup("env", "ANSIBLE_MODULE_UTILS").split(":") + + ["../../module_utils", "../module_utils"] + + lookup("config", "DEFAULT_MODULE_UTILS_PATH")) | + select | unique | list + }} + +- name: Create Tar file + command: > + tar -cvf {{ temptar.path }} --exclude "*.pyc" + --exclude "__pycache__" + -C {{ __network_tar_paths.stdout_lines[0] }} + {{ __network_tar_paths.stdout_lines[1] }} + -C {{ __network_tar_paths.stdout_lines[2] }} + {{ __network_tar_paths.stdout_lines[3] }} + -C {{ __network_tar_paths.stdout_lines[4] }} + {{ __network_tar_paths.stdout_lines[5] }} + -C {{ __network_tar_paths.stdout_lines[6] }} + {{ __network_tar_paths.stdout_lines[7] }} + # noqa command-instead-of-module + delegate_to: localhost + changed_when: false + +- name: Copy testrepo.tar to the remote system + copy: + src: "{{ temptar.path }}" + dest: "{{ _rundir.path }}" + mode: preserve + +- name: Untar testrepo.tar + unarchive: + src: "{{ _rundir.path }}/{{ temptar.path | basename }}" + dest: "{{ _rundir.path }}" + remote_src: true + +# only needed for legacy role - ignored for collection case +- name: Create subdirectory './ansible' under '{{ _rundir.path }}' + file: + state: directory + path: "{{ _rundir.path }}/ansible" + mode: "0755" + +# only needed for legacy role - ignored for collection case +- name: Move module_utils to ansible directory + shell: + executable: /bin/bash + cmd: | + set -euo pipefail + if [ -d {{ _rundir.path }}/module_utils ]; then + mv {{ _rundir.path }}/module_utils {{ _rundir.path }}/ansible + elif [ ! -d {{ _rundir.path }}/ansible/module_utils ]; then + mkdir -p {{ _rundir.path }}/ansible/module_utils + fi + changed_when: false + +- name: Fake out python module directories, primarily for python2 + shell: + executable: /bin/bash + cmd: | + set -euo pipefail + for dir in $(find {{ _rundir.path }} -type d -print); do + if [ ! -f "$dir/__init__.py" ]; then + touch "$dir/__init__.py" + fi + done + changed_when: false + +- name: Set _lsr_python_path + set_fact: + _lsr_python_path: "{{ __paths | unique | join(':') }}" + vars: + __module_utils_path: "{{ __network_tar_paths.stdout_lines[7] | dirname }}" + __plugins_path: "{{ __module_utils_path | dirname }}" + __paths: + - "{{ _rundir.path ~ '/' ~ __plugins_path ~ '/modules' }}" + - "{{ _rundir.path ~ '/ansible/' ~ __module_utils_path }}" + - "{{ _rundir.path ~ '/' ~ __module_utils_path }}" + - "{{ _rundir.path }}" + +- name: Show _lsr_python_path + debug: + msg: path {{ _lsr_python_path }} + +- name: List the files in {{ _rundir.path }} + command: ls -alrtFR {{ _rundir.path }} + changed_when: false diff --git a/tests/tests_unit.yml b/tests/tests_unit.yml index b62917b..168d7d9 100644 --- a/tests/tests_unit.yml +++ b/tests/tests_unit.yml @@ -26,100 +26,8 @@ tasks: - name: Execute python unit tests block: - - name: Create tempdir for code to test - tempfile: - state: directory - prefix: lsrtest_ - register: _rundir - - - name: Get tempfile for tar - tempfile: - prefix: lsrtest_ - suffix: ".tar" - register: temptar - delegate_to: localhost - - - name: Include the task 'get_modules_and_utils_paths.yml' - include_tasks: tasks/get_modules_and_utils_paths.yml - - # TODO: using tar and copying the file is a workaround for the - # synchronize module that does not work in test-harness. Related issue: - # https://github.com/linux-system-roles/test-harness/issues/102 - # - - name: Create Tar file - command: > - tar -cvf {{ temptar.path }} --exclude "*.pyc" - --exclude "__pycache__" - -C {{ modules_parent_and_dir.stdout_lines[0] }} - {{ modules_parent_and_dir.stdout_lines[1] }} - -C {{ module_utils_parent_and_dir.stdout_lines[0] }} - {{ module_utils_parent_and_dir.stdout_lines[1] }} - # noqa command-instead-of-module - delegate_to: localhost - changed_when: false - - - name: Copy testrepo.tar to the remote system - copy: - src: "{{ temptar.path }}" - dest: "{{ _rundir.path }}" - mode: preserve - - - name: Untar testrepo.tar - unarchive: - src: "{{ _rundir.path }}/{{ temptar.path | basename }}" - dest: "{{ _rundir.path }}" - remote_src: true - - - name: Create subdirectory '/ansible/module_utils' under - '{{ _rundir.path }}' - file: - state: directory - path: "{{ item }}" - mode: "0755" - loop: - - "{{ _rundir.path }}/ansible" - - "{{ _rundir.path }}/ansible/module_utils" - - - name: Move module_utils to ansible directory - shell: | - if [ -d {{ _rundir.path }}/module_utils ]; then - mv {{ _rundir.path }}/module_utils {{ _rundir.path }}/ansible - fi - changed_when: false - - - name: Fake out python module directories, primarily for python2 - shell: | - for dir in $(find {{ _rundir.path }} -type d -print); do - if [ ! -f "$dir/__init__.py" ]; then - touch "$dir/__init__.py" - fi - done - changed_when: false - - - name: Copy unit test to remote system - copy: - src: unit/test_network_connections.py - dest: "{{ _rundir.path }}" - mode: preserve - - - name: Set _lsr_python_path - set_fact: - _lsr_python_path: "{{ - _rundir.path ~ '/' ~ - modules_parent_and_dir.stdout_lines[1] ~ ':' ~ - _rundir.path ~ '/' ~ 'ansible' ~ '/' ~ - module_utils_parent_and_dir.stdout_lines[1] ~ ':' ~ - _rundir.path ~ '/' ~ - module_utils_parent_and_dir.stdout_lines[1] ~ ':' ~ - _rundir.path - }}" - - - name: "List the files in {{ _rundir.path }}" - command: ls -alrtFR {{ _rundir.path }} - changed_when: false - - name: Debug the lsr Python path - debug: - msg: path {{ _lsr_python_path }} + - name: Copy python files to the remote system + include_tasks: tasks/setup_remote_pytest.yml - name: Check if python2 is available command: python2 --version @@ -130,7 +38,7 @@ - name: Run python2 unit tests command: > - python2 {{ _rundir.path }}/test_network_connections.py --verbose + python2 {{ _rundir.path }}/unit/test_network_connections.py --verbose environment: PYTHONPATH: "{{ _lsr_python_path }}" when: > @@ -147,7 +55,7 @@ - name: Run python3 unit tests command: > - python3 {{ _rundir.path }}/test_network_connections.py --verbose + python3 {{ _rundir.path }}/unit/test_network_connections.py --verbose environment: PYTHONPATH: "{{ _lsr_python_path }}" when: python3_available is succeeded