From 7f0f07599a680258092a7790fb3046856f486f2f Mon Sep 17 00:00:00 2001 From: Radostin Stoyanov Date: Sun, 4 Dec 2022 16:27:09 +0000 Subject: [PATCH] crit: fix compatibility with Python 3.12 Python 3.12 includes a few breaking changes, such as the removal of the distutils module [1] and the deprecation of `setup.py install` in favour of pip install [2]. This patch updates the installation script for crit to reflect these changes by replacing the use of `setup.py install` with `pip install` and `distutils` with `setuptools`. In addition, a minimal pyproject.toml file has been added as it is required by the new version of pip [3]. It is worth noting that with this change we are switching from the egg packaging format to wheel [4] and add pip as a build dependency. [1] https://www.python.org/downloads/release/python-3120a2/ [2] https://github.com/pypa/setuptools/pull/2824 [3] https://pip.pypa.io/en/stable/reference/build-system/pyproject-toml/ [4] https://packaging.python.org/en/latest/discussions/wheel-vs-egg/ Signed-off-by: Radostin Stoyanov --- .cirrus.yml | 4 +- .gitignore | 1 - Makefile | 3 +- crit/.gitignore | 2 + crit/pyproject.toml | 2 + crit/setup.py | 29 +++++++++++ lib/Makefile | 12 ++--- scripts/build/Dockerfile.alpine | 1 + scripts/build/Dockerfile.archlinux | 1 + scripts/ci/prepare-for-fedora-rawhide.sh | 2 + scripts/ci/run-ci-tests.sh | 2 +- scripts/ci/vagrant.sh | 2 +- scripts/crit-setup.py | 25 --------- scripts/uninstall_module.py | 65 ++++++++++++++++++++++++ 14 files changed, 114 insertions(+), 37 deletions(-) create mode 100644 crit/.gitignore create mode 100644 crit/pyproject.toml create mode 100644 crit/setup.py delete mode 100644 scripts/crit-setup.py create mode 100755 scripts/uninstall_module.py diff --git a/.cirrus.yml b/.cirrus.yml index 914ceb72c..bd4799fd0 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -36,7 +36,7 @@ task: ln -sf /usr/include/google/protobuf/descriptor.proto images/google/protobuf/descriptor.proto dnf config-manager --set-enabled crb # Same as CentOS 8 powertools dnf -y install epel-release epel-next-release - dnf -y install --allowerasing asciidoc gcc git gnutls-devel libaio-devel libasan libcap-devel libnet-devel libnl3-devel libbsd-devel libselinux-devel make protobuf-c-devel protobuf-devel python-devel python-PyYAML python-future python-protobuf python-junit_xml python-flake8 xmlto + dnf -y install --allowerasing asciidoc gcc git gnutls-devel libaio-devel libasan libcap-devel libnet-devel libnl3-devel libbsd-devel libselinux-devel make protobuf-c-devel protobuf-devel python-devel python-PyYAML python-future python-protobuf python-junit_xml python3-importlib-metadata python-flake8 xmlto systemctl stop sssd # Even with selinux in permissive mode the selinux tests will be executed. # The Cirrus CI user runs as a service from selinux point of view and is @@ -108,7 +108,7 @@ task: yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm || : yum install -y dnf-plugins-core yum config-manager --set-enabled powertools - yum install -y --allowerasing asciidoc gcc git gnutls-devel libaio-devel libasan libcap-devel libnet-devel libnl3-devel libbsd-devel libselinux-devel make protobuf-c-devel protobuf-devel python3-devel python3-flake8 python3-PyYAML python3-future python3-protobuf python3-junit_xml xmlto + yum install -y --allowerasing asciidoc gcc git gnutls-devel libaio-devel libasan libcap-devel libnet-devel libnl3-devel libbsd-devel libselinux-devel make protobuf-c-devel protobuf-devel python3-devel python3-flake8 python3-PyYAML python3-future python3-protobuf python3-importlib-metadata python3-junit_xml xmlto alternatives --set python /usr/bin/python3 systemctl stop sssd # Even with selinux in permissive mode the selinux tests will be executed diff --git a/.gitignore b/.gitignore index 23894d631..1ea828bbc 100644 --- a/.gitignore +++ b/.gitignore @@ -38,7 +38,6 @@ criu/pie/parasite-blob.h criu/protobuf-desc-gen.h lib/build/ lib/c/criu.pc -lib/.crit-setup.files compel/include/asm include/common/asm include/common/config.h diff --git a/Makefile b/Makefile index 7d0f2350a..8061a42c4 100644 --- a/Makefile +++ b/Makefile @@ -428,7 +428,8 @@ lint: flake8 --config=scripts/flake8.cfg lib/py/images/pb2dict.py flake8 --config=scripts/flake8.cfg lib/py/images/images.py flake8 --config=scripts/flake8.cfg scripts/criu-ns - flake8 --config=scripts/flake8.cfg scripts/crit-setup.py + flake8 --config=scripts/flake8.cfg crit/setup.py + flake8 --config=scripts/flake8.cfg scripts/uninstall_module.py flake8 --config=scripts/flake8.cfg coredump/ shellcheck --version shellcheck scripts/*.sh diff --git a/crit/.gitignore b/crit/.gitignore new file mode 100644 index 000000000..810661179 --- /dev/null +++ b/crit/.gitignore @@ -0,0 +1,2 @@ +crit.egg-info/ +build/ diff --git a/crit/pyproject.toml b/crit/pyproject.toml new file mode 100644 index 000000000..b1e1a4650 --- /dev/null +++ b/crit/pyproject.toml @@ -0,0 +1,2 @@ +[build-system] +requires = ["setuptools"] diff --git a/crit/setup.py b/crit/setup.py new file mode 100644 index 000000000..1aaa73a13 --- /dev/null +++ b/crit/setup.py @@ -0,0 +1,29 @@ +import os +from setuptools import setup, find_packages + + +def get_version(): + version = '0.0.1' + env = os.environ + if 'CRIU_VERSION_MAJOR' in env and 'CRIU_VERSION_MINOR' in env: + version = '{}.{}'.format( + env['CRIU_VERSION_MAJOR'], + env['CRIU_VERSION_MINOR'] + ) + if 'CRIU_VERSION_SUBLEVEL' in env and env['CRIU_VERSION_SUBLEVEL']: + version += '.' + env['CRIU_VERSION_SUBLEVEL'] + return version + + +setup( + name='crit', + version=get_version(), + description='CRiu Image Tool', + author='CRIU team', + author_email='criu@openvz.org', + license='GPLv2', + url='https://github.com/checkpoint-restore/criu', + packages=find_packages('.'), + scripts=['crit'], + install_requires=[], +) diff --git a/lib/Makefile b/lib/Makefile index 575a7bad3..ff540fb75 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -2,10 +2,6 @@ CRIU_SO := libcriu.so CRIU_A := libcriu.a UAPI_HEADERS := lib/c/criu.h images/rpc.proto images/rpc.pb-c.h criu/include/version.h -# -# File to keep track of files installed by setup.py -CRIT_SETUP_FILES := lib/.crit-setup.files - all-y += lib-c lib-a lib-py # @@ -58,8 +54,10 @@ install: lib-c lib-a lib-py crit/crit lib/c/criu.pc.in $(Q) mkdir -p $(DESTDIR)$(LIBDIR)/pkgconfig $(Q) sed -e 's,@version@,$(CRIU_VERSION),' -e 's,@libdir@,$(LIBDIR),' -e 's,@includedir@,$(dir $(INCLUDEDIR)/criu/),' lib/c/criu.pc.in > lib/c/criu.pc $(Q) install -m 644 lib/c/criu.pc $(DESTDIR)$(LIBDIR)/pkgconfig +ifeq ($(PYTHON),python3) $(E) " INSTALL " crit - $(Q) $(PYTHON) scripts/crit-setup.py install --prefix=$(DESTDIR)$(PREFIX) --record $(CRIT_SETUP_FILES) + $(Q) $(PYTHON) -m pip install --upgrade --force-reinstall --prefix=$(DESTDIR)$(PREFIX) ./crit +endif .PHONY: install uninstall: @@ -71,6 +69,8 @@ uninstall: $(Q) $(RM) $(addprefix $(DESTDIR)$(INCLUDEDIR)/criu/,$(notdir $(UAPI_HEADERS))) $(E) " UNINSTALL" pkgconfig/criu.pc $(Q) $(RM) $(addprefix $(DESTDIR)$(LIBDIR)/pkgconfig/,criu.pc) +ifeq ($(PYTHON),python3) $(E) " UNINSTALL" crit - $(Q) while read -r file; do $(RM) "$$file"; done < $(CRIT_SETUP_FILES) + $(Q) $(PYTHON) ./scripts/uninstall_module.py --prefix=$(DESTDIR)$(PREFIX) crit +endif .PHONY: uninstall diff --git a/scripts/build/Dockerfile.alpine b/scripts/build/Dockerfile.alpine index 19b08315f..af1858ab5 100644 --- a/scripts/build/Dockerfile.alpine +++ b/scripts/build/Dockerfile.alpine @@ -40,6 +40,7 @@ RUN apk add \ e2fsprogs \ py-yaml \ py3-flake8 \ + py3-importlib-metadata \ asciidoctor # The rpc test cases are running as user #1000, let's add the user diff --git a/scripts/build/Dockerfile.archlinux b/scripts/build/Dockerfile.archlinux index ce2a38bd4..f2bce1e5b 100644 --- a/scripts/build/Dockerfile.archlinux +++ b/scripts/build/Dockerfile.archlinux @@ -34,6 +34,7 @@ RUN pacman -Syu --noconfirm \ flake8 \ asciidoctor \ python-junit-xml \ + python-importlib-metadata \ diffutils COPY . /criu diff --git a/scripts/ci/prepare-for-fedora-rawhide.sh b/scripts/ci/prepare-for-fedora-rawhide.sh index f4d3155f9..7c62aaaa2 100755 --- a/scripts/ci/prepare-for-fedora-rawhide.sh +++ b/scripts/ci/prepare-for-fedora-rawhide.sh @@ -27,6 +27,8 @@ dnf install -y \ python3-future \ python3-protobuf \ python3-junit_xml \ + python3-pip \ + python3-importlib-metadata \ python-unversioned-command \ redhat-rpm-config \ sudo \ diff --git a/scripts/ci/run-ci-tests.sh b/scripts/ci/run-ci-tests.sh index 5b9f6d929..229de97c1 100755 --- a/scripts/ci/run-ci-tests.sh +++ b/scripts/ci/run-ci-tests.sh @@ -6,7 +6,7 @@ CI_PKGS=(protobuf-c-compiler libprotobuf-c-dev libaio-dev libgnutls28-dev libnl-3-dev gdb bash libnet-dev util-linux asciidoctor libnl-route-3-dev time flake8 libbsd-dev python3-yaml libperl-dev pkg-config python3-future python3-protobuf - python3-junit.xml) + python3-pip python3-importlib-metadata python3-junit.xml) X86_64_PKGS=(gcc-multilib) diff --git a/scripts/ci/vagrant.sh b/scripts/ci/vagrant.sh index f0996b01d..5cc842442 100755 --- a/scripts/ci/vagrant.sh +++ b/scripts/ci/vagrant.sh @@ -38,7 +38,7 @@ setup() { ssh default sudo dnf upgrade -y ssh default sudo dnf install -y gcc git gnutls-devel nftables-devel libaio-devel \ libasan libcap-devel libnet-devel libnl3-devel libbsd-devel make protobuf-c-devel \ - protobuf-devel python3-flake8 python3-future python3-protobuf \ + protobuf-devel python3-flake8 python3-future python3-protobuf python3-importlib-metadata \ python3-junit_xml rubygem-asciidoctor iptables libselinux-devel libbpf-devel # Disable sssd to avoid zdtm test failures in pty04 due to sssd socket ssh default sudo systemctl mask sssd diff --git a/scripts/crit-setup.py b/scripts/crit-setup.py deleted file mode 100644 index 13df03e3b..000000000 --- a/scripts/crit-setup.py +++ /dev/null @@ -1,25 +0,0 @@ -import os -from distutils.core import setup - -criu_version = "0.0.1" -env = os.environ - -if 'CRIU_VERSION_MAJOR' in env and 'CRIU_VERSION_MINOR' in env: - criu_version = '{}.{}'.format( - env['CRIU_VERSION_MAJOR'], - env['CRIU_VERSION_MINOR'] - ) - - if 'CRIU_VERSION_SUBLEVEL' in env and env['CRIU_VERSION_SUBLEVEL']: - criu_version += '.' + env['CRIU_VERSION_SUBLEVEL'] - -setup(name="crit", - version=criu_version, - description="CRiu Image Tool", - author="CRIU team", - author_email="criu@openvz.org", - license="GPLv2", - url="https://github.com/checkpoint-restore/criu", - package_dir={'pycriu': 'lib/py'}, - packages=["pycriu", "pycriu.images"], - scripts=["crit/crit"]) diff --git a/scripts/uninstall_module.py b/scripts/uninstall_module.py new file mode 100755 index 000000000..439fca18a --- /dev/null +++ b/scripts/uninstall_module.py @@ -0,0 +1,65 @@ +#!/usr/bin/python3 +""" +`pip uninstall` doesn't support `--prefix`. +https://github.com/pypa/pip/issues/11213 +""" +import argparse +import os +import shutil +import site +import subprocess +import sys + +import importlib_metadata + + +def add_site_dir(prefix: str): + """ + Add site directory with prefix to sys.path and update PYTHONPATH. + """ + # If prefix is used, we need to make sure that we + # do not uninstall other packages from the system paths. + sys.path = [] + site.PREFIXES = [prefix] + pkgs = site.getsitepackages() + for path in pkgs: + site.addsitedir(path) + if 'dist-packages' in path: + # Ubuntu / Debian might use both dist- and site- packages. + site.addsitedir(path.replace('dist-packages', 'site-packages')) + os.environ['PYTHONPATH'] = os.pathsep.join(sys.path) + + +def uninstall_module(package_name: str, prefix=None): + """ + Enable support for '--prefix' with 'pip uninstall'. + """ + dist_info_path = None + if prefix: + add_site_dir(prefix) + try: + dist_info_path = str(importlib_metadata.distribution(package_name)._path) + except importlib_metadata.PackageNotFoundError: + print(f"Skipping {package_name} as it is not installed.") + sys.exit(0) + + command = [sys.executable, '-m', 'pip', 'uninstall', '-y', package_name] + try: + subprocess.check_call(command, env=os.environ) + if dist_info_path and os.path.isdir(dist_info_path): + # .dist-info files are not cleaned up when the package + # has been installed with --prefix. + # https://github.com/pypa/pip/issues/5573 + shutil.rmtree(dist_info_path) + if 'dist-packages' in dist_info_path: + shutil.rmtree(dist_info_path.replace('dist-packages', 'site-packages')) + except subprocess.CalledProcessError as err: + print(f'Error uninstalling package {package_name}: {err}') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('module_name', help='The name of the module to uninstall') + parser.add_argument('--prefix', help='The prefix where the module was installed') + args = parser.parse_args() + uninstall_module(args.module_name, args.prefix)