From 95479ba41fd38330f66b16f406869a2c6a9cc722 Mon Sep 17 00:00:00 2001 From: Filippo Squillace Date: Mon, 21 Dec 2020 08:58:23 +0100 Subject: [PATCH 1/6] Use arch-travis in CI --- .travis.yml | 91 +++++++++++++++++++++++++++++++++++------------ lib/core/build.sh | 10 ++++-- 2 files changed, 77 insertions(+), 24 deletions(-) diff --git a/.travis.yml b/.travis.yml index b4126d8..e09143c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,31 +1,78 @@ -language: bash - sudo: required +os: linux -env: - - TRAVIS_BASH_VERSION="4.0" +cache: + directories: + - ~/.ccache + - ~/.pkg-cache -before_install: - - ./tests/integ-tests/install-bash.sh "$TRAVIS_BASH_VERSION" +services: +- docker -install: - - PATH=$PWD/bin:$PATH - - junest setup - - junest -- echo "Installing JuNest (\$(uname -m))" - - JUNEST_HOME=~/.junest-arm junest setup --arch arm - - JUNEST_HOME=~/.junest-arm junest proot --fakeroot -- echo "Installing JuNest (\$(uname -m))" +archlinux: + mount: + - ~/.ccache:~/.ccache + - ~/.pkg-cache:/var/cache/pacman/pkg + repos: + - bartus=https://github.com/bartoszek/AUR-repo/raw/master + packages: + # pacman packages + - git + - ccache + - haveged + - aws-cli + + before_install: +# 1.Override `package-cleanup.hook` to preserve cache for travis. +# 2.Enable ccache +# 3.Multithreaded build and compress +# 4.Suppress all gcc warnings + - | + sudo mkdir /etc/pacman.d/hooks/ + sudo ln -s /dev/null /etc/pacman.d/hooks/package-cleanup.hook + sudo sed -i '/^BUILDENV/s/\!ccache/ccache/' /etc/makepkg.conf + sudo sed -i '/#MAKEFLAGS=/c MAKEFLAGS="-j2"' /etc/makepkg.conf + sudo sed -i '/^COMPRESSXZ/s/\xz/xz -T 2/' /etc/makepkg.conf + sudo sed -i '$a CFLAGS="$CFLAGS -w"' /etc/makepkg.conf + sudo sed -i '$a CXXFLAGS="$CXXFLAGS -w"' /etc/makepkg.conf + # TODO this is not used + script: + - "./bin/junest build -n" script: - - bash --version - - bash ./tests/checkstyle/checkstyle.sh - - bash ./tests/unit-tests/unit-tests.sh +- "curl -s https://raw.githubusercontent.com/bartoszek/arch-travis/master/arch-travis.sh | bash" +- "echo pacman pkg cache size: $(du -h ~/.pkg-cache|cut -f1) in $(ls ~/.pkg-cache|wc -l) files" - - export JUNEST_HOME=~/.junest - - ${PWD}/lib/checks/check_all.sh - - yes | junest setup --delete - # ARM with qemu does seem to work properly. Disabling integ tests for ARM for now. - #- export JUNEST_HOME=~/.junest-arm - #- junest proot --fakeroot -- ${PWD}/lib/checks/check.sh --skip-aur-tests - #- junest proot -- ${PWD}/lib/checks/check.sh --skip-aur-tests --use-sudo + +#language: bash + +#sudo: required + +#env: + #- TRAVIS_BASH_VERSION="4.0" + +#before_install: + #- ./tests/integ-tests/install-bash.sh "$TRAVIS_BASH_VERSION" + +#install: + #- PATH=$PWD/bin:$PATH + #- junest setup + #- junest -- echo "Installing JuNest (\$(uname -m))" + #- JUNEST_HOME=~/.junest-arm junest setup --arch arm + #- JUNEST_HOME=~/.junest-arm junest proot --fakeroot -- echo "Installing JuNest (\$(uname -m))" + +#script: + #- bash --version + #- bash ./tests/checkstyle/checkstyle.sh + #- bash ./tests/unit-tests/unit-tests.sh + + #- export JUNEST_HOME=~/.junest + #- ${PWD}/lib/checks/check_all.sh #- yes | junest setup --delete + + ## ARM with qemu does seem to work properly. Disabling integ tests for ARM for now. + ##- export JUNEST_HOME=~/.junest-arm + ##- junest proot --fakeroot -- ${PWD}/lib/checks/check.sh --skip-aur-tests + ##- junest proot -- ${PWD}/lib/checks/check.sh --skip-aur-tests --use-sudo + ##- yes | junest setup --delete diff --git a/lib/core/build.sh b/lib/core/build.sh index af0c5bb..a3ed8bd 100644 --- a/lib/core/build.sh +++ b/lib/core/build.sh @@ -80,6 +80,7 @@ function build_image_env(){ sudo install -d -m 755 "${maindir}/root/etc/${CMD}" sudo bash -c "echo 'JUNEST_ARCH=$ARCH' > ${maindir}/root/etc/${CMD}/info" + set -x info "Generating the locales..." # sed command is required for locale-gen but it is required by fakeroot # and cannot be removed @@ -87,14 +88,18 @@ function build_image_env(){ sudo pacman --noconfirm --root ${maindir}/root -S sed gzip sudo ln -sf /usr/share/zoneinfo/posix/UTC ${maindir}/root/etc/localtime sudo bash -c "echo 'en_US.UTF-8 UTF-8' >> ${maindir}/root/etc/locale.gen" - sudo ${maindir}/root/bin/groot ${maindir}/root locale-gen + sudo ${maindir}/root/bin/groot --no-umount ${maindir}/root locale-gen + #sudo mount --bind ${maindir}/root ${maindir}/root + #sudo arch-chroot ${maindir}/root locale-gen sudo bash -c "echo LANG=\"en_US.UTF-8\" >> ${maindir}/root/etc/locale.conf" sudo pacman --noconfirm --root ${maindir}/root -Rsn gzip info "Setting up the pacman keyring (this might take a while!)..." # gawk command is required for pacman-key sudo pacman --noconfirm --root ${maindir}/root -S gawk - sudo ${maindir}/root/bin/groot -b /dev ${maindir}/root bash -c ' + #TODO sudo mount --bind /dev ${maindir}/root/dev + #sudo ${maindir}/root/bin/groot --no-umount -b /dev ${maindir}/root bash -c ' + sudo ${maindir}/root/bin/groot --no-umount ${maindir}/root bash -c ' pacman-key --init; for keyring_file in /usr/share/pacman/keyrings/*.gpg; do @@ -102,6 +107,7 @@ function build_image_env(){ pacman-key --populate $keyring; done; [ -e /etc/pacman.d/gnupg/S.gpg-agent ] && gpg-connect-agent -S /etc/pacman.d/gnupg/S.gpg-agent killagent /bye' + sudo umount ${maindir}/root sudo pacman --noconfirm --root ${maindir}/root -Rsn gawk sudo rm ${maindir}/root/var/cache/pacman/pkg/* From b75faac48fccba542b82f7cfd786d2f0984853bd Mon Sep 17 00:00:00 2001 From: Filippo Squillace Date: Sat, 26 Dec 2020 12:33:48 +0100 Subject: [PATCH 2/6] Add remaining junest tests in travis --- .travis.yml | 81 ++++++++++++++++++++++------------------- lib/checks/check.sh | 6 ++- lib/checks/check_all.sh | 11 +++--- lib/core/build.sh | 11 ++---- 4 files changed, 58 insertions(+), 51 deletions(-) diff --git a/.travis.yml b/.travis.yml index e09143c..cfd7148 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,20 +13,18 @@ archlinux: mount: - ~/.ccache:~/.ccache - ~/.pkg-cache:/var/cache/pacman/pkg - repos: - - bartus=https://github.com/bartoszek/AUR-repo/raw/master packages: - # pacman packages - - git - - ccache - - haveged + # Pacman packages - aws-cli + - ccache + - git + - haveged before_install: -# 1.Override `package-cleanup.hook` to preserve cache for travis. -# 2.Enable ccache -# 3.Multithreaded build and compress -# 4.Suppress all gcc warnings + # 1.Override `package-cleanup.hook` to preserve cache for travis. + # 2.Enable ccache + # 3.Multithreaded build and compress + # 4.Suppress all gcc warnings - | sudo mkdir /etc/pacman.d/hooks/ sudo ln -s /dev/null /etc/pacman.d/hooks/package-cleanup.hook @@ -35,44 +33,51 @@ archlinux: sudo sed -i '/^COMPRESSXZ/s/\xz/xz -T 2/' /etc/makepkg.conf sudo sed -i '$a CFLAGS="$CFLAGS -w"' /etc/makepkg.conf sudo sed -i '$a CXXFLAGS="$CXXFLAGS -w"' /etc/makepkg.conf - # TODO this is not used script: - - "./bin/junest build -n" + - ./bin/junest build -n + + +env: + - TRAVIS_BASH_VERSION="4.0" + +before_install: + - ./tests/integ-tests/install-bash.sh "$TRAVIS_BASH_VERSION" + +install: + - PATH=$PWD/bin:$PATH script: -- "curl -s https://raw.githubusercontent.com/bartoszek/arch-travis/master/arch-travis.sh | bash" -- "echo pacman pkg cache size: $(du -h ~/.pkg-cache|cut -f1) in $(ls ~/.pkg-cache|wc -l) files" + ####################### + # Unit Tests + ####################### + - bash --version + - bash ./tests/checkstyle/checkstyle.sh + - bash ./tests/unit-tests/unit-tests.sh + # ARM with qemu does seem to work properly. Disabling integ tests for ARM for now. + #- export JUNEST_HOME=~/.junest-arm + #- junest proot --fakeroot -- ${PWD}/lib/checks/check.sh --skip-aur-tests + #- junest proot -- ${PWD}/lib/checks/check.sh --skip-aur-tests --use-sudo + #- yes | junest setup --delete + ####################### + # Build and validation tests + ####################### + - "curl -s https://raw.githubusercontent.com/bartoszek/arch-travis/master/arch-travis.sh | bash" + - "echo pacman pkg cache size: $(du -h ~/.pkg-cache|cut -f1) in $(ls ~/.pkg-cache|wc -l) files" + - ls -l + - pwd -#language: bash - -#sudo: required - -#env: - #- TRAVIS_BASH_VERSION="4.0" - -#before_install: - #- ./tests/integ-tests/install-bash.sh "$TRAVIS_BASH_VERSION" - -#install: - #- PATH=$PWD/bin:$PATH + # TODO Test the new created JuNest image against Ubuntu host #- junest setup #- junest -- echo "Installing JuNest (\$(uname -m))" #- JUNEST_HOME=~/.junest-arm junest setup --arch arm #- JUNEST_HOME=~/.junest-arm junest proot --fakeroot -- echo "Installing JuNest (\$(uname -m))" + - export JUNEST_HOME=~/.junest + - junest setup -i junest-x86_64.tar.gz + - ${PWD}/lib/checks/check_all.sh + - yes | junest setup --delete -#script: - #- bash --version - #- bash ./tests/checkstyle/checkstyle.sh - #- bash ./tests/unit-tests/unit-tests.sh + # TODO test deployed image can be downloaded correctly - #- export JUNEST_HOME=~/.junest - #- ${PWD}/lib/checks/check_all.sh - #- yes | junest setup --delete - ## ARM with qemu does seem to work properly. Disabling integ tests for ARM for now. - ##- export JUNEST_HOME=~/.junest-arm - ##- junest proot --fakeroot -- ${PWD}/lib/checks/check.sh --skip-aur-tests - ##- junest proot -- ${PWD}/lib/checks/check.sh --skip-aur-tests --use-sudo - ##- yes | junest setup --delete diff --git a/lib/checks/check.sh b/lib/checks/check.sh index 57dea5f..6ae3e6a 100755 --- a/lib/checks/check.sh +++ b/lib/checks/check.sh @@ -80,7 +80,11 @@ $SUDO pacman $PACMAN_OPTIONS -Rsn ${repo_package1} repo_package2=iftop info "Checking ${repo_package2} package from official repo..." $SUDO pacman $PACMAN_OPTIONS -S ${repo_package2} -$RUN_ROOT_TESTS && $SUDO iftop -t -s 5 +if $RUN_ROOT_TESTS +then + # Time it out given that sometimes it gets stuck after few seconds. + $SUDO timeout 10 iftop -t -s 5 || true +fi $SUDO pacman $PACMAN_OPTIONS -Rsn ${repo_package2} if ! $SKIP_AUR_TESTS diff --git a/lib/checks/check_all.sh b/lib/checks/check_all.sh index 6e28ee5..91bc058 100755 --- a/lib/checks/check_all.sh +++ b/lib/checks/check_all.sh @@ -8,10 +8,11 @@ set -ex JUNEST_BASE="${JUNEST_BASE:-$(readlink -f $(dirname $(readlink -f "$0"))/../..)}" JUNEST_SCRIPT=${JUNEST_SCRIPT:-${JUNEST_BASE}/bin/junest} + CHECK_SCRIPT=${JUNEST_BASE}/lib/checks/check.sh -$JUNEST_SCRIPT proot --fakeroot -- "$CHECK_SCRIPT" --skip-aur-tests -$JUNEST_SCRIPT proot -- "$CHECK_SCRIPT" --skip-aur-tests --use-sudo -$JUNEST_SCRIPT ns --fakeroot -- "$CHECK_SCRIPT" --skip-aur-tests -$JUNEST_SCRIPT ns -- "$CHECK_SCRIPT" --use-sudo -sudo -E $JUNEST_SCRIPT groot -- "$CHECK_SCRIPT" --run-root-tests --skip-aur-tests +$JUNEST_SCRIPT proot --fakeroot --backend-args "-b ${JUNEST_BASE}" -- "$CHECK_SCRIPT" --skip-aur-tests +$JUNEST_SCRIPT proot --backend-args "-b ${JUNEST_BASE}" -- "$CHECK_SCRIPT" --skip-aur-tests --use-sudo +$JUNEST_SCRIPT ns --backend-args "--bind ${JUNEST_BASE} ${JUNEST_BASE}" --fakeroot -- "$CHECK_SCRIPT" --skip-aur-tests +$JUNEST_SCRIPT ns --backend-args "--bind ${JUNEST_BASE} ${JUNEST_BASE}" -- "$CHECK_SCRIPT" --use-sudo +sudo -E $JUNEST_SCRIPT groot --backend-args "--bind ${JUNEST_BASE}" -- "$CHECK_SCRIPT" --run-root-tests --skip-aur-tests diff --git a/lib/core/build.sh b/lib/core/build.sh index a3ed8bd..fb41ede 100644 --- a/lib/core/build.sh +++ b/lib/core/build.sh @@ -88,18 +88,14 @@ function build_image_env(){ sudo pacman --noconfirm --root ${maindir}/root -S sed gzip sudo ln -sf /usr/share/zoneinfo/posix/UTC ${maindir}/root/etc/localtime sudo bash -c "echo 'en_US.UTF-8 UTF-8' >> ${maindir}/root/etc/locale.gen" - sudo ${maindir}/root/bin/groot --no-umount ${maindir}/root locale-gen - #sudo mount --bind ${maindir}/root ${maindir}/root - #sudo arch-chroot ${maindir}/root locale-gen + sudo ${maindir}/root/bin/groot ${maindir}/root locale-gen sudo bash -c "echo LANG=\"en_US.UTF-8\" >> ${maindir}/root/etc/locale.conf" sudo pacman --noconfirm --root ${maindir}/root -Rsn gzip info "Setting up the pacman keyring (this might take a while!)..." # gawk command is required for pacman-key sudo pacman --noconfirm --root ${maindir}/root -S gawk - #TODO sudo mount --bind /dev ${maindir}/root/dev - #sudo ${maindir}/root/bin/groot --no-umount -b /dev ${maindir}/root bash -c ' - sudo ${maindir}/root/bin/groot --no-umount ${maindir}/root bash -c ' + sudo ${maindir}/root/bin/groot --no-umount --avoid-bind -b /dev ${maindir}/root bash -c ' pacman-key --init; for keyring_file in /usr/share/pacman/keyrings/*.gpg; do @@ -107,7 +103,8 @@ function build_image_env(){ pacman-key --populate $keyring; done; [ -e /etc/pacman.d/gnupg/S.gpg-agent ] && gpg-connect-agent -S /etc/pacman.d/gnupg/S.gpg-agent killagent /bye' - sudo umount ${maindir}/root + sudo umount --force --recursive --lazy ${maindir}/root/dev + sudo umount --force --recursive ${maindir}/root sudo pacman --noconfirm --root ${maindir}/root -Rsn gawk sudo rm ${maindir}/root/var/cache/pacman/pkg/* From 9d2e6e4bbf09b0630b7a4f388dab2abfe6bb18df Mon Sep 17 00:00:00 2001 From: Filippo Squillace Date: Mon, 28 Dec 2020 10:22:13 +0100 Subject: [PATCH 3/6] Add deploy feature --- .travis.yml | 31 ++++++++------ ci/deploy.sh | 50 +++++++++++++++++++++++ {tests/integ-tests => ci}/install-bash.sh | 0 lib/checks/check.sh | 2 +- lib/checks/check_all.sh | 10 ++--- 5 files changed, 75 insertions(+), 18 deletions(-) create mode 100755 ci/deploy.sh rename {tests/integ-tests => ci}/install-bash.sh (100%) diff --git a/.travis.yml b/.travis.yml index cfd7148..648a0cb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,6 @@ archlinux: - ~/.pkg-cache:/var/cache/pacman/pkg packages: # Pacman packages - - aws-cli - ccache - git - haveged @@ -34,14 +33,23 @@ archlinux: sudo sed -i '$a CFLAGS="$CFLAGS -w"' /etc/makepkg.conf sudo sed -i '$a CXXFLAGS="$CXXFLAGS -w"' /etc/makepkg.conf script: + # Here do not make any validation (-n) because it will be done later on in the Ubuntu host directly - ./bin/junest build -n env: + matrix: - TRAVIS_BASH_VERSION="4.0" + global: + # AWS_ACCESS_KEY_ID + - secure: "ZotyKKWH5ZrBXDdEnVmV22gbn86BBSiqDZn2d2jVAApgUQdDc3wa7/uYAZP1bts6oQ897nnkUSFHk3M3QAcIoPJerUITTU5D7yjKcFDejgHdpJ4t9XSajmpY9CgKftWapwliWG4wolAKwyAp5GnYqz4GGltHyGxbF/VzUNRF3lw=" + # AWS_SECRET_ACCESS_KEY + - secure: "AWixvJmhr6+rfF4cspMWMjkvLuOsdfNanLK5wrqkgx/0ezDGBBThH0qVhn5Mp1QFM6wVF+LRA6UESNnj0wNwByZHdM6LddkJWlWHb/qkVK+AO4RKUsXJWNyPyOkCNj/WEFpZHQKKUAlEtC8m8AmAcuoi90cr6ih0PXIePRyPFrM=" before_install: - - ./tests/integ-tests/install-bash.sh "$TRAVIS_BASH_VERSION" + - ./ci/install-bash.sh "$TRAVIS_BASH_VERSION" + - sudo apt-get update + - sudo apt-get -y install awscli install: - PATH=$PWD/bin:$PATH @@ -56,28 +64,27 @@ script: # ARM with qemu does seem to work properly. Disabling integ tests for ARM for now. #- export JUNEST_HOME=~/.junest-arm + #- junest setup --arch arm + #- junest proot --fakeroot -- echo "Installing JuNest (\$(uname -m))" #- junest proot --fakeroot -- ${PWD}/lib/checks/check.sh --skip-aur-tests #- junest proot -- ${PWD}/lib/checks/check.sh --skip-aur-tests --use-sudo #- yes | junest setup --delete ####################### - # Build and validation tests + # Build and validation ####################### - "curl -s https://raw.githubusercontent.com/bartoszek/arch-travis/master/arch-travis.sh | bash" - "echo pacman pkg cache size: $(du -h ~/.pkg-cache|cut -f1) in $(ls ~/.pkg-cache|wc -l) files" - ls -l - - pwd - - # TODO Test the new created JuNest image against Ubuntu host - #- junest setup - #- junest -- echo "Installing JuNest (\$(uname -m))" - #- JUNEST_HOME=~/.junest-arm junest setup --arch arm - #- JUNEST_HOME=~/.junest-arm junest proot --fakeroot -- echo "Installing JuNest (\$(uname -m))" + # Test the newly created JuNest image against Ubuntu host - export JUNEST_HOME=~/.junest - junest setup -i junest-x86_64.tar.gz - ${PWD}/lib/checks/check_all.sh - yes | junest setup --delete - # TODO test deployed image can be downloaded correctly - +after_success: + ####################### + # Deploy and validation + ####################### + - ./ci/deploy.sh ${PWD}/junest-x86_64.tar.gz diff --git a/ci/deploy.sh b/ci/deploy.sh new file mode 100755 index 0000000..fa2ff75 --- /dev/null +++ b/ci/deploy.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +set -e + +IMG_PATH=$1 + +set -u + +MAX_OLD_IMAGES=30 + +# ARCH can be one of: x86, x86_64, arm +HOST_ARCH=$(uname -m) +if [ $HOST_ARCH == "i686" ] || [ $HOST_ARCH == "i386" ] +then + ARCH="x86" +elif [ $HOST_ARCH == "x86_64" ] +then + ARCH="x86_64" +elif [[ $HOST_ARCH =~ .*(arm).* ]] +then + ARCH="arm" +else + echo "Unknown architecture ${HOST_ARCH}" >&2 + exit 11 +fi + +if [[ "$TRAVIS_BRANCH" == "master" ]] +then + + export AWS_DEFAULT_REGION=eu-west-1 + # Upload image + # The put is done via a temporary filename in order to prevent outage on the + # production file for a longer period of time. + cp ${IMG_PATH} ${IMG_PATH}.temp + aws s3 cp ${IMG_PATH}.temp s3://junest-repo/junest/ + aws s3 mv s3://junest-repo/junest/${IMG_PATH}.temp s3://junest-repo/junest/${IMG_PATH} + aws s3api put-object-acl --acl public-read --bucket junest-repo --key junest/${IMG_PATH} + + DATE=$(date +'%Y-%m-%d-%H-%M-%S') + + aws s3 cp ${IMG_PATH} s3://junest-repo/junest/${IMG_PATH}.${DATE} + + # Cleanup old images + aws s3 ls s3://junest-repo/junest/junest-${ARCH}.tar.gz. | awk '{print $4}' | head -n -${MAX_OLD_IMAGES} | xargs -I {} s3 rm "s3://junest-repo/junest/{}" + + # Test the newly deployed image can be downloaded correctly + junest setup + junest -- echo "Installed JuNest (\$(uname -m))" + yes | junest setup --delete +fi diff --git a/tests/integ-tests/install-bash.sh b/ci/install-bash.sh similarity index 100% rename from tests/integ-tests/install-bash.sh rename to ci/install-bash.sh diff --git a/lib/checks/check.sh b/lib/checks/check.sh index 6ae3e6a..957156b 100755 --- a/lib/checks/check.sh +++ b/lib/checks/check.sh @@ -9,7 +9,7 @@ # # vim: ft=sh -set -e +set -ex RUN_ROOT_TESTS=false diff --git a/lib/checks/check_all.sh b/lib/checks/check_all.sh index 91bc058..cc1c4df 100755 --- a/lib/checks/check_all.sh +++ b/lib/checks/check_all.sh @@ -11,8 +11,8 @@ JUNEST_SCRIPT=${JUNEST_SCRIPT:-${JUNEST_BASE}/bin/junest} CHECK_SCRIPT=${JUNEST_BASE}/lib/checks/check.sh -$JUNEST_SCRIPT proot --fakeroot --backend-args "-b ${JUNEST_BASE}" -- "$CHECK_SCRIPT" --skip-aur-tests -$JUNEST_SCRIPT proot --backend-args "-b ${JUNEST_BASE}" -- "$CHECK_SCRIPT" --skip-aur-tests --use-sudo -$JUNEST_SCRIPT ns --backend-args "--bind ${JUNEST_BASE} ${JUNEST_BASE}" --fakeroot -- "$CHECK_SCRIPT" --skip-aur-tests -$JUNEST_SCRIPT ns --backend-args "--bind ${JUNEST_BASE} ${JUNEST_BASE}" -- "$CHECK_SCRIPT" --use-sudo -sudo -E $JUNEST_SCRIPT groot --backend-args "--bind ${JUNEST_BASE}" -- "$CHECK_SCRIPT" --run-root-tests --skip-aur-tests +$JUNEST_SCRIPT proot --fakeroot -- "$CHECK_SCRIPT" --skip-aur-tests +$JUNEST_SCRIPT proot -- "$CHECK_SCRIPT" --skip-aur-tests --use-sudo +$JUNEST_SCRIPT ns --fakeroot -- "$CHECK_SCRIPT" --skip-aur-tests +$JUNEST_SCRIPT ns -- "$CHECK_SCRIPT" --use-sudo +sudo -E $JUNEST_SCRIPT groot -- "$CHECK_SCRIPT" --run-root-tests --skip-aur-tests From 3d16ee2583de6fc071e787422d9d990238053acb Mon Sep 17 00:00:00 2001 From: Filippo Squillace Date: Tue, 29 Dec 2020 10:46:49 +0100 Subject: [PATCH 4/6] First commit for wrappers --- bin/junest | 43 +++++++++++++++++++++++++++++++++++++++++++ lib/core/wrappers.sh | 31 +++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 lib/core/wrappers.sh diff --git a/bin/junest b/bin/junest index 17d32bc..a19e550 100755 --- a/bin/junest +++ b/bin/junest @@ -17,6 +17,7 @@ source "${JUNEST_BASE}/lib/core/setup.sh" source "${JUNEST_BASE}/lib/core/chroot.sh" source "${JUNEST_BASE}/lib/core/namespace.sh" source "${JUNEST_BASE}/lib/core/proot.sh" +source "${JUNEST_BASE}/lib/core/wrappers.sh" ################################### @@ -67,6 +68,12 @@ usage() { echo -e " b[uild] Build a $NAME image (must run in ArchLinux)" echo -e " -n, --disable-check Disable the $NAME image check" echo + echo -e " c[reate-wrappers] Create wrappers in ${JUNEST_HOME}/usr/bin_wrappers of the executables under" + echo -e " ${JUNEST_HOME}/usr/bin to be run directly from the host." + echo -e " Use the variable JUNEST_ARGS to define how the wrapper will call $NAME (default "ns --fakeroot")." + echo -e " Use PATH variable to point directly to the bin_wrappers directory from the host." + echo -e " -r, --recreate-existing Instead of skipping existing wrappers recreate them" + echo } version() { @@ -80,6 +87,7 @@ function parse_arguments(){ ACT_NAMESPACE=false ACT_PROOT=false ACT_GROOT=false + ACT_WRAPPERS=false ACT_ROOT=false ACT_HELP=false ACT_VERSION=false @@ -91,6 +99,7 @@ function parse_arguments(){ p|proot) ACT_PROOT=true ; shift ;; g|groot) ACT_GROOT=true ; shift ;; r|root) ACT_ROOT=true ; shift ;; + c|create-wrappers) ACT_WRAPPERS=true ; shift ;; -h|--help) ACT_HELP=true ; shift ;; -V|--version) ACT_VERSION=true ; shift ;; *) ACT_NAMESPACE=true ;; @@ -114,9 +123,33 @@ function parse_arguments(){ elif $ACT_ROOT then _parse_root_opts "$@" + elif $ACT_WRAPPERS + then + _parse_wrappers_opts "$@" fi } +function _parse_wrappers_opts() { + # Options: + OPT_RECREATE_EXISTING=false + + while [[ -n "$1" ]] + do + case "$1" in + -r|--recreate-existing) OPT_RECREATE_EXISTING=true ; shift ;; + --) shift ; break ;; + -*) die "Invalid option $1" ;; + *) break ;; + esac + done + + ARGS=() + for arg in "$@" + do + ARGS+=("$arg") + done +} + function _parse_root_opts() { # Options: BACKEND_ARGS="" @@ -247,6 +280,7 @@ function execute_operation() { else setup_env $ARCH_ARG fi + create_wrappers false fi return @@ -258,6 +292,11 @@ function execute_operation() { die "Error: The image is still not installed in $JUNEST_HOME. Run this first: $CMD setup" fi + if $ACT_WRAPPERS; then + create_wrappers $OPT_RECREATE_EXISTING + return + fi + local run_env if $ACT_NAMESPACE; then if $OPT_FAKEROOT; then @@ -279,6 +318,10 @@ function execute_operation() { $run_env "$BACKEND_COMMAND" "${BACKEND_ARGS}" $OPT_NO_COPY_FILES "${ARGS[@]}" + # TODO use the run_env exit status + + # Call create_wrappers in case new bin files have been created + create_wrappers false } function main() { diff --git a/lib/core/wrappers.sh b/lib/core/wrappers.sh new file mode 100644 index 0000000..184fca5 --- /dev/null +++ b/lib/core/wrappers.sh @@ -0,0 +1,31 @@ + + +function create_wrappers() { + mkdir -p ${JUNEST_HOME}/usr/bin_wrappers + + cd ${JUNEST_HOME}/usr/bin + for file in * + do + [[ -x $file ]] || continue + if [[ -e ${JUNEST_HOME}/usr/bin_wrappers/$file ]] + then + continue + fi + cat < ${JUNEST_HOME}/usr/bin_wrappers/${file} +#!/usr/bin/env bash + +JUNEST_ARGS=\${JUNEST_ARGS:-ns --fakeroot} + +junest \${JUNEST_ARGS} -- ${file} "\$@" +EOF + chmod +x ${JUNEST_HOME}/usr/bin_wrappers/${file} + done + + # Remove wrappers no longer needed + cd ${JUNEST_HOME}/usr/bin_wrappers + for file in * + do + [[ -e ${JUNEST_HOME}/usr/bin/$file ]] || rm -f $file + done + +} From 2aeb23b88283cd710855c6919203b4fa8ef71aa3 Mon Sep 17 00:00:00 2001 From: Filippo Squillace Date: Tue, 29 Dec 2020 13:09:35 +0100 Subject: [PATCH 5/6] Add tests for wrappers --- bin/junest | 46 ++------------------- lib/checks/check_all.sh | 3 ++ tests/unit-tests/test-wrappers.sh | 67 +++++++++++++++++++++++++++++++ tests/utils/utils.sh | 1 + 4 files changed, 74 insertions(+), 43 deletions(-) create mode 100755 tests/unit-tests/test-wrappers.sh diff --git a/bin/junest b/bin/junest index a19e550..2671e86 100755 --- a/bin/junest +++ b/bin/junest @@ -68,12 +68,6 @@ usage() { echo -e " b[uild] Build a $NAME image (must run in ArchLinux)" echo -e " -n, --disable-check Disable the $NAME image check" echo - echo -e " c[reate-wrappers] Create wrappers in ${JUNEST_HOME}/usr/bin_wrappers of the executables under" - echo -e " ${JUNEST_HOME}/usr/bin to be run directly from the host." - echo -e " Use the variable JUNEST_ARGS to define how the wrapper will call $NAME (default "ns --fakeroot")." - echo -e " Use PATH variable to point directly to the bin_wrappers directory from the host." - echo -e " -r, --recreate-existing Instead of skipping existing wrappers recreate them" - echo } version() { @@ -87,7 +81,6 @@ function parse_arguments(){ ACT_NAMESPACE=false ACT_PROOT=false ACT_GROOT=false - ACT_WRAPPERS=false ACT_ROOT=false ACT_HELP=false ACT_VERSION=false @@ -99,7 +92,6 @@ function parse_arguments(){ p|proot) ACT_PROOT=true ; shift ;; g|groot) ACT_GROOT=true ; shift ;; r|root) ACT_ROOT=true ; shift ;; - c|create-wrappers) ACT_WRAPPERS=true ; shift ;; -h|--help) ACT_HELP=true ; shift ;; -V|--version) ACT_VERSION=true ; shift ;; *) ACT_NAMESPACE=true ;; @@ -123,33 +115,9 @@ function parse_arguments(){ elif $ACT_ROOT then _parse_root_opts "$@" - elif $ACT_WRAPPERS - then - _parse_wrappers_opts "$@" fi } -function _parse_wrappers_opts() { - # Options: - OPT_RECREATE_EXISTING=false - - while [[ -n "$1" ]] - do - case "$1" in - -r|--recreate-existing) OPT_RECREATE_EXISTING=true ; shift ;; - --) shift ; break ;; - -*) die "Invalid option $1" ;; - *) break ;; - esac - done - - ARGS=() - for arg in "$@" - do - ARGS+=("$arg") - done -} - function _parse_root_opts() { # Options: BACKEND_ARGS="" @@ -280,7 +248,7 @@ function execute_operation() { else setup_env $ARCH_ARG fi - create_wrappers false + create_wrappers fi return @@ -292,11 +260,6 @@ function execute_operation() { die "Error: The image is still not installed in $JUNEST_HOME. Run this first: $CMD setup" fi - if $ACT_WRAPPERS; then - create_wrappers $OPT_RECREATE_EXISTING - return - fi - local run_env if $ACT_NAMESPACE; then if $OPT_FAKEROOT; then @@ -316,12 +279,9 @@ function execute_operation() { run_env=run_env_as_chroot fi - $run_env "$BACKEND_COMMAND" "${BACKEND_ARGS}" $OPT_NO_COPY_FILES "${ARGS[@]}" - - # TODO use the run_env exit status - # Call create_wrappers in case new bin files have been created - create_wrappers false + trap "create_wrappers" EXIT QUIT TERM KILL + $run_env "$BACKEND_COMMAND" "${BACKEND_ARGS}" $OPT_NO_COPY_FILES "${ARGS[@]}" } function main() { diff --git a/lib/checks/check_all.sh b/lib/checks/check_all.sh index cc1c4df..9e5f0a6 100755 --- a/lib/checks/check_all.sh +++ b/lib/checks/check_all.sh @@ -16,3 +16,6 @@ $JUNEST_SCRIPT proot -- "$CHECK_SCRIPT" --skip-aur-tests --use-sudo $JUNEST_SCRIPT ns --fakeroot -- "$CHECK_SCRIPT" --skip-aur-tests $JUNEST_SCRIPT ns -- "$CHECK_SCRIPT" --use-sudo sudo -E $JUNEST_SCRIPT groot -- "$CHECK_SCRIPT" --run-root-tests --skip-aur-tests + +# Test the wrappers work +$JUNEST_HOME/usr/bin_wrappers/pacman --help diff --git a/tests/unit-tests/test-wrappers.sh b/tests/unit-tests/test-wrappers.sh new file mode 100755 index 0000000..a78c81f --- /dev/null +++ b/tests/unit-tests/test-wrappers.sh @@ -0,0 +1,67 @@ +#!/bin/bash +source "$(dirname $0)/../utils/utils.sh" + +source "$(dirname $0)/../../lib/core/wrappers.sh" + +# Disable the exiterr +set +e + +function oneTimeSetUp(){ + setUpUnitTests +} + +function setUp(){ + junestSetUp +} + +function tearDown(){ + junestTearDown +} + +function test_create_wrappers_empty_bin(){ + assertCommandSuccess create_wrappers + assertEquals "" "$(cat $STDOUTF)" + assertTrue "bin_wrappers does not exist" "[ -e $JUNEST_HOME/usr/bin_wrappers ]" +} + +function test_create_wrappers_not_executable_file(){ + touch $JUNEST_HOME/usr/bin/myfile + assertCommandSuccess create_wrappers + assertEquals "" "$(cat $STDOUTF)" + assertTrue "bin_wrappers should exist" "[ -e $JUNEST_HOME/usr/bin_wrappers ]" + assertTrue "myfile wrapper should not exist" "[ ! -x $JUNEST_HOME/usr/bin_wrappers/myfile ]" +} + +function test_create_wrappers_executable_file(){ + touch $JUNEST_HOME/usr/bin/myfile + chmod +x $JUNEST_HOME/usr/bin/myfile + assertCommandSuccess create_wrappers + assertEquals "" "$(cat $STDOUTF)" + assertTrue "bin_wrappers should exist" "[ -e $JUNEST_HOME/usr/bin_wrappers ]" + assertTrue "myfile wrapper should exist" "[ -x $JUNEST_HOME/usr/bin_wrappers/myfile ]" +} + +function test_create_wrappers_already_exist(){ + touch $JUNEST_HOME/usr/bin/myfile + chmod +x $JUNEST_HOME/usr/bin/myfile + mkdir -p $JUNEST_HOME/usr/bin_wrappers + touch $JUNEST_HOME/usr/bin_wrappers/myfile + chmod +x $JUNEST_HOME/usr/bin_wrappers/myfile + assertCommandSuccess create_wrappers + assertEquals "" "$(cat $STDOUTF)" + assertTrue "bin_wrappers should exist" "[ -e $JUNEST_HOME/usr/bin_wrappers ]" + assertTrue "myfile wrapper should exist" "[ -x $JUNEST_HOME/usr/bin_wrappers/myfile ]" + assertEquals "" "$(touch $JUNEST_HOME/usr/bin_wrappers/myfile)" +} + +function test_create_wrappers_executable_no_longer_exist(){ + mkdir -p $JUNEST_HOME/usr/bin_wrappers + touch $JUNEST_HOME/usr/bin_wrappers/myfile + chmod +x $JUNEST_HOME/usr/bin_wrappers/myfile + assertCommandSuccess create_wrappers + assertEquals "" "$(cat $STDOUTF)" + assertTrue "bin_wrappers should exist" "[ -e $JUNEST_HOME/usr/bin_wrappers ]" + assertTrue "myfile wrapper should not exist" "[ ! -x $JUNEST_HOME/usr/bin_wrappers/myfile ]" +} + +source $(dirname $0)/../utils/shunit2 diff --git a/tests/utils/utils.sh b/tests/utils/utils.sh index ed7cb8c..542af68 100644 --- a/tests/utils/utils.sh +++ b/tests/utils/utils.sh @@ -11,6 +11,7 @@ function cwdTearDown(){ function junestSetUp(){ JUNEST_HOME=$(TMPDIR=/tmp mktemp -d -t junest-home.XXXXXXXXXX) + mkdir -p ${JUNEST_HOME}/usr/bin mkdir -p ${JUNEST_HOME}/etc/junest echo "JUNEST_ARCH=x86_64" > ${JUNEST_HOME}/etc/junest/info mkdir -p ${JUNEST_HOME}/etc/ca-certificates From d25ae301588a9cdad51f4cd571032c3da17952ac Mon Sep 17 00:00:00 2001 From: Filippo Squillace Date: Tue, 29 Dec 2020 14:42:41 +0100 Subject: [PATCH 6/6] Update README --- README.md | 75 +++++++++++++++++++++------------ VERSION | 2 +- lib/checks/check.sh | 8 ---- tests/unit-tests/test-junest.sh | 3 ++ 4 files changed, 51 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index fc9f320..4779250 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ The lightweight Arch Linux based distro that runs upon any Linux distros without Description =========== **JuNest** (Jailed User NEST) is a lightweight Arch Linux based distribution -that allows to have disposable and isolated GNU/Linux environments +that allows to have disposable and partial isolated GNU/Linux environments within any generic GNU/Linux host OS and without the need to have root privileges for installing packages. @@ -41,7 +41,7 @@ The main advantages on using JuNest are: - Install packages without root privileges. - Partial isolated environment which you can install packages without affecting a production system. -- Access to a wide range of packages in particular on GNU/Linux distros that may contain limited repositories (such as CentOS and RedHat). +- Access to a wide range of packages, in particular on GNU/Linux distros that may contain limited repositories (such as CentOS and RedHat). - Available for `x86_64` and `arm` architectures but you can build your own image from scratch too! - Run on a different architecture from the host OS via QEMU - All Arch Linux lovers can have their favourite distro everywhere! @@ -57,7 +57,7 @@ build a complete isolated environment but, conversely, is the ability to run programs as they were running natively from the host OS. Almost everything is shared between host OS and the JuNest sandbox (kernel, process subtree, network, mounting, etc) and only the root filesystem gets isolated -(as the programs installed in JuNest need to reside elsewhere). +(since the programs installed in JuNest need to reside elsewhere). This allows interaction between processes belonging to both host OS and JuNest. For instance, you can install `top` command in JuNest in order to monitor @@ -107,6 +107,27 @@ There are multiple backend programs, each with its own pros/cons. To know more about the JuNest execution modes depending on the backend program used, see the [Usage](#usage) section below. +Run commands installed in JuNest directly from host +--------------------------------------- + +Installed programs can be accessible directly from host. +For instance, supposing the host OS is an Ubuntu distro you can directly +run `pacman` by simply updating the `PATH` variable: + +```sh +export PATH="$PATH:~/.junest/usr/bin_wrappers" +pacman -S htop +htop +``` + +By default the wrappers use `"ns --fakeroot"` but you can change it via `JUNEST_ARGS`. +For instance, if you want to run `iftop` with real root privileges: + +``` +pacman -S iftop +sudo JUNEST_ARGS="groot" iftop +``` + Have fun! --------- @@ -149,12 +170,22 @@ section below. ## Installation from git repository ## Just clone the JuNest repo somewhere (for example in ~/.local/share/junest): - git clone git://github.com/fsquillace/junest ~/.local/share/junest - export PATH=~/.local/share/junest/bin:$PATH +```sh +git clone git://github.com/fsquillace/junest ~/.local/share/junest +export PATH=~/.local/share/junest/bin:$PATH +``` + +Optionally you want to use the wrappers to run commands +installed in JuNest directly from host: + +```sh +export PATH="$PATH:~/.junest/usr/bin_wrappers" +``` +Update your `~/.bashrc` or `~/.zshrc` to get always the wrappers available. ### Installation using AUR (Arch Linux only) ### If you are using an Arch Linux system you can, alternatively, install JuNest from the [AUR repository](https://aur.archlinux.org/packages/junest-git/). -After installing junest will be located in `/opt/junest/` +JuNest will be located in `/opt/junest/` Usage ===== @@ -189,7 +220,8 @@ This mode is based on the fantastic PRoot based ----------- [Proot](https://wiki.archlinux.org/index.php/Proot) represents a portable -solution that works well in most of GNU/Linux distros available. +solution which allows unprivileged users to execute programs inside a sandbox +and works well in most of GNU/Linux distros available. One of the major drawbacks is the fact that Proot is not officially supported anymore, therefore, Proot bugs may no longer be fixed. @@ -203,7 +235,9 @@ Chroot based ------------ This solution suits only for privileged users. JuNest provides the possibility to run the environment via `chroot` program. -In particular, it uses a special program called `GRoot`, an enhanced `chroot` +In particular, it uses a special program called `GRoot`, a small and portable +version of +[arch-chroot](https://wiki.archlinux.org/index.php/Chroot) wrapper, that allows to bind mount directories specified by the user, such as `/proc`, `/sys`, `/dev`, `/tmp` and `$HOME`, before executing any programs inside the JuNest sandbox. In case the mounting will not @@ -275,7 +309,7 @@ To bind a host directory to a guest location, you can use proot arguments: junest proot -b "-b /mnt/mydata:/home/user/mydata" ``` -The option `-b` to provide options to the backeng program will work with PRoot, Namespace and GRoot backend programs. +The option `-b` to provide options to the backend program will work with PRoot, Namespace and GRoot backend programs. Check out the backend program options by passing `--help` option: ```sh @@ -303,21 +337,6 @@ Related wiki page: Internals ========= - -There are two main chroot jail used in JuNest. -The main one is [proot](https://wiki.archlinux.org/index.php/Proot) which -allows unprivileged users to execute programs inside a sandbox and -GRoot, a small and portable version of -[arch-chroot](https://wiki.archlinux.org/index.php/Chroot) which is an -enhanced chroot for privileged users that mounts the primary directories -(i.e. `/proc`, `/sys`, `/dev` and `/run`) before executing any programs inside -the sandbox. - -## Automatic fallback to classic chroot ## -If GRoot fails for some reasons in the host system (i.e. it is not able to -mount one of the directories), -JuNest automatically tries to fallback to the classic chroot. - ## Automatic fallback for all the dependent host OS executables ## JuNest attempts first to run the executables in the host OS located in different positions (`/usr/bin`, `/bin`, `/usr/sbin` and `/sbin`). @@ -325,10 +344,10 @@ As a fallback it tries to run the same executable if it is available in the JuNe environment. ## Automatic building of the JuNest images ## -There is not periodic automation build of the JuNest images yet. -This was due to the difficulty to automate builds for arm architecture. -The JuNest image for the `x86_64` is built periodically every once every three -months. +There is a periodic automation build of the JuNest images for `x86_64` arch +only. +The JuNest image for `arm` architecture may not be always up to date because +the build is performed manually. ## Static QEMU binaries ## There are static QEMU binaries included in JuNest image that allows to run JuNest diff --git a/VERSION b/VERSION index 77f5bec..1502020 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -7.2.2 +7.3.0 diff --git a/lib/checks/check.sh b/lib/checks/check.sh index 957156b..7af5832 100755 --- a/lib/checks/check.sh +++ b/lib/checks/check.sh @@ -95,12 +95,4 @@ then $SUDO pacman $PACMAN_OPTIONS -Rsn ${aur_package} fi -# The following ensures that the gpg agent gets killed (if exists) -# otherwise it is not possible to exit from the session -if [[ -e /etc/pacman.d/gnupg/S.gpg-agent ]] -then - gpg-connect-agent -S /etc/pacman.d/gnupg/S.gpg-agent killagent /bye || echo "GPG agent did not close properly" - echo "GPG agent closed" -fi - exit 0 diff --git a/tests/unit-tests/test-junest.sh b/tests/unit-tests/test-junest.sh index e3c7ee1..0ba7cf5 100755 --- a/tests/unit-tests/test-junest.sh +++ b/tests/unit-tests/test-junest.sh @@ -31,6 +31,9 @@ function build_image_env(){ function delete_env(){ echo "delete_env" } +function create_wrappers(){ + : +} function setup_env_from_file(){ echo "setup_env_from_file($1)" }