diff --git a/.travis.yml b/.travis.yml index b4126d8..648a0cb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,31 +1,90 @@ -language: bash - sudo: required +os: linux + +cache: + directories: + - ~/.ccache + - ~/.pkg-cache + +services: +- docker + +archlinux: + mount: + - ~/.ccache:~/.ccache + - ~/.pkg-cache:/var/cache/pacman/pkg + packages: + # Pacman packages + - ccache + - git + - haveged + + before_install: + # 1.Override `package-cleanup.hook` to preserve cache for travis. + # 2.Enable ccache + # 3.Multithreaded build and compress + # 4.Suppress all gcc warnings + - | + sudo mkdir /etc/pacman.d/hooks/ + sudo ln -s /dev/null /etc/pacman.d/hooks/package-cleanup.hook + sudo sed -i '/^BUILDENV/s/\!ccache/ccache/' /etc/makepkg.conf + sudo sed -i '/#MAKEFLAGS=/c MAKEFLAGS="-j2"' /etc/makepkg.conf + sudo sed -i '/^COMPRESSXZ/s/\xz/xz -T 2/' /etc/makepkg.conf + sudo sed -i '$a CFLAGS="$CFLAGS -w"' /etc/makepkg.conf + sudo sed -i '$a CXXFLAGS="$CXXFLAGS -w"' /etc/makepkg.conf + script: + # Here do not make any validation (-n) because it will be done later on in the Ubuntu host directly + - ./bin/junest build -n + env: + matrix: - TRAVIS_BASH_VERSION="4.0" + global: + # AWS_ACCESS_KEY_ID + - secure: "ZotyKKWH5ZrBXDdEnVmV22gbn86BBSiqDZn2d2jVAApgUQdDc3wa7/uYAZP1bts6oQ897nnkUSFHk3M3QAcIoPJerUITTU5D7yjKcFDejgHdpJ4t9XSajmpY9CgKftWapwliWG4wolAKwyAp5GnYqz4GGltHyGxbF/VzUNRF3lw=" + # AWS_SECRET_ACCESS_KEY + - secure: "AWixvJmhr6+rfF4cspMWMjkvLuOsdfNanLK5wrqkgx/0ezDGBBThH0qVhn5Mp1QFM6wVF+LRA6UESNnj0wNwByZHdM6LddkJWlWHb/qkVK+AO4RKUsXJWNyPyOkCNj/WEFpZHQKKUAlEtC8m8AmAcuoi90cr6ih0PXIePRyPFrM=" before_install: - - ./tests/integ-tests/install-bash.sh "$TRAVIS_BASH_VERSION" + - ./ci/install-bash.sh "$TRAVIS_BASH_VERSION" + - sudo apt-get update + - sudo apt-get -y install awscli install: - PATH=$PWD/bin:$PATH - - junest setup - - junest -- echo "Installing JuNest (\$(uname -m))" - - JUNEST_HOME=~/.junest-arm junest setup --arch arm - - JUNEST_HOME=~/.junest-arm junest proot --fakeroot -- echo "Installing JuNest (\$(uname -m))" script: + ####################### + # Unit Tests + ####################### - bash --version - bash ./tests/checkstyle/checkstyle.sh - bash ./tests/unit-tests/unit-tests.sh - - export JUNEST_HOME=~/.junest - - ${PWD}/lib/checks/check_all.sh - - yes | junest setup --delete - # ARM with qemu does seem to work properly. Disabling integ tests for ARM for now. #- export JUNEST_HOME=~/.junest-arm + #- junest setup --arch arm + #- junest proot --fakeroot -- echo "Installing JuNest (\$(uname -m))" #- junest proot --fakeroot -- ${PWD}/lib/checks/check.sh --skip-aur-tests #- junest proot -- ${PWD}/lib/checks/check.sh --skip-aur-tests --use-sudo #- yes | junest setup --delete + + ####################### + # Build and validation + ####################### + - "curl -s https://raw.githubusercontent.com/bartoszek/arch-travis/master/arch-travis.sh | bash" + - "echo pacman pkg cache size: $(du -h ~/.pkg-cache|cut -f1) in $(ls ~/.pkg-cache|wc -l) files" + - ls -l + # Test the newly created JuNest image against Ubuntu host + - export JUNEST_HOME=~/.junest + - junest setup -i junest-x86_64.tar.gz + - ${PWD}/lib/checks/check_all.sh + - yes | junest setup --delete + + +after_success: + ####################### + # Deploy and validation + ####################### + - ./ci/deploy.sh ${PWD}/junest-x86_64.tar.gz diff --git a/README.md b/README.md index ce38b1c..285d160 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ The lightweight Arch Linux based distro that runs upon any Linux distros without Description =========== **JuNest** (Jailed User NEST) is a lightweight Arch Linux based distribution -that allows to have disposable and isolated GNU/Linux environments +that allows to have disposable and partial isolated GNU/Linux environments within any generic GNU/Linux host OS and without the need to have root privileges for installing packages. @@ -41,7 +41,7 @@ The main advantages on using JuNest are: - Install packages without root privileges. - Partial isolated environment which you can install packages without affecting a production system. -- Access to a wide range of packages in particular on GNU/Linux distros that may contain limited repositories (such as CentOS and RedHat). +- Access to a wide range of packages, in particular on GNU/Linux distros that may contain limited repositories (such as CentOS and RedHat). - Available for `x86_64` and `arm` architectures but you can build your own image from scratch too! - Run on a different architecture from the host OS via QEMU - All Arch Linux lovers can have their favourite distro everywhere! @@ -57,7 +57,7 @@ build a complete isolated environment but, conversely, is the ability to run programs as they were running natively from the host OS. Almost everything is shared between host OS and the JuNest sandbox (kernel, process subtree, network, mounting, etc) and only the root filesystem gets isolated -(as the programs installed in JuNest need to reside elsewhere). +(since the programs installed in JuNest need to reside elsewhere). This allows interaction between processes belonging to both host OS and JuNest. For instance, you can install `top` command in JuNest in order to monitor @@ -107,6 +107,27 @@ There are multiple backend programs, each with its own pros/cons. To know more about the JuNest execution modes depending on the backend program used, see the [Usage](#usage) section below. +Run commands installed in JuNest directly from host +--------------------------------------- + +Installed programs can be accessible directly from host. +For instance, supposing the host OS is an Ubuntu distro you can directly +run `pacman` by simply updating the `PATH` variable: + +```sh +export PATH="$PATH:~/.junest/usr/bin_wrappers" +pacman -S htop +htop +``` + +By default the wrappers use `"ns --fakeroot"` but you can change it via `JUNEST_ARGS`. +For instance, if you want to run `iftop` with real root privileges: + +``` +pacman -S iftop +sudo JUNEST_ARGS="groot" iftop +``` + Have fun! --------- @@ -149,12 +170,22 @@ section below. ## Installation from git repository ## Just clone the JuNest repo somewhere (for example in ~/.local/share/junest): - git clone git://github.com/fsquillace/junest ~/.local/share/junest - export PATH=~/.local/share/junest/bin:$PATH +```sh +git clone git://github.com/fsquillace/junest ~/.local/share/junest +export PATH=~/.local/share/junest/bin:$PATH +``` + +Optionally you want to use the wrappers to run commands +installed in JuNest directly from host: + +```sh +export PATH="$PATH:~/.junest/usr/bin_wrappers" +``` +Update your `~/.bashrc` or `~/.zshrc` to get always the wrappers available. ### Installation using AUR (Arch Linux only) ### If you are using an Arch Linux system you can, alternatively, install JuNest from the [AUR repository](https://aur.archlinux.org/packages/junest-git/). -After installing junest will be located in `/opt/junest/` +JuNest will be located in `/opt/junest/` Usage ===== @@ -189,7 +220,8 @@ This mode is based on the fantastic PRoot based ----------- [Proot](https://wiki.archlinux.org/index.php/Proot) represents a portable -solution that works well in most of GNU/Linux distros available. +solution which allows unprivileged users to execute programs inside a sandbox +and works well in most of GNU/Linux distros available. One of the major drawbacks is the fact that Proot is not officially supported anymore, therefore, Proot bugs may no longer be fixed. @@ -203,7 +235,9 @@ Chroot based ------------ This solution suits only for privileged users. JuNest provides the possibility to run the environment via `chroot` program. -In particular, it uses a special program called `GRoot`, an enhanced `chroot` +In particular, it uses a special program called `GRoot`, a small and portable +version of +[arch-chroot](https://wiki.archlinux.org/index.php/Chroot) wrapper, that allows to bind mount directories specified by the user, such as `/proc`, `/sys`, `/dev`, `/tmp` and `$HOME`, before executing any programs inside the JuNest sandbox. In case the mounting will not @@ -281,7 +315,7 @@ Or using proot arguments: junest proot -b "-b /mnt/mydata:/home/user/mydata" ``` -The option `-b` to provide options to the backeng program will work with PRoot, Namespace and GRoot backend programs. +The option `-b` to provide options to the backend program will work with PRoot, Namespace and GRoot backend programs. Check out the backend program options by passing `--help` option: ```sh @@ -309,21 +343,6 @@ Related wiki page: Internals ========= - -There are two main chroot jail used in JuNest. -The main one is [proot](https://wiki.archlinux.org/index.php/Proot) which -allows unprivileged users to execute programs inside a sandbox and -GRoot, a small and portable version of -[arch-chroot](https://wiki.archlinux.org/index.php/Chroot) which is an -enhanced chroot for privileged users that mounts the primary directories -(i.e. `/proc`, `/sys`, `/dev` and `/run`) before executing any programs inside -the sandbox. - -## Automatic fallback to classic chroot ## -If GRoot fails for some reasons in the host system (i.e. it is not able to -mount one of the directories), -JuNest automatically tries to fallback to the classic chroot. - ## Automatic fallback for all the dependent host OS executables ## JuNest attempts first to run the executables in the host OS located in different positions (`/usr/bin`, `/bin`, `/usr/sbin` and `/sbin`). @@ -331,10 +350,10 @@ As a fallback it tries to run the same executable if it is available in the JuNe environment. ## Automatic building of the JuNest images ## -There is not periodic automation build of the JuNest images yet. -This was due to the difficulty to automate builds for arm architecture. -The JuNest image for the `x86_64` is built periodically every once every three -months. +There is a periodic automation build of the JuNest images for `x86_64` arch +only. +The JuNest image for `arm` architecture may not be always up to date because +the build is performed manually. ## Static QEMU binaries ## There are static QEMU binaries included in JuNest image that allows to run JuNest diff --git a/VERSION b/VERSION index 77f5bec..1502020 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -7.2.2 +7.3.0 diff --git a/bin/junest b/bin/junest index 17d32bc..2671e86 100755 --- a/bin/junest +++ b/bin/junest @@ -17,6 +17,7 @@ source "${JUNEST_BASE}/lib/core/setup.sh" source "${JUNEST_BASE}/lib/core/chroot.sh" source "${JUNEST_BASE}/lib/core/namespace.sh" source "${JUNEST_BASE}/lib/core/proot.sh" +source "${JUNEST_BASE}/lib/core/wrappers.sh" ################################### @@ -247,6 +248,7 @@ function execute_operation() { else setup_env $ARCH_ARG fi + create_wrappers fi return @@ -277,8 +279,9 @@ function execute_operation() { run_env=run_env_as_chroot fi + # Call create_wrappers in case new bin files have been created + trap "create_wrappers" EXIT QUIT TERM KILL $run_env "$BACKEND_COMMAND" "${BACKEND_ARGS}" $OPT_NO_COPY_FILES "${ARGS[@]}" - } function main() { diff --git a/ci/deploy.sh b/ci/deploy.sh new file mode 100755 index 0000000..fa2ff75 --- /dev/null +++ b/ci/deploy.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +set -e + +IMG_PATH=$1 + +set -u + +MAX_OLD_IMAGES=30 + +# ARCH can be one of: x86, x86_64, arm +HOST_ARCH=$(uname -m) +if [ $HOST_ARCH == "i686" ] || [ $HOST_ARCH == "i386" ] +then + ARCH="x86" +elif [ $HOST_ARCH == "x86_64" ] +then + ARCH="x86_64" +elif [[ $HOST_ARCH =~ .*(arm).* ]] +then + ARCH="arm" +else + echo "Unknown architecture ${HOST_ARCH}" >&2 + exit 11 +fi + +if [[ "$TRAVIS_BRANCH" == "master" ]] +then + + export AWS_DEFAULT_REGION=eu-west-1 + # Upload image + # The put is done via a temporary filename in order to prevent outage on the + # production file for a longer period of time. + cp ${IMG_PATH} ${IMG_PATH}.temp + aws s3 cp ${IMG_PATH}.temp s3://junest-repo/junest/ + aws s3 mv s3://junest-repo/junest/${IMG_PATH}.temp s3://junest-repo/junest/${IMG_PATH} + aws s3api put-object-acl --acl public-read --bucket junest-repo --key junest/${IMG_PATH} + + DATE=$(date +'%Y-%m-%d-%H-%M-%S') + + aws s3 cp ${IMG_PATH} s3://junest-repo/junest/${IMG_PATH}.${DATE} + + # Cleanup old images + aws s3 ls s3://junest-repo/junest/junest-${ARCH}.tar.gz. | awk '{print $4}' | head -n -${MAX_OLD_IMAGES} | xargs -I {} s3 rm "s3://junest-repo/junest/{}" + + # Test the newly deployed image can be downloaded correctly + junest setup + junest -- echo "Installed JuNest (\$(uname -m))" + yes | junest setup --delete +fi diff --git a/tests/integ-tests/install-bash.sh b/ci/install-bash.sh similarity index 100% rename from tests/integ-tests/install-bash.sh rename to ci/install-bash.sh diff --git a/lib/checks/check.sh b/lib/checks/check.sh index 57dea5f..7af5832 100755 --- a/lib/checks/check.sh +++ b/lib/checks/check.sh @@ -9,7 +9,7 @@ # # vim: ft=sh -set -e +set -ex RUN_ROOT_TESTS=false @@ -80,7 +80,11 @@ $SUDO pacman $PACMAN_OPTIONS -Rsn ${repo_package1} repo_package2=iftop info "Checking ${repo_package2} package from official repo..." $SUDO pacman $PACMAN_OPTIONS -S ${repo_package2} -$RUN_ROOT_TESTS && $SUDO iftop -t -s 5 +if $RUN_ROOT_TESTS +then + # Time it out given that sometimes it gets stuck after few seconds. + $SUDO timeout 10 iftop -t -s 5 || true +fi $SUDO pacman $PACMAN_OPTIONS -Rsn ${repo_package2} if ! $SKIP_AUR_TESTS @@ -91,12 +95,4 @@ then $SUDO pacman $PACMAN_OPTIONS -Rsn ${aur_package} fi -# The following ensures that the gpg agent gets killed (if exists) -# otherwise it is not possible to exit from the session -if [[ -e /etc/pacman.d/gnupg/S.gpg-agent ]] -then - gpg-connect-agent -S /etc/pacman.d/gnupg/S.gpg-agent killagent /bye || echo "GPG agent did not close properly" - echo "GPG agent closed" -fi - exit 0 diff --git a/lib/checks/check_all.sh b/lib/checks/check_all.sh index 6e28ee5..9e5f0a6 100755 --- a/lib/checks/check_all.sh +++ b/lib/checks/check_all.sh @@ -8,6 +8,7 @@ set -ex JUNEST_BASE="${JUNEST_BASE:-$(readlink -f $(dirname $(readlink -f "$0"))/../..)}" JUNEST_SCRIPT=${JUNEST_SCRIPT:-${JUNEST_BASE}/bin/junest} + CHECK_SCRIPT=${JUNEST_BASE}/lib/checks/check.sh $JUNEST_SCRIPT proot --fakeroot -- "$CHECK_SCRIPT" --skip-aur-tests @@ -15,3 +16,6 @@ $JUNEST_SCRIPT proot -- "$CHECK_SCRIPT" --skip-aur-tests --use-sudo $JUNEST_SCRIPT ns --fakeroot -- "$CHECK_SCRIPT" --skip-aur-tests $JUNEST_SCRIPT ns -- "$CHECK_SCRIPT" --use-sudo sudo -E $JUNEST_SCRIPT groot -- "$CHECK_SCRIPT" --run-root-tests --skip-aur-tests + +# Test the wrappers work +$JUNEST_HOME/usr/bin_wrappers/pacman --help diff --git a/lib/core/build.sh b/lib/core/build.sh index af0c5bb..fb41ede 100644 --- a/lib/core/build.sh +++ b/lib/core/build.sh @@ -80,6 +80,7 @@ function build_image_env(){ sudo install -d -m 755 "${maindir}/root/etc/${CMD}" sudo bash -c "echo 'JUNEST_ARCH=$ARCH' > ${maindir}/root/etc/${CMD}/info" + set -x info "Generating the locales..." # sed command is required for locale-gen but it is required by fakeroot # and cannot be removed @@ -94,7 +95,7 @@ function build_image_env(){ info "Setting up the pacman keyring (this might take a while!)..." # gawk command is required for pacman-key sudo pacman --noconfirm --root ${maindir}/root -S gawk - sudo ${maindir}/root/bin/groot -b /dev ${maindir}/root bash -c ' + sudo ${maindir}/root/bin/groot --no-umount --avoid-bind -b /dev ${maindir}/root bash -c ' pacman-key --init; for keyring_file in /usr/share/pacman/keyrings/*.gpg; do @@ -102,6 +103,8 @@ function build_image_env(){ pacman-key --populate $keyring; done; [ -e /etc/pacman.d/gnupg/S.gpg-agent ] && gpg-connect-agent -S /etc/pacman.d/gnupg/S.gpg-agent killagent /bye' + sudo umount --force --recursive --lazy ${maindir}/root/dev + sudo umount --force --recursive ${maindir}/root sudo pacman --noconfirm --root ${maindir}/root -Rsn gawk sudo rm ${maindir}/root/var/cache/pacman/pkg/* diff --git a/lib/core/wrappers.sh b/lib/core/wrappers.sh new file mode 100644 index 0000000..184fca5 --- /dev/null +++ b/lib/core/wrappers.sh @@ -0,0 +1,31 @@ + + +function create_wrappers() { + mkdir -p ${JUNEST_HOME}/usr/bin_wrappers + + cd ${JUNEST_HOME}/usr/bin + for file in * + do + [[ -x $file ]] || continue + if [[ -e ${JUNEST_HOME}/usr/bin_wrappers/$file ]] + then + continue + fi + cat < ${JUNEST_HOME}/usr/bin_wrappers/${file} +#!/usr/bin/env bash + +JUNEST_ARGS=\${JUNEST_ARGS:-ns --fakeroot} + +junest \${JUNEST_ARGS} -- ${file} "\$@" +EOF + chmod +x ${JUNEST_HOME}/usr/bin_wrappers/${file} + done + + # Remove wrappers no longer needed + cd ${JUNEST_HOME}/usr/bin_wrappers + for file in * + do + [[ -e ${JUNEST_HOME}/usr/bin/$file ]] || rm -f $file + done + +} diff --git a/tests/unit-tests/test-junest.sh b/tests/unit-tests/test-junest.sh index e3c7ee1..0ba7cf5 100755 --- a/tests/unit-tests/test-junest.sh +++ b/tests/unit-tests/test-junest.sh @@ -31,6 +31,9 @@ function build_image_env(){ function delete_env(){ echo "delete_env" } +function create_wrappers(){ + : +} function setup_env_from_file(){ echo "setup_env_from_file($1)" } diff --git a/tests/unit-tests/test-wrappers.sh b/tests/unit-tests/test-wrappers.sh new file mode 100755 index 0000000..a78c81f --- /dev/null +++ b/tests/unit-tests/test-wrappers.sh @@ -0,0 +1,67 @@ +#!/bin/bash +source "$(dirname $0)/../utils/utils.sh" + +source "$(dirname $0)/../../lib/core/wrappers.sh" + +# Disable the exiterr +set +e + +function oneTimeSetUp(){ + setUpUnitTests +} + +function setUp(){ + junestSetUp +} + +function tearDown(){ + junestTearDown +} + +function test_create_wrappers_empty_bin(){ + assertCommandSuccess create_wrappers + assertEquals "" "$(cat $STDOUTF)" + assertTrue "bin_wrappers does not exist" "[ -e $JUNEST_HOME/usr/bin_wrappers ]" +} + +function test_create_wrappers_not_executable_file(){ + touch $JUNEST_HOME/usr/bin/myfile + assertCommandSuccess create_wrappers + assertEquals "" "$(cat $STDOUTF)" + assertTrue "bin_wrappers should exist" "[ -e $JUNEST_HOME/usr/bin_wrappers ]" + assertTrue "myfile wrapper should not exist" "[ ! -x $JUNEST_HOME/usr/bin_wrappers/myfile ]" +} + +function test_create_wrappers_executable_file(){ + touch $JUNEST_HOME/usr/bin/myfile + chmod +x $JUNEST_HOME/usr/bin/myfile + assertCommandSuccess create_wrappers + assertEquals "" "$(cat $STDOUTF)" + assertTrue "bin_wrappers should exist" "[ -e $JUNEST_HOME/usr/bin_wrappers ]" + assertTrue "myfile wrapper should exist" "[ -x $JUNEST_HOME/usr/bin_wrappers/myfile ]" +} + +function test_create_wrappers_already_exist(){ + touch $JUNEST_HOME/usr/bin/myfile + chmod +x $JUNEST_HOME/usr/bin/myfile + mkdir -p $JUNEST_HOME/usr/bin_wrappers + touch $JUNEST_HOME/usr/bin_wrappers/myfile + chmod +x $JUNEST_HOME/usr/bin_wrappers/myfile + assertCommandSuccess create_wrappers + assertEquals "" "$(cat $STDOUTF)" + assertTrue "bin_wrappers should exist" "[ -e $JUNEST_HOME/usr/bin_wrappers ]" + assertTrue "myfile wrapper should exist" "[ -x $JUNEST_HOME/usr/bin_wrappers/myfile ]" + assertEquals "" "$(touch $JUNEST_HOME/usr/bin_wrappers/myfile)" +} + +function test_create_wrappers_executable_no_longer_exist(){ + mkdir -p $JUNEST_HOME/usr/bin_wrappers + touch $JUNEST_HOME/usr/bin_wrappers/myfile + chmod +x $JUNEST_HOME/usr/bin_wrappers/myfile + assertCommandSuccess create_wrappers + assertEquals "" "$(cat $STDOUTF)" + assertTrue "bin_wrappers should exist" "[ -e $JUNEST_HOME/usr/bin_wrappers ]" + assertTrue "myfile wrapper should not exist" "[ ! -x $JUNEST_HOME/usr/bin_wrappers/myfile ]" +} + +source $(dirname $0)/../utils/shunit2 diff --git a/tests/utils/utils.sh b/tests/utils/utils.sh index ed7cb8c..542af68 100644 --- a/tests/utils/utils.sh +++ b/tests/utils/utils.sh @@ -11,6 +11,7 @@ function cwdTearDown(){ function junestSetUp(){ JUNEST_HOME=$(TMPDIR=/tmp mktemp -d -t junest-home.XXXXXXXXXX) + mkdir -p ${JUNEST_HOME}/usr/bin mkdir -p ${JUNEST_HOME}/etc/junest echo "JUNEST_ARCH=x86_64" > ${JUNEST_HOME}/etc/junest/info mkdir -p ${JUNEST_HOME}/etc/ca-certificates