Merge branch 'master' into dev

.travis.yml

@@ -1,31 +1,90 @@
-language: bash
-
 sudo: required
+os: linux
+
+cache:
+  directories:
+  - ~/.ccache
+  - ~/.pkg-cache
+
+services:
+- docker
+
+archlinux:
+  mount:
+  - ~/.ccache:~/.ccache
+  - ~/.pkg-cache:/var/cache/pacman/pkg
+  packages:
+  # Pacman packages
+  - ccache
+  - git
+  - haveged
+
+  before_install:
+  # 1.Override `package-cleanup.hook` to preserve cache for travis.
+  # 2.Enable ccache
+  # 3.Multithreaded build and compress
+  # 4.Suppress all gcc warnings
+  - |
+     sudo mkdir /etc/pacman.d/hooks/
+     sudo ln -s /dev/null /etc/pacman.d/hooks/package-cleanup.hook
+     sudo sed -i '/^BUILDENV/s/\!ccache/ccache/' /etc/makepkg.conf
+     sudo sed -i '/#MAKEFLAGS=/c MAKEFLAGS="-j2"' /etc/makepkg.conf
+     sudo sed -i '/^COMPRESSXZ/s/\xz/xz -T 2/' /etc/makepkg.conf
+     sudo sed -i '$a   CFLAGS="$CFLAGS -w"'   /etc/makepkg.conf
+     sudo sed -i '$a CXXFLAGS="$CXXFLAGS -w"' /etc/makepkg.conf
+  script:
+  # Here do not make any validation (-n) because it will be done later on in the Ubuntu host directly
+  - ./bin/junest build -n
+
 
 env:
+  matrix:
   - TRAVIS_BASH_VERSION="4.0"
+  global:
+    # AWS_ACCESS_KEY_ID
+    - secure: "ZotyKKWH5ZrBXDdEnVmV22gbn86BBSiqDZn2d2jVAApgUQdDc3wa7/uYAZP1bts6oQ897nnkUSFHk3M3QAcIoPJerUITTU5D7yjKcFDejgHdpJ4t9XSajmpY9CgKftWapwliWG4wolAKwyAp5GnYqz4GGltHyGxbF/VzUNRF3lw="
+    # AWS_SECRET_ACCESS_KEY
+    - secure: "AWixvJmhr6+rfF4cspMWMjkvLuOsdfNanLK5wrqkgx/0ezDGBBThH0qVhn5Mp1QFM6wVF+LRA6UESNnj0wNwByZHdM6LddkJWlWHb/qkVK+AO4RKUsXJWNyPyOkCNj/WEFpZHQKKUAlEtC8m8AmAcuoi90cr6ih0PXIePRyPFrM="
 
 before_install:
-  - ./tests/integ-tests/install-bash.sh "$TRAVIS_BASH_VERSION"
+  - ./ci/install-bash.sh "$TRAVIS_BASH_VERSION"
+  - sudo apt-get update
+  - sudo apt-get -y install awscli
 
 install:
   - PATH=$PWD/bin:$PATH
-  - junest setup
-  - junest -- echo "Installing JuNest (\$(uname -m))"
-  - JUNEST_HOME=~/.junest-arm junest setup --arch arm
-  - JUNEST_HOME=~/.junest-arm junest proot --fakeroot -- echo "Installing JuNest (\$(uname -m))"
 
 script:
+  #######################
+  # Unit Tests
+  #######################
   - bash --version
   - bash ./tests/checkstyle/checkstyle.sh
   - bash ./tests/unit-tests/unit-tests.sh
 
-  - export JUNEST_HOME=~/.junest
-  - ${PWD}/lib/checks/check_all.sh
-  - yes | junest setup --delete
-
   # ARM with qemu does seem to work properly. Disabling integ tests for ARM for now.
   #- export JUNEST_HOME=~/.junest-arm
+  #- junest setup --arch arm
+  #- junest proot --fakeroot -- echo "Installing JuNest (\$(uname -m))"
   #- junest proot --fakeroot -- ${PWD}/lib/checks/check.sh --skip-aur-tests
   #- junest proot -- ${PWD}/lib/checks/check.sh --skip-aur-tests --use-sudo
   #- yes | junest setup --delete
+
+  #######################
+  # Build and validation
+  #######################
+  - "curl -s https://raw.githubusercontent.com/bartoszek/arch-travis/master/arch-travis.sh | bash"
+  - "echo pacman pkg cache size: $(du -h ~/.pkg-cache|cut -f1) in $(ls ~/.pkg-cache|wc -l) files"
+  - ls -l
+  # Test the newly created JuNest image against Ubuntu host
+  - export JUNEST_HOME=~/.junest
+  - junest setup -i junest-x86_64.tar.gz
+  - ${PWD}/lib/checks/check_all.sh
+  - yes | junest setup --delete
+
+
+after_success:
+  #######################
+  # Deploy and validation
+  #######################
+  - ./ci/deploy.sh ${PWD}/junest-x86_64.tar.gz

README.md

@@ -29,7 +29,7 @@ The lightweight Arch Linux based distro that runs upon any Linux distros without
 Description
 ===========
 **JuNest** (Jailed User NEST) is a lightweight Arch Linux based distribution
-that allows to have disposable and isolated GNU/Linux environments
+that allows to have disposable and partial isolated GNU/Linux environments
 within any generic GNU/Linux host OS
 and without the need to have root privileges for installing packages.
 
@@ -41,7 +41,7 @@ The main advantages on using JuNest are:
 
 - Install packages without root privileges.
 - Partial isolated environment which you can install packages without affecting a production system.
-- Access to a wide range of packages in particular on GNU/Linux distros that may contain limited repositories (such as CentOS and RedHat).
+- Access to a wide range of packages, in particular on GNU/Linux distros that may contain limited repositories (such as CentOS and RedHat).
 - Available for `x86_64` and `arm` architectures but you can build your own image from scratch too!
 - Run on a different architecture from the host OS via QEMU
 - All Arch Linux lovers can have their favourite distro everywhere!
@@ -57,7 +57,7 @@ build a complete isolated environment but, conversely, is the ability to run
 programs as they were running natively from the host OS. Almost everything is shared
 between host OS and the JuNest sandbox (kernel, process subtree, network, mounting, etc)
 and only the root filesystem gets isolated
-(as the programs installed in JuNest need to reside elsewhere).
+(since the programs installed in JuNest need to reside elsewhere).
 
 This allows interaction between processes belonging to both host OS and JuNest.
 For instance, you can install `top` command in JuNest in order to monitor
@@ -107,6 +107,27 @@ There are multiple backend programs, each with its own pros/cons.
 To know more about the JuNest execution modes depending on the backend program
 used, see the [Usage](#usage) section below.
 
+Run commands installed in JuNest directly from host
+---------------------------------------
+
+Installed programs can be accessible directly from host.
+For instance, supposing the host OS is an Ubuntu distro you can directly
+run `pacman` by simply updating the `PATH` variable:
+
+```sh
+export PATH="$PATH:~/.junest/usr/bin_wrappers"
+pacman -S htop
+htop
+```
+
+By default the wrappers use `"ns --fakeroot"` but you can change it via `JUNEST_ARGS`.
+For instance, if you want to run `iftop` with real root privileges:
+
+```
+pacman -S iftop
+sudo JUNEST_ARGS="groot" iftop
+```
+
 
 Have fun!
 ---------
@@ -149,12 +170,22 @@ section below.
 ## Installation from git repository ##
 Just clone the JuNest repo somewhere (for example in ~/.local/share/junest):
 
-    git clone git://github.com/fsquillace/junest ~/.local/share/junest
-    export PATH=~/.local/share/junest/bin:$PATH
+```sh
+git clone git://github.com/fsquillace/junest ~/.local/share/junest
+export PATH=~/.local/share/junest/bin:$PATH
+```
+
+Optionally you want to use the wrappers to run commands
+installed in JuNest directly from host:
+
+```sh
+export PATH="$PATH:~/.junest/usr/bin_wrappers"
+```
+Update your `~/.bashrc` or `~/.zshrc` to get always the wrappers available.
 
 ### Installation using AUR (Arch Linux only) ###
 If you are using an Arch Linux system you can, alternatively, install JuNest from the [AUR repository](https://aur.archlinux.org/packages/junest-git/).
-After installing junest will be located in `/opt/junest/`
+JuNest will be located in `/opt/junest/`
 
 Usage
 =====
@@ -189,7 +220,8 @@ This mode is based on the fantastic
 PRoot based
 -----------
 [Proot](https://wiki.archlinux.org/index.php/Proot) represents a portable
-solution that works well in most of GNU/Linux distros available.
+solution which allows unprivileged users to execute programs inside a sandbox
+and works well in most of GNU/Linux distros available.
 One of the major drawbacks is the fact that Proot is not officially
 supported anymore, therefore, Proot bugs may no longer be fixed.
 
@@ -203,7 +235,9 @@ Chroot based
 ------------
 This solution suits only for privileged users. JuNest provides the possibility
 to run the environment via `chroot` program.
-In particular, it uses a special program called `GRoot`, an enhanced `chroot`
+In particular, it uses a special program called `GRoot`, a small and portable
+version of
+[arch-chroot](https://wiki.archlinux.org/index.php/Chroot)
 wrapper, that allows to bind mount directories specified by the user, such as
 `/proc`, `/sys`, `/dev`, `/tmp` and `$HOME`, before
 executing any programs inside the JuNest sandbox. In case the mounting will not
@@ -281,7 +315,7 @@ Or using proot arguments:
 junest proot -b "-b /mnt/mydata:/home/user/mydata"
 ```
 
-The option `-b` to provide options to the backeng program will work with PRoot, Namespace and GRoot backend programs.
+The option `-b` to provide options to the backend program will work with PRoot, Namespace and GRoot backend programs.
 Check out the backend program options by passing `--help` option:
 
 ```sh
@@ -309,21 +343,6 @@ Related wiki page:
 
 Internals
 =========
-
-There are two main chroot jail used in JuNest.
-The main one is [proot](https://wiki.archlinux.org/index.php/Proot) which
-allows unprivileged users to execute programs inside a sandbox and
-GRoot, a small and portable version of
-[arch-chroot](https://wiki.archlinux.org/index.php/Chroot) which is an
-enhanced chroot for privileged users that mounts the primary directories
-(i.e. `/proc`, `/sys`, `/dev` and `/run`) before executing any programs inside
-the sandbox.
-
-## Automatic fallback to classic chroot ##
-If GRoot fails for some reasons in the host system (i.e. it is not able to
-mount one of the directories),
-JuNest automatically tries to fallback to the classic chroot.
-
 ## Automatic fallback for all the dependent host OS executables ##
 JuNest attempts first to run the executables in the host OS located in different
 positions (`/usr/bin`, `/bin`, `/usr/sbin` and `/sbin`).
@@ -331,10 +350,10 @@ As a fallback it tries to run the same executable if it is available in the JuNe
 environment.
 
 ## Automatic building of the JuNest images ##
-There is not periodic automation build of the JuNest images yet.
-This was due to the difficulty to automate builds for arm architecture.
-The JuNest image for the `x86_64` is built periodically every once every three
-months.
+There is a periodic automation build of the JuNest images for `x86_64` arch
+only.
+The JuNest image for `arm` architecture may not be always up to date because
+the build is performed manually.
 
 ## Static QEMU binaries ##
 There are static QEMU binaries included in JuNest image that allows to run JuNest

VERSION

@@ -1 +1 @@
-7.2.2
+7.3.0

bin/junest

@@ -17,6 +17,7 @@ source "${JUNEST_BASE}/lib/core/setup.sh"
 source "${JUNEST_BASE}/lib/core/chroot.sh"
 source "${JUNEST_BASE}/lib/core/namespace.sh"
 source "${JUNEST_BASE}/lib/core/proot.sh"
+source "${JUNEST_BASE}/lib/core/wrappers.sh"
 
 
 ###################################
@@ -247,6 +248,7 @@ function execute_operation() {
             else
                 setup_env $ARCH_ARG
             fi
+            create_wrappers
         fi
 
         return
@@ -277,8 +279,9 @@ function execute_operation() {
         run_env=run_env_as_chroot
     fi
 
+    # Call create_wrappers in case new bin files have been created
+    trap "create_wrappers" EXIT QUIT TERM KILL
     $run_env "$BACKEND_COMMAND" "${BACKEND_ARGS}" $OPT_NO_COPY_FILES "${ARGS[@]}"
-
 }
 
 function main() {

ci/deploy.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+set -e
+
+IMG_PATH=$1
+
+set -u
+
+MAX_OLD_IMAGES=30
+
+# ARCH can be one of: x86, x86_64, arm
+HOST_ARCH=$(uname -m)
+if [ $HOST_ARCH == "i686" ] || [ $HOST_ARCH == "i386" ]
+then
+    ARCH="x86"
+elif [ $HOST_ARCH == "x86_64" ]
+then
+    ARCH="x86_64"
+elif [[ $HOST_ARCH =~ .*(arm).* ]]
+then
+    ARCH="arm"
+else
+    echo "Unknown architecture ${HOST_ARCH}" >&2
+    exit 11
+fi
+
+if [[ "$TRAVIS_BRANCH" == "master" ]]
+then
+
+    export AWS_DEFAULT_REGION=eu-west-1
+    # Upload image
+    # The put is done via a temporary filename in order to prevent outage on the
+    # production file for a longer period of time.
+    cp ${IMG_PATH} ${IMG_PATH}.temp
+    aws s3 cp ${IMG_PATH}.temp s3://junest-repo/junest/
+    aws s3 mv s3://junest-repo/junest/${IMG_PATH}.temp s3://junest-repo/junest/${IMG_PATH}
+    aws s3api put-object-acl --acl public-read --bucket junest-repo --key junest/${IMG_PATH}
+
+    DATE=$(date +'%Y-%m-%d-%H-%M-%S')
+
+    aws s3 cp ${IMG_PATH} s3://junest-repo/junest/${IMG_PATH}.${DATE}
+
+    # Cleanup old images
+    aws s3 ls s3://junest-repo/junest/junest-${ARCH}.tar.gz. | awk '{print $4}' | head -n -${MAX_OLD_IMAGES} | xargs -I {} s3 rm "s3://junest-repo/junest/{}"
+
+    # Test the newly deployed image can be downloaded correctly
+    junest setup
+    junest -- echo "Installed JuNest (\$(uname -m))"
+    yes | junest setup --delete
+fi

lib/checks/check.sh

@@ -9,7 +9,7 @@
 #
 # vim: ft=sh
 
-set -e
+set -ex
 
 
 RUN_ROOT_TESTS=false
@@ -80,7 +80,11 @@ $SUDO pacman $PACMAN_OPTIONS -Rsn ${repo_package1}
 repo_package2=iftop
 info "Checking ${repo_package2} package from official repo..."
 $SUDO pacman $PACMAN_OPTIONS -S ${repo_package2}
-$RUN_ROOT_TESTS && $SUDO iftop -t -s 5
+if $RUN_ROOT_TESTS
+then
+    # Time it out given that sometimes it gets stuck after few seconds.
+    $SUDO timeout 10 iftop -t -s 5 || true
+fi
 $SUDO pacman $PACMAN_OPTIONS -Rsn ${repo_package2}
 
 if ! $SKIP_AUR_TESTS
@@ -91,12 +95,4 @@ then
     $SUDO pacman $PACMAN_OPTIONS -Rsn ${aur_package}
 fi
 
-# The following ensures that the gpg agent gets killed (if exists)
-# otherwise it is not possible to exit from the session
-if [[ -e /etc/pacman.d/gnupg/S.gpg-agent ]]
-then
-    gpg-connect-agent -S /etc/pacman.d/gnupg/S.gpg-agent killagent /bye || echo "GPG agent did not close properly"
-    echo "GPG agent closed"
-fi
-
 exit 0

lib/checks/check_all.sh

@@ -8,6 +8,7 @@ set -ex
 JUNEST_BASE="${JUNEST_BASE:-$(readlink -f $(dirname $(readlink -f "$0"))/../..)}"
 
 JUNEST_SCRIPT=${JUNEST_SCRIPT:-${JUNEST_BASE}/bin/junest}
+
 CHECK_SCRIPT=${JUNEST_BASE}/lib/checks/check.sh
 
 $JUNEST_SCRIPT proot --fakeroot -- "$CHECK_SCRIPT" --skip-aur-tests
@@ -15,3 +16,6 @@ $JUNEST_SCRIPT proot -- "$CHECK_SCRIPT" --skip-aur-tests --use-sudo
 $JUNEST_SCRIPT ns --fakeroot -- "$CHECK_SCRIPT" --skip-aur-tests
 $JUNEST_SCRIPT ns -- "$CHECK_SCRIPT" --use-sudo
 sudo -E $JUNEST_SCRIPT groot -- "$CHECK_SCRIPT" --run-root-tests --skip-aur-tests
+
+# Test the wrappers work
+$JUNEST_HOME/usr/bin_wrappers/pacman --help

lib/core/build.sh

@@ -80,6 +80,7 @@ function build_image_env(){
     sudo install -d -m 755 "${maindir}/root/etc/${CMD}"
     sudo bash -c "echo 'JUNEST_ARCH=$ARCH' > ${maindir}/root/etc/${CMD}/info"
 
+    set -x
     info "Generating the locales..."
     # sed command is required for locale-gen but it is required by fakeroot
     # and cannot be removed
@@ -94,7 +95,7 @@ function build_image_env(){
     info "Setting up the pacman keyring (this might take a while!)..."
     # gawk command is required for pacman-key
     sudo pacman --noconfirm --root ${maindir}/root -S gawk
-    sudo ${maindir}/root/bin/groot -b /dev ${maindir}/root bash -c '
+    sudo ${maindir}/root/bin/groot --no-umount --avoid-bind -b /dev ${maindir}/root bash -c '
     pacman-key --init;
     for keyring_file in /usr/share/pacman/keyrings/*.gpg;
     do
@@ -102,6 +103,8 @@ function build_image_env(){
         pacman-key --populate $keyring;
     done;
     [ -e /etc/pacman.d/gnupg/S.gpg-agent ] && gpg-connect-agent -S /etc/pacman.d/gnupg/S.gpg-agent killagent /bye'
+    sudo umount --force --recursive --lazy ${maindir}/root/dev
+    sudo umount --force --recursive ${maindir}/root
     sudo pacman --noconfirm --root ${maindir}/root -Rsn gawk
 
     sudo rm ${maindir}/root/var/cache/pacman/pkg/*

lib/core/wrappers.sh

@@ -0,0 +1,31 @@
+
+
+function create_wrappers() {
+    mkdir -p ${JUNEST_HOME}/usr/bin_wrappers
+
+    cd ${JUNEST_HOME}/usr/bin
+    for file in *
+    do
+        [[ -x $file ]] || continue
+        if [[ -e ${JUNEST_HOME}/usr/bin_wrappers/$file ]]
+        then
+            continue
+        fi
+        cat <<EOF > ${JUNEST_HOME}/usr/bin_wrappers/${file}
+#!/usr/bin/env bash
+
+JUNEST_ARGS=\${JUNEST_ARGS:-ns --fakeroot}
+
+junest \${JUNEST_ARGS} -- ${file} "\$@"
+EOF
+        chmod +x ${JUNEST_HOME}/usr/bin_wrappers/${file}
+    done
+
+    # Remove wrappers no longer needed
+    cd ${JUNEST_HOME}/usr/bin_wrappers
+    for file in *
+    do
+        [[ -e ${JUNEST_HOME}/usr/bin/$file ]] || rm -f $file
+    done
+
+}

tests/unit-tests/test-junest.sh

@@ -31,6 +31,9 @@ function build_image_env(){
 function delete_env(){
     echo "delete_env"
 }
+function create_wrappers(){
+    :
+}
 function setup_env_from_file(){
     echo "setup_env_from_file($1)"
 }

tests/unit-tests/test-wrappers.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+source "$(dirname $0)/../utils/utils.sh"
+
+source "$(dirname $0)/../../lib/core/wrappers.sh"
+
+# Disable the exiterr
+set +e
+
+function oneTimeSetUp(){
+    setUpUnitTests
+}
+
+function setUp(){
+    junestSetUp
+}
+
+function tearDown(){
+    junestTearDown
+}
+
+function test_create_wrappers_empty_bin(){
+    assertCommandSuccess create_wrappers
+    assertEquals "" "$(cat $STDOUTF)"
+    assertTrue "bin_wrappers does not exist" "[ -e $JUNEST_HOME/usr/bin_wrappers ]"
+}
+
+function test_create_wrappers_not_executable_file(){
+    touch $JUNEST_HOME/usr/bin/myfile
+    assertCommandSuccess create_wrappers
+    assertEquals "" "$(cat $STDOUTF)"
+    assertTrue "bin_wrappers should exist" "[ -e $JUNEST_HOME/usr/bin_wrappers ]"
+    assertTrue "myfile wrapper should not exist" "[ ! -x $JUNEST_HOME/usr/bin_wrappers/myfile ]"
+}
+
+function test_create_wrappers_executable_file(){
+    touch $JUNEST_HOME/usr/bin/myfile
+    chmod +x $JUNEST_HOME/usr/bin/myfile
+    assertCommandSuccess create_wrappers
+    assertEquals "" "$(cat $STDOUTF)"
+    assertTrue "bin_wrappers should exist" "[ -e $JUNEST_HOME/usr/bin_wrappers ]"
+    assertTrue "myfile wrapper should exist" "[ -x $JUNEST_HOME/usr/bin_wrappers/myfile ]"
+}
+
+function test_create_wrappers_already_exist(){
+    touch $JUNEST_HOME/usr/bin/myfile
+    chmod +x $JUNEST_HOME/usr/bin/myfile
+    mkdir -p $JUNEST_HOME/usr/bin_wrappers
+    touch $JUNEST_HOME/usr/bin_wrappers/myfile
+    chmod +x $JUNEST_HOME/usr/bin_wrappers/myfile
+    assertCommandSuccess create_wrappers
+    assertEquals "" "$(cat $STDOUTF)"
+    assertTrue "bin_wrappers should exist" "[ -e $JUNEST_HOME/usr/bin_wrappers ]"
+    assertTrue "myfile wrapper should exist" "[ -x $JUNEST_HOME/usr/bin_wrappers/myfile ]"
+    assertEquals "" "$(touch $JUNEST_HOME/usr/bin_wrappers/myfile)"
+}
+
+function test_create_wrappers_executable_no_longer_exist(){
+    mkdir -p $JUNEST_HOME/usr/bin_wrappers
+    touch $JUNEST_HOME/usr/bin_wrappers/myfile
+    chmod +x $JUNEST_HOME/usr/bin_wrappers/myfile
+    assertCommandSuccess create_wrappers
+    assertEquals "" "$(cat $STDOUTF)"
+    assertTrue "bin_wrappers should exist" "[ -e $JUNEST_HOME/usr/bin_wrappers ]"
+    assertTrue "myfile wrapper should not exist" "[ ! -x $JUNEST_HOME/usr/bin_wrappers/myfile ]"
+}
+
+source $(dirname $0)/../utils/shunit2

tests/utils/utils.sh

@@ -11,6 +11,7 @@ function cwdTearDown(){
 
 function junestSetUp(){
     JUNEST_HOME=$(TMPDIR=/tmp mktemp -d -t junest-home.XXXXXXXXXX)
+    mkdir -p ${JUNEST_HOME}/usr/bin
     mkdir -p ${JUNEST_HOME}/etc/junest
     echo "JUNEST_ARCH=x86_64" > ${JUNEST_HOME}/etc/junest/info
     mkdir -p ${JUNEST_HOME}/etc/ca-certificates