From 81e38b00597be6880d232d8807cde33159b3dd68 Mon Sep 17 00:00:00 2001 From: Felix Abecassis Date: Thu, 30 Nov 2017 22:51:38 -0800 Subject: [PATCH 1/5] lxc-oci: cleanup temporary download directory if umoci is interrupted Signed-off-by: Felix Abecassis --- templates/lxc-oci.in | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/templates/lxc-oci.in b/templates/lxc-oci.in index f98c38bcd..1c7e0721f 100755 --- a/templates/lxc-oci.in +++ b/templates/lxc-oci.in @@ -38,8 +38,11 @@ LXC_TEMPLATE_CONFIG="@LXCTEMPLATECONFIG@" # Some useful functions cleanup() { - if [ -d "$DOWNLOAD_TEMP" ]; then - rm -Rf $DOWNLOAD_TEMP + if [ -d "${DOWNLOAD_TEMP}" ]; then + rm -Rf "${DOWNLOAD_TEMP}" + fi + if [ -d "${LXC_ROOTFS}.tmp" ]; then + rm -Rf "${LXC_ROOTFS}.tmp" fi } @@ -244,7 +247,6 @@ fi umoci unpack ${umoci_args[@]} --image "${DOWNLOAD_TEMP}:latest" "${LXC_ROOTFS}.tmp" rmdir "${LXC_ROOTFS}" mv "${LXC_ROOTFS}.tmp/rootfs" "${LXC_ROOTFS}" -rm -rf "${LXC_ROOTFS}.tmp" OCI_CONF_FILE=$(getconfigpath ${DOWNLOAD_TEMP} latest) LXC_CONF_FILE="${LXC_PATH}/config" From 4b42266dc6f08ef11475fd9fb2ba9a8e3b7cd1bf Mon Sep 17 00:00:00 2001 From: Felix Abecassis Date: Thu, 30 Nov 2017 22:51:47 -0800 Subject: [PATCH 2/5] lxc-oci: remove unhelpful comment Signed-off-by: Felix Abecassis --- templates/lxc-oci.in | 2 -- 1 file changed, 2 deletions(-) diff --git a/templates/lxc-oci.in b/templates/lxc-oci.in index 1c7e0721f..8e6df1f5b 100755 --- a/templates/lxc-oci.in +++ b/templates/lxc-oci.in @@ -237,9 +237,7 @@ if [ -n "$OCI_USERNAME" ]; then fi skopeo copy ${skopeo_args[@]} "${OCI_URL}" "oci:${DOWNLOAD_TEMP}:latest" -# Unpack the rootfs echo "Unpacking the rootfs" - umoci_args=("") if [ -n "$LXC_MAPPED_UID" ] && [ "$LXC_MAPPED_UID" != "-1" ]; then umoci_args+=(--rootless) From ca1280fea4236f67d69920103804b1a0cb6a5645 Mon Sep 17 00:00:00 2001 From: Felix Abecassis Date: Thu, 30 Nov 2017 22:51:53 -0800 Subject: [PATCH 3/5] lxc-oci: rely on jq instead of sed to transform values Signed-off-by: Felix Abecassis --- templates/lxc-oci.in | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/templates/lxc-oci.in b/templates/lxc-oci.in index 8e6df1f5b..b3a65e9a1 100755 --- a/templates/lxc-oci.in +++ b/templates/lxc-oci.in @@ -63,7 +63,7 @@ getconfigpath() { basedir="$1" q="$2" - digest=`cat "${basedir}/index.json" | jq --arg q "$q" '.manifests[] | if .annotations."org.opencontainers.image.ref.name" == $q then .digest else null end' | sed -e 's/"//g'` + digest=`cat "${basedir}/index.json" | jq -c -r --arg q "$q" '.manifests[] | if .annotations."org.opencontainers.image.ref.name" == $q then .digest else null end'` if [ -z "${digest}" ]; then echo "$q not found in index.json" >&2 return @@ -71,7 +71,7 @@ getconfigpath() { # Ok we have the image config digest, now get the config from that, d=${digest:7} - cdigest=`cat "${basedir}/blobs/sha256/${d}" | jq '.config.digest' | sed -e 's/"//g'` + cdigest=`cat "${basedir}/blobs/sha256/${d}" | jq -c -r '.config.digest'` if [ -z "${cdigest}" ]; then echo "container config not found" >&2 return @@ -91,22 +91,18 @@ getep() { configpath="$1" - ep=`cat "${configpath}" | jq -c '.config.Entrypoint' | sed -e 's/^\[//; s/\]$//; s/","/" "/'` - cmd=`cat "${configpath}" | jq -c '.config.Cmd' | sed -e 's/^\[//; s/\]$//; s/","/" "/'` - if [ "${ep}" = "null" ]; then + ep=`cat "${configpath}" | jq -c -r '.config.Entrypoint[]?'` + cmd=`cat "${configpath}" | jq -c -r '.config.Cmd[]?'` + if [ -z "${ep}" ]; then ep="${cmd}" - if [ "${ep}" = "null" ]; then + if [ -z "${ep}" ]; then ep="/bin/sh" fi - elif [ "${cmd}" != "null" ]; then + elif [ -n "${cmd}" ]; then ep="${ep} ${cmd}" fi - if [ -z "${ep}" ]; then - echo "/bin/sh" - return - fi - echo "${ep}" + echo ${ep} return } @@ -118,8 +114,7 @@ getenv() { configpath="$1" - cat "${configpath}" > /tmp/config - env=`cat "${configpath}" | jq -c '.config.Env[]'` + env=`cat "${configpath}" | jq -c -r '.config.Env[]'` echo "${env}" return From 0fd2b679727bfcff36b7e08d87ccfcca744aa602 Mon Sep 17 00:00:00 2001 From: Felix Abecassis Date: Thu, 30 Nov 2017 22:51:57 -0800 Subject: [PATCH 4/5] lxc-oci: support index files with multiple manifests Previously, the output would contain "null" strings in this case. Signed-off-by: Felix Abecassis --- templates/lxc-oci.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/lxc-oci.in b/templates/lxc-oci.in index b3a65e9a1..6689f053c 100755 --- a/templates/lxc-oci.in +++ b/templates/lxc-oci.in @@ -63,7 +63,7 @@ getconfigpath() { basedir="$1" q="$2" - digest=`cat "${basedir}/index.json" | jq -c -r --arg q "$q" '.manifests[] | if .annotations."org.opencontainers.image.ref.name" == $q then .digest else null end'` + digest=`cat "${basedir}/index.json" | jq -c -r --arg q "$q" '.manifests[] | if .annotations."org.opencontainers.image.ref.name" == $q then .digest else empty end'` if [ -z "${digest}" ]; then echo "$q not found in index.json" >&2 return From 52e31c07c8451074528bff5646c24cf93c2fd8d7 Mon Sep 17 00:00:00 2001 From: Felix Abecassis Date: Fri, 1 Dec 2017 11:04:34 -0800 Subject: [PATCH 5/5] lxc-oci: support skopeo layer caching This requires skopeo version 0.1.25 Signed-off-by: Felix Abecassis --- templates/lxc-oci.in | 51 ++++++++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/templates/lxc-oci.in b/templates/lxc-oci.in index 6689f053c..5bd8edbf4 100755 --- a/templates/lxc-oci.in +++ b/templates/lxc-oci.in @@ -34,6 +34,7 @@ for bin in skopeo umoci jq; do fi done +LOCALSTATEDIR="@LOCALSTATEDIR@" LXC_TEMPLATE_CONFIG="@LXCTEMPLATECONFIG@" # Some useful functions @@ -145,7 +146,7 @@ EOF return 0 } -options=$(getopt -o u:h -l help,url:,username:,password:,\ +options=$(getopt -o u:h -l help,url:,username:,password:,no-cache,\ name:,path:,rootfs:,mapped-uid:,mapped-gid: -- "$@") if [ $? -ne 0 ]; then @@ -157,6 +158,7 @@ eval set -- "$options" OCI_URL="" OCI_USERNAME= OCI_PASSWORD= +OCI_USE_CACHE="true" LXC_MAPPED_GID= LXC_MAPPED_UID= @@ -170,6 +172,7 @@ while :; do -u|--url) OCI_URL=$2; shift 2;; --username) OCI_USERNAME=$2; shift 2;; --password) OCI_PASSWORD=$2; shift 2;; + --no-cache) OCI_USE_CACHE="false"; shift 1;; --name) LXC_NAME=$2; shift 2;; --path) LXC_PATH=$2; shift 2;; --rootfs) LXC_ROOTFS=$2; shift 2;; @@ -195,33 +198,43 @@ if [ -n "$OCI_PASSWORD" ] && [ -z "$OCI_USERNAME" ]; then exit 1 fi +if [ "${OCI_USE_CACHE}" = "true" ]; then + if ! skopeo copy --help | grep -q 'dest-shared-blob-dir'; then + echo "INFO: skopeo doesn't support blob caching" + OCI_USE_CACHE="false" + fi +fi + USERNS=$(in_userns) -if [ "$USERNS" != "no" ]; then - if [ "$USERNS" = "yes" ]; then - if [ -z "$LXC_MAPPED_UID" ] || [ "$LXC_MAPPED_UID" = "-1" ]; then - echo "ERROR: In a user namespace without a map." 1>&2 - exit 1 - fi - DOWNLOAD_MODE="user" - DOWNLOAD_TARGET="user" - else - DOWNLOAD_MODE="user" - DOWNLOAD_TARGET="system" +if [ "$USERNS" = "yes" ]; then + if [ -z "$LXC_MAPPED_UID" ] || [ "$LXC_MAPPED_UID" = "-1" ]; then + echo "ERROR: In a user namespace without a map." 1>&2 + exit 1 fi fi +if [ "${OCI_USE_CACHE}" = "true" ]; then + if [ "$USERNS" = "yes" ]; then + DOWNLOAD_BASE="${HOME}/.cache/lxc" + else + DOWNLOAD_BASE="${LOCALSTATEDIR}/cache/lxc" + fi +else + DOWNLOAD_BASE=/tmp +fi + # Trap all exit signals trap cleanup EXIT HUP INT TERM if ! type mktemp >/dev/null 2>&1; then - DOWNLOAD_TEMP=/tmp/lxc-oci.$$ + DOWNLOAD_TEMP="${DOWNLOAD_BASE}/lxc-oci.$$" mkdir -p $DOWNLOAD_TEMP else - DOWNLOAD_TEMP=$(mktemp -d) + DOWNLOAD_TEMP=$(mktemp -d -p "${DOWNLOAD_BASE}") fi -# Download the image - TODO - cache +# Download the image skopeo_args=("") if [ -n "$OCI_USERNAME" ]; then CREDENTIALS="${OCI_USERNAME}" @@ -230,7 +243,13 @@ if [ -n "$OCI_USERNAME" ]; then fi skopeo_args+=(--src-creds "${CREDENTIALS}") fi -skopeo copy ${skopeo_args[@]} "${OCI_URL}" "oci:${DOWNLOAD_TEMP}:latest" +if [ "${OCI_USE_CACHE}" = "true" ]; then + skopeo_args+=(--dest-shared-blob-dir "${DOWNLOAD_BASE}") + skopeo copy ${skopeo_args[@]} "${OCI_URL}" "oci:${DOWNLOAD_TEMP}:latest" + ln -s "${DOWNLOAD_BASE}/sha256" "${DOWNLOAD_TEMP}/blobs/sha256" +else + skopeo copy ${skopeo_args[@]} "${OCI_URL}" "oci:${DOWNLOAD_TEMP}:latest" +fi echo "Unpacking the rootfs" umoci_args=("")