From 4f3322ed1137c4a20d42cf7ac5a0fd0028e5b03b Mon Sep 17 00:00:00 2001
From: zawz <zawz@hotmail.fr>
Date: Tue, 26 May 2020 17:31:37 +0200
Subject: [PATCH] Implement compression configuration

---
 .config.example                   | 17 ++++++++--
 README.md                         |  3 +-
 server_deploy.sh                  | 11 +++++--
 server_scripts/database_update.sh | 16 ++++++++--
 server_scripts/gen_install.sh     | 27 ++++++++++++----
 src/config.sh                     | 13 ++++++--
 src/deploy.sh                     | 53 ++++++++++++++++---------------
 src/fetch.sh                      |  4 +--
 src/install.sh                    |  6 ++--
 src/main.sh                       |  2 +-
 src/print.sh                      |  8 ++---
 src/remove.sh                     |  6 ++--
 src/view.sh                       |  6 ++--
 13 files changed, 114 insertions(+), 58 deletions(-)

diff --git a/.config.example b/.config.example
index bba4fda..6b9f46d 100644
--- a/.config.example
+++ b/.config.example
@@ -1,5 +1,18 @@
+# zpkg server config
+
+# address of the SSH server
 SSH_ADDR=example.com
+# user for the SSH server
 SSH_USER=zpkg
-HTTP_ADDR=example.com
-HTTP_PATH=zpkg
+# path to packages for the zpkg user
 PKG_PATH=pkg
+
+# public address of the HTTP server
+HTTP_ADDR=example.com
+# path on server
+HTTP_PATH=zpkg
+
+# Compression to use for packages
+# format:
+#   extension:binary:parallel_binary:arguments
+COMPRESSION=xz:xz:pxz:-1
diff --git a/README.md b/README.md
index 4ebd53f..7fffab3 100644
--- a/README.md
+++ b/README.md
@@ -13,9 +13,10 @@ Requirements:
 - sudo
 - wget
 - tar
+- xz/gz
 
 Optional:
-- pv
+- pxz/pigz (faster compression/decompression)
 
 ### Installing
 
diff --git a/server_deploy.sh b/server_deploy.sh
index af16fde..ab10fe3 100755
--- a/server_deploy.sh
+++ b/server_deploy.sh
@@ -4,6 +4,13 @@
 
 ssh="$SSH_USER@$SSH_ADDR"
 
+[ -z "$COMPRESSION" ] && COMPRESSION="xz:xz:pxz"
+extension=$(echo "$COMPRESSION" | cut -d':' -f1)
+compress=$(echo "$COMPRESSION" | cut -d':' -f2)
+pcompress=$(echo "$COMPRESSION" | cut -d':' -f3)
+which $pcompress >/dev/null 2>&1 || pcompress=$compress
+[ -z "$pcompress" ] && pcompress=$compress
+which $compress >/dev/null 2>&1 || { echo "Compression '$compress' not installed" && exit 12; }
 
 random_string()
 {
@@ -30,9 +37,9 @@ mv zpkg "$fullpath$DEST" || exit $?
 # create and send package
 (
   cd "$tmpdir/$PKG" || exit $?
-  tar -cvJf zpkg.tar.xz * || exit $?
+  tar -cf - * | $pcompress > zpkg.tar.$extension || exit $?
   # send package
-  scp zpkg.tar.xz "$ssh":~/"$PKG_PATH" || exit $?
+  scp zpkg.tar.$extension "$ssh":~/"$PKG_PATH" || exit $?
 )
 # cleanup
 rm -rd "$tmpdir"
diff --git a/server_scripts/database_update.sh b/server_scripts/database_update.sh
index f1274d5..caffa5b 100755
--- a/server_scripts/database_update.sh
+++ b/server_scripts/database_update.sh
@@ -2,14 +2,24 @@
 
 . "$(pwd)/.config"
 
+# resolve compression
+[ -z "$COMPRESSION" ] && COMPRESSION="xz:xz:pxz"
+extension=$(echo "$COMPRESSION" | cut -d':' -f1)
+compress=$(echo "$COMPRESSION" | cut -d':' -f2)
+pcompress=$(echo "$COMPRESSION" | cut -d':' -f3)
+which $pcompress >/dev/null 2>&1 || pcompress=$compress
+[ -z "$pcompress" ] && pcompress=$compress
+which $compress >/dev/null 2>&1 || { echo "Compression '$compress' not installed" && exit 12; }
+
+# iterate packages
 cd "$HOME/$PKG_PATH" || exit $?
-PKGLIST="$(ls ./*.tar.xz)"
+PKGLIST="$(ls ./*.tar.$extension)"
 {
 for I in $PKGLIST
 do
-  NAME=$(echo "$I" | sed 's|\.tar\.xz||g;s|^\./||g')
+  NAME=$(echo "$I" | sed 's|\.tar\..*$||g;s|^\./||g')
   TIME=$(stat -c "%Y" "$I")
-  DEPS=$(tar -xOf "$I" DEPS 2>/dev/null | tr -s '\n\t ' ' ')
+  DEPS=$($pcompress -dc "$I" | tar -xOf - DEPS 2>/dev/null | tr -s '\n\t ' ' ')
   echo "$NAME $TIME $DEPS"
 done
 } > pkglist
diff --git a/server_scripts/gen_install.sh b/server_scripts/gen_install.sh
index a453457..814ed27 100755
--- a/server_scripts/gen_install.sh
+++ b/server_scripts/gen_install.sh
@@ -11,6 +11,15 @@ cat .config >> install.sh
 # body
 echo '
 
+# resolve compression
+[ -z "$COMPRESSION" ] && COMPRESSION="xz:xz:pxz"
+extension=$(echo "$COMPRESSION" | cut -d":" -f1)
+compress=$(echo "$COMPRESSION" | cut -d":" -f2)
+pcompress=$(echo "$COMPRESSION" | cut -d":" -f3)
+which $pcompress >/dev/null 2>&1 || pcompress=$compress
+[ -z "$pcompress" ] && pcompress=$compress
+which $compress >/dev/null 2>&1 || { echo "Compression $compress not installed" && exit 12; }
+
 usage()
 {
   echo "$(basename "$0")" [option...]
@@ -47,13 +56,16 @@ done
 
 shift $((OPTIND-1))
 
-if [ "$(id | cut -d"=" -f2 | cut -d"(" -f1)" -eq 0 ] && [ "$1" != "force" ]
+unset sudo
+if [ "$(id | cut -d"=" -f2 | cut -d"(" -f1)" -eq 0 ]
 then
-  echo "Cannot run as root" >&2
-  echo "Use '"'"'$(basename "$0") force'"'"' to force running as root"
-  exit 10
+  if [ "$1" != "force" ] ; then
+    echo "Cannot run as root" >&2
+    echo "Use '"'"'$(basename "$0") force'"'"' to force running as root"
+    exit 10
+  fi
 else
-  which sudo >/dev/null || { echo "sudo not installed" >&2 && exit 11; }
+  which sudo >/dev/null 2>&1 || { echo "sudo not installed" >&2 && exit 11; }
   sudo=sudo
 fi
 
@@ -61,6 +73,7 @@ fi
 $sudo sh -c "{
   echo SSH_ADDRESS=$SSH_USER@$SSH_ADDR
   echo HTTP_ADDRESS=$HTTP_ADDR/$HTTP_PATH
+  echo COMPRESSION=$COMPRESSION
   echo PKG_PATH=pkg
 } > zpkg.conf"
 
@@ -73,12 +86,12 @@ tmpdir=/tmp/zpkg$(random_string 5)
 mkdir -p "$tmpdir" || exit $?
 (
   cd "$tmpdir" || exit $?
-  if ! wget "$HTTP_ADDR/$HTTP_PATH/zpkg.tar.xz" -q -O "zpkg.tar.xz"
+  if ! wget "$HTTP_ADDR/$HTTP_PATH/zpkg.tar.$extension" -q -O "zpkg.tar.$extension"
   then
     echo "Cannot reach $HTTP_ADDR/$HTTP_PATH" > /dev/stderr
     exit 1
   fi
-  tar -xf zpkg.tar.xz || exit $?
+  $pcompress -dc "zpkg.tar.$extension" | tar -xf - || exit $?
 
 # install zpkg package
   ROOT/usr/local/bin/zpkg -f install zpkg || exit $?
diff --git a/src/config.sh b/src/config.sh
index 8ea00a3..f292dc2 100644
--- a/src/config.sh
+++ b/src/config.sh
@@ -17,7 +17,7 @@ config_file="$config_path/zpkg.conf"
 # setup sudo prefix
 unset sudo
 if ! root_check ; then
-  which sudo >/dev/null || { echo "sudo not installed" && exit 11; }
+  which sudo >/dev/null 2>&1 || { echo "sudo not installed" && exit 11; }
   sudo=sudo
 fi
 
@@ -31,5 +31,14 @@ PKG_PATH="$(resolve_path "$PKG_PATH" "$config_path")"
 
 root_check && [ -z "$opt_f" ] && [ "$ALLOW_ROOT" != "true" ] && echo "Cannot run as root" >&2 && exit 10
 
-[ ! -d "$PKG_PATH" ] $sudo mkdir -p "$PKG_PATH" 2>/dev/null
+[ ! -d "$PKG_PATH" ] && $sudo mkdir -p "$PKG_PATH"
 
+# resolve compression
+[ -z "$COMPRESSION" ] && COMPRESSION="xz:xz:pxz"
+extension=$(echo "$COMPRESSION" | cut -d':' -f1)
+compress=$(echo "$COMPRESSION" | cut -d':' -f2)
+pcompress=$(echo "$COMPRESSION" | cut -d':' -f3)
+comparg=$(echo "$COMPRESSION" | cut -d':' -f4-)
+which $pcompress >/dev/null 2>&1 || pcompress=$compress
+[ -z "$pcompress" ] && pcompress=$compress
+which $compress >/dev/null 2>&1 || { echo "Compression '$compress' not installed" && exit 12; }
diff --git a/src/deploy.sh b/src/deploy.sh
index eab2022..0d10fc6 100644
--- a/src/deploy.sh
+++ b/src/deploy.sh
@@ -1,55 +1,58 @@
 #!/bin/sh
 
+
+# $1 = source , $2 = package , $3 = name
 package()
 {
-  unset clean_needed
   src="$1"
   pkg="$2"
-  echo "Packaging $(getname "$src"): $(du -sh "$src" | awk '{print $1}')iB"
 
-  tmpdir="/tmp/zpkg_$(random_string 5)"
-  mkdir -p "$tmpdir"
-  if [ ! -d "$src/ROOT" ] && [ ! -d "$src/HOME" ] && [ ! -f "$src/DEPS" ]
+  unset tmpdir
+  if [ ! -d "$src/ROOT" ] && [ ! -d "$src/HOME" ] && [ ! -f "$src/DEPS" ] && [ ! -f "$src/DESC" ]
   then
-    mkdir -p "$tmpdir/package"
-    cp -r "$src" "$tmpdir/package/ROOT"
-  else
-    cp -r "$src" "$tmpdir/package"
+    tmpdir="/tmp/zpkg_$(random_string 5)"
+    mkdir -p "$tmpdir"
+    cp -r "$src" "$tmpdir/ROOT"
+    src="$tmpdir"
   fi
   (
-  cd "$tmpdir/package"
+  cd "$src"
   unset list
   [ -f DEPS ] && list=DEPS
   [ -f DESC ] && list="$list DESC"
   [ -d HOME ] && list="$list HOME"
   [ -d ROOT ] && list="$list ROOT"
-  if which pv >/dev/null 2>&1
-  then
-    tar -cf - --owner=0 --group=0 -P $list | pv -s "$(du -sb . | awk '{print $1}')" | xz > "../$pkg"
-  else
-    tar -cvJf - --owner=0 --group=0 $list > "../$pkg"
-  fi
+  size=$(du -sb $list | awk '{print $1}' | paste -sd+ | bc)
+  echo "Packaging $(basename "$pkg"): $(echo "$size" | numfmt --to=iec-i)B"
+  cc=$compress
+  [ $size -gt 1048576 ] && cc=$pcompress
+  tar -cf - --owner=0 --group=0 $list | $cc $comparg > "$pkg"
   )
-  mv "$tmpdir/$pkg" ./
-  rm -rd "$tmpdir"
+  [ -n "$tmpdir" ] && rm -rd "$tmpdir"
+  return 0
 }
 
+# $1 = file
 deploy_package()
 {
-  echo "Deploying $1: $(du -sh "$1" | awk '{print $1}')iB"
+  echo "Deploying $(basename "$1"): $(du -sh "$1" | awk '{print $1}')iB"
   scp "$1" $SSH_ADDRESS:~/'$(grep "PKG_PATH=" .config | cut -d"=" -f2-)'
 }
 
 deploy_folder()
 {
-  archive="$(getname "$1").tar.xz"
-  if [ -n "$(echo "$1" | grep '\.tar\.xz$' )" ]
+  if [ -f "$1" ] && echo "$1" | grep -q '\.tar\.'"$extension\$" # file and valid extension
   then
-    deploy_package "$1" || return 1
+    $pcompress -dc >/dev/null 2>&1 | tar -tf - >/dev/null 2>&1|| { echo "File '$1' is not a valid archive" && return 1; }
+    deploy_package "$1" "$1" || return $?
+  elif [ -d "$1" ]
+  then
+    archive="$(getname "$1").tar.$extension"
+    package "$1" "/tmp/$archive" || return $?
+    deploy_package "/tmp/$archive" || return $?
+    rm "/tmp/$archive"
   else
-    package "$1" "$archive" || return 1
-    deploy_package "$archive" || return 1
-    rm "$archive" 2> /dev/null
+    echo "Target '$1' doesn't exist"
   fi
 }
 
diff --git a/src/fetch.sh b/src/fetch.sh
index bd0998b..98fb9e4 100644
--- a/src/fetch.sh
+++ b/src/fetch.sh
@@ -4,8 +4,8 @@
 fetch_package()
 {
   out="$2"
-  [ -z "$out" ] && out="$1.tar.xz"
-  wget "$HTTP_ADDRESS/$1.tar.xz" -q --show-progress -O "$out" 2>&1
+  [ -z "$out" ] && out="$1.tar.$extension"
+  wget "$HTTP_ADDRESS/$1.tar.$extension" -q --show-progress -O "$out" 2>&1
 }
 
 # $1 = prefix
diff --git a/src/install.sh b/src/install.sh
index 2cc7f38..addf213 100644
--- a/src/install.sh
+++ b/src/install.sh
@@ -3,7 +3,7 @@
 unpack()
 {
   echo "Unpacking $1"
-  tar -xf "$1"
+  $pcompress -dc "$1" | tar -xf -
 }
 
 # $1 = package , $2 = prefix
@@ -46,8 +46,8 @@ install_package()
   (
     cd "$tmpdir"
     fetch_package "$1" || { echo "Package '$1' not found" >&2 && return 1; }
-    $2 cp "$1.tar.xz" "$PKG_PATH"
-    unpack "$1.tar.xz" || return $?
+    $2 cp "$1.tar.$extension" "$PKG_PATH"
+    unpack "$1.tar.$extension" || return $?
     move_files ROOT / $2 2>/dev/null
     move_files HOME "$HOME" 2>/dev/null
     add_package_entry "$1" $2
diff --git a/src/main.sh b/src/main.sh
index a58647a..c94716b 100644
--- a/src/main.sh
+++ b/src/main.sh
@@ -28,7 +28,7 @@ show)
       then
         view_package "$I"
       else
-        wget "$HTTP_ADDRESS/$1.tar.xz" -q -O - 2>/dev/null | view_package_file - || { echo "Could not fetch package '$I'" >&2 ; return 1 ; }
+        wget "$HTTP_ADDRESS/$1.tar.$extension" -q -O - 2>/dev/null | view_package_file - || { echo "Could not fetch package '$I'" >&2 ; return 1 ; }
       fi
     done
   fi
diff --git a/src/print.sh b/src/print.sh
index 28f7d06..46c0a35 100644
--- a/src/print.sh
+++ b/src/print.sh
@@ -42,20 +42,20 @@ package_info() {
   status="not installed"
   grep -wq "^$1" "$PKG_PATH/pkglist" 2>/dev/null || { echo "Package '$I' not found" && return 1; }
   grep -wq "^$1" "$PKG_PATH/installed" 2>/dev/null && status=installed
-  if [ "$status" = "installed" ] && [ -f "$PKG_PATH/$1.tar.xz" ]
+  if [ "$status" = "installed" ] && [ -f "$PKG_PATH/$1.tar.$extension" ]
   then
-    pkg="$PKG_PATH/$1.tar.xz"
+    pkg="$PKG_PATH/$1.tar.$extension"
   else
     tmpdir="/tmp/zpkg_$(random_string 5)"
     pwd=$(pwd)
     mkdir "$tmpdir"
     fetch_package "$1" >/dev/null 2>&1 || { echo "Error fetching package" >&2 && ret=$?; }
-    pkg="$1.tar.xz"
+    pkg="$1.tar.$extension"
   fi
   deps=$(deps "$1")
   desc=$(desc "$pkg" 2>/dev/null)
   csize=$(stat -c '%s' "$pkg" | numfmt --to=iec-i --suffix=B --padding 6)
-  isize=$(xz -dc "$pkg" | wc -c | numfmt --to=iec-i --suffix=B --padding 6)
+  isize=$($pcompress -dc "$pkg" | wc -c | numfmt --to=iec-i --suffix=B --padding 6)
   [ -n "$cleanup" ] && { cd "$pwd"; rm -rd "$tmpdir"; }
 
   [ -n "$ret" ] && return $ret
diff --git a/src/remove.sh b/src/remove.sh
index bbacc43..f7fce64 100644
--- a/src/remove.sh
+++ b/src/remove.sh
@@ -14,7 +14,7 @@ delete_files()
 remove_package()
 {
   cd "$PKG_PATH"
-  archive="$(pwd)/$1.tar.xz"
+  archive="$(pwd)/$1.tar.$extension"
   if [ ! -f "$archive" ] || ! grep -q -w "^$1" installed
   then
     echo "Package '$1' not installed" >&2
@@ -24,11 +24,11 @@ remove_package()
 
   ( # delete root files
     cd /
-    tar -tf "$archive" ROOT 2>/dev/null | sed 's|^ROOT/||g' | tac | delete_files $2
+    $pcompress -dc "$archive" | tar -tf - ROOT 2>/dev/null | sed 's|^ROOT/||g' | tac | delete_files $2
   )
   ( # delete home files
     cd "$HOME"
-    tar -tf "$archive" HOME 2>/dev/null | sed 's|^HOME/||g' | tac | delete_files
+    $pcompress -dc "$archive" | tar -tf - HOME 2>/dev/null | sed 's|^HOME/||g' | tac | delete_files
   )
 
   $2 rm "$archive" 2>/dev/null
diff --git a/src/view.sh b/src/view.sh
index 176cc12..981ebbb 100644
--- a/src/view.sh
+++ b/src/view.sh
@@ -9,7 +9,7 @@ deps()
 
 # $1 = pkg file
 desc() {
-  tar -xOf "$1" DESC
+  $pcompress -dc "$1" | tar -xOf - DESC
 }
 
 resolve_packages()
@@ -52,13 +52,13 @@ is_installed()
 
 # $1 = file
 view_package_file() {
-  tree=$(tar -tJf "$1" 2>/dev/null) || exit $?
+  tree=$($pcompress -dc "$1" | tar -tf - 2>/dev/null) || exit $?
   echo "$tree" | grep -E '^ROOT/|^HOME/' | sed "/\/$/d ; s|^ROOT/|/|g ; s|^HOME/|$HOME/|g" 2>/dev/null
 }
 
 # $1 = package name
 view_package() {
-  cd "$PKG_PATH" && view_package_file "$1.tar.xz"
+  cd "$PKG_PATH" && view_package_file "$1.tar.$extension"
 }
 
 removed_packages()