zawz/zpkg
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Implement compression configuration

Browse source
This commit is contained in:
zawz 2020-05-26 17:31:37 +02:00
parent 5d087a66f2
commit 4f3322ed11
13 changed files with 114 additions and 58 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
.config.example
View file

@ -1,5 +1,18 @@
# zpkg server config
# address of the SSH server
SSH_ADDR=example.com SSH_ADDR=example.com
# user for the SSH server
SSH_USER=zpkg SSH_USER=zpkg
HTTP_ADDR=example.com # path to packages for the zpkg user
HTTP_PATH=zpkg
PKG_PATH=pkg PKG_PATH=pkg
# public address of the HTTP server
HTTP_ADDR=example.com
# path on server
HTTP_PATH=zpkg
# Compression to use for packages
# format:
# extension:binary:parallel_binary:arguments
COMPRESSION=xz:xz:pxz:-1

3
README.md
View file

@ -13,9 +13,10 @@ Requirements:
- sudo - sudo
- wget - wget
- tar - tar
- xz/gz
Optional: Optional:
- pv - pxz/pigz (faster compression/decompression)
### Installing ### Installing

11
server_deploy.sh
View file

@ -4,6 +4,13 @@
ssh="$SSH_USER@$SSH_ADDR" ssh="$SSH_USER@$SSH_ADDR"
[ -z "$COMPRESSION" ] && COMPRESSION="xz:xz:pxz"
extension=$(echo "$COMPRESSION" | cut -d':' -f1)
compress=$(echo "$COMPRESSION" | cut -d':' -f2)
pcompress=$(echo "$COMPRESSION" | cut -d':' -f3)
which $pcompress >/dev/null 2>&1 || pcompress=$compress
[ -z "$pcompress" ] && pcompress=$compress
which $compress >/dev/null 2>&1 || { echo "Compression '$compress' not installed" && exit 12; }
random_string() random_string()
{ {
@ -30,9 +37,9 @@ mv zpkg "$fullpath$DEST" || exit $?
# create and send package # create and send package
( (
cd "$tmpdir/$PKG" || exit $? cd "$tmpdir/$PKG" || exit $?
tar -cvJf zpkg.tar.xz * || exit $? tar -cf - * | $pcompress > zpkg.tar.$extension || exit $?
# send package # send package
scp zpkg.tar.xz "$ssh":~/"$PKG_PATH" || exit $? scp zpkg.tar.$extension "$ssh":~/"$PKG_PATH" || exit $?
) )
# cleanup # cleanup
rm -rd "$tmpdir" rm -rd "$tmpdir"

16
server_scripts/database_update.sh
View file

@ -2,14 +2,24 @@
. "$(pwd)/.config" . "$(pwd)/.config"
# resolve compression
[ -z "$COMPRESSION" ] && COMPRESSION="xz:xz:pxz"
extension=$(echo "$COMPRESSION" | cut -d':' -f1)
compress=$(echo "$COMPRESSION" | cut -d':' -f2)
pcompress=$(echo "$COMPRESSION" | cut -d':' -f3)
which $pcompress >/dev/null 2>&1 || pcompress=$compress
[ -z "$pcompress" ] && pcompress=$compress
which $compress >/dev/null 2>&1 || { echo "Compression '$compress' not installed" && exit 12; }
# iterate packages
cd "$HOME/$PKG_PATH" || exit $? cd "$HOME/$PKG_PATH" || exit $?
PKGLIST="$(ls ./*.tar.xz)" PKGLIST="$(ls ./*.tar.$extension)"
{ {
for I in $PKGLIST for I in $PKGLIST
do do
NAME=$(echo "$I" | sed 's|\.tar\.xz||g;s|^\./||g') NAME=$(echo "$I" | sed 's|\.tar\..*$||g;s|^\./||g')
TIME=$(stat -c "%Y" "$I") TIME=$(stat -c "%Y" "$I")
DEPS=$(tar -xOf "$I" DEPS 2>/dev/null | tr -s '\n\t ' ' ') DEPS=$($pcompress -dc "$I" | tar -xOf - DEPS 2>/dev/null | tr -s '\n\t ' ' ')
echo "$NAME $TIME $DEPS" echo "$NAME $TIME $DEPS"
done done
} > pkglist } > pkglist

21
server_scripts/gen_install.sh
View file

@ -11,6 +11,15 @@ cat .config >> install.sh
# body # body
echo ' echo '
# resolve compression
[ -z "$COMPRESSION" ] && COMPRESSION="xz:xz:pxz"
extension=$(echo "$COMPRESSION" | cut -d":" -f1)
compress=$(echo "$COMPRESSION" | cut -d":" -f2)
pcompress=$(echo "$COMPRESSION" | cut -d":" -f3)
which $pcompress >/dev/null 2>&1 || pcompress=$compress
[ -z "$pcompress" ] && pcompress=$compress
which $compress >/dev/null 2>&1 || { echo "Compression $compress not installed" && exit 12; }
usage() usage()
{ {
echo "$(basename "$0")" [option...] echo "$(basename "$0")" [option...]
@ -47,13 +56,16 @@ done
shift $((OPTIND-1)) shift $((OPTIND-1))
if [ "$(id | cut -d"=" -f2 | cut -d"(" -f1)" -eq 0 ] && [ "$1" != "force" ] unset sudo
if [ "$(id | cut -d"=" -f2 | cut -d"(" -f1)" -eq 0 ]
then then
if [ "$1" != "force" ] ; then
echo "Cannot run as root" >&2 echo "Cannot run as root" >&2
echo "Use '"'"'$(basename "$0") force'"'"' to force running as root" echo "Use '"'"'$(basename "$0") force'"'"' to force running as root"
exit 10 exit 10
fi
else else
which sudo >/dev/null || { echo "sudo not installed" >&2 && exit 11; } which sudo >/dev/null 2>&1 || { echo "sudo not installed" >&2 && exit 11; }
sudo=sudo sudo=sudo
fi fi
@ -61,6 +73,7 @@ fi
$sudo sh -c "{ $sudo sh -c "{
echo SSH_ADDRESS=$SSH_USER@$SSH_ADDR echo SSH_ADDRESS=$SSH_USER@$SSH_ADDR
echo HTTP_ADDRESS=$HTTP_ADDR/$HTTP_PATH echo HTTP_ADDRESS=$HTTP_ADDR/$HTTP_PATH
echo COMPRESSION=$COMPRESSION
echo PKG_PATH=pkg echo PKG_PATH=pkg
} > zpkg.conf" } > zpkg.conf"
@ -73,12 +86,12 @@ tmpdir=/tmp/zpkg$(random_string 5)
mkdir -p "$tmpdir" || exit $? mkdir -p "$tmpdir" || exit $?
( (
cd "$tmpdir" || exit $? cd "$tmpdir" || exit $?
if ! wget "$HTTP_ADDR/$HTTP_PATH/zpkg.tar.xz" -q -O "zpkg.tar.xz" if ! wget "$HTTP_ADDR/$HTTP_PATH/zpkg.tar.$extension" -q -O "zpkg.tar.$extension"
then then
echo "Cannot reach $HTTP_ADDR/$HTTP_PATH" > /dev/stderr echo "Cannot reach $HTTP_ADDR/$HTTP_PATH" > /dev/stderr
exit 1 exit 1
fi fi
tar -xf zpkg.tar.xz || exit $? $pcompress -dc "zpkg.tar.$extension" | tar -xf - || exit $?
# install zpkg package # install zpkg package
ROOT/usr/local/bin/zpkg -f install zpkg || exit $? ROOT/usr/local/bin/zpkg -f install zpkg || exit $?

13
src/config.sh
View file

@ -17,7 +17,7 @@ config_file="$config_path/zpkg.conf"
# setup sudo prefix # setup sudo prefix
unset sudo unset sudo
if ! root_check ; then if ! root_check ; then
which sudo >/dev/null || { echo "sudo not installed" && exit 11; } which sudo >/dev/null 2>&1 || { echo "sudo not installed" && exit 11; }
sudo=sudo sudo=sudo
fi fi
@ -31,5 +31,14 @@ PKG_PATH="$(resolve_path "$PKG_PATH" "$config_path")"
root_check && [ -z "$opt_f" ] && [ "$ALLOW_ROOT" != "true" ] && echo "Cannot run as root" >&2 && exit 10 root_check && [ -z "$opt_f" ] && [ "$ALLOW_ROOT" != "true" ] && echo "Cannot run as root" >&2 && exit 10
[ ! -d "$PKG_PATH" ] $sudo mkdir -p "$PKG_PATH" 2>/dev/null [ ! -d "$PKG_PATH" ] && $sudo mkdir -p "$PKG_PATH"
# resolve compression
[ -z "$COMPRESSION" ] && COMPRESSION="xz:xz:pxz"
extension=$(echo "$COMPRESSION" | cut -d':' -f1)
compress=$(echo "$COMPRESSION" | cut -d':' -f2)
pcompress=$(echo "$COMPRESSION" | cut -d':' -f3)
comparg=$(echo "$COMPRESSION" | cut -d':' -f4-)
which $pcompress >/dev/null 2>&1 || pcompress=$compress
[ -z "$pcompress" ] && pcompress=$compress
which $compress >/dev/null 2>&1 || { echo "Compression '$compress' not installed" && exit 12; }

51
src/deploy.sh
View file

@ -1,55 +1,58 @@
#!/bin/sh #!/bin/sh
# $1 = source , $2 = package , $3 = name
package() package()
{ {
unset clean_needed
src="$1" src="$1"
pkg="$2" pkg="$2"
echo "Packaging $(getname "$src"): $(du -sh "$src" | awk '{print $1}')iB"
unset tmpdir
if [ ! -d "$src/ROOT" ] && [ ! -d "$src/HOME" ] && [ ! -f "$src/DEPS" ] && [ ! -f "$src/DESC" ]
then
tmpdir="/tmp/zpkg_$(random_string 5)" tmpdir="/tmp/zpkg_$(random_string 5)"
mkdir -p "$tmpdir" mkdir -p "$tmpdir"
if [ ! -d "$src/ROOT" ] && [ ! -d "$src/HOME" ] && [ ! -f "$src/DEPS" ] cp -r "$src" "$tmpdir/ROOT"
then src="$tmpdir"
mkdir -p "$tmpdir/package"
cp -r "$src" "$tmpdir/package/ROOT"
else
cp -r "$src" "$tmpdir/package"
fi fi
( (
cd "$tmpdir/package" cd "$src"
unset list unset list
[ -f DEPS ] && list=DEPS [ -f DEPS ] && list=DEPS
[ -f DESC ] && list="$list DESC" [ -f DESC ] && list="$list DESC"
[ -d HOME ] && list="$list HOME" [ -d HOME ] && list="$list HOME"
[ -d ROOT ] && list="$list ROOT" [ -d ROOT ] && list="$list ROOT"
if which pv >/dev/null 2>&1 size=$(du -sb $list | awk '{print $1}' | paste -sd+ | bc)
then echo "Packaging $(basename "$pkg"): $(echo "$size" | numfmt --to=iec-i)B"
tar -cf - --owner=0 --group=0 -P $list | pv -s "$(du -sb . | awk '{print $1}')" | xz > "../$pkg" cc=$compress
else [ $size -gt 1048576 ] && cc=$pcompress
tar -cvJf - --owner=0 --group=0 $list > "../$pkg" tar -cf - --owner=0 --group=0 $list | $cc $comparg > "$pkg"
fi
) )
mv "$tmpdir/$pkg" ./ [ -n "$tmpdir" ] && rm -rd "$tmpdir"
rm -rd "$tmpdir" return 0
} }
# $1 = file
deploy_package() deploy_package()
{ {
echo "Deploying $1: $(du -sh "$1" | awk '{print $1}')iB" echo "Deploying $(basename "$1"): $(du -sh "$1" | awk '{print $1}')iB"
scp "$1" $SSH_ADDRESS:~/'$(grep "PKG_PATH=" .config | cut -d"=" -f2-)' scp "$1" $SSH_ADDRESS:~/'$(grep "PKG_PATH=" .config | cut -d"=" -f2-)'
} }
deploy_folder() deploy_folder()
{ {
archive="$(getname "$1").tar.xz" if [ -f "$1" ] && echo "$1" | grep -q '\.tar\.'"$extension\$" # file and valid extension
if [ -n "$(echo "$1" | grep '\.tar\.xz$' )" ]
then then
deploy_package "$1" || return 1 $pcompress -dc >/dev/null 2>&1 | tar -tf - >/dev/null 2>&1|| { echo "File '$1' is not a valid archive" && return 1; }
deploy_package "$1" "$1" || return $?
elif [ -d "$1" ]
then
archive="$(getname "$1").tar.$extension"
package "$1" "/tmp/$archive" || return $?
deploy_package "/tmp/$archive" || return $?
rm "/tmp/$archive"
else else
package "$1" "$archive" || return 1 echo "Target '$1' doesn't exist"
deploy_package "$archive" || return 1
rm "$archive" 2> /dev/null
fi fi
} }

4
src/fetch.sh
View file

@ -4,8 +4,8 @@
fetch_package() fetch_package()
{ {
out="$2" out="$2"
[ -z "$out" ] && out="$1.tar.xz" [ -z "$out" ] && out="$1.tar.$extension"
wget "$HTTP_ADDRESS/$1.tar.xz" -q --show-progress -O "$out" 2>&1 wget "$HTTP_ADDRESS/$1.tar.$extension" -q --show-progress -O "$out" 2>&1
} }
# $1 = prefix # $1 = prefix

6
src/install.sh
View file

@ -3,7 +3,7 @@
unpack() unpack()
{ {
echo "Unpacking $1" echo "Unpacking $1"
tar -xf "$1" $pcompress -dc "$1" | tar -xf -
} }
# $1 = package , $2 = prefix # $1 = package , $2 = prefix
@ -46,8 +46,8 @@ install_package()
( (
cd "$tmpdir" cd "$tmpdir"
fetch_package "$1" || { echo "Package '$1' not found" >&2 && return 1; } fetch_package "$1" || { echo "Package '$1' not found" >&2 && return 1; }
$2 cp "$1.tar.xz" "$PKG_PATH" $2 cp "$1.tar.$extension" "$PKG_PATH"
unpack "$1.tar.xz" || return $? unpack "$1.tar.$extension" || return $?
move_files ROOT / $2 2>/dev/null move_files ROOT / $2 2>/dev/null
move_files HOME "$HOME" 2>/dev/null move_files HOME "$HOME" 2>/dev/null
add_package_entry "$1" $2 add_package_entry "$1" $2

2
src/main.sh
View file

@ -28,7 +28,7 @@ show)
then then
view_package "$I" view_package "$I"
else else
wget "$HTTP_ADDRESS/$1.tar.xz" -q -O - 2>/dev/null | view_package_file - || { echo "Could not fetch package '$I'" >&2 ; return 1 ; } wget "$HTTP_ADDRESS/$1.tar.$extension" -q -O - 2>/dev/null | view_package_file - || { echo "Could not fetch package '$I'" >&2 ; return 1 ; }
fi fi
done done
fi fi

8
src/print.sh
View file

@ -42,20 +42,20 @@ package_info() {
status="not installed" status="not installed"
grep -wq "^$1" "$PKG_PATH/pkglist" 2>/dev/null || { echo "Package '$I' not found" && return 1; } grep -wq "^$1" "$PKG_PATH/pkglist" 2>/dev/null || { echo "Package '$I' not found" && return 1; }
grep -wq "^$1" "$PKG_PATH/installed" 2>/dev/null && status=installed grep -wq "^$1" "$PKG_PATH/installed" 2>/dev/null && status=installed
if [ "$status" = "installed" ] && [ -f "$PKG_PATH/$1.tar.xz" ] if [ "$status" = "installed" ] && [ -f "$PKG_PATH/$1.tar.$extension" ]
then then
pkg="$PKG_PATH/$1.tar.xz" pkg="$PKG_PATH/$1.tar.$extension"
else else
tmpdir="/tmp/zpkg_$(random_string 5)" tmpdir="/tmp/zpkg_$(random_string 5)"
pwd=$(pwd) pwd=$(pwd)
mkdir "$tmpdir" mkdir "$tmpdir"
fetch_package "$1" >/dev/null 2>&1 || { echo "Error fetching package" >&2 && ret=$?; } fetch_package "$1" >/dev/null 2>&1 || { echo "Error fetching package" >&2 && ret=$?; }
pkg="$1.tar.xz" pkg="$1.tar.$extension"
fi fi
deps=$(deps "$1") deps=$(deps "$1")
desc=$(desc "$pkg" 2>/dev/null) desc=$(desc "$pkg" 2>/dev/null)
csize=$(stat -c '%s' "$pkg" | numfmt --to=iec-i --suffix=B --padding 6) csize=$(stat -c '%s' "$pkg" | numfmt --to=iec-i --suffix=B --padding 6)
isize=$(xz -dc "$pkg" | wc -c | numfmt --to=iec-i --suffix=B --padding 6) isize=$($pcompress -dc "$pkg" | wc -c | numfmt --to=iec-i --suffix=B --padding 6)
[ -n "$cleanup" ] && { cd "$pwd"; rm -rd "$tmpdir"; } [ -n "$cleanup" ] && { cd "$pwd"; rm -rd "$tmpdir"; }
[ -n "$ret" ] && return $ret [ -n "$ret" ] && return $ret

6
src/remove.sh
View file

@ -14,7 +14,7 @@ delete_files()
remove_package() remove_package()
{ {
cd "$PKG_PATH" cd "$PKG_PATH"
archive="$(pwd)/$1.tar.xz" archive="$(pwd)/$1.tar.$extension"
if [ ! -f "$archive" ] || ! grep -q -w "^$1" installed if [ ! -f "$archive" ] || ! grep -q -w "^$1" installed
then then
echo "Package '$1' not installed" >&2 echo "Package '$1' not installed" >&2
@ -24,11 +24,11 @@ remove_package()
( # delete root files ( # delete root files
cd / cd /
tar -tf "$archive" ROOT 2>/dev/null | sed 's|^ROOT/||g' | tac | delete_files $2 $pcompress -dc "$archive" | tar -tf - ROOT 2>/dev/null | sed 's|^ROOT/||g' | tac | delete_files $2
) )
( # delete home files ( # delete home files
cd "$HOME" cd "$HOME"
tar -tf "$archive" HOME 2>/dev/null | sed 's|^HOME/||g' | tac | delete_files $pcompress -dc "$archive" | tar -tf - HOME 2>/dev/null | sed 's|^HOME/||g' | tac | delete_files
) )
$2 rm "$archive" 2>/dev/null $2 rm "$archive" 2>/dev/null

6
src/view.sh
View file

@ -9,7 +9,7 @@ deps()
# $1 = pkg file # $1 = pkg file
desc() { desc() {
tar -xOf "$1" DESC $pcompress -dc "$1" | tar -xOf - DESC
} }
resolve_packages() resolve_packages()
@ -52,13 +52,13 @@ is_installed()
# $1 = file # $1 = file
view_package_file() { view_package_file() {
tree=$(tar -tJf "$1" 2>/dev/null) || exit $? tree=$($pcompress -dc "$1" | tar -tf - 2>/dev/null) || exit $?
echo "$tree" | grep -E '^ROOT/|^HOME/' | sed "/\/$/d ; s|^ROOT/|/|g ; s|^HOME/|$HOME/|g" 2>/dev/null echo "$tree" | grep -E '^ROOT/|^HOME/' | sed "/\/$/d ; s|^ROOT/|/|g ; s|^HOME/|$HOME/|g" 2>/dev/null
} }
# $1 = package name # $1 = package name
view_package() { view_package() {
cd "$PKG_PATH" && view_package_file "$1.tar.xz" cd "$PKG_PATH" && view_package_file "$1.tar.$extension"
} }
removed_packages() removed_packages()