zawz/zpkg
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Implement compression configuration

Browse source
This commit is contained in:
zawz 2020-05-26 17:31:37 +02:00
parent 5d087a66f2
commit 4f3322ed11
13 changed files with 114 additions and 58 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
.config.example
View file

@ -1,5 +1,18 @@
# zpkg server config
# address of the SSH server
SSH_ADDR=example.com
# user for the SSH server
SSH_USER=zpkg
HTTP_ADDR=example.com
HTTP_PATH=zpkg
# path to packages for the zpkg user
PKG_PATH=pkg
# public address of the HTTP server
HTTP_ADDR=example.com
# path on server
HTTP_PATH=zpkg
# Compression to use for packages
# format:
# extension:binary:parallel_binary:arguments
COMPRESSION=xz:xz:pxz:-1

3
README.md
View file

@ -13,9 +13,10 @@ Requirements:
- sudo
- wget
- tar
- xz/gz
Optional:
- pv
- pxz/pigz (faster compression/decompression)
### Installing

11
server_deploy.sh
View file

@ -4,6 +4,13 @@
ssh="$SSH_USER@$SSH_ADDR"
[ -z "$COMPRESSION" ] && COMPRESSION="xz:xz:pxz"
extension=$(echo "$COMPRESSION" | cut -d':' -f1)
compress=$(echo "$COMPRESSION" | cut -d':' -f2)
pcompress=$(echo "$COMPRESSION" | cut -d':' -f3)
which $pcompress >/dev/null 2>&1 || pcompress=$compress
[ -z "$pcompress" ] && pcompress=$compress
which $compress >/dev/null 2>&1 || { echo "Compression '$compress' not installed" && exit 12; }
random_string()
{
@ -30,9 +37,9 @@ mv zpkg "$fullpath$DEST" || exit $?
# create and send package
(
cd "$tmpdir/$PKG" || exit $?
tar -cvJf zpkg.tar.xz * || exit $?
tar -cf - * | $pcompress > zpkg.tar.$extension || exit $?
# send package
scp zpkg.tar.xz "$ssh":~/"$PKG_PATH" || exit $?
scp zpkg.tar.$extension "$ssh":~/"$PKG_PATH" || exit $?
)
# cleanup
rm -rd "$tmpdir"

16
server_scripts/database_update.sh
View file

@ -2,14 +2,24 @@
. "$(pwd)/.config"
# resolve compression
[ -z "$COMPRESSION" ] && COMPRESSION="xz:xz:pxz"
extension=$(echo "$COMPRESSION" | cut -d':' -f1)
compress=$(echo "$COMPRESSION" | cut -d':' -f2)
pcompress=$(echo "$COMPRESSION" | cut -d':' -f3)
which $pcompress >/dev/null 2>&1 || pcompress=$compress
[ -z "$pcompress" ] && pcompress=$compress
which $compress >/dev/null 2>&1 || { echo "Compression '$compress' not installed" && exit 12; }
# iterate packages
cd "$HOME/$PKG_PATH" || exit $?
PKGLIST="$(ls ./*.tar.xz)"
PKGLIST="$(ls ./*.tar.$extension)"
{
for I in $PKGLIST
do
NAME=$(echo "$I" | sed 's|\.tar\.xz||g;s|^\./||g')
NAME=$(echo "$I" | sed 's|\.tar\..*$||g;s|^\./||g')
TIME=$(stat -c "%Y" "$I")
DEPS=$(tar -xOf "$I" DEPS 2>/dev/null | tr -s '\n\t ' ' ')
DEPS=$($pcompress -dc "$I" | tar -xOf - DEPS 2>/dev/null | tr -s '\n\t ' ' ')
echo "$NAME $TIME $DEPS"
done
} > pkglist

27
server_scripts/gen_install.sh
View file

@ -11,6 +11,15 @@ cat .config >> install.sh
# body
echo '
# resolve compression
[ -z "$COMPRESSION" ] && COMPRESSION="xz:xz:pxz"
extension=$(echo "$COMPRESSION" | cut -d":" -f1)
compress=$(echo "$COMPRESSION" | cut -d":" -f2)
pcompress=$(echo "$COMPRESSION" | cut -d":" -f3)
which $pcompress >/dev/null 2>&1 || pcompress=$compress
[ -z "$pcompress" ] && pcompress=$compress
which $compress >/dev/null 2>&1 || { echo "Compression $compress not installed" && exit 12; }
usage()
{
echo "$(basename "$0")" [option...]
@ -47,13 +56,16 @@ done
shift $((OPTIND-1))
if [ "$(id | cut -d"=" -f2 | cut -d"(" -f1)" -eq 0 ] && [ "$1" != "force" ]
unset sudo
if [ "$(id | cut -d"=" -f2 | cut -d"(" -f1)" -eq 0 ]
then
echo "Cannot run as root" >&2
echo "Use '"'"'$(basename "$0") force'"'"' to force running as root"
exit 10
if [ "$1" != "force" ] ; then
echo "Cannot run as root" >&2
echo "Use '"'"'$(basename "$0") force'"'"' to force running as root"
exit 10
fi
else
which sudo >/dev/null || { echo "sudo not installed" >&2 && exit 11; }
which sudo >/dev/null 2>&1 || { echo "sudo not installed" >&2 && exit 11; }
sudo=sudo
fi
@ -61,6 +73,7 @@ fi
$sudo sh -c "{
echo SSH_ADDRESS=$SSH_USER@$SSH_ADDR
echo HTTP_ADDRESS=$HTTP_ADDR/$HTTP_PATH
echo COMPRESSION=$COMPRESSION
echo PKG_PATH=pkg
} > zpkg.conf"
@ -73,12 +86,12 @@ tmpdir=/tmp/zpkg$(random_string 5)
mkdir -p "$tmpdir" || exit $?
(
cd "$tmpdir" || exit $?
if ! wget "$HTTP_ADDR/$HTTP_PATH/zpkg.tar.xz" -q -O "zpkg.tar.xz"
if ! wget "$HTTP_ADDR/$HTTP_PATH/zpkg.tar.$extension" -q -O "zpkg.tar.$extension"
then
echo "Cannot reach $HTTP_ADDR/$HTTP_PATH" > /dev/stderr
exit 1
fi
tar -xf zpkg.tar.xz || exit $?
$pcompress -dc "zpkg.tar.$extension" | tar -xf - || exit $?
# install zpkg package
ROOT/usr/local/bin/zpkg -f install zpkg || exit $?

13
src/config.sh
View file

@ -17,7 +17,7 @@ config_file="$config_path/zpkg.conf"
# setup sudo prefix
unset sudo
if ! root_check ; then
which sudo >/dev/null || { echo "sudo not installed" && exit 11; }
which sudo >/dev/null 2>&1 || { echo "sudo not installed" && exit 11; }
sudo=sudo
fi
@ -31,5 +31,14 @@ PKG_PATH="$(resolve_path "$PKG_PATH" "$config_path")"
root_check && [ -z "$opt_f" ] && [ "$ALLOW_ROOT" != "true" ] && echo "Cannot run as root" >&2 && exit 10
[ ! -d "$PKG_PATH" ] $sudo mkdir -p "$PKG_PATH" 2>/dev/null
[ ! -d "$PKG_PATH" ] && $sudo mkdir -p "$PKG_PATH"
# resolve compression
[ -z "$COMPRESSION" ] && COMPRESSION="xz:xz:pxz"
extension=$(echo "$COMPRESSION" | cut -d':' -f1)
compress=$(echo "$COMPRESSION" | cut -d':' -f2)
pcompress=$(echo "$COMPRESSION" | cut -d':' -f3)
comparg=$(echo "$COMPRESSION" | cut -d':' -f4-)
which $pcompress >/dev/null 2>&1 || pcompress=$compress
[ -z "$pcompress" ] && pcompress=$compress
which $compress >/dev/null 2>&1 || { echo "Compression '$compress' not installed" && exit 12; }

53
src/deploy.sh
View file

@ -1,55 +1,58 @@
#!/bin/sh
# $1 = source , $2 = package , $3 = name
package()
{
unset clean_needed
src="$1"
pkg="$2"
echo "Packaging $(getname "$src"): $(du -sh "$src" | awk '{print $1}')iB"
tmpdir="/tmp/zpkg_$(random_string 5)"
mkdir -p "$tmpdir"
if [ ! -d "$src/ROOT" ] && [ ! -d "$src/HOME" ] && [ ! -f "$src/DEPS" ]
unset tmpdir
if [ ! -d "$src/ROOT" ] && [ ! -d "$src/HOME" ] && [ ! -f "$src/DEPS" ] && [ ! -f "$src/DESC" ]
then
mkdir -p "$tmpdir/package"
cp -r "$src" "$tmpdir/package/ROOT"
else
cp -r "$src" "$tmpdir/package"
tmpdir="/tmp/zpkg_$(random_string 5)"
mkdir -p "$tmpdir"
cp -r "$src" "$tmpdir/ROOT"
src="$tmpdir"
fi
(
cd "$tmpdir/package"
cd "$src"
unset list
[ -f DEPS ] && list=DEPS
[ -f DESC ] && list="$list DESC"
[ -d HOME ] && list="$list HOME"
[ -d ROOT ] && list="$list ROOT"
if which pv >/dev/null 2>&1
then
tar -cf - --owner=0 --group=0 -P $list | pv -s "$(du -sb . | awk '{print $1}')" | xz > "../$pkg"
else
tar -cvJf - --owner=0 --group=0 $list > "../$pkg"
fi
size=$(du -sb $list | awk '{print $1}' | paste -sd+ | bc)
echo "Packaging $(basename "$pkg"): $(echo "$size" | numfmt --to=iec-i)B"
cc=$compress
[ $size -gt 1048576 ] && cc=$pcompress
tar -cf - --owner=0 --group=0 $list | $cc $comparg > "$pkg"
)
mv "$tmpdir/$pkg" ./
rm -rd "$tmpdir"
[ -n "$tmpdir" ] && rm -rd "$tmpdir"
return 0
}
# $1 = file
deploy_package()
{
echo "Deploying $1: $(du -sh "$1" | awk '{print $1}')iB"
echo "Deploying $(basename "$1"): $(du -sh "$1" | awk '{print $1}')iB"
scp "$1" $SSH_ADDRESS:~/'$(grep "PKG_PATH=" .config | cut -d"=" -f2-)'
}
deploy_folder()
{
archive="$(getname "$1").tar.xz"
if [ -n "$(echo "$1" | grep '\.tar\.xz$' )" ]
if [ -f "$1" ] && echo "$1" | grep -q '\.tar\.'"$extension\$" # file and valid extension
then
deploy_package "$1" || return 1
$pcompress -dc >/dev/null 2>&1 | tar -tf - >/dev/null 2>&1|| { echo "File '$1' is not a valid archive" && return 1; }
deploy_package "$1" "$1" || return $?
elif [ -d "$1" ]
then
archive="$(getname "$1").tar.$extension"
package "$1" "/tmp/$archive" || return $?
deploy_package "/tmp/$archive" || return $?
rm "/tmp/$archive"
else
package "$1" "$archive" || return 1
deploy_package "$archive" || return 1
rm "$archive" 2> /dev/null
echo "Target '$1' doesn't exist"
fi
}

4
src/fetch.sh
View file

@ -4,8 +4,8 @@
fetch_package()
{
out="$2"
[ -z "$out" ] && out="$1.tar.xz"
wget "$HTTP_ADDRESS/$1.tar.xz" -q --show-progress -O "$out" 2>&1
[ -z "$out" ] && out="$1.tar.$extension"
wget "$HTTP_ADDRESS/$1.tar.$extension" -q --show-progress -O "$out" 2>&1
}
# $1 = prefix

6
src/install.sh
View file

@ -3,7 +3,7 @@
unpack()
{
echo "Unpacking $1"
tar -xf "$1"
$pcompress -dc "$1" | tar -xf -
}
# $1 = package , $2 = prefix
@ -46,8 +46,8 @@ install_package()
(
cd "$tmpdir"
fetch_package "$1" || { echo "Package '$1' not found" >&2 && return 1; }
$2 cp "$1.tar.xz" "$PKG_PATH"
unpack "$1.tar.xz" || return $?
$2 cp "$1.tar.$extension" "$PKG_PATH"
unpack "$1.tar.$extension" || return $?
move_files ROOT / $2 2>/dev/null
move_files HOME "$HOME" 2>/dev/null
add_package_entry "$1" $2

2
src/main.sh
View file

@ -28,7 +28,7 @@ show)
then
view_package "$I"
else
wget "$HTTP_ADDRESS/$1.tar.xz" -q -O - 2>/dev/null | view_package_file - || { echo "Could not fetch package '$I'" >&2 ; return 1 ; }
wget "$HTTP_ADDRESS/$1.tar.$extension" -q -O - 2>/dev/null | view_package_file - || { echo "Could not fetch package '$I'" >&2 ; return 1 ; }
fi
done
fi

8
src/print.sh
View file

@ -42,20 +42,20 @@ package_info() {
status="not installed"
grep -wq "^$1" "$PKG_PATH/pkglist" 2>/dev/null || { echo "Package '$I' not found" && return 1; }
grep -wq "^$1" "$PKG_PATH/installed" 2>/dev/null && status=installed
if [ "$status" = "installed" ] && [ -f "$PKG_PATH/$1.tar.xz" ]
if [ "$status" = "installed" ] && [ -f "$PKG_PATH/$1.tar.$extension" ]
then
pkg="$PKG_PATH/$1.tar.xz"
pkg="$PKG_PATH/$1.tar.$extension"
else
tmpdir="/tmp/zpkg_$(random_string 5)"
pwd=$(pwd)
mkdir "$tmpdir"
fetch_package "$1" >/dev/null 2>&1 || { echo "Error fetching package" >&2 && ret=$?; }
pkg="$1.tar.xz"
pkg="$1.tar.$extension"
fi
deps=$(deps "$1")
desc=$(desc "$pkg" 2>/dev/null)
csize=$(stat -c '%s' "$pkg" | numfmt --to=iec-i --suffix=B --padding 6)
isize=$(xz -dc "$pkg" | wc -c | numfmt --to=iec-i --suffix=B --padding 6)
isize=$($pcompress -dc "$pkg" | wc -c | numfmt --to=iec-i --suffix=B --padding 6)
[ -n "$cleanup" ] && { cd "$pwd"; rm -rd "$tmpdir"; }
[ -n "$ret" ] && return $ret

6
src/remove.sh
View file

@ -14,7 +14,7 @@ delete_files()
remove_package()
{
cd "$PKG_PATH"
archive="$(pwd)/$1.tar.xz"
archive="$(pwd)/$1.tar.$extension"
if [ ! -f "$archive" ] || ! grep -q -w "^$1" installed
then
echo "Package '$1' not installed" >&2
@ -24,11 +24,11 @@ remove_package()
( # delete root files
cd /
tar -tf "$archive" ROOT 2>/dev/null | sed 's|^ROOT/||g' | tac | delete_files $2
$pcompress -dc "$archive" | tar -tf - ROOT 2>/dev/null | sed 's|^ROOT/||g' | tac | delete_files $2
)
( # delete home files
cd "$HOME"
tar -tf "$archive" HOME 2>/dev/null | sed 's|^HOME/||g' | tac | delete_files
$pcompress -dc "$archive" | tar -tf - HOME 2>/dev/null | sed 's|^HOME/||g' | tac | delete_files
)
$2 rm "$archive" 2>/dev/null

6
src/view.sh
View file

@ -9,7 +9,7 @@ deps()
# $1 = pkg file
desc() {
tar -xOf "$1" DESC
$pcompress -dc "$1" | tar -xOf - DESC
}
resolve_packages()
@ -52,13 +52,13 @@ is_installed()
# $1 = file
view_package_file() {
tree=$(tar -tJf "$1" 2>/dev/null) || exit $?
tree=$($pcompress -dc "$1" | tar -tf - 2>/dev/null) || exit $?
echo "$tree" | grep -E '^ROOT/|^HOME/' | sed "/\/$/d ; s|^ROOT/|/|g ; s|^HOME/|$HOME/|g" 2>/dev/null
}
# $1 = package name
view_package() {
cd "$PKG_PATH" && view_package_file "$1.tar.xz"
cd "$PKG_PATH" && view_package_file "$1.tar.$extension"
}
removed_packages()