diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..034ebbc --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +zsync +Zmakefile +zmakefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..4891981 --- /dev/null +++ b/Makefile @@ -0,0 +1,20 @@ + +var_exclude = ZPASS_.* XDG_.* REMOTE_.* DISPLAY CONFIGFILE TMPDIR +fct_exclude = _tty_on + +zsync: src/* + lxsh -o zsync -M --exclude-var "$(var_exclude)" --exclude-fct "$(fct_exclude)" src/main.sh + +debug: src/* + lxsh -o zsync src/main.sh + +build: zsync + +install: build + mv zsync /usr/local/bin + +uninstall: + rm /usr/local/bin/zpass + +clear: + rm zsync diff --git a/src/config.sh b/src/config.sh new file mode 100644 index 0000000..d1ef2c1 --- /dev/null +++ b/src/config.sh @@ -0,0 +1,49 @@ + +# globals +syncdir=".zsync" +lock_file=".zsync/lock" +ignore_file=".zsync/ignore" +tree_full=".zsync/tree_full" +tree_hash=".zsync/tree_hash" +config_file=".zsync/config" + +# NO -t OPTION FOR RSYNC +rsync_opts='-rvlpE' + +tmpdir=${TMPDIR-/tmp} + +## CONFIG + +init_config() { + mkdir -p "$syncdir" || return 2 + which rsync >/dev/null 2>&1 || { echo rsync not installed on server >&2 ; return 3; } + touch "$config_file" || return 5 +} + +get_server() { + [ ! -f "$config_file" ] && return 1 + servconf=$(sed 's|^[ \t]*||g' "$config_file" | grep -E '^server[ \t]' | sed 's|^server[ \t]*||g' | tail -n1) + raddr=$(echo "$servconf" | cut -d ':' -f1) + rdir=$(echo "$servconf" | cut -d ':' -f2-) +} + +# $1 = server arg +setup_server() +{ + init_config || return $? + [ -z "$1" ] && echo "$fname server user@host:path" && return 1 + sed -i '/^[ \t]*server[ \t]/d' "$config_file" + echo "server $1" >> "$config_file" +} + +ignores='\.zsync' +get_ignores() { + if [ -f "$ignore_file" ] + then + ignores="($(tr '\n' '|' < "$ignore_file"))" + else + ignores='(^$)' + fi + ignores=$(echo "$ignores" | sed ' s/|)/)/g ; s/^()$/^$/g ') + [ -n "$ignores" ] || ignores='\.zsync' +} diff --git a/src/diff.sh b/src/diff.sh new file mode 100644 index 0000000..74ec071 --- /dev/null +++ b/src/diff.sh @@ -0,0 +1,24 @@ + +## DIFFERENCES + +# find changes from list +# $1 = list file , $@ = targets +# requisite: file contains both hash and filename and is sorted +list_diff() +{ + file=$1 + shift 1 + [ ! -f "$tree_hash" ] && { cut -c34- "$file" ; return 0; } + diff --old-line-format="" --unchanged-line-format="" "$tree_hash" "$file" | cut -c34- | merge "$@" +} + +# find deleted from list +# $1 = list file , $@ = targets +# requisite: file contains only filename and is sorted +get_deleted() +{ + file=$1 + shift 1 + [ ! -f "$tree_full" ] && return 0 + diff --new-line-format="" --unchanged-line-format="" "$tree_full" "$file" | reduce_list | grep -vE "$ignores" | merge "$@" +} diff --git a/src/filter.sh b/src/filter.sh new file mode 100644 index 0000000..6dc22b2 --- /dev/null +++ b/src/filter.sh @@ -0,0 +1,54 @@ + +## FILTERS + +run_ignore() { + [ -n "$ignores" ] && grep -vE "$ignores" "$@" +} + +# $1 = regex , $@ = args +grep_after_sum() +{ + reg=$1 + shift 1 + grep --color=never -E "^[0-9a-f]{32} $reg" "$@" +} + +# $@ = match these +merge() +{ + if [ $# -gt 0 ] + then + re="$1" + shift 1 + for N + do + re="$re|$N" + done + grep -E "^($re)" + return 0 + else # don't change input + cat + fi +} + +# reduce list to only subsets +# kind of like "uniq" but matches lines that are supersets, only outputting the subset +# needs a sorted list in input +# $1 = separator for subset-superset (default="/") +reduce_list() +{ + awk 'BEGIN{ FS="\n"; RS="\0" ; i = 0 } { + i=1 + while(i<=NF) + { + print $i + val=$i "'"${1-"/"}"'" + i++; + while(substr($i, 0, length(val)) == val) + { + i++; + } + } + } + ' +} diff --git a/src/list.sh b/src/list.sh new file mode 100644 index 0000000..2381dd7 --- /dev/null +++ b/src/list.sh @@ -0,0 +1,60 @@ + +## LIST + +local_hash_list() +{ + { ( set -e + find . -type f ! -regex "^./$syncdir/.*" | sed 's|^./||g' | tr '\n' '\0' | xargs -0 md5sum | cut -c1-33,35- | grep -vE "$ignores" + find . -type l | sed 's|^./||g' | while read -r ln + do + find "$ln" -maxdepth 0 -printf '%l' | md5sum | sed "s|-|$ln|g" + done | cut -c1-33,35- | grep -vE "$ignores" + ) || return $?; } | sort +} + +server_hash_list() +{ + ssh_exec '#LXSH_PARSE_MINIFY + set -e + cd "$1" + find . -type f ! -regex "^./$2/.*" | sed "s|^./||g" | tr "\n" "\0" | xargs -0 md5sum | cut -c1-33,35- | grep -vE "$3" + find . -type l | sed "s|^./||g" | while read -r ln + do + find "$ln" -maxdepth 0 -printf "%l" | md5sum | sed "s|-|$ln|g" + done | cut -c1-33,35- | grep -vE "$3" + ' "$rdir" "$syncdir" "$ignores" | sort +} + +local_full_list() { + find . -mindepth 1 ! -regex "^./$syncdir\$" ! -regex "^./$syncdir/.*" | sed 's|^./||g' | grep -vE "$ignores" | sort +} + +server_full_list() { + ssh_exec '#LXSH_PARSE_MINIFY + set -e + cd "$1" + find . -mindepth 1 ! -regex "^./$2\$" ! -regex "^./$2/.*" | sed "s|^./||g" | grep -vE "$3" + ' "$rdir" "$syncdir" "$ignores" | sort +} + +server_both_list() { + ssh_exec '#LXSH_PARSE_MINIFY + set -e + cd "$1" + find . -mindepth 1 ! -regex "^./$2\$" ! -regex "^./$2/.*" | sed "s|^./||g" | grep -vE "$3" + printf "\0" + { + find . -type f ! -regex "^./$2/.*" | sed "s|^./||g" | tr "\n" "\0" | xargs -0 md5sum | cut -c1-33,35- | grep -vE "$3" + find . -type l | sed "s|^./||g" | while read -r ln + do + find "$ln" -maxdepth 0 -printf "%l" | md5sum | sed "s|-|$ln|g" + done | cut -c1-33,35- | grep -vE "$3" + } + ' "$rdir" "$syncdir" "$ignores" +} + +write_lists() +{ + local_full_list > "$tree_full" + local_hash_list > "$tree_hash" +} diff --git a/src/lock.sh b/src/lock.sh new file mode 100644 index 0000000..e46b87f --- /dev/null +++ b/src/lock.sh @@ -0,0 +1,41 @@ + +## LOCK + +lock_local() { touch "$lock_file"; } +unlock_local() { rm "$lock_file"; } +lock_server() { + ssh_exec 'touch "$1"' "$rdir/$lock_file"; +} +unlock_server() { + ssh_exec 'rm "$1"' "$rdir/$lock_file"; +} +lock_all() { lock_local && lock_server ; } +unlock_all() { ret=0; unlock_local || ret=$? ; unlock_server || ret=$?; return $ret ; } + +local_lock_check() { + [ ! -f "$lock_file" ] || { echo "Local sync locked, wait for sync completion" >&2 && return 1; } +} +server_lock_check() { + ssh_exec ' + cd "$1" || return 0 + [ ! -f "$2" ] + ' "$rdir" "$lock_file" || { echo "Server sync locked, wait for sync completion" >&2 && return 1; } +} + +# init +init_local() { + mkdir -p "$syncdir" || return 2 + which rsync >/dev/null 2>&1 || { echo rsync not installed on server >&2 ; return 3; } + local_lock_check || return 4 + touch "$lock_file" || return 5 +} + +init_server() { + ssh_exec ' + cd "$1" || exit 1 + mkdir -p "$2" || exit 2 + which rsync >/dev/null 2>&1 || { echo rsync not installed on server >&2 ; exit 3; } + [ -f "$2" ] && { echo Server sync locked, wait for sync completion ; exit 4; } + touch "$3" || exit 5 + ' "$rdir" "$syncdir" "$lock_file" +} diff --git a/src/main.sh b/src/main.sh new file mode 100644 index 0000000..7593f58 --- /dev/null +++ b/src/main.sh @@ -0,0 +1,22 @@ +#!/bin/lxsh + +set -e + +%include config.sh options.sh *.sh + +arg=$1 +shift 1 + +# actions +case $arg in + server) setup_server "$@" ;; + run) sync "" "" "$@" ;; + pull) sync pull "" "$@" ;; + push) sync push "" "$@" ;; + dry) sync "" dry "$@" ;; + drypush) sync push dry "$@" ;; + drypull) sync pull dry "$@" ;; + forcepush) forcepush ;; + forcepull) forcepull ;; + *) usage && exit 1 ;; +esac diff --git a/src/options.sh b/src/options.sh new file mode 100644 index 0000000..a8411d1 --- /dev/null +++ b/src/options.sh @@ -0,0 +1,44 @@ + +# usage +fname=$(basename "$0") +usage() +{ + echo "$fname [operation] + +Operations: + server Setup sync on current folder with server target + run [file...] Run sync with server + push [file...] Regular run but push all conflicts + pull [file...] Regular run but pull all conflicts + dry [file...] Run a simulated sync but do not perform any action + drypush [file...] Dry run as push + drypull [file...] Dry run as pull + forcepush Push by force the entire tree. Will replace and delete remote files + forcepull Pull by force the entire tree. Will replace and delete local files" +} + +# options +unset arg_c +while getopts ":hC:" opt; +do + case $opt in + C) + [ -z "$OPTARG" ] && echo "Option -C requires an argument" >&2 && exit 1 + arg_c="$OPTARG" + ;; + h) usage && exit 1 ;; + *) echo "Unknown option: $OPTARG" >&2 && usage && exit 1 ;; + esac +done +shift $((OPTIND-1)) + +# raddr=zawz@zawz.net +# rdir=sync/tmp +[ -f "$server_file" ] && get_server + +[ -n "$arg_c" ] && { cd "$arg_c" || exit $?; } # -C opt + +[ $# -lt 1 ] && usage && exit 1 + +which rsync >/dev/null || { echo "rsync not installed" >&2 && exit 1; } +/usr/bin/printf %q >/dev/null || { echo "printf does not support %q" >&2 && exit 1; } diff --git a/src/sync.sh b/src/sync.sh new file mode 100644 index 0000000..3a970fb --- /dev/null +++ b/src/sync.sh @@ -0,0 +1,87 @@ + +# $1 = method (null/'push'/'pull') , $2 = dry (null/'dry') , $@ = files +sync() +{ + method=$1 + dry=$2 + shift 2 + + check_paths "$@" || return $? + + get_server || { echo "Server not configured on this instance" >&2 && return 1; } + get_ignores + + # init and check local + init_local || return $? + + # init, check, and lock server + init_server || { + ret=$? + unlock_local + return $ret + } + + tdir=$(tmpdir) + mkdir -p "$tdir" + + + local_full_list > "$tdir/local_full" + local_hash_list > "$tdir/local_hash" + server_both_list | tee >( + head -z -n1 | tr -d '\0' | sort > "$tdir/server_full" + ) | tail -z -n1 | sort > "$tdir/server_hash" + + # get changed on both sides + local_newer=$( list_diff "$tdir/local_hash" "$@") || { rm -rf "$tdir" ; unlock_all ; return 1; } + server_newer=$(list_diff "$tdir/server_hash" "$@") || { rm -rf "$tdir" ; unlock_all ; return 1; } + # get deleted on both sides + deleted_local=$( get_deleted "$tdir/local_full" "$@") || { rm -rf "$tdir" ; unlock_all ; return 1; } + deleted_server=$(get_deleted "$tdir/server_full" "$@") || { rm -rf "$tdir" ; unlock_all ; return 1; } + + # get collisions + collisions=$(printf "%s\n%s\n" "$local_newer" "$server_newer" | sort | uniq -d) + [ -n "$collisions" ] && [ "$method" != push ] && [ "$method" != pull ] && { + echo "-- There are file collisions" >&2 + echo "$collisions" + rm -rf "$tdir" + unlock_all + return 100 + } + + # remove collisions from opposing method + [ -n "$collisions" ] && { + if [ "$method" = "pull" ] + then + local_newer=$(printf "%s\n%s\n" "$collisions" "$local_newer" | sort | uniq -u) + else + server_newer=$(printf "%s\n%s\n" "$collisions" "$server_newer" | sort | uniq -u) + fi + } + + if [ -n "$local_newer" ] || [ -n "$server_newer" ] || [ -n "$deleted_local" ] || [ -n "$deleted_server" ] + then + # operations + if [ "$method" = "pull" ] + then + [ -n "$server_newer" ] && echo "$server_newer" | recieve "$dry" + [ -n "$local_newer" ] && echo "$local_newer" | send "$dry" + else + [ -n "$local_newer" ] && echo "$local_newer" | send "$dry" + [ -n "$server_newer" ] && echo "$server_newer" | recieve "$dry" + fi + + # delete has no impact on timestamps + [ -n "$deleted_local" ] && echo "$deleted_local" | delete_server "$dry" + [ -n "$deleted_server" ] && echo "$deleted_server" | delete_local "$dry" + + # real run + [ "$dry" != "dry" ] && { + # update lists + write_lists + } + fi + + rm -rf "$tdir" + + unlock_all +} diff --git a/src/transaction.sh b/src/transaction.sh new file mode 100644 index 0000000..a09888e --- /dev/null +++ b/src/transaction.sh @@ -0,0 +1,92 @@ + +## TRANSACTIONS + +# read list from stdin +# $1 = dry mode +send() { + if [ "$1" = "dry" ] + then + echo "* files to send" + cat + else + printf '* ' + rsync $rsync_opts --files-from=- --exclude=".zsync" -e ssh "$(pwd)" "$raddr:$rdir" || return $? + fi +} + +# read list from stdin +# $1 = dry mode +recieve() { + if [ "$1" = "dry" ] + then + echo "* files to recieve" + cat + else + printf '* ' + rsync $rsync_opts --files-from=- -e ssh "$raddr:$rdir" "$(pwd)" || return $? + fi +} + + +# read list from stdin +# $1 = dry mode +delete_server() { + if [ "$1" = "dry" ] + then + echo "* deleted to send" + cat + else + echo "* sending deleted" + ssh_exec '# LXSH_PARSE_MINIFY + cd "$1" || exit 1 + shift 1 + trashutil="gio trash" + which trash-put >/dev/null 2>&1 && trashutil=trash-put + for N + do + $trashutil "$N" && echo "$N" || exit $? + done + ' "$rdir" $(xargs -d "\n" /usr/bin/printf "%q ") + fi +} +# read delete from stdin +# $1 = dry mode +delete_local() { + if [ "$1" = "dry" ] + then + echo "* deleted to recieve" + cat + else + echo "* recieving deleted" + trashutil='gio trash' + which trash-put >/dev/null 2>&1 && trashutil=trash-put + while read -r ln + do + $trashutil "$ln" && echo "$ln" || return $? + done + fi +} + +forcepull() +{ + local ret=0 + get_server || return $? + init_local || return $? + init_server || { unlock_local ; return $?; } + rsync $rsync_opts -r --delete -e ssh "$raddr:$rdir" "$(pwd)/." || ret=$? + unlock_all + write_lists + return $ret +} + +forcepush() +{ + local ret=0 + get_server || return $? + init_local || return $? + init_server || { unlock_local ; return $?; } + rsync $rsync_opts -r --delete -e ssh "$(pwd)/." "$raddr:$rdir" || ret=$? + unlock_all + write_lists + return $ret +} diff --git a/src/util.sh b/src/util.sh new file mode 100644 index 0000000..5675716 --- /dev/null +++ b/src/util.sh @@ -0,0 +1,24 @@ + +## generic tools + +# $@ = paths +check_paths() +{ + for N + do + echo "$N" | grep "^/" && echo "Path cannot start with /" >&2 && return 1 + echo "$N" | grep -Fw ".." && echo "Path cannot contain .." >&2 && return 1 + done + return 0 +} + +tmpdir() { + echo "$tmpdir/zsync_$(tr -dc '[:alnum:]'