From 5ba8eaddf3b95fdaa0d06e16fbb4a68b424e8200 Mon Sep 17 00:00:00 2001 From: zawz Date: Fri, 22 May 2020 02:01:25 +0200 Subject: [PATCH] Add zsync --- README.md | 1 + zsync/zsync | 292 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 293 insertions(+) create mode 100755 zsync/zsync diff --git a/README.md b/README.md index 8eb9aca..9320070 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ Various CLI Shell tools - [zpass](/zpass) : Simple CLI password manager - [zrct2](/zrct2) : OpenRCT2 install/server utilities - [zsmc](/zmc) : Minecraft server utilities +- [zsync](/zmc) : Simple server/client file sync - [ztr](/ztr) : Configured text replacing - [zumask](/zumask) : Do umask operations recursively - [zupdate](/zupdate) : Tools to go with [zupdate](https://github.com/zawwz/zupdate) diff --git a/zsync/zsync b/zsync/zsync new file mode 100755 index 0000000..334bf23 --- /dev/null +++ b/zsync/zsync @@ -0,0 +1,292 @@ +#!/bin/sh + +## +## NO T OPTION FOR RSYNC + +# globals +syncdir=".zsync" +timestamp_file=".zsync/timestamp" +lock_file=".zsync/lock" +tree_file=".zsync/tree" +server_file=".zsync/server" + +rsync_opts='-rvlpE' + +# usage +fname=$(basename "$0") +usage() +{ + echo "$fname [operation] + +Operations: + server Setup sync on current folder with server target + run Run sync with server + push Regular run but push all conflicts + pull Regular run but pull all conflits + dry Run a simulated sync but do not perform any action + drypush Dry run as push + drypull Dry run as pull + forcepush Push by force the entire tree. Will replace and delete remote files + forcepull Pull by force the entire tree. Will replace and delete local files" +} + +# generic tools + +# read list from stdin +reduce_list() +{ + list="$(cat /dev/stdin)" + I=1 + while true + do + ln=$(echo "$list" | sed -n "$I"p) # get nth line + [ -z "$ln" ] && break + list=$(echo "$list" | grep -v "^$ln/") + I=$((I+1)) + done + echo "$list" +} + + +lock_local() { touch "$lock_file"; } +unlock_local() { rm "$lock_file"; } +lock_server() { ssh "$raddr" "cd '$rdir' && touch '$lock_file'"; } +unlock_server() { ssh "$raddr" "cd '$rdir' && rm '$lock_file'"; } +lock_all() { lock_local && lock_server; } +unlock_all() { unlock_local && unlock_server; } + +local_lock_check() { + [ ! -f "$lock_file" ] || { echo "Local sync locked, wait for sync completion" >&2 && return 1; } +} +server_lock_check() { + ssh "$raddr" "cd '$rdir' && [ ! -f '$lock_file' ]" || { echo "Server is busy, wait for sync completion" >&2 && return 1; } +} + +set_timestamp_local() { touch -m "$timestamp_file"; } + +get_newer_local_files() +{ + if [ -f "$timestamp_file" ] + then + find . ! -type d ! -regex "^./$syncdir/.*" -newer "$timestamp_file" + else + find . ! -type d ! -regex "^./$syncdir/.*" + fi +} +get_newer_server_files() +{ + if [ -f "$timestamp_file" ] + then + TIME=$(stat -c "%Y" .zsync/timestamp 2>/dev/null) + ssh $raddr "cd '$rdir' && find . ! -type d ! -regex '^./$syncdir/.*' -newermt @$TIME" + else + ssh $raddr "cd '$rdir' && find . ! -type d ! -regex '^./$syncdir/.*'" + fi +} + +# full list +get_server_list() { + ssh $raddr "cd '$rdir' || exit 1 + find . ! -regex '^./$syncdir.*'" | sort +} +get_local_list() { + find . ! -regex "^./$syncdir.*" | sort +} + +# find deleted from list +# $1 = full list +get_deleted() +{ + [ ! -f "$tree_file" ] && return 0 + echo "$1" | diff --new-line-format="" --unchanged-line-format="" "$tree_file" - | reduce_list +} + +# init +init_local(){ + mkdir -p "$syncdir" || exit $? # create syncdir +} +init_server() { + ssh $raddr "mkdir -p '$rdir/$syncdir'" +} + +# read file list from stdin +# $1 = list of files +send() { + if [ "$1" = "dry" ] + then + echo "* files to send" + sed 's|\./||g' + else + printf '* ' + rsync $rsync_opts --files-from=- --exclude=".zsync" -e ssh "$(pwd)" "$raddr:$rdir" + fi +} + +# read file list from stdin +recieve() { + if [ "$1" = "dry" ] + then + echo "* files to recieve" + sed 's|\./||g' + else + printf '* ' + rsync $rsync_opts --files-from=- -e ssh "$raddr:$rdir" "$(pwd)" + fi +} + + +# read delete from stdin +delete_server() { + if [ "$1" = "dry" ] + then + echo "* deleted to send" + sed 's|\./||g' + else + echo "* sending deleted" + ssh $raddr "cd '$rdir' || exit 1 + while read -r ln + do + gio trash \"\$ln\" && echo \"\$ln\" + done + " + fi +} +# read delete from stdin +delete_local() { + if [ "$1" = "dry" ] + then + echo "* deleted to recieve" + sed 's|\./||g' + else + echo "* recieving deleted" + while read -r ln + do + gio trash "$ln" && echo "$ln" + done + fi +} + +get_server() { + [ ! -f "$server_file" ] && return 1 + raddr=$(cut -d ':' -f1 "$server_file") + rdir=$(cut -d ':' -f2- "$server_file") +} + +setup_server() +{ + init_local || return $? + [ -z "$1" ] && echo "$fname server user@host:path" && return 1 + echo "$1" > "$server_file" +} + +forcepull() +{ + rsync $rsync_opts -r --delete -e ssh "$raddr:$rdir" "$(pwd)/." + sleep 1 + set_timestamp_local +} + +forcepush() +{ + rsync $rsync_opts -r --delete -e ssh "$(pwd)/." "$raddr:$rdir" + sleep 1 + set_timestamp_local +} + +# $1 = print only +sync() +{ + get_server || { echo "Server not configured on this instance" >&2 && return 1; } + init_local || return $? + init_server || return $? + + # quit if local or server locked + local_lock_check || return $? + server_lock_check || return $? + + # lock + lock_all || { unlock_all ; return 1; } + + # retrieve lists + local_newer=$(get_newer_local_files) || { unlock_all ; return 1; } + server_newer=$(get_newer_server_files) || { unlock_all ; return 1; } + local_list=$(get_local_list) || { unlock_all ; return 1; } + server_list=$(get_server_list) || { unlock_all ; return 1; } + + # get collisions + collisions=$(printf "%s\n%s" "$local_newer" "$server_newer" | sort | uniq -d) + [ -n "$collisions" ] && [ -z "$1" ] && { + echo "There are file collisions" >&2 + echo "$collisions" | sed 's|^\./||g' + unlock_all + return 100 + } + + # get deleted on both sides + deleted_local=$(get_deleted "$local_list") || { unlock_all ; return 1; } + deleted_server=$(get_deleted "$server_list") || { unlock_all ; return 1; } + + if [ -n "$local_newer" ] || [ -n "$server_newer" ] || [ -n "$deleted_local" ] || [ -n "$deleted_server" ] + then + # operations + if [ "$1" = "pull" ] + then + [ -n "$server_newer" ] && echo "$server_newer" | recieve $2 | sed 's|^\./||g' + [ -n "$local_newer" ] && echo "$local_newer" | send $2 | sed 's|^\./||g' + else + [ -n "$local_newer" ] && echo "$local_newer" | send $2 | sed 's|^\./||g' + [ -n "$server_newer" ] && echo "$server_newer" | recieve $2 | sed 's|^\./||g' + fi + [ -n "$deleted_local" ] && echo "$deleted_local" | delete_server $2 | sed 's|^\./||g' + [ -n "$deleted_server" ] && echo "$deleted_server" | delete_local $2 | sed 's|^\./||g' + # update tree + [ "$2" != "dry" ] && { + sleep 1 & # wait 1s to make sure, for timestamp + get_local_list > "$tree_file" + wait $(jobs -p) + # set timestamp + set_timestamp_local + } + fi + + unlock_all +} + +which rsync >/dev/null || { echo "rsync not installed" >&2 && exit 1; } + +# options +unset arg_c +while getopts ":hC:" opt; +do + case $opt in + C) + [ -z "$OPTARG" ] && echo "Option -C requires an argument" >&2 && exit 1 + arg_c="$OPTARG" + ;; + h) usage && exit 1 ;; + \?) echo "Uknown option: $OPTARG" >&2 && usage && exit 1 ;; + esac +done +shift $((OPTIND-1)) + +# raddr=zawz@zawz.net +# rdir=sync/tmp +[ -f "$server_file" ] && get_server + + +# preprocess +[ -n "$arg_c" ] && { cd "$arg_c" || exit $?; } # -C opt + +# actions +case $1 in + server) setup_server "$2" ;; + run) sync ;; + pull) sync pull ;; + push) sync push ;; + dry) sync "" dry ;; + drypush) sync push dry ;; + drypull) sync pull dry ;; + forcepush) forcepush ;; + forcepull) forcepull ;; + *) usage ;; +esac