zsync: major revision
+ Add file selection ~ Change timestamp method ~ Optimize
This commit is contained in:
zawz
parent
496e0572bb
commit
bb003b2200
1 changed files with 144 additions and 61 deletions
205
zsync/zsync
205
zsync/zsync
|
|
@ -20,12 +20,12 @@ usage()
|
||||||
|
|
||||||
Operations:
|
Operations:
|
||||||
server <server> Setup sync on current folder with server target
|
server <server> Setup sync on current folder with server target
|
||||||
run Run sync with server
|
run [file...] Run sync with server
|
||||||
push Regular run but push all conflicts
|
push [file...] Regular run but push all conflicts
|
||||||
pull Regular run but pull all conflits
|
pull [file...] Regular run but pull all conflicts
|
||||||
dry Run a simulated sync but do not perform any action
|
dry [file...] Run a simulated sync but do not perform any action
|
||||||
drypush Dry run as push
|
drypush [file...] Dry run as push
|
||||||
drypull Dry run as pull
|
drypull [file...] Dry run as pull
|
||||||
forcepush Push by force the entire tree. Will replace and delete remote files
|
forcepush Push by force the entire tree. Will replace and delete remote files
|
||||||
forcepull Pull by force the entire tree. Will replace and delete local files"
|
forcepull Pull by force the entire tree. Will replace and delete local files"
|
||||||
}
|
}
|
||||||
|
|
@ -52,8 +52,8 @@ lock_local() { touch "$lock_file"; }
|
||||||
unlock_local() { rm "$lock_file"; }
|
unlock_local() { rm "$lock_file"; }
|
||||||
lock_server() { ssh "$raddr" "cd '$rdir' && touch '$lock_file'"; }
|
lock_server() { ssh "$raddr" "cd '$rdir' && touch '$lock_file'"; }
|
||||||
unlock_server() { ssh "$raddr" "cd '$rdir' && rm '$lock_file'"; }
|
unlock_server() { ssh "$raddr" "cd '$rdir' && rm '$lock_file'"; }
|
||||||
lock_all() { lock_local && lock_server; }
|
lock_all() { lock_local && lock_server ; }
|
||||||
unlock_all() { unlock_local && unlock_server; }
|
unlock_all() { ret=0; unlock_local || ret=$? ; unlock_server || ret=$?; return $ret ; }
|
||||||
|
|
||||||
local_lock_check() {
|
local_lock_check() {
|
||||||
[ ! -f "$lock_file" ] || { echo "Local sync locked, wait for sync completion" >&2 && return 1; }
|
[ ! -f "$lock_file" ] || { echo "Local sync locked, wait for sync completion" >&2 && return 1; }
|
||||||
|
|
@ -62,52 +62,103 @@ server_lock_check() {
|
||||||
ssh "$raddr" "cd '$rdir' && [ ! -f '$lock_file' ]" || { echo "Server is busy, wait for sync completion" >&2 && return 1; }
|
ssh "$raddr" "cd '$rdir' && [ ! -f '$lock_file' ]" || { echo "Server is busy, wait for sync completion" >&2 && return 1; }
|
||||||
}
|
}
|
||||||
|
|
||||||
set_timestamp_local() { touch -m "$timestamp_file"; }
|
set_timestamp_local() { date +%s > "$timestamp_file" ; }
|
||||||
|
|
||||||
|
# $@ = match these
|
||||||
|
merge()
|
||||||
|
{
|
||||||
|
if [ $# -gt 0 ]
|
||||||
|
then
|
||||||
|
re="^\./$1"
|
||||||
|
shift 1
|
||||||
|
for N
|
||||||
|
do
|
||||||
|
re="$re|^\./$N"
|
||||||
|
done
|
||||||
|
grep -E "($re)"
|
||||||
|
else # don't change input
|
||||||
|
cat
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
get_newer_local_files()
|
get_newer_local_files()
|
||||||
{
|
{
|
||||||
if [ -f "$timestamp_file" ]
|
TIME=$(cat "$timestamp_file" 2>/dev/null)
|
||||||
|
if [ "$TIME" -gt 0 ] 2>/dev/null
|
||||||
then
|
then
|
||||||
find . ! -type d ! -regex "^./$syncdir/.*" -newer "$timestamp_file"
|
find . ! -type d ! -regex "^./$syncdir/.*" -newermt @$TIME | merge "$@"
|
||||||
else
|
else
|
||||||
find . ! -type d ! -regex "^./$syncdir/.*"
|
find . ! -type d ! -regex "^./$syncdir/.*" | merge "$@"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
get_newer_server_files()
|
get_newer_server_files()
|
||||||
{
|
{
|
||||||
if [ -f "$timestamp_file" ]
|
TIME=$(cat "$timestamp_file" 2>/dev/null)
|
||||||
|
if [ "$TIME" -ge 0 ] 2>/dev/null
|
||||||
then
|
then
|
||||||
TIME=$(stat -c "%Y" .zsync/timestamp 2>/dev/null)
|
ssh $raddr "cd '$rdir' && find . ! -type d ! -regex '^\./$syncdir/.*' -newermt @$TIME" | merge "$@"
|
||||||
ssh $raddr "cd '$rdir' && find . ! -type d ! -regex '^./$syncdir/.*' -newermt @$TIME"
|
|
||||||
else
|
else
|
||||||
ssh $raddr "cd '$rdir' && find . ! -type d ! -regex '^./$syncdir/.*'"
|
ssh $raddr "cd '$rdir' && find . ! -type d ! -regex '^\./$syncdir/.*'" | merge "$@"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
# full list
|
# full list
|
||||||
get_server_list() {
|
get_server_list() {
|
||||||
ssh $raddr "cd '$rdir' || exit 1
|
ssh $raddr "cd '$rdir' || exit 1
|
||||||
find . ! -regex '^./$syncdir.*'" | sort
|
find . ! -regex '^\./$syncdir.*'" | sort | merge "$@"
|
||||||
}
|
}
|
||||||
get_local_list() {
|
get_local_list() {
|
||||||
find . ! -regex "^./$syncdir.*" | sort
|
find . ! -regex "^\./$syncdir.*" | sort | merge "$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
get_server_composed_list()
|
||||||
|
{
|
||||||
|
TIME=$(cat "$timestamp_file" 2>/dev/null)
|
||||||
|
[ "$TIME" -ge 0 ] 2>/dev/null || TIME=0
|
||||||
|
{ ssh $raddr "cd '$rdir' || exit 1
|
||||||
|
{
|
||||||
|
find . ! -regex '^\./$syncdir.*'
|
||||||
|
find . ! -type d ! -regex '^\./$syncdir/.*' -newermt @$TIME
|
||||||
|
} | sort" || return $? ; } | merge "$@"
|
||||||
}
|
}
|
||||||
|
|
||||||
# find deleted from list
|
# find deleted from list
|
||||||
# $1 = full list
|
# $1 = full list , $@ = merge
|
||||||
get_deleted()
|
get_deleted()
|
||||||
{
|
{
|
||||||
[ ! -f "$tree_file" ] && return 0
|
[ ! -f "$tree_file" ] && return 0
|
||||||
echo "$1" | diff --new-line-format="" --unchanged-line-format="" "$tree_file" - | reduce_list
|
arg=$1
|
||||||
|
shift 1
|
||||||
|
echo "$arg" | diff --new-line-format="" --unchanged-line-format="" "$tree_file" - | reduce_list | merge "$@"
|
||||||
}
|
}
|
||||||
|
|
||||||
# init
|
# init
|
||||||
init_local() {
|
init_local() {
|
||||||
mkdir -p "$syncdir" || exit $? # create syncdir
|
mkdir -p "$syncdir" || exit $?
|
||||||
}
|
}
|
||||||
init_server() {
|
init_server() {
|
||||||
ssh $raddr "mkdir -p '$rdir/$syncdir'" || return $?
|
ssh $raddr "mkdir -p '$rdir/$syncdir' && { which rsync >/dev/null 2>&1 || { echo \"rsync not found on server\" >&2 && exit 1; } ; }" || return $?
|
||||||
ssh $raddr "which rsync >/dev/null 2>&1" || { echo "rsync not found on server" >&2 && return 1; }
|
# ssh $raddr "which rsync >/dev/null 2>&1" || { echo "rsync not found on server" >&2 && return 1; }
|
||||||
|
}
|
||||||
|
|
||||||
|
initandcheck_server() {
|
||||||
|
ssh $raddr "mkdir -p '$rdir/$syncdir' && cd '$rdir' {
|
||||||
|
which rsync >/dev/null 2>&1 || { echo \"rsync not found on server\" >&2 ; exit 1; } ;
|
||||||
|
} && {
|
||||||
|
[ ! -f '$lock_file' ] || { echo \"Server is busy, wait for sync completion\" ; exit 1; }
|
||||||
|
} && exit 0" || return $?
|
||||||
|
# ssh $raddr "which rsync >/dev/null 2>&1" || { echo "rsync not found on server" >&2 && return 1; }
|
||||||
|
}
|
||||||
|
full_prep_server() {
|
||||||
|
ssh $raddr "
|
||||||
|
mkdir -p '$rdir/$syncdir' || exit 1
|
||||||
|
cd '$rdir' || exit 2
|
||||||
|
which rsync >/dev/null 2>&1 || { echo rsync not installed on server >&2 ; exit 3; }
|
||||||
|
[ -f '$lock_file' ] && { echo Server is busy, wait for sync completion ; exit 4; }
|
||||||
|
touch '$lock_file' || exit 5
|
||||||
|
exit 0"
|
||||||
|
# ssh $raddr "which rsync >/dev/null 2>&1" || { echo "rsync not found on server" >&2 && return 1; }
|
||||||
}
|
}
|
||||||
|
|
||||||
# read file list from stdin
|
# read file list from stdin
|
||||||
|
|
@ -198,62 +249,91 @@ forcepush()
|
||||||
set_timestamp_local
|
set_timestamp_local
|
||||||
}
|
}
|
||||||
|
|
||||||
# $1 = print only
|
# $1 = method (null/'push'/'pull') , $2 = dry (null/'dry') , $@ = files
|
||||||
sync()
|
sync()
|
||||||
{
|
{
|
||||||
get_server || { echo "Server not configured on this instance" >&2 && return 1; }
|
method=$1
|
||||||
init_local || return $?
|
dry=$2
|
||||||
init_server || return $?
|
shift 2
|
||||||
|
|
||||||
# quit if local or server locked
|
get_server || { echo "Server not configured on this instance" >&2 && return 1; }
|
||||||
|
|
||||||
|
# init and check local
|
||||||
|
init_local || return $?
|
||||||
local_lock_check || return $?
|
local_lock_check || return $?
|
||||||
server_lock_check || return $?
|
|
||||||
|
# init, check, and lock server
|
||||||
|
full_prep_server || {
|
||||||
|
case $? in
|
||||||
|
5) ret=$? ; unlock_server ; return $ret ;;
|
||||||
|
*) return $? ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
# lock
|
# lock
|
||||||
lock_all || { unlock_all ; return 1; }
|
lock_local || { unlock_all ; return 1; }
|
||||||
|
|
||||||
|
# retrieve local lists
|
||||||
|
local_list=$(get_local_list "$@") || { unlock_all ; return 1; }
|
||||||
|
local_newer=$(get_newer_local_files "$@") || { unlock_all ; return 1; }
|
||||||
|
|
||||||
|
# retrieve server lists
|
||||||
|
server_composed_list=$(get_server_composed_list "$@") || { unlock_all; return 1; }
|
||||||
|
server_list=$(echo "$server_composed_list" | uniq)
|
||||||
|
server_newer=$(echo "$server_composed_list" | uniq -d)
|
||||||
|
|
||||||
# retrieve lists
|
|
||||||
local_newer=$(get_newer_local_files) || { unlock_all ; return 1; }
|
|
||||||
server_newer=$(get_newer_server_files) || { unlock_all ; return 1; }
|
|
||||||
local_list=$(get_local_list) || { unlock_all ; return 1; }
|
|
||||||
server_list=$(get_server_list) || { unlock_all ; return 1; }
|
|
||||||
|
|
||||||
# get collisions
|
# get collisions
|
||||||
collisions=$(printf "%s\n%s" "$local_newer" "$server_newer" | sort | uniq -d)
|
collisions=$(printf "%s\n%s" "$local_newer" "$server_newer" | sort | uniq -d)
|
||||||
[ -n "$collisions" ] && [ -z "$1" ] && {
|
[ -n "$collisions" ] && [ "$method" != "push" ] && [ "$method" != pull ] && {
|
||||||
echo "There are file collisions" >&2
|
echo "-- There are file collisions" >&2
|
||||||
echo "$collisions" | sed 's|^\./||g'
|
echo "$collisions" | sed 's|^\./||g'
|
||||||
unlock_all
|
unlock_all
|
||||||
return 100
|
return 100
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# remove collisions from opposing method
|
||||||
|
[ -n "$collisions" ] && {
|
||||||
|
if [ "$method" = "pull" ]
|
||||||
|
then
|
||||||
|
local_newer=$(printf "%s\n%s\n" "$collisions" "$local_newer" | sort | uniq -u)
|
||||||
|
else
|
||||||
|
server_newer=$(printf "%s\n%s\n" "$collisions" "$server_newer" | sort | uniq -u)
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
# get deleted on both sides
|
# get deleted on both sides
|
||||||
deleted_local=$(get_deleted "$local_list") || { unlock_all ; return 1; }
|
deleted_local=$(get_deleted "$local_list" "$@") || { unlock_all ; return 1; }
|
||||||
deleted_server=$(get_deleted "$server_list") || { unlock_all ; return 1; }
|
deleted_server=$(get_deleted "$server_list" "$@") || { unlock_all ; return 1; }
|
||||||
|
|
||||||
if [ -n "$local_newer" ] || [ -n "$server_newer" ] || [ -n "$deleted_local" ] || [ -n "$deleted_server" ]
|
if [ -n "$local_newer" ] || [ -n "$server_newer" ] || [ -n "$deleted_local" ] || [ -n "$deleted_server" ]
|
||||||
then
|
then
|
||||||
# operations
|
# operations
|
||||||
if [ "$1" = "pull" ]
|
if [ "$method" = "pull" ]
|
||||||
then
|
then
|
||||||
[ -n "$server_newer" ] && echo "$server_newer" | recieve $2 | sed 's|^\./||g'
|
[ -n "$server_newer" ] && echo "$server_newer" | recieve "$dry" | sed 's|^\./||g'
|
||||||
[ -n "$local_newer" ] && echo "$local_newer" | send $2 | sed 's|^\./||g'
|
[ -n "$local_newer" ] && echo "$local_newer" | send "$dry" | sed 's|^\./||g'
|
||||||
else
|
else
|
||||||
[ -n "$local_newer" ] && echo "$local_newer" | send $2 | sed 's|^\./||g'
|
[ -n "$local_newer" ] && echo "$local_newer" | send "$dry" | sed 's|^\./||g'
|
||||||
[ -n "$server_newer" ] && echo "$server_newer" | recieve $2 | sed 's|^\./||g'
|
[ -n "$server_newer" ] && echo "$server_newer" | recieve "$dry" | sed 's|^\./||g'
|
||||||
fi
|
fi
|
||||||
[ -n "$deleted_local" ] && echo "$deleted_local" | delete_server $2 | sed 's|^\./||g'
|
# wait 1s to make sure, for timestamp
|
||||||
[ -n "$deleted_server" ] && echo "$deleted_server" | delete_local $2 | sed 's|^\./||g'
|
sleep 1 &
|
||||||
# update tree
|
|
||||||
[ "$2" != "dry" ] && {
|
# delete has no impact on timestamps
|
||||||
sleep 1 & # wait 1s to make sure, for timestamp
|
[ -n "$deleted_local" ] && echo "$deleted_local" | delete_server "$dry" | sed 's|^\./||g'
|
||||||
|
[ -n "$deleted_server" ] && echo "$deleted_server" | delete_local "$dry" | sed 's|^\./||g'
|
||||||
|
|
||||||
|
# real run
|
||||||
|
[ "$dry" != "dry" ] && {
|
||||||
|
# update tree
|
||||||
get_local_list > "$tree_file"
|
get_local_list > "$tree_file"
|
||||||
wait $(jobs -p)
|
wait
|
||||||
# set timestamp
|
# set timestamp
|
||||||
set_timestamp_local
|
set_timestamp_local
|
||||||
}
|
}
|
||||||
fi
|
fi
|
||||||
|
|
||||||
unlock_all
|
unlock_all
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -278,20 +358,23 @@ shift $((OPTIND-1))
|
||||||
# rdir=sync/tmp
|
# rdir=sync/tmp
|
||||||
[ -f "$server_file" ] && get_server
|
[ -f "$server_file" ] && get_server
|
||||||
|
|
||||||
|
|
||||||
# preprocess
|
# preprocess
|
||||||
[ -n "$arg_c" ] && { cd "$arg_c" || exit $?; } # -C opt
|
[ -n "$arg_c" ] && { cd "$arg_c" || exit $?; } # -C opt
|
||||||
|
|
||||||
|
[ $# -lt 1 ] && usage && exit 1
|
||||||
|
arg=$1
|
||||||
|
shift 1
|
||||||
|
|
||||||
# actions
|
# actions
|
||||||
case $1 in
|
case $arg in
|
||||||
server) setup_server "$2" ;;
|
server) setup_server "$@" ;;
|
||||||
run) sync ;;
|
run) sync "" "" "$@" ;;
|
||||||
pull) sync pull ;;
|
pull) sync pull "" "$@" ;;
|
||||||
push) sync push ;;
|
push) sync push "" "$@" ;;
|
||||||
dry) sync "" dry ;;
|
dry) sync "" dry "$@" ;;
|
||||||
drypush) sync push dry ;;
|
drypush) sync push dry "$@" ;;
|
||||||
drypull) sync pull dry ;;
|
drypull) sync pull dry "$@" ;;
|
||||||
forcepush) forcepush ;;
|
forcepush) forcepush ;;
|
||||||
forcepull) forcepull ;;
|
forcepull) forcepull ;;
|
||||||
*) usage ;;
|
*) usage && exit 1 ;;
|
||||||
esac
|
esac
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue