init

.gitignore

@@ -0,0 +1,3 @@
+zsync
+Zmakefile
+zmakefile

Makefile

@@ -0,0 +1,20 @@
+
+var_exclude = ZPASS_.* XDG_.* REMOTE_.* DISPLAY CONFIGFILE TMPDIR
+fct_exclude = _tty_on
+
+zsync: src/*
+	lxsh -o zsync -M --exclude-var "$(var_exclude)" --exclude-fct "$(fct_exclude)" src/main.sh
+
+debug: src/*
+	lxsh -o zsync src/main.sh
+
+build: zsync
+
+install: build
+	mv zsync /usr/local/bin
+
+uninstall:
+	rm /usr/local/bin/zpass
+
+clear:
+	rm zsync

src/config.sh

@@ -0,0 +1,49 @@
+
+# globals
+syncdir=".zsync"
+lock_file=".zsync/lock"
+ignore_file=".zsync/ignore"
+tree_full=".zsync/tree_full"
+tree_hash=".zsync/tree_hash"
+config_file=".zsync/config"
+
+# NO -t OPTION FOR RSYNC
+rsync_opts='-rvlpE'
+
+tmpdir=${TMPDIR-/tmp}
+
+## CONFIG
+
+init_config() {
+  mkdir -p "$syncdir" || return 2
+  which rsync >/dev/null 2>&1 || { echo rsync not installed on server >&2 ; return 3; }
+  touch "$config_file" || return 5
+}
+
+get_server() {
+  [ ! -f "$config_file" ] && return 1
+  servconf=$(sed 's|^[ \t]*||g' "$config_file" | grep -E '^server[ \t]' | sed 's|^server[ \t]*||g' | tail -n1)
+  raddr=$(echo "$servconf" | cut -d ':' -f1)
+  rdir=$(echo "$servconf" | cut -d ':' -f2-)
+}
+
+# $1 = server arg
+setup_server()
+{
+  init_config || return $?
+  [ -z "$1" ] && echo "$fname server user@host:path" && return 1
+  sed -i '/^[ \t]*server[ \t]/d' "$config_file"
+  echo "server $1" >> "$config_file"
+}
+
+ignores='\.zsync'
+get_ignores() {
+  if [ -f "$ignore_file" ]
+  then
+    ignores="($(tr '\n' '|' < "$ignore_file"))"
+  else
+    ignores='(^$)'
+  fi
+  ignores=$(echo "$ignores" | sed ' s/|)/)/g ; s/^()$/^$/g ')
+  [ -n "$ignores" ] || ignores='\.zsync'
+}

src/diff.sh

@@ -0,0 +1,24 @@
+
+## DIFFERENCES
+
+# find changes from list
+# $1 = list file , $@ = targets
+# requisite: file contains both hash and filename and is sorted
+list_diff()
+{
+  file=$1
+  shift 1
+  [ ! -f "$tree_hash" ] && { cut -c34- "$file" ; return 0; }
+  diff --old-line-format="" --unchanged-line-format="" "$tree_hash" "$file" | cut -c34- | merge "$@"
+}
+
+# find deleted from list
+# $1 = list file , $@ = targets
+# requisite: file contains only filename and is sorted
+get_deleted()
+{
+  file=$1
+  shift 1
+  [ ! -f "$tree_full" ] && return 0
+  diff --new-line-format="" --unchanged-line-format="" "$tree_full" "$file" | reduce_list | grep -vE "$ignores" | merge "$@"
+}

src/filter.sh

@@ -0,0 +1,54 @@
+
+## FILTERS
+
+run_ignore() {
+  [ -n "$ignores" ] && grep -vE "$ignores" "$@"
+}
+
+# $1 = regex , $@ = args
+grep_after_sum()
+{
+  reg=$1
+  shift 1
+  grep --color=never -E "^[0-9a-f]{32} $reg" "$@"
+}
+
+# $@ = match these
+merge()
+{
+  if [ $# -gt 0 ]
+  then
+    re="$1"
+    shift 1
+    for N
+    do
+      re="$re|$N"
+    done
+    grep -E "^($re)"
+    return 0
+  else # don't change input
+    cat
+  fi
+}
+
+# reduce list to only subsets
+# kind of like "uniq" but matches lines that are supersets, only outputting the subset
+# needs a sorted list in input
+# $1 = separator for subset-superset (default="/")
+reduce_list()
+{
+  awk 'BEGIN{ FS="\n"; RS="\0" ; i = 0 } {
+      i=1
+      while(i<=NF)
+      {
+        print $i
+        val=$i "'"${1-"/"}"'"
+        i++;
+        while(substr($i, 0, length(val)) == val)
+        {
+          i++;
+        }
+      }
+    }
+  '
+}

src/list.sh

@@ -0,0 +1,60 @@
+
+## LIST
+
+local_hash_list()
+{
+  { ( set -e
+  find . -type f ! -regex "^./$syncdir/.*" | sed 's|^./||g' | tr '\n' '\0' | xargs -0 md5sum | cut -c1-33,35- | grep -vE "$ignores"
+  find . -type l | sed 's|^./||g' | while read -r ln
+  do
+    find "$ln" -maxdepth 0 -printf '%l' | md5sum | sed "s|-|$ln|g"
+  done | cut -c1-33,35- | grep -vE "$ignores"
+  ) || return $?; } | sort
+}
+
+server_hash_list()
+{
+  ssh_exec '#LXSH_PARSE_MINIFY
+    set -e
+    cd "$1"
+    find . -type f ! -regex "^./$2/.*" | sed "s|^./||g" | tr "\n" "\0" | xargs -0 md5sum | cut -c1-33,35- | grep -vE "$3"
+    find . -type l | sed "s|^./||g" | while read -r ln
+    do
+      find "$ln" -maxdepth 0 -printf "%l" | md5sum | sed "s|-|$ln|g"
+    done | cut -c1-33,35- | grep -vE "$3"
+  ' "$rdir" "$syncdir" "$ignores" | sort
+}
+
+local_full_list() {
+  find . -mindepth 1 ! -regex "^./$syncdir\$" ! -regex "^./$syncdir/.*" | sed 's|^./||g' | grep -vE "$ignores" | sort
+}
+
+server_full_list() {
+  ssh_exec '#LXSH_PARSE_MINIFY
+    set -e
+    cd "$1"
+    find . -mindepth 1 ! -regex "^./$2\$" ! -regex "^./$2/.*" | sed "s|^./||g" | grep -vE "$3"
+  ' "$rdir" "$syncdir" "$ignores" | sort
+}
+
+server_both_list() {
+  ssh_exec '#LXSH_PARSE_MINIFY
+    set -e
+    cd "$1"
+    find . -mindepth 1 ! -regex "^./$2\$" ! -regex "^./$2/.*" | sed "s|^./||g" | grep -vE "$3"
+    printf "\0"
+    {
+      find . -type f ! -regex "^./$2/.*" | sed "s|^./||g" | tr "\n" "\0" | xargs -0 md5sum | cut -c1-33,35- | grep -vE "$3"
+      find . -type l | sed "s|^./||g" | while read -r ln
+      do
+        find "$ln" -maxdepth 0 -printf "%l" | md5sum | sed "s|-|$ln|g"
+      done | cut -c1-33,35- | grep -vE "$3"
+    }
+  ' "$rdir" "$syncdir" "$ignores"
+}
+
+write_lists()
+{
+  local_full_list > "$tree_full"
+  local_hash_list > "$tree_hash"
+}

src/lock.sh

@@ -0,0 +1,41 @@
+
+## LOCK
+
+lock_local() { touch "$lock_file"; }
+unlock_local() { rm "$lock_file"; }
+lock_server() {
+  ssh_exec 'touch "$1"' "$rdir/$lock_file";
+}
+unlock_server() {
+  ssh_exec 'rm "$1"' "$rdir/$lock_file";
+}
+lock_all() { lock_local && lock_server ; }
+unlock_all() { ret=0; unlock_local || ret=$? ; unlock_server || ret=$?; return $ret ; }
+
+local_lock_check() {
+  [ ! -f "$lock_file" ] || { echo "Local sync locked, wait for sync completion" >&2 && return 1; }
+}
+server_lock_check() {
+  ssh_exec '
+    cd "$1" || return 0
+    [ ! -f "$2" ]
+  ' "$rdir" "$lock_file" || { echo "Server sync locked, wait for sync completion" >&2 && return 1; }
+}
+
+# init
+init_local() {
+  mkdir -p "$syncdir" || return 2
+  which rsync >/dev/null 2>&1 || { echo rsync not installed on server >&2 ; return 3; }
+  local_lock_check || return 4
+  touch "$lock_file" || return 5
+}
+
+init_server() {
+  ssh_exec '
+    cd "$1" || exit 1
+    mkdir -p "$2" || exit 2
+    which rsync >/dev/null 2>&1 || { echo rsync not installed on server >&2 ; exit 3; }
+    [ -f "$2" ] && { echo Server sync locked, wait for sync completion ; exit 4; }
+    touch "$3" || exit 5
+  ' "$rdir" "$syncdir" "$lock_file"
+}

src/main.sh

@@ -0,0 +1,22 @@
+#!/bin/lxsh
+
+set -e
+
+%include config.sh options.sh *.sh
+
+arg=$1
+shift 1
+
+# actions
+case $arg in
+  server) setup_server "$@" ;;
+  run)     sync ""   ""  "$@" ;;
+  pull)    sync pull ""  "$@" ;;
+  push)    sync push ""  "$@" ;;
+  dry)     sync ""   dry "$@" ;;
+  drypush) sync push dry "$@" ;;
+  drypull) sync pull dry "$@" ;;
+  forcepush) forcepush ;;
+  forcepull) forcepull ;;
+  *) usage && exit 1 ;;
+esac

src/options.sh

@@ -0,0 +1,44 @@
+
+# usage
+fname=$(basename "$0")
+usage()
+{
+  echo "$fname [operation]
+
+Operations:
+  server <server>     Setup sync on current folder with server target
+  run [file...]       Run sync with server
+  push [file...]      Regular run but push all conflicts
+  pull [file...]      Regular run but pull all conflicts
+  dry [file...]       Run a simulated sync but do not perform any action
+  drypush [file...]   Dry run as push
+  drypull [file...]   Dry run as pull
+  forcepush           Push by force the entire tree. Will replace and delete remote files
+  forcepull           Pull by force the entire tree. Will replace and delete local files"
+}
+
+# options
+unset arg_c
+while getopts ":hC:" opt;
+do
+  case $opt in
+    C)
+      [ -z "$OPTARG" ] && echo "Option -C requires an argument" >&2 && exit 1
+      arg_c="$OPTARG"
+      ;;
+    h) usage && exit 1 ;;
+    *) echo "Unknown option: $OPTARG" >&2 && usage && exit 1 ;;
+  esac
+done
+shift $((OPTIND-1))
+
+# raddr=zawz@zawz.net
+# rdir=sync/tmp
+[ -f "$server_file" ] && get_server
+
+[ -n "$arg_c" ] && { cd "$arg_c" || exit $?; }   # -C opt
+
+[ $# -lt 1 ] && usage && exit 1
+
+which rsync >/dev/null || { echo "rsync not installed" >&2 && exit 1; }
+/usr/bin/printf %q >/dev/null || { echo "printf does not support %q" >&2 && exit 1; }

src/sync.sh

@@ -0,0 +1,87 @@
+
+# $1 = method (null/'push'/'pull') , $2 = dry (null/'dry') , $@ = files
+sync()
+{
+  method=$1
+  dry=$2
+  shift 2
+
+  check_paths "$@" || return $?
+
+  get_server || { echo "Server not configured on this instance" >&2 && return 1; }
+  get_ignores
+
+  # init and check local
+  init_local || return $?
+
+  # init, check, and lock server
+  init_server || {
+    ret=$?
+    unlock_local
+    return $ret
+  }
+
+  tdir=$(tmpdir)
+  mkdir -p "$tdir"
+
+
+  local_full_list  > "$tdir/local_full"
+  local_hash_list  > "$tdir/local_hash"
+  server_both_list | tee >(
+      head -z -n1  | tr -d '\0' | sort > "$tdir/server_full"
+  ) | tail -z -n1 | sort > "$tdir/server_hash"
+
+  # get changed on both sides
+  local_newer=$( list_diff "$tdir/local_hash"  "$@") || { rm -rf "$tdir" ; unlock_all ; return 1; }
+  server_newer=$(list_diff "$tdir/server_hash" "$@") || { rm -rf "$tdir" ; unlock_all ; return 1; }
+  # get deleted on both sides
+  deleted_local=$( get_deleted "$tdir/local_full"  "$@") || { rm -rf "$tdir" ; unlock_all ; return 1; }
+  deleted_server=$(get_deleted "$tdir/server_full" "$@") || { rm -rf "$tdir" ; unlock_all ; return 1; }
+
+  # get collisions
+  collisions=$(printf "%s\n%s\n" "$local_newer" "$server_newer" | sort | uniq -d)
+  [ -n "$collisions" ] && [ "$method" != push ] && [ "$method" != pull ] && {
+    echo "-- There are file collisions" >&2
+    echo "$collisions"
+    rm -rf "$tdir"
+    unlock_all
+    return 100
+  }
+
+  # remove collisions from opposing method
+  [ -n "$collisions" ] && {
+    if [ "$method" = "pull" ]
+    then
+      local_newer=$(printf "%s\n%s\n" "$collisions" "$local_newer" | sort | uniq -u)
+    else
+      server_newer=$(printf "%s\n%s\n" "$collisions" "$server_newer" | sort | uniq -u)
+    fi
+  }
+
+  if [ -n "$local_newer" ] || [ -n "$server_newer" ] || [ -n "$deleted_local" ] || [ -n "$deleted_server" ]
+  then
+    # operations
+    if [ "$method" = "pull" ]
+    then
+      [ -n "$server_newer" ] && echo "$server_newer" | recieve "$dry"
+      [ -n "$local_newer"  ] && echo "$local_newer"  | send    "$dry"
+    else
+      [ -n "$local_newer"  ] && echo "$local_newer"  | send    "$dry"
+      [ -n "$server_newer" ] && echo "$server_newer" | recieve "$dry"
+    fi
+
+    # delete has no impact on timestamps
+    [ -n "$deleted_local"  ] && echo "$deleted_local"  | delete_server "$dry"
+    [ -n "$deleted_server" ] && echo "$deleted_server" | delete_local  "$dry"
+
+    # real run
+    [ "$dry" != "dry" ] && {
+      # update lists
+      write_lists
+    }
+  fi
+
+  rm -rf "$tdir"
+
+  unlock_all
+}

src/transaction.sh

@@ -0,0 +1,92 @@
+
+## TRANSACTIONS
+
+# read list from stdin
+# $1 = dry mode
+send() {
+  if [ "$1" = "dry" ]
+  then
+    echo "* files to send"
+    cat
+  else
+    printf '* '
+    rsync $rsync_opts --files-from=- --exclude=".zsync" -e ssh "$(pwd)" "$raddr:$rdir" || return $?
+  fi
+}
+
+# read list from stdin
+# $1 = dry mode
+recieve() {
+  if [ "$1" = "dry" ]
+  then
+    echo "* files to recieve"
+    cat
+  else
+    printf '* '
+    rsync $rsync_opts --files-from=- -e ssh "$raddr:$rdir" "$(pwd)" || return $?
+  fi
+}
+
+
+# read list from stdin
+# $1 = dry mode
+delete_server() {
+  if [ "$1" = "dry" ]
+  then
+    echo "* deleted to send"
+    cat
+  else
+    echo "* sending deleted"
+    ssh_exec '# LXSH_PARSE_MINIFY
+      cd "$1" || exit 1
+      shift 1
+      trashutil="gio trash"
+      which trash-put >/dev/null 2>&1 && trashutil=trash-put
+      for N
+      do
+        $trashutil "$N" && echo "$N" || exit $?
+      done
+    ' "$rdir" $(xargs -d "\n" /usr/bin/printf "%q ")
+  fi
+}
+# read delete from stdin
+# $1 = dry mode
+delete_local() {
+  if [ "$1" = "dry" ]
+  then
+    echo "* deleted to recieve"
+    cat
+  else
+    echo "* recieving deleted"
+    trashutil='gio trash'
+    which trash-put >/dev/null 2>&1 && trashutil=trash-put
+    while read -r ln
+    do
+      $trashutil "$ln" && echo "$ln" || return $?
+    done
+  fi
+}
+
+forcepull()
+{
+  local ret=0
+  get_server  || return $?
+  init_local  || return $?
+  init_server || { unlock_local ; return $?; }
+  rsync $rsync_opts -r --delete -e ssh "$raddr:$rdir" "$(pwd)/." || ret=$?
+  unlock_all
+  write_lists
+  return $ret
+}
+
+forcepush()
+{
+  local ret=0
+  get_server  || return $?
+  init_local  || return $?
+  init_server || { unlock_local ; return $?; }
+  rsync $rsync_opts -r --delete -e ssh "$(pwd)/." "$raddr:$rdir" || ret=$?
+  unlock_all
+  write_lists
+  return $ret
+}

src/util.sh

@@ -0,0 +1,24 @@
+
+## generic tools
+
+# $@ = paths
+check_paths()
+{
+  for N
+  do
+    echo "$N" | grep "^/" && echo "Path cannot start with /" >&2 && return 1
+    echo "$N" | grep -Fw ".." && echo "Path cannot contain .." >&2 && return 1
+  done
+  return 0
+}
+
+tmpdir() {
+  echo "$tmpdir/zsync_$(tr -dc '[:alnum:]' </dev/urandom | head -c20)"
+}
+
+# $1 = code , $@ = arguments
+ssh_exec() {
+  code=$1
+  shift 1
+  ssh "$raddr" sh -c "$(/usr/bin/printf "%q " "$code")" sh "$(/usr/bin/printf "%q " "$@")"
+}