aboutsummaryrefslogtreecommitdiffstats
path: root/bitpocket
diff options
context:
space:
mode:
authorDaniel Lange <DLange@git.local>2016-02-23 08:50:37 +0100
committerDaniel Lange <DLange@git.local>2016-02-23 08:50:37 +0100
commit507c7bbc4f74e8ca972d1b248e7469bc4315046d (patch)
treee983711ade5c20a23045911fa7d84bc5a1fbaf2b /bitpocket
downloadbitpocket-507c7bbc4f74e8ca972d1b248e7469bc4315046d.tar.gz
bitpocket-507c7bbc4f74e8ca972d1b248e7469bc4315046d.tar.bz2
bitpocket-507c7bbc4f74e8ca972d1b248e7469bc4315046d.zip
Initial commit of cleaned up github:sickill/bitpocket repository
Diffstat (limited to 'bitpocket')
-rwxr-xr-xbitpocket386
1 files changed, 386 insertions, 0 deletions
diff --git a/bitpocket b/bitpocket
new file mode 100755
index 0000000..a0dbbd6
--- /dev/null
+++ b/bitpocket
@@ -0,0 +1,386 @@
+#!/bin/bash
+
+LANG=$(locale | grep LANG= | sed 's:LANG=::')
+if [ -z "$LANG" ]; then
+ LANG="C"
+fi
+
+export LC_ALL=$LANG # for stable "sort" output
+
+# Paths
+DOT_DIR=.bitpocket
+CFG_FILE="$DOT_DIR/config"
+TMP_DIR="$DOT_DIR/tmp"
+STATE_DIR="$DOT_DIR/state"
+LOCK_DIR="$TMP_DIR/lock" # Use a lock directory for atomic locks. See the Bash FAQ http://mywiki.wooledge.org/BashFAQ/045
+
+# Default settings
+SLOW_SYNC_TIME=10
+SLOW_SYNC_FILE="$TMP_DIR/slow"
+RSYNC_RSH="ssh"
+
+# Load config file
+[ -f "$CFG_FILE" ] && . "$CFG_FILE"
+
+# Test for GNU versions of core utils. Bail if non-GNU.
+sed --version >/dev/null 2>/dev/null
+if [ $? -ne 0 ]; then
+ echo "fatal: It seems like you are running non-GNU versions of coreutils."
+ echo " It is currently unsafe to use bitpocket with this setup,"
+ echo " so I'll have to stop here. Sorry ..."
+ exit 1
+fi
+
+# Decide on runner (ssh / bash -c)
+if [ -n "$REMOTE_HOST" ]; then
+ REMOTE_RUNNER="$RSYNC_RSH $REMOTE_HOST"
+ REMOTE="$REMOTE_HOST:$REMOTE_PATH"
+else
+ REMOTE_RUNNER="bash -c"
+ REMOTE="$REMOTE_PATH"
+fi
+
+REMOTE_TMP_DIR="$REMOTE_PATH/$DOT_DIR/tmp"
+
+# Don't sync user excluded files
+if [ -f "$DOT_DIR/exclude" ]; then
+ user_exclude="--exclude-from $DOT_DIR/exclude"
+fi
+
+# Specify certain files to include
+if [ -f "$DOT_DIR/include" ]; then
+ user_include="--include-from $DOT_DIR/include"
+fi
+
+# Specify rsync filter rules
+if [ -f "$DOT_DIR/filter" ]; then
+ # The underscore (_) is required for correct operation
+ user_filter="--filter merge_$DOT_DIR/filter"
+fi
+
+USER_RULES="$user_filter $user_include $user_exclude"
+
+TIMESTAMP=$(date "+%Y-%m-%d.%H%M%S")
+
+export RSYNC_RSH
+
+function init {
+ if [[ -d "$DOT_DIR" || -f "$CFG_FILE" ]]; then
+ echo "fatal: Current directory already initialized for bitpocket"
+ exit 128
+ fi
+
+ if [[ $# != 2 ]]; then
+ echo "usage: bitpocket init <REMOTE_HOST> <REMOTE_PATH>"
+ exit 128
+ fi
+
+ mkdir "$DOT_DIR"
+
+ cat <<EOF > "$CFG_FILE"
+## Host and path of central storage
+REMOTE_HOST=$1
+REMOTE_PATH="$2"
+
+## SSH command with options for connecting to \$REMOTE
+# RSYNC_RSH="ssh -p 22 -i $DOT_DIR/id_rsa"
+
+## Uncomment following line to follow symlinks (transform it into referent file/dir)
+# RSYNC_OPTS="-L"
+
+## Uncomment following lines to get sync notifications
+# SLOW_SYNC_TIME=10
+# SLOW_SYNC_START_CMD="notify-send 'BitPocket sync in progress...'"
+# SLOW_SYNC_STOP_CMD="notify-send 'BitPocket sync finished'"
+EOF
+
+ echo "Initialized bitpocket directory at `pwd`"
+ echo "Please have a look at the config file ($DOT_DIR/config)"
+}
+
+function log {
+ assert_dotdir
+ tail -f "$DOT_DIR/log"
+}
+
+# Do the actual synchronization
+function sync {
+ assert_dotdir
+ acquire_lock
+ acquire_remote_lock
+
+ echo
+ echo -e "\x1b\x5b1;32mbitpocket started\x1b\x5b0m at `date`."
+ echo
+
+ # Fire off slow sync start notifier in background
+ on_slow_sync_start
+
+ # Check what has changed
+ touch "$STATE_DIR/tree-prev"
+ touch "$STATE_DIR/added-prev"
+
+ # Save before-sync state
+ # Must be done with rsync itself (rather than find) to respect includes/excludes
+ # Order of includes/excludes/filters is EXTREMELY important
+ echo "# Saving current state and backing up files (if needed)"
+ echo " | Root dir: $(pwd)"
+ rsync -av --list-only --exclude "/$DOT_DIR" $RSYNC_OPTS $USER_RULES . | grep "^-\|^d" \
+ | sed "s:^\S*\s*\S*\s*\S*\s*\S*\s*:/:" | sed "s:^/\.::" | sort > "$STATE_DIR/tree-current"
+
+ # Prevent bringing back locally deleted files or removing new local files
+ cp -f "$STATE_DIR/added-prev" "$TMP_DIR/fetch-exclude"
+ sort "$STATE_DIR/tree-prev" "$STATE_DIR/tree-current" | uniq -u >> "$TMP_DIR/fetch-exclude"
+
+ # It is difficult to only create the backup directory if needed; instead
+ # we always create it, but remove it if it is empty afterwards.
+ mkdir --parents $DOT_DIR/backups/$TIMESTAMP
+
+ # Determine what will be fetched from server and make backup
+ # copies of any local files to be deleted or overwritten.
+ # Order of includes/excludes/filters is EXTREMELY important
+ rsync --dry-run \
+ -auvzxi --delete $RSYNC_OPTS --exclude "/$DOT_DIR" --exclude-from "$TMP_DIR/fetch-exclude" $USER_RULES $REMOTE/ . \
+ | grep "^[ch<>\.\*][f]\|\*deleting" | sed "s:^\S*\s*::" | sed 's:\d96:\\\`:g' | sed "s:\(.*\):if [ -f \"\1\" ]; then cp --parents \"\1\" $DOT_DIR/backups/$TIMESTAMP; fi:" | sh || die "BACKUP"
+ [ "$(ls -A $DOT_DIR/backups/$TIMESTAMP)" ] && echo " | Some files were backed up to $DOT_DIR/backups/$TIMESTAMP"
+ [ "$(ls -A $DOT_DIR/backups/$TIMESTAMP)" ] || rmdir $DOT_DIR/backups/$TIMESTAMP
+
+ # Actual fetch
+ # Pulling changes from server
+ # Order of includes/excludes/filters is EXTREMELY important
+ echo
+ echo "# Pulling changes from server"
+ rsync -auvzxi --delete $RSYNC_OPTS --exclude "/$DOT_DIR" --exclude-from "$TMP_DIR/fetch-exclude" $USER_RULES $REMOTE/ . | sed "s/^/ | /" || die "PULL"
+
+ # Actual push
+ # Send new and updated, remotely remove files deleted locally
+ # Order of includes/excludes/filters is EXTREMELY important
+ echo
+ echo "# Pushing changes to server"
+ rsync -auvzxi --delete $RSYNC_OPTS --exclude "/$DOT_DIR" $USER_RULES . $REMOTE/ | sed "s/^/ | /" || die "PUSH"
+
+ # Save after-sync state
+ # Must be done with rsync itself (rather than find) to respect includes/excludes
+ # Order of includes/excludes/filters is EXTREMELY important
+ echo
+ echo "# Saving after-sync state and cleaning up"
+ rsync -av --list-only --exclude "/$DOT_DIR" $USER_RULES . | grep "^-\|^d" \
+ | sed "s:^\S*\s*\S*\s*\S*\s*\S*\s*:/:" | sed "s:^/\.::" | sort > "$TMP_DIR/tree-after"
+
+ # Save all newly created files for next run (to prevent deletion of them)
+ # This includes files created by user in parallel to sync and files fetched from remote
+ comm -23 "$TMP_DIR/tree-after" "$STATE_DIR/tree-current" >"$STATE_DIR/added-prev"
+
+ # Save new tree state for next run
+ cat "$STATE_DIR/tree-current" "$STATE_DIR/added-prev" >"$STATE_DIR/tree-prev"
+
+ # Fire off slow sync stop notifier in background
+ on_slow_sync_stop
+
+ cleanup
+ echo
+ echo -e "\x1b\x5b1;32mbitpocket finished\x1b\x5b0m at `date`."
+ echo
+
+}
+
+
+# Pack backups into a git repository
+function pack {
+ assert_dotdir
+
+ # Git is required for backup packing
+ if [ ! `builtin type -p git` ]; then
+ echo "fatal: For backup packing, git must be installed"
+ exit 128
+ fi
+
+ # If pack directory is missing, create it and prepare git repo
+ if [ ! -d "$DOT_DIR/pack" ]
+ then
+ mkdir $DOT_DIR/pack
+ git init $DOT_DIR/pack
+ touch $DOT_DIR/pack/.git-init-marker
+ (cd $DOT_DIR/pack && git add .)
+ (cd $DOT_DIR/pack && git commit -a -q -m "INIT")
+ fi
+
+ # If any backups exist, pack them into the repo
+ if [ -d "$DOT_DIR/backups" ] && [ "$(ls -A $DOT_DIR/backups)" ]
+ then
+ for DIR in $DOT_DIR/backups/*
+ do
+ TSTAMP=$(echo $DIR | sed "s|.*/||")
+ if [ "$(ls -A $DIR)" ]
+ then
+ echo -n "Processing: $TSTAMP ... "
+ echo -n "Moving ... "
+ (cp -rfl $DIR/* $DOT_DIR/pack && rm -rf $DIR) || die MV
+ echo -n "Adding ... "
+ (cd $DOT_DIR/pack && git add .) || die ADD
+ echo -n "Committing ... "
+ # Commit only if repository has uncommitted changes
+ (cd $DOT_DIR/pack \
+ && git diff-index --quiet HEAD \
+ || git commit -a -q -m "$TSTAMP" ) || die COMMIT
+ echo "Done."
+ else
+ echo "Removing empty dir $DIR ..."
+ rmdir $DIR
+ fi
+ done
+ echo "Running 'git gc' on pack dir"
+ du -hs $DOT_DIR/pack
+ (cd $DOT_DIR/pack && git gc) || die GC
+ du -hs $DOT_DIR/pack
+ echo "All snapshots packed successfully."
+ else
+ echo "No unpacked backups found ..."
+ fi
+
+}
+
+function on_slow_sync_start {
+ if [ -n "$SLOW_SYNC_START_CMD" ]; then
+ rm -rf "$SLOW_SYNC_FILE"
+ (sleep $SLOW_SYNC_TIME && touch "$SLOW_SYNC_FILE" && eval "$SLOW_SYNC_START_CMD" ; wait) &
+ disown
+ shell_pid=$!
+ fi
+}
+
+function on_slow_sync_stop {
+ if [ -n "$shell_pid" ]; then
+ kill $shell_pid &>/dev/null
+
+ if [[ -n "$SLOW_SYNC_STOP_CMD" && -f "$SLOW_SYNC_FILE" ]]; then
+ (eval "$SLOW_SYNC_STOP_CMD") &
+ fi
+ fi
+}
+
+function cron {
+ DISPLAY=:0.0 sync 2>&1 | timestamp >>"$DOT_DIR/log"
+}
+
+function timestamp {
+ while read data
+ do
+ echo "[$(date +"%D %T")] $data"
+ done
+}
+
+function acquire_lock {
+ if ! mkdir "$LOCK_DIR" 2>/dev/null ; then
+ kill -0 $(cat "$LOCK_DIR/pid") &>/dev/null
+
+ if [[ $? == 0 ]]; then
+ echo "There's already an instance of BitPocket syncing this directory. Exiting."
+ exit 1
+ else
+ echo -e "\x1b\x5b1;31mbitpocket error:\x1b\x5b0m Bitpocket found a stale lock directory:"
+ echo " | Root dir: $(pwd)"
+ echo " | Lock dir: $LOCK_DIR"
+ echo " | Command: LOCK_PATH=$(pwd)/$LOCK_DIR && rm \$LOCK_PATH/pid && rmdir \$LOCK_PATH"
+ echo "Please remove the lock directory and try again."
+ exit 2
+ fi
+ fi
+
+ echo $$ > "$LOCK_DIR/pid"
+}
+
+function release_lock {
+ rm "$LOCK_DIR/pid" &>/dev/null && rmdir "$LOCK_DIR" &>/dev/null
+}
+
+function acquire_remote_lock {
+ $REMOTE_RUNNER "mkdir -p \"$REMOTE_TMP_DIR\"; cd \"$REMOTE_PATH\" && mkdir \"$LOCK_DIR\" 2>/dev/null"
+
+ if [[ $? != 0 ]]; then
+ echo "Couldn't acquire remote lock. Another client is syncing with $REMOTE or lock file couldn't be created. Exiting."
+ release_lock
+ exit 3
+ fi
+}
+
+function release_remote_lock {
+ $REMOTE_RUNNER "cd \"$REMOTE_PATH\" && rmdir \"$LOCK_DIR\" &>/dev/null"
+}
+
+function assert_dotdir {
+ if [ ! -d "$DOT_DIR" ]; then
+ echo "fatal: Not a bitpocket directory. Try 'bitpocket help' for usage."
+ exit 128
+ fi
+ mkdir -p "$TMP_DIR"
+ mkdir -p "$STATE_DIR"
+}
+
+function cleanup {
+ release_lock
+ release_remote_lock
+}
+
+function bring_the_children_let_me_kill_them {
+ if [ -n "$shell_pid" ]; then
+ pkill -P $shell_pid &>/dev/null
+ kill $shell_pid &>/dev/null
+ fi
+}
+
+function die {
+ cleanup
+ bring_the_children_let_me_kill_them
+
+ echo "fatal: command failed $1"
+ exit 128
+}
+
+# List all files in the sync set
+function list {
+ echo -e "\x1b\x5b1;32mbitpocket\x1b\x5b0m will sync the following files:"
+ rsync -av --list-only --exclude "/$DOT_DIR" $USER_RULES . | grep "^-\|^d" \
+ | sed "s:^\S*\s*\S*\s*\S*\s*\S*\s*:/:" | sed "s:^/\.::" | sort
+}
+
+function usage {
+ echo "usage: bitpocket [sync|help|pack|log|cron|list|init <REMOTE_HOST> <REMOTE_PATH>]"
+ echo ""
+ echo "Available commands:"
+ echo " sync Run the sync process. If no command is specified, sync is run by default."
+ echo " init Initialize a new bitpocket folder. Requires remote host and path params."
+ echo " pack Pack any existing (automatic) backups into a git repository."
+ echo " cron Run sync optimized for cron, logging output to file instead of stdout."
+ echo " log Display the log generated by the cron command"
+ echo " list List all files in the sync set (honoring include/exclude/filter config)."
+ echo " help Show this message."
+ echo ""
+ echo "Note: All commands (apart from help), must be run in the root of a"
+ echo " new or existing bitpocket directory structure."
+ echo ""
+
+}
+
+if [ "$1" = "init" ]; then
+ # Initialize bitpocket directory
+ init $2 $3 $4
+elif [ "$1" = "pack" ]; then
+ # Pack backups using git
+ pack
+elif [ "$1" = "log" ]; then
+ # Display log file
+ log
+elif [ "$1" = "cron" ]; then
+ # Run through cron?
+ cron
+elif [ "$1" = "list" ]; then
+ # List all file in sync set (honoring .bitpocket/include & .bitpocket/exclude)
+ list
+elif [ "$1" != "" ] && [ "$1" != "sync" ]; then
+ # Show help
+ usage
+else
+ # By default, run the sync process
+ sync
+fi

© 2014-2022 Faster IT GmbH | imprint | privacy policy