Rsync 同步文件 client&server 配置

#!/bin/bash

BASE_DIR=/data/rsync-data
SCRIPT_DIR=$BASE_DIR/script
DATA_DIR=$BASE_DIR/data

JOB_NAME="rsync_data"_"$1"

# max run 1 day
MAX_RUN_TIME=86400
if [[ $# > 1 ]];then
    MAX_RUN_TIME=$[ $2 * 60 ]
fi

#function to create file lock to indicate that the job is now currently running
function set_file_lock() {
    JOB_NAME=$1
    echo "lock name: " $JOB_NAME
    if [ -f $SCRIPT_DIR/$JOB_NAME.lock ]; then
        local epgid=`cat $SCRIPT_DIR/$JOB_NAME.lock`
        # Check the epgid empty or not
        # Generate one flag file if empty to alarm and exit current job
        if [ ! -n $epgid ]; then
          echo "[$(date)] [CHECK]PID_EMPTY_ERROR. Exit $JOB_NAME and remove lock file"
          remove_file_lock $JOB_NAME
          exit 3
        fi
        if [ "`ps aux | awk '$2=='$epgid' {print "RUNNING"}'`" == "RUNNING" ]; then
            local modify_time=`stat --format "%Z" $SCRIPT_DIR/$JOB_NAME.lock`
            local time_now=`date +"%s"`
            local run_time=$(($time_now - $modify_time))
            if [ $run_time -ge  $MAX_RUN_TIME ];then
                echo "[$(date)] last $JOB_NAME exists too long: $run_time, kill it."
                # kill all processes in the group
                kill -9 -- -$epgid
            else
                echo "[$(date)] $JOB_NAME is now running, will not start a new job."
                exit 2
            fi
        fi
    fi
    # Get process group id and save it.
    # crontab will fork a shell, and run rsync.
    # if we want to control rsync process, we should get rsync pid or pgid.
    # 2019-03-25: Add retry mechanism to avoid empty pid or pgid
    is_success=0
    for ((i=1;i<=10;i++)) do
      pgid=$(ps -p $$ -o pgid | sed -n '2p')
      echo -n $pgid > $SCRIPT_DIR/$JOB_NAME.lock
      local pid=`cat $SCRIPT_DIR/$JOB_NAME.lock`
      if [ -n $pid ]; then
        is_success=1
        break
      else
        sleep 1
        echo "The pid of [$(date)] $JOB_NAME is empty. Retry for $i times"
      fi
    done
    # Fail to generate pid file
    if [ ${is_success} -eq 0 ]; then
      echo "[$(date)] [GENERATE]PID_EMPTY_ERROR. Exit $JOB_NAME and remove lock file"
      remove_file_lock $JOB_NAME
      exit 3
    fi
}

#function to remove file lock to indicate that the job is now restartable
function remove_file_lock() {
    JOB_NAME=$1
    echo "lock name: " $JOB_NAME
    if [ -f $SCRIPT_DIR/$JOB_NAME.lock ]; then
        echo "remove the lock file: $JOB_NAME.lock."
        rm $SCRIPT_DIR/$JOB_NAME.lock
    fi
}

function get_rsync_server_host() {
    server_host=10.128.243.124
    echo ${server_host}
}

set_file_lock $JOB_NAME

echo "==============[Time now is: `date`] $JOB_NAME job started. =============="

# default rsync params
REMOTE_FILE_NAME=""
LOCAL_FILE_NAME=""
# rsyncd.passwd内容为server端rsyncd.passwd中配置的密码，注意要去掉"user:"，权限同样设为600
RSYNC_PASSWORD_FILE="/etc/rsyncd.passwd"
RSYNC_ACCOUNT=""
RSYNC_PORT="888"
RSYNC_REMOTE_PATH=""
# decide which file to rsync
if [ "$1" == "base.data" ]; then
    RSYNC_REMOTE_PATH="$(get_rsync_server_host)::testmodule"
    RSYNC_ACCOUNT="testuser"
    REMOTE_FILE_NAME="base.data"
    LOCAL_FILE_NAME="base.data"
elif [ "$1" == "base2.data" ]; then
    RSYNC_REMOTE_PATH="$(get_rsync_server_host)::testmodule"
    RSYNC_ACCOUNT="testuser"
    REMOTE_FILE_NAME="base2.data"
    LOCAL_FILE_NAME="base2.data"
else
    echo "wrong args provided: "$1
    remove_file_lock $JOB_NAME
    exit 1
fi

# remove temp file (last downloading), the rsync process may be killed by function set_file_lock.
# file data will not be removed if the rsync process is alive, so it is safe.
rm -vf ${DATA_DIR}/.${LOCAL_FILE_NAME}.*

START_TIME=$(date)
# Almost the whole file changes, so use -W option to disable delta-rsync, just rsync the whole file.
# This will reduce cpu consumption. We also make rsync lower priority of process schedule using nice.
# We may also remove -z option to disable compression for reducing more cpu consumption if needed.
echo "rsync -avzP -W --partial --partial-dir=${DATA_DIR}/rsync_partial/ --password-file=${RSYNC_PASSWORD_FILE} ${RSYNC_ACCOUNT}@${RSYNC_REMOTE_PATH}/${REMOTE_FILE_NAME} ${DATA_DIR}/${LOCAL_FILE_NAME}"
/bin/nice -n19 rsync -avzP -W --port=${RSYNC_PORT} --exclude='.*' --partial --partial-dir=${DATA_DIR}/rsync_partial/ --password-file=${RSYNC_PASSWORD_FILE} ${RSYNC_ACCOUNT}@${RSYNC_REMOTE_PATH}/${REMOTE_FILE_NAME} ${DATA_DIR}/${LOCAL_FILE_NAME}

remove_file_lock $JOB_NAME

echo "==============[Start Time: $START_TIME, End Time: `date`] $JOB_NAME job finished, now quit. ===================="
echo
echo