Skip to content

Instantly share code, notes, and snippets.

@evil5hadow

evil5hadow/personal-backup.sh

Forked from erincerys/personal-backup.sh
Created August 6, 2018 13:49
Show Gist options
  • Save evil5hadow/1112f652c87f2edf3c5a59fb0b0cb3d8 to your computer and use it in GitHub Desktop.
Save evil5hadow/1112f652c87f2edf3c5a59fb0b0cb3d8 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. @erincerys erincerys revised this gist Nov 22, 2017. 1 changed file with 6 additions and 0 deletions.
    6 changes: 6 additions & 0 deletions personal-backup.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -190,6 +190,12 @@ ContainerChecksumFile="${ScriptWorkingPath}/${ContainerMapperName}.sha256sum"
    [ -f ] ; mv $ContainerChecksumFile ${ContainerChecksumFile}.old
    echo $ContainerChecksum > $ContainerChecksumFile

    # if the container hash hasnt changed since last time, dont sync anything
    if [[ -e ${ContainerChecksumFile} && "$(cat ${ContainerChecksumFile}.old)" == "$ContainerChecksum" ]] ; then
    echo "No changes since last sync!"
    exit 0
    fi

    # Sync working directory to S3
    # Effectively copies file lists but NOT container or logs
    AwsResponse=`aws s3 sync --exclude "*.log" --exclude "$ContainerFileName" --size-only $AwsCliOptions $ScriptWorkingPath s3://${S3BucketName}/${S3Prefix}/ 2>&1`
  2. @erincerys erincerys revised this gist Oct 25, 2017. 1 changed file with 76 additions and 37 deletions.
    113 changes: 76 additions & 37 deletions personal-backup.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -1,44 +1,68 @@
    #!/bin/bash
    #!/bin/bash

    # Dependencies:
    # - cryptsetup (and a LUKS container already created)
    # - rsync
    # - aws-cli
    ## Description:
    # Sync files to and create lists of directory contents in a LUKS container volume and upload it to S3
    # Good for periodic backup jobs
    # Supports rate limiting, encryption in transit and at rest and file path exclusions

    ## Usage:
    # bash $0

    ## Dependencies

    # Packages:
    # - awscli
    # - s3cmd
    # - cryptsetup
    # - gawk
    # - sha256sum
    # - grep
    # - rsync

    # Setup:
    # - luks file container
    # - configuration parameters below set properly
    # - if this will be used in cron, youll need to set environment variables for awscli in the crontab

    # WARNING:
    # If you experience bitrot or otherwise encounter corruption, these is no detecting of this.
    # You will overwrite the good data with the bad
    # This can be mitigated by using versioning, either implemented through this script or enabled in the S3 bucket.

    ## ---

    ## Configuration

    Debug=0

    ScriptWorkingPath='/path/to/$0' # without filename
    ContainerFile="${ScriptWorkingPath}/cloudsync.img"
    # encrypted container
    ScriptWorkingPath='/media/data/backups/syncops'
    ContainerFileName='cloudsync.img'
    ContainerFile="${ScriptWorkingPath}/${ContainerFileName}"
    ContainerMapperName='cloudsync'
    ContainerMountPath='/media/tmpcrypt'
    ContainerKeyFile='/path/to/cloudsync.pem'
    ContainerKeyFile='/media/data/personal/keyfiles/cloudsync.pem'

    # Escape slashes and use globs
    ExcludedPaths=( \
    "tech\/virtualmachines\/*" \
    "tech\/caches\/*" \
    "dev\/web\/packages\/*" \
    'tech/virtualmachines' \
    'tech/caches' \
    )

    # Do not leave trailing slash on the end of directory sources
    SourcePaths=( \
    '/media/raid/backups/configuration' \
    '/media/raid/backups/drive-order-in-case.txt' \
    '/media/raid/misc/papers' \
    '/media/raid/misc/research' \
    '/media/raid/personal/text' \
    '/media/raid/personal/resumes' \
    '/media/raid/personal/keyfiles' \
    '/media/raid/personal/finances' \
    '/media/raid/games/strategy' \
    '/media/raid/games/saves' \
    '/media/raid/tech' \
    '/home/ian/scripts' \
    )

    DestinationPath="${ContainerMountPath}/"
    @@ -49,24 +73,35 @@ FileListSources=( \
    '/media/data/backups' \
    '/media/data/misc/applications' \
    '/media/data/misc/ebooks' \
    '/media/data/misc/lingual_studies' \
    '/media/data/music' \
    )

    AwsProfile='personal'
    # SNS ARN for notifications
    SnsTopic='arn:aws:sns:us-west-1:183912708525:Ian'

    # awscli / s3cmd
    AwsProfile='default'
    S3CmdConfigFile='/home/ian/.s3cfg'
    AwsConfigFile='/home/ian/.aws/config'
    AwsCredentialFile='/home/ian/.aws/credentials'
    AwsRegion='us-west-1'
    ThrottleLimit=500 # how fast to upload the file to s3 in kilobytes

    # s3 paths
    S3BucketName='ians-backups'
    S3Prefix='personal-workstation'
    S3Options='--storage-class STANDARD_IA'

    ## END CONFIGURATION

    # set awscli and s3cmd arguments given configuration
    AwsCliOptions="--storage-class=STANDARD_IA --profile=${AwsProfile} --region=${AwsRegion}"
    S3CmdOptions="--storage-class=STANDARD_IA --region=${AwsRegion} --config=${S3CmdConfigFile} --limit-rate=${ThrottleLimit}k"
    if [ $Debug -eq 1 ] ; then
    S3Options="${S3Options} --debug"
    AwsCliOptions="${AwsCliOptions} --debug"
    S3CmdOptions="${S3CmdOptions} --debug"
    #else
    # S3Options="${S3Options} --only-show-errors"
    # AwsCliOptions="${AwsCliOptions} --only-show-errors"
    fi
    S3Options="${S3Options} --profile ${AwsProfile}"
    AwsRegion='us-west-1'
    SnsTopic='arn:aws:sns:us-west-1:*:*'

    ## Stuf begins

    @@ -78,13 +113,13 @@ export AWS_SHARED_CREDENTIALS_FILE=$AwsCredentialFile
    function SnsPublication () {
    local ErrorCode=$1

    if [ $ErrorCode -gt 0 ] ; then
    if [ $ErrorCode -gt 0 ] ; then
    local ScriptResultCode='Failure'
    else
    local ScriptResultCode='Success'
    local ScriptPostMessage="Successfully synchronized file changes to S3. See $LogFile for incremental changes"
    fi

    if [ $ErrorCode -eq 1 ] ; then
    local ScriptPostMessage="Failed to complete rsync operation."
    elif [ $ErrorCode -eq 2 ] ; then
    @@ -152,35 +187,39 @@ done
    ContainerFileList=`ls -Rla $ContainerMountPath`
    ContainerChecksum=`echo $ContainerFileList | sha256sum | awk '{ print $1 }'`
    ContainerChecksumFile="${ScriptWorkingPath}/${ContainerMapperName}.sha256sum"
    [ -f ] ; mv $ContainerChecksumFile ${ContainerChecksumFile}.old
    [ -f ] ; mv $ContainerChecksumFile ${ContainerChecksumFile}.old
    echo $ContainerChecksum > $ContainerChecksumFile

    # Sync working directory to S3
    # Effectively copies file lists, logs, but NOT container
    AwsResponse=`aws s3 sync --region $AwsRegion --size-only $S3Options $ScriptWorkingPath s3://${S3BucketName}/${S3Prefix}/ 2>&1`
    # Effectively copies file lists but NOT container or logs
    AwsResponse=`aws s3 sync --exclude "*.log" --exclude "$ContainerFileName" --size-only $AwsCliOptions $ScriptWorkingPath s3://${S3BucketName}/${S3Prefix}/ 2>&1`
    AwsReturnCode=$?
    echo $AwsResponse >> $LogFile
    [ $AwsReturnCode -ne 0 ] && { echo 'Error copying file lists to S3!' >> $LogFile ; Errors=1 ; }

    # Sync logfile
    AwsResponse=`aws s3 cp $AwsCliOptions $LogFile s3://${S3BucketName}/${S3Prefix}/ 2>&1`
    AwsReturnCode=$?
    echo $AwsResponse >> $LogFile
    [ $AwsReturnCode -ne 0 ] && echo 'Error copying file lists or logs to S3!' >> $LogFile
    [ $AwsReturnCode -ne 0 ] && { echo 'Error copying log file to S3!' >> $LogFile ; Errors=1 ; }

    # Copy container, if necessary
    if [ "$(cat ${ContainerChecksumFile}.old)" != "$ContainerChecksum" ] ; then
    if [[ ! -e ${ContainerChecksumFile} || "$(cat ${ContainerChecksumFile}.old)" != "$ContainerChecksum" ]] ; then
    AwsTempLog=`mktemp`
    aws s3 cp --region $AwsRegion $S3Options ${ScriptWorkingPath}/${ContainerMapperName}.img s3://${S3BucketName}/${S3Prefix}/ 2>&1 > $AwsTempLog | throttle -K 350
    s3cmd $S3CmdOptions put ${ScriptWorkingPath}/${ContainerFileName} s3://${S3BucketName}/${S3Prefix}/ 2>&1 > $AwsTempLog
    AwsReturnCode=$?
    echo $AwsResponse >> $LogFile
    [ $AwsReturnCode -ne 0 ] && echo 'Error copying container to S3!' >> $LogFile
    fi
    [ $AwsReturnCode -ne 0 ] && { echo 'Error copying container to S3!' >> $LogFile ; Errors=1 ; }

    # Catch s3 errors
    if [ `grep -Pic "(Error copying|BadDigest|Client error|stacktrace|unable|does not match MD5 checksum|exit status is '1')" $AwsTempLog` -gt 0 ] ; then
    echo "There were errors copying to S3. Review the log at ${LogFile}"
    SnsPublication 2
    CloseContainer
    exit 1
    # Catch s3 errors
    if [ `grep -Pic "(Error copying|BadDigest|Client error|stacktrace|unable|does not match MD5 checksum|exit status is '1')" $AwsTempLog` -gt 0 ] ; then
    echo "There were errors copying to S3. Review the log at ${LogFile}"
    SnsPublication 2
    fi
    fi

    # We got this far, so things must be OK, right?
    SnsPublication 0
    CloseContainer
    [ "$Errors" == 1 ] && exit 1 || SnsPublication 0

    echo 'All done!'
    exit 0
  3. @erincerys erincerys revised this gist Jun 26, 2017. 1 changed file with 1 addition and 0 deletions.
    1 change: 1 addition & 0 deletions personal-backup.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -2,6 +2,7 @@

    # Dependencies:
    # - cryptsetup (and a LUKS container already created)
    # - rsync
    # - aws-cli
    # - gawk
    # - grep
  4. @erincerys erincerys created this gist Jun 26, 2017.
    185 changes: 185 additions & 0 deletions personal-backup.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,185 @@
    #!/bin/bash

    # Dependencies:
    # - cryptsetup (and a LUKS container already created)
    # - aws-cli
    # - gawk
    # - grep

    # WARNING:
    # If you experience bitrot or otherwise encounter corruption, these is no detecting of this.
    # You will overwrite the good data with the bad
    # This can be mitigated by using versioning, either implemented through this script or enabled in the S3 bucket.

    ## Configuration

    Debug=0

    ScriptWorkingPath='/path/to/$0' # without filename
    ContainerFile="${ScriptWorkingPath}/cloudsync.img"
    ContainerMapperName='cloudsync'
    ContainerMountPath='/media/tmpcrypt'
    ContainerKeyFile='/path/to/cloudsync.pem'

    # Escape slashes and use globs
    ExcludedPaths=( \
    "tech\/virtualmachines\/*" \
    "tech\/caches\/*" \
    "dev\/web\/packages\/*" \
    )

    # Do not leave trailing slash on the end of directory sources
    SourcePaths=( \
    '/media/raid/backups/configuration' \
    '/media/raid/misc/papers' \
    '/media/raid/misc/research' \
    '/media/raid/personal/text' \
    '/media/raid/personal/resumes' \
    '/media/raid/games/strategy' \
    '/media/raid/games/saves' \
    '/media/raid/tech' \
    )

    DestinationPath="${ContainerMountPath}/"
    LogFile="${ScriptWorkingPath}/`date '+%Y%m%d'`.log"

    # Pathes to create file lists for the contents of to be included in backup
    FileListSources=( \
    '/media/data/backups' \
    '/media/data/misc/applications' \
    '/media/data/misc/ebooks' \
    '/media/data/misc/lingual_studies' \
    '/media/data/music' \
    )

    AwsProfile='personal'
    AwsConfigFile='/home/ian/.aws/config'
    AwsCredentialFile='/home/ian/.aws/credentials'
    S3BucketName='ians-backups'
    S3Prefix='personal-workstation'
    S3Options='--storage-class STANDARD_IA'
    if [ $Debug -eq 1 ] ; then
    S3Options="${S3Options} --debug"
    #else
    # S3Options="${S3Options} --only-show-errors"
    fi
    S3Options="${S3Options} --profile ${AwsProfile}"
    AwsRegion='us-west-1'
    SnsTopic='arn:aws:sns:us-west-1:*:*'

    ## Stuf begins

    # Configure environment vars for Aws CLI
    #export AWS_DEFAULT_PROFILE=$AwsProfile
    export AWS_CONFIG_FILE=$AwsConfigFile
    export AWS_SHARED_CREDENTIALS_FILE=$AwsCredentialFile

    function SnsPublication () {
    local ErrorCode=$1

    if [ $ErrorCode -gt 0 ] ; then
    local ScriptResultCode='Failure'
    else
    local ScriptResultCode='Success'
    local ScriptPostMessage="Successfully synchronized file changes to S3. See $LogFile for incremental changes"
    fi

    if [ $ErrorCode -eq 1 ] ; then
    local ScriptPostMessage="Failed to complete rsync operation."
    elif [ $ErrorCode -eq 2 ] ; then
    local ScriptPostMessage="Failed to copy to S3."
    fi

    # Publish to SNS topic to notify admin
    aws sns publish \
    --region $AwsRegion \
    --topic-arn $SnsTopic \
    --subject "Workstation backup job notification ($ScriptResultCode)" \
    --message "$ScriptPostMessage" \
    2>&1 >> $LogFile
    }

    function CloseContainer () {
    # Immediately flush pending cache writes to disk
    sync -f ${ContainerMountPath}

    # Unmount container
    umount $ContainerMountPath

    # Close container
    cryptsetup luksClose $ContainerMapperName
    }

    # Open the crypted container
    cryptsetup --key-file $ContainerKeyFile luksOpen $ContainerFile $ContainerMapperName

    # Mount it
    mount /dev/mapper/$ContainerMapperName $ContainerMountPath

    # Assemble the one-liner
    CommandPrefix="rsync --archive --delete --checksum --verbose --log-file=${LogFile}"
    if [ $Debug -eq 1 ] ; then
    CommandPrefix="${CommandPrefix} --msgs2stderr --debug=ALL"
    fi
    for e in "${ExcludedPaths[@]}" ; do
    CommandExclusions="${CommandExclusions} --exclude=$e"
    done
    for i in "${SourcePaths[@]}" ; do
    CommandSources="${CommandSources} $i"
    done
    Command="$CommandPrefix $CommandExclusions $CommandSources $DestinationPath"

    # rsync
    RsyncTempLog=`mktemp`
    $Command 2>&1 > $RsyncTempLog
    if [[ `grep -ic error $RsyncTempLog` -gt 0 ]] ; then
    echo "There was an error. Check $LogFile for more info. Response is below."
    echo $RsyncResponse
    SnsPublication 1
    CloseContainer
    exit 1
    fi
    cat $RsyncTempLog >> $LogFile

    # Create file list for each source
    for fl in "${FileListSources[@]}" ; do
    FlName=`echo $fl | grep -Po '[a-zA-Z_]+$'`
    ls -Rla $fl 2>&1 > "${ScriptWorkingPath}/${FlName}.filelist"
    done

    # Get hash of container contents
    ContainerFileList=`ls -Rla $ContainerMountPath`
    ContainerChecksum=`echo $ContainerFileList | sha256sum | awk '{ print $1 }'`
    ContainerChecksumFile="${ScriptWorkingPath}/${ContainerMapperName}.sha256sum"
    [ -f ] ; mv $ContainerChecksumFile ${ContainerChecksumFile}.old
    echo $ContainerChecksum > $ContainerChecksumFile

    # Sync working directory to S3
    # Effectively copies file lists, logs, but NOT container
    AwsResponse=`aws s3 sync --region $AwsRegion --size-only $S3Options $ScriptWorkingPath s3://${S3BucketName}/${S3Prefix}/ 2>&1`
    AwsReturnCode=$?
    echo $AwsResponse >> $LogFile
    [ $AwsReturnCode -ne 0 ] && echo 'Error copying file lists or logs to S3!' >> $LogFile

    # Copy container, if necessary
    if [ "$(cat ${ContainerChecksumFile}.old)" != "$ContainerChecksum" ] ; then
    AwsTempLog=`mktemp`
    aws s3 cp --region $AwsRegion $S3Options ${ScriptWorkingPath}/${ContainerMapperName}.img s3://${S3BucketName}/${S3Prefix}/ 2>&1 > $AwsTempLog | throttle -K 350
    AwsReturnCode=$?
    echo $AwsResponse >> $LogFile
    [ $AwsReturnCode -ne 0 ] && echo 'Error copying container to S3!' >> $LogFile
    fi

    # Catch s3 errors
    if [ `grep -Pic "(Error copying|BadDigest|Client error|stacktrace|unable|does not match MD5 checksum|exit status is '1')" $AwsTempLog` -gt 0 ] ; then
    echo "There were errors copying to S3. Review the log at ${LogFile}"
    SnsPublication 2
    CloseContainer
    exit 1
    fi

    # We got this far, so things must be OK, right?
    SnsPublication 0
    CloseContainer
    echo 'All done!'
    exit 0