#!/bin/sh
#
# Copyright (c) 2002, 2023 Nick Holland <nick@holland-consulting.net>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
# Additional (non-license) usage notes:
# Really, using this code "as is" without carefully examining the assumptions
# made by me about your needs is highly discouraged.  
#
# I would love to hear if you are using this in your world in any way
# (even if as a counter example!).  Drop me an email and tell me about it
# at nick@holland-consulting.net

# Change history:
# 2023-10-21 -- added .latest symlink
#    fix a few bugs
#    found checking rc of "ssh rsync" was not reliable on some old
#      solaris systems.  Looking at the data returned is more reliable.
#      Better POSIX compatibility with read usage.

# Cheap and easy error checking.  Use if something shouldn't fail, but might
# and you don't feel like real error handling.
yell() { echo "$0: $*" >&2; }
die() { yell "$*"; rm -f /tmp/$$.*; exit 111; }
try() { "$@" || die "cannot $*"; }

umask 022 # sets permissions for backup directories.  I like avoiding use
          # of root when possible.  Tighter might be appropriate.

# CONFIGDIR is a directory holding various IBS configuration
# and data files.  $CONFIG is sourced to override any of these
# settings.
# Types of files in $CONFIGDIR:
#   ibs.config: sets operational variables.
#   *.bufilter:  Backup list, one  for each system
#   *.list: lists of systems to backup
CONFIGDIR=/etc/ibs
CONFIG=$CONFIGDIR/ibs.config

MEG=1048576 # bytes in a MB.  Used to turn big numbers into not-so-big numbers.

COMMANDLINE="$0 $*" # save for end of report line.

if [[ "$1" == "-template" ]]; then
    cat <<- __ENDCONFIGTEMPLATE >$(basename $CONFIG)
	# IBSBASE is where the backups appear to be -- may be a bunch
	# of symlinks to other places.
	IBSBASE=/ibs
	
	# STORAGERE is a RegEx for matching the location where physical
	# storage of backups is stored.
	STORAGERE="^/v/ibs[0-9]"
	
	# LOGS is where the log files are (surprise!).  Putting them
	# in IBSBASE may not be wise, but it is handy.
	# LOGS=/var/dbulogs
	LOGS=/ibs/z-logs

	# MINDAILY is minimum number of daily backups,
	# MINME is minimum number of ME backups in the rotation
	# If this number does not exist in the backup target directory
	# that backup won't be run.
	# DEFAULTEXTRA is how many more ME and daily backups will be created
	# by the prep process.
	MINDAILY=5
	MINME=3
	DEFAULTEXTRA=2

	# rsync options
	#   Recommended:
	#       -H : preserve hard links (Expensive, but worth it)
	#       -a : General archive (-rlptgoD)
	#       --stats : required for reporting
	#       --verbose : required for File Alteration Reporting Tool
	#       --force : Important if a directory becomes a file or file becomes a directory
	#	--numeric-ids : since your user base on the backup server will be different than
	#	   that on the backed-up systems, this will retain the numeric ids of source files.
	#   Consider:
	#       -W : Whole files, disable delta transfer.  Often works better
	#       -z : Compression of files during transfer.  May help, may hurt.
	#       -X : Extended attributes. Probably won't work between different OSs.
	#       --rsh=ssh : this is the default, probably not needed.
	RSOPTS="-HWza --rsh=ssh --stats --force --verbose --numeric-ids"

	# Maximum number of rsyncs to have running at any time.
	# If more than this are running when a new job is to start, hold off
	# starting the next backup until below this number.  Each "bu"
	# will have at least two rsyncs associated with it.
	MAXRSYNCS=10

	# Delay between starts, in seconds.
	DELAY=20

	__ENDCONFIGTEMPLATE
    echo "Template ibs.config file has been generated in current directory."
    echo "Populate it, and copy it to $CONFIG"
    exit
fi

if [[ -f $CONFIG ]]; then
    . $CONFIG
else
    echo "No config file found.  Run "ibs -template" to generate a template file"
    echo "in the current directory.  Populate and place in $CONFIGDIR"
    exit
fi

# Date format has to easily sort by date.  Change at your own risk.
TODAY=$(date "+%Y-%m-%d")
DATEFMT="2[0-9][0-9][0-9]-[01][0-9]-[0-3][0-9]"
MEFMT="$DATEFMT-ME"

# Variable check.  Files are checked for existence, other things are checked
# just for populated variables.  Could be improved, but hopefully will be rarely
# an issue.
if [[ ! -d $IBSBASE ]]; then
    echo "  Error! IBSBASE variable set to '$IBSBASE' does not exist.  Exiting"
    exit
fi

if [[ -z $STORAGERE ]]; then
    echo "  STOREAGERE not set"
    exit
fi

if [[ ! -d $LOGS ]]; then
    echo "  Error!  LOGS variable set to '$LOGS';  directory doesn't exist.  Exiting"
    exit
fi

if [[ -z $MINDAILY ]]; then
    echo "  MINDAILY not set"
    exit
fi

if [[ -z $MINME ]]; then
    echo "  MINME not set"
    exit
fi

if [[ -z $DEFAULTEXTRA ]]; then
    echo "  DEFAULTEXTRA not set"
    exit
fi

if [[ -z $RSOPTS ]]; then
    echo "  MINDAILY not set"
    exit
fi

if [[ -z $MAXRSYNCS ]]; then
    echo "  STOREAGERE not set"
    exit
fi

if [[ -z $DELAY ]]; then
    echo "  DELAY not set"
    exit
fi


function preflightcheck {
## Checks various failure points before launching an individual backup.
## Assumption here is the basic IBS config is good, a failure here should
## not be fatal to the entire backup process.
    typeset TARGET DCOUNT MECOUNT
    TARGET=$1

    if [[ -n $VERBOSE ]]; then
	echo
	echo "IBSBASE=$IBSBASE  LOGS=$LOGS  $TODAY "
	echo "TARGET=$TARGET"
    fi

    if [[ -z $TARGET ]]; then
	echo "Preflightcheck called without parameter.  oops"
	return 100 # exit? tbd.
    fi

    if [[ ! -d $IBSBASE/$TARGET ]]; then
	echo "  $IBSBASE/$TARGET does not exist.  Skipping"
	return 10
    fi

    # Note this catches DNS issues, host file issues, SSH issues, key issues.
    # in a commercial product, you would want to narrow down the problem a
    # lot more.
    ssh $TARGET rsync --version 2>/dev/null|grep -qi "protocol version" 
    rc=$?
    if [[ $rc != 0 ]]; then
	echo "  Error $TARGET : can't run rsync via ssh to $TARGET. Skipping.  rc=$rc"
	return 11
    fi

    # Test for ME count
    MECOUNT=$(ls -1 $IBSBASE/$TARGET|grep "^$MEFMT$" |wc -w)
    if [[ $MECOUNT -lt $MINME ]]; then
    #if [[ $(ls -1d $IBSBASE/$TARGET/$MEFMT|wc -w) -lt $MINME ]]; then
	echo "  Error $TARGET : too few Month End backups ($MECOUNT), skipping"
        return 12
    fi

    # Test daily backup count
    DCOUNT=$(ls -1 $IBSBASE/$TARGET | grep "^$DATEFMT$" |wc -w)
    if [[ $DCOUNT -lt $MINDAILY ]]; then
	echo "  Error $TARGET : too few daily backups ($DCOUNT), skipping."
	return 13
    fi
    if [[ -n $VERBOSE ]]; then
	echo "Found $DCOUNT daily backups and"
	echo "      $MECOUNT ME backups in rotation"
    fi

} # end preflightcheck

function prep {
## Build a backup directory, link to $IBSBASE, prepopulate with a set of
## daily and monthly backups.
    typeset -l N TARGET TARGDIR # The -l (lowercase) is highly subjective.
    typeset -l YN
    TARGDIR=$1
    TARGET=$(basename $TARGDIR)

    if ! echo $TARGDIR |egrep -q "$STORAGERE"; then
	echo "  Error: must specify absolute path of actual storage location"
	return 10
    fi

    if [[ -z $1 ]]; then
	echo "  Error: must specify backup directory with hostname, i.e.:"
	echo "   /v/dbu1/fluffy"
	return 11
    fi

    if [[ ! -d $(dirname $TARGDIR) ]]; then
	echo "  Error: target volume $(dirname $TARGDIR) doesn't exist."
        return 12
    fi

    if ! ssh -oConnectTimeout=2 $TARGET rsync --version >/dev/null; then
	echo
	echo -n "Unable to run rsync on $TARGET;  Continue? ->"
	read YN
	if [[ $YN != y* ]]; then
	    echo "Aborting prep"
	    return 13
	fi
    fi

    try mkdir -pm 755 $TARGDIR

    try ln -sf $TARGDIR /$IBSBASE/$TARGET

    for N in $(seq 1 $(($MINDAILY + $DEFAULTEXTRA)) ); do
	mkdir -m755 -p $TARGDIR/2001-00-$(printf "%02d" $N)
    done

    for N in $(seq 1 $(($MINME + $DEFAULTEXTRA)) ); do
	mkdir -m755 -p $TARGDIR/2000-00-$(printf "%02d" $N)-ME
    done

    if [[ -n $2 ]]; then # cheezy recursion if more than one to prep
	shift
	prep $*
    fi
    return 0

}
    
function addibsdir {
    TARGDIR=$IBSBASE/$1
    if [[ ! -d $TARGDIR ]]; then
	echo "Invalid hostname $1"
	return 1
    fi
    for DAY in $(seq 1 99); do
	NEWDAY=$TARGDIR/2000-00-$(printf "%02d" $DAY)$MONTHLY
	if [[ ! -d $NEWDAY ]]; then
	    mkdir $NEWDAY
	    echo "Added $NEWDAY"
	    return
	fi
    done
    echo "Could not add a new backup to the rotation for $1"
    return 2
}

function buhost {
## Run a backup on one host, log to a file.
    typeset TARGET MONTHLY HOSTDIR PREVIOUS OLDEST SKIPDELETE BUFILTER LOGFILE
    TARGET=$1
    HOSTDIR=$IBSBASE/$TARGET
    LOGFILE=$LOGS/$TARGET-$TODAY

    if ! preflightcheck $TARGET; then
	echo "  $TARGET failed preflight check" 
	return
    fi

    # Month-end check.  Could look for last-day of the month,
    # but this is easier, both in code and in cron.
    DAY=$(date +%e)
    if [[ $DAY -le 1 ]]; then # this is a month-end check.
	MONTHLY="-ME"
	echo "Will save as a Month-End (ME) backup" >>$LOGFILE
    else
	MONTHLY=""
    fi

    OLDEST=$(ls -d1 $HOSTDIR/$DATEFMT$MONTHLY|head -1)

    if ! echo $OLDEST|grep -q "^$IBSBASE/$TARGET/$DATEFMT"; then
	echo "something went horribly wrong with the deletion process. aborting!"|tee -a $LOGFILE
	exit
    fi

    if [[ -d $HOSTDIR/$TODAY$MONTHLY ]]; then
	SKIPDELETE="y" # directory already existed, this is a re-run
    else
	try mkdir $HOSTDIR/$TODAY$MONTHLY 
	rm -f $HOSTDIR/.latest
	ln -sf $HOSTDIR/$TODAY$MONTHLY $HOSTDIR/.latest
    fi

    PREVIOUS=$(ls -d1 $HOSTDIR/$DATEFMT*| tail -2 | head -1)
    test -z "$PREVIOUS" && exit


    # Build a backup list.  Here, we are using static files.
    # Could be a dynamcially computed filter, though.
    if [[ -f $CONFIGDIR/$TARGET.bufilter ]]; then
	BUFILTER=$CONFIGDIR/$TARGET.bufilter
    else
	BUFILTER=$CONFIGDIR/default.bufilter
    fi

    # Did we get a good backup list file?
    if [[ ! -f $BUFILTER ]]; then
	echo "$BUFILTER doesn't exist, skipping" |tee -a $LOGFILE
	return
    fi
    echo "= Backing up $TARGET to $HOSTDIR/$TODAY$MONTHLY using $(basename $BUFILTER)"

    echo $TARGET $HOSTDIR $PREVIOUS $TODAY$MONTHLY >>$LOGFILE
    echo "==== $BUFILTER" >>$LOGFILE
    cat $BUFILTER >>$LOGFILE

    # Delete first. Hopefully at this point, the backup will work.
    # if this is not a good assumption, deletion code should be AFTER
    # a good backup.
    if [[ -z $SKIPDELETE ]]; then
	echo "Deleting $OLDEST" >>$LOGFILE
	rm -rf $OLDEST &
    fi
    rsync $RSOPTS --filter="merge $BUFILTER" --link-dest=$PREVIOUS $TARGET:/ $HOSTDIR/$TODAY$MONTHLY/ >>$LOGFILE 2>&1

    echo "==== BACKUP COMPLETE rc=$? ====" >>$LOGFILE
    }


function reportheader {
## Something to report on the RAID and disk status of the backup server
## This will have to be reimplemented for whatever OS it is running 
## 
## This ends up producing an output that looks like this:
##    Volume      Status               Size Device  
##    softraid0 0 Online      4000786726912 sd3     RAID1 
##              0 Online      4000786726912 0:0.0   noencl <sd0a>
##              1 Online      4000786726912 0:1.0   noencl <sd1a>
##    softraid0 1 Online      8001562918912 sd4     CRYPTO
##              0 Online      8001562918912 1:0.0   noencl <sd2p>
##    
##    Disk Space: /v/1 /v/3
##    Mounted on              Size     Used    Avail Capacity
##    /v/1                    2.1T     1.8T     198G   90%
##    /v/3                    7.2T     5.5T     1.4T   80%
##     System             MostRecent   Oldest   BUs    ME-Recent ME-Oldest  MEs  TotSize   IncSize     vol  rc
##    =======================================================================================================
## This will vary based on your needs and the options of the OS you are
## running it on.
    typeset BLOCKS USED FREE PUSED MOUNTEDON
    bioctl softraid0
    echo

    echo Disk Space:  $STORAGERE
    df -h $(mount |cut -f3 -d' '|grep "$STORAGERE") |\
    while read MOUNT BLOCKS USED FREE PUSED MOUNTEDON; do # make a header.
	if [[ $mount = "Filesystem" ]]; then
	    BLOCKS="Size GB"
	    USED="Used"
	    FREE="Free"
	    PUSED="%Used"
	    MOUNTEDON="Mounted"
	fi
	printf "%-19s %8s %8s %8s %5s\n" "$MOUNTEDON" "$BLOCKS" "$USED" "$FREE" "$PUSED"
    done
    echo
    echo "Last and finished time:"
    # this may require tuning.  OpenBSD -c35-, Linux -c34-
    ls -lt $LOGS/ |head -2 |tail -1|cut -c35-


    # Semi-standardized stuff.
    echo
    echo "System             MostRecent   Oldest   BUs    ME-Recent ME-Oldest  MEs  TotSize   IncSize     vol  rc"
    echo "======================================================================================================="
    }

function bureport {
## Backup result summary, by machine.
## imperfect error checking here.  Where problems have been seen, code has been
## fluffed out with error checking, but otherwise kept simple for readability.
##
## Looks for key bits of info in the backup directories and the rsync log file,
## and pretty-prints it.
##  Likely to need revisions as rsync and other things get updated.
##
    typeset TARGET VOL TOTFSIZE INCFSIZE OLDEST RECENT MEOLDEST MERECENT COUNT MECOUNT LOGFILE
    TARGET=$1

    VOL=$(dirname $(ls -dl $IBSBASE/$TARGET|cut -f2 -d'>'))
    TOTFSIZE=0
    INCFSIZE=0
    if [[ ! -d $IBSBASE/$TARGET ]]; then
	echo "$TARGET  -- no backup directory"
	return 10 # skip if no such directory
    fi
    try cd $IBSBASE/$TARGET
    if ! ls -d $DATEFMT>/dev/null 2>&1 ; then
	echo  "$TARGET -- no daily backups"
	return 11
    fi
    OLDEST=$(ls -1d $DATEFMT 2>/dev/null|head -1)
    RECENT=$(ls -1d $DATEFMT 2>/dev/null|tail -1)
    MEOLDEST=$(ls -1d $MEFMT 2>/dev/null|head -1|cut -f1-3 -d"-")
    MERECENT=$(ls -1d $MEFMT 2>/dev/null|tail -1|cut -f1-3 -d"-")
    COUNT=$(ls -1d $DATEFMT 2>/dev/null|wc -l)
    MECOUNT=$(ls -1d $MEFMT 2>/dev/null|wc -l)
    LOGFILE=$(ls $LOGS/$TARGET-$DATEFMT 2>/dev/null|tail -1)
    if [[ -z $LOGFILE ]]; then
	echo "$TARGET -- no log file"
	return 12
    fi
    TOTFSIZE=$(grep "Total file size:" $LOGFILE|tail -1 |tr -d "," | cut -d" " -f4)
    if [[ -z $TOTFSIZE ]]; then
	TOTFSIZE="-----"
	INCFSIZE="-----"
    else
	TOTFSIZE=$(echo $TOTFSIZE'/'$MEG |bc)
	INCFSIZE=$(grep "^Total transferred file size:" $LOGFILE |tail -1| tr -d "," |cut -d" " -f5)
	INCFSIZE=$(echo $INCFSIZE'/'$MEG |bc)
    fi

    RC=$(tail -1 $LOGFILE|grep BACKUP |sed -e "s/.* rc=//" -e "s/ ====//")

    printf "%-18s %10s %10s %2d    %10s %10s %2d %8sM %8sM %7s %3s\n" "$(echo $TARGET|cut -c1-18)" "$RECENT" "$OLDEST" "$COUNT" "$MERECENT" "$MEOLDEST" "$MECOUNT" "$TOTFSIZE" "$INCFSIZE" "$VOL" "$RC"

   }


#### Main ####
if [[ -z $1 ]]; then # if no parameters, show help screen.
    cat <<-__ENDHELP
	Incremental Backup System

	ibs -check host [ host ...] 
	   Does a "preflight" check on designated backup targets,
	   does not actually do a backup

	ibs -prep /targ/host
	   Creates the backup directory specified, then symlinks it to
	   the IBSBASE directory.
	   Pre-populates the directory with daily and ME backups

	ibs [-v] hostname
	   runs a backup of the given host.  -v causes it to display the
	   log it creates to the console in addition to the log file.

	ibs -l hostname|listname [...]
	   runs a backup of given hosts or lists of hosts

	ibs -r hostname|listname [...]
	   runs a backup report agasinst a given host or list of hosts

	ibs -template 
	   generates a template ibs.config directory in the current working
	   directory for population and copying to /etc/ibs

	ibs -add hostname|listname [...]
	   adds another "blank" backup directory, to increase the number of
	   retained backups.

	ibs -addme hostname|listname [...]
	   adds another "blank" backup Month-End directory, to increase the
	   number of retained Month End (-ME) backups.

	__ENDHELP
    exit
fi

CMD=buhost # default, if just a list of hostnames are given.
BACKGROUND='y'

while [[ -n $1 ]]; do
    case $1 in 
	-c | -check ) CMD=preflightcheck
	    shift
	    BACKGROUND=""
	    ;;
	-v | -verbose ) VERBOSE="y"
	    shift
	    ;;
	-p | -prep )
	    shift # get rid of the -p(rep)
	    prep $* # prep needs entire path, not just a name.
	    exit
	    ;;
	-r | -report ) CMD=bureport
	    reportheader
	    shift
	    BACKGROUND=""
	    ;;
	-add ) CMD="addibsdir"
	    BACKGROUND=""
	    shift
	    ;;
	-addme ) CMD="addibsdir"
	    BACKGROUND=""
	    MONTHLY="-ME"
	    shift
	    ;;
	-* ) echo "invalid option $1.  Exiting"
	    exit
	    ;;
	* ) break
	    ;; # Assumption is it's a hostname.
    esac
done

try cd $IBSBASE

# At this point, we've removed the commands from the command line, all
# we have left are the hosts to run that command against.

# Expand *.list files into their individual backup targets.
BUTARGETS=""
for H in $*; do
    if [[ $H = *.list ]]; then
	if [[ -f $CONFIGDIR/$H ]]; then
	    BUTARGETS="$BUTARGETS $(sed "s/#.*//" $CONFIGDIR/$H | grep -v "^\s*$")"
        else
	    echo "List file $H doesn't exist"
        fi 
    else 
	BUTARGETS="$BUTARGETS $(basename $H)"
    fi
done


if [[ -n "$VERBOSE" ]]; then
    echo "=Work list: " $BUTARGETS;
    echo
fi

# Main loop: process all hosts listed.
set -a $BUTARGETS
while [[ -n $1 ]]; do
    if [[ -z $BACKGROUND ]]; then
	$CMD $1 
    else
	$CMD $1 & # backups should be paralled.
    fi

    shift
    # If we are doing a backup, delay between starts and don't run too many at once
    if [[ $CMD = "buhost" && -n $1 ]]; then
	sleep $DELAY
	RSRUNNING=$(pgrep rsync |wc -l)
	while [[ $RSRUNNING -gt $MAXRSYNCS ]]; do
	    echo "...Waiting -- $RSRUNNING rsync processes already running"
	    sleep $DELAY
	    RSRUNNING=$(pgrep rsync |wc -l)
	done
    fi
done

# The following keeps the command prompt from coming in the middle of outwise
# interesting output.  Backgrounding tasks that spew output can be ugly.
test -n "$BACKGROUND" && sleep 2

echo
echo "EOT generated from $COMMANDLINE by $(whoami) on $(hostname)"