#!/bin/bash
# Script to rebuild rpm databases (when the DB is beyond repair)

# written by Bob Fulton of BRAEWORKS, January 2015.

# This script came into being after a update disaster befell several production servers running on baremetal and VMs.

# The basics:-
# This script by default generates another bash script (a dry run) that contains the commands to rebuild the rpm DB
# from the yum repositories available packages and comparing the files installed on the system.
# It typically take 30 to 60 minutes to run to generate the dry run script.
# The generated script has it's execution bit tuned off by default (stopping inadvertent execution). This script can
# viewed or executed (by turning on the execution bit).

# This script and the generated script both record command output into log and error log files.
# Typically if all goes well, the error log file has just start, datetime record and finish entries.

# This script puts all output and backup file into a work directory based on the system hostname helping to stop
# inadvertant application of the scripts on the wrong host. The generated script also has a further
# check to match hostname to the generated script, to stop cross host corruption.

# This script has been run on Redhat 6 and CentOS 6 servers.
# And also on local desktops, which revealed a suprising number of rpm DB errors.

# This script has various options, use the --help or -h command line argument to see options.
# The default options (dry run) have been tested, but for non default options regard with caution.

# NOTE: Most of the hosts I am responsible for have custom repositories. And because of this the package signature
# checking has been turned off. See the RPM_OPTS and YUM_OPTS variables.

# Provided as is. Hope this is helpful.
# feedback to netadmin@braeworks.net

VERSION="V0.01"
PROG="$(basename $0)"
PDIR="$(dirname $0)"
cd "$PDIR"	# keep it all together
PDIR="$(pwd)"	# keep it all together as dirname may be ./ use absolute reference

# check for root user
if [ "$USER" != "root" ]; then
	echo "Must be the root user to run $PROG."
	exit 3
fi

# some useful variables
DATETIME=$(date '+%Y%m%d-%H%M%S')
WORK_DIR="$(echo "${PDIR}/${HOSTNAME%%.*}")"	# work/outpu dir
DISTRO=$(sed -n 's/^distroverpkg=//p' /etc/yum.conf)
RELEASEVER=$(rpm -q --qf "%{version}" -f /etc/$DISTRO)
BASEARCH=$(rpm -q --qf "%{arch}" -f /etc/$DISTRO)
RPM_OPTS="--nosignature"	# used on all rpm ops
YUM_OPTS="--releasever=$RELEASEVER --nogpgcheck"	# used on all yum ops except repoquery

LOG="${WORK_DIR}/$PROG.log"	#-$DATETIME.log"
ERR_LOG="${WORK_DIR}/$PROG-error.log"	#-$DATETIME.log"

PKG_CNT=0
PKG_CHK_CNT=0
PKG_NAMES=""

DRY_RUN_LIST="${WORK_DIR}/dry-run-list-${PROG}"
DRY_RUN=1

DO_POST_CHKS=0

RPMS_AVAIL_FILE="${WORK_DIR}/rpms-available-list.txt"
RPM_BD_BAK="${WORK_DIR}/rpm-DB-$HOSTNAME-$DATETIME.tgz"
INSTALL_RPM=0	# set 0 to just add to rpm DB or set 1 to install the package
REBUILD_DB=0	# set 0 normal or set 1 to dump the old rpm DBs
MUST_INSTALL_PKGS=""	# must have these packages installed to run this script
DB_ONLY_PKGS="kernel hal glibc glibc-common binutils bash nss-softokn nss-softokn-freebl yum yum-utils rpm openssh openssh-clients openssh-server openssl perl python mc"	# packages where only the RPM DB is updated

USAGE=" \n\
Usage: $PROG [--dry-run|--run-direct] [--db-only|--install] [--clear-db] [--do-checks] \n\
Options;- \n\
--dry-run\tDRY RUN (default, generates \"$DRY_RUN_LIST\" script of suggested changes) \n\
--run-direct\trun the additions or installs directly \n\
--db-only\tadd package info to rpm DB only (default, no system files changes) \n\
--install\tinstall packages (and dependencies) \n\
--clear-db\tclear rpm DB to empty first (CAUTION) \n\
--do-checks\tdo post processing package checks \n\
 \n\
\tINFO: Found $DISTRO, version $RELEASEVER, basearch $BASEARCH\n"

# get options
while [ ! -z "$1" ]
do
	case "$1" in
	"--install")
		INSTALL_RPM=1
		;;
	"--db-only")
		INSTALL_RPM=0
		;;
	"--clear-db")
		REBUILD_DB=1
		;;
	"--dry-run")
		DRY_RUN=1
		;;
	"--run-direct")
		DRY_RUN=0
		;;
	"--do-checks")
		DO_POST_CHKS=1
		;;
	*)
		echo -e $USAGE
		exit 0
		;;
	esac
	shift
done

# functions
addRpm2DB() {	# $1=NAME $2=LOC $3=ARCH $4=VER $5=REPO
	# echo "addPkg2DB()"	# test
	local NAME="$1"
	local LOC="$2"
	local ARCH="$3"
	local VER="$4"
	local REPO="$5"
	local RET=100

	echo "Adding \"${NAME}.${ARCH}\" to rpm DB from \"$REPO\""	# test
	if [ $DRY_RUN -ne 0 ]; then
		echo "# rpm add \"${NAME}.${ARCH}\" package to DB from \"$REPO\"" >> "$DRY_RUN_LIST"
		echo "rpm -i -v --nodeps --noscripts --notriggers $RPM_OPTS --excludepath / -p \"$LOC\" >> \"$LOG\" 2>> \"$ERR_LOG\""  >> "$DRY_RUN_LIST"
		echo "" >> "$DRY_RUN_LIST"
		PKG_NAMES="${PKG_NAMES}${NAME}.${ARCH}\n"	# assume: package name and arch for a yum cleanup/reinstall
		return 0
	fi

	rpm -i -v --nodeps --noscripts --notriggers $RPM_OPT --excludepath / -p "$LOC" >> "$LOG" 2>> "$ERR_LOG"
	RET=$?
	if [ $RET -eq 0 ]; then
		local PV=$(rpm  $RPM_OPT -V "${NAME}.${ARCH}")
		if [ -z "$PV" ]; then
			echo "SUCCESS: Added \"${NAME}.${ARCH}\" to RPM DB from \"$REPO\"." >> "$LOG"
			PKG_NAMES="${PKG_NAMES}${NAME}.${ARCH}\n"	# package name and arch for a yum cleanup/reinstall
		else
			echo "VERIFY FAILED: Added \"${NAME}.${ARCH}\" to RPM DB from \"$REPO\", but failed to verify." >> "$LOG"
			yum $YUM_OPTS install "${NAME}.${ARCH}" >> "$LOG"
			RET=$?
			if [ $RET -eq 0 ]; then
				echo "SUCCESS: Installed \"${NAME}.${ARCH}\" and dependencies from verify failure." >> "$LOG"
				PKG_NAMES="${PKG_NAMES}${NAME}.${ARCH}\n"	# package name and arch for a yum cleanup/reinstall
			else
				echo "FAILED: Install \"${NAME}.${ARCH}\" or dependencies from verify failure." >> "$LOG"
			fi
		fi
	else
		echo "FAILED: Add \"${NAME}.${ARCH}\" to RPM DB from \"$REPO\"." >> "$LOG"
	fi
	return $RET
} # addRpm2DB()

addYum2DB() {	# $1=NAME $2=LOC $3=ARCH $4=VER $5=REPO
	# echo "addPkg2DB()"	# test
	local NAME="$1"
	local LOC="$2"
	local ARCH="$3"
	local VER="$4"
	local REPO="$5"
	local RET=100

	echo "Installing \"${NAME}.${ARCH}\" from \"$REPO\"."	# test

	if [ $DRY_RUN -ne 0 ]; then
		echo "# yum install \"${NAME}.${ARCH}\" from \"$REPO\"" >> "$DRY_RUN_LIST"
		echo "yum $YUM_OPTS install \"${NAME}.${ARCH}\" >> \"$LOG\" 2>> \"$ERR_LOG\"" >> "$DRY_RUN_LIST"
		echo "" >> "$DRY_RUN_LIST"
		PKG_NAMES="${PKG_NAMES}${NAME}.${ARCH}\n"	# assume: package name and arch for a yum cleanup/reinstall
		return 0
	fi
	
	yum $YUM_OPTS install "${NAME}.${ARCH}" >> "$LOG" 2>> "$ERR_LOG"
	RET=$?
	if [ $RET -eq 0 ]; then
		echo "SUCCESS: Installed \"${NAME}.${ARCH}\" and dependencies from \"$REPO\"." >> "$LOG"
		PKG_NAMES="${PKG_NAMES}${NAME}.${ARCH}\n"	# package name and arch for a yum cleanup/reinstall
	else
		echo "FAILED: Install \"${NAME}.${ARCH}\" or dependencies from \"$REPO\"." >> "$LOG"
	fi
	return $RET
} # addYum2DB()

addPkg2DB() {	# $1=NAME $2=LOC $3=ARCH $4=VER $5=REPO
	# echo "addPkg2DB()"	# test
	local NAME="$1"
	local LOC="$2"
	local ARCH="$3"
	local VER="$4"
	local REPO="$5"
	local RET=100

	local CHK=$(rpm -q "$NAME" | grep "$ARCH" | grep "$VER")
	if [ ! -z "$CHK" ]; then return 0; fi 	# done already

	local EX=$(echo "$DB_ONLY_PKGS" | grep -i -w "$NAME")
	if [ $INSTALL_RPM -eq 1 -a -z "EX" ]; then
		addYum2DB  "$NAME" "$LOC" "$ARCH" "$VER" "$REPO"
		RET=$?
	else
		addRpm2DB  "$NAME" "$LOC" "$ARCH" "$VER" "$REPO"
		RET=$?
	fi
	echo "" >> "$LOG"	# blank line
	return $RET
} # addPkg2DB()

hasRPMfiles() {	# $1=NAME $2=LOC $3=ARCH $4=VER $5=REPO
	# echo "hasRPMfiles()"	# test
	# exit	# test
	local NAME="$1"
	local LOC="$2"
	local ARCH="$3"
	local VER="$4"
	local REPO="$5"
	local FILE_CNT=0
	local FOUND_CNT=0

# 	local CHK=$(rpm -q "$NAME" | grep "$ARCH" | grep "$VER")
# 	if [ ! -z "$CHK" ]; then return 10; fi 	# done already

	for F in $(rpm -q -l $RPM_OPT -p "$LOC" 2>> "$ERR_LOG")
	do
		(( FILE_CNT += 1))
		# echo "Checking: $F"	# test
		if [ -e "$F" ]; then	# file found
			# echo "Found: $F"	# test
			(( FOUND_CNT += 1))
		fi
	done

	# make an estimation (due to files having version numbers)
	local FILE_CHK=$FILE_CNT
	# must have the same files (( FILE_CHK /= 2 ))	# rough
	if [ $FILE_CNT -gt 0 -a $FOUND_CNT -ge $FILE_CHK ]; then
		echo ""	# info
		echo "\"${NAME}.${ARCH}\" from \"$REPO\" has $FOUND_CNT of $FILE_CNT files present."	# info
		echo "\"${NAME}.${ARCH}\" from \"$REPO\" has $FOUND_CNT of $FILE_CNT files present." >> "$LOG"
		return 0	# enough files found
	fi
	return 1	# not enough , wrong version ???
} # hasRPMfiles()

# the code
echo "Ready to check RPM DB and packages for $HOSTNAME. Release=$RELEASEVER, Basearch=$BASEARCH"
echo "Running $PROG is a major undertaking and should only be run in extreme circumstances."
echo "$PROG can take upto an hour to do a dry run, allow plenty of time."
echo "A log of operations will be available in \"$LOG\"."
echo ""

if [ $DRY_RUN -ne 0 ]; then
	echo "NOTE: This is a dry run."
	echo "Changes are NOT applied."
	echo "Sample execution code placed \"$DRY_RUN_LIST\""
else
	echo "WARNING: This is a LIVE run."
	echo "Changes WILL BE applied."
fi
echo ""

# check if a total rebuild
if [ $REBUILD_DB -eq 1 -a $DRY_RUN -eq 0 ]; then
	echo "WARNING: You are about to remove all rpm DB package info from the system (including any special extra package info)."
	echo "Type 'yes' to continue or other to exit."
	read TMP
	if [ "$TMP" != 'yes' -a "$TMP" != 'YES' ]; then echo "Exiting"; echo -e $USAGE; exit 101; fi

	OLDDIR="${WORK_DIR}/var-lib-rpm-old-$DATETIME"
	mv "/var/lib/rpm" "$OLDDIR" >> "$LOG" 2>> "$ERR_LOG"
	mkdir "/var/lib/rpm" >> "$LOG" 2>> "$ERR_LOG"
	rpm --initdb >> "$LOG" 2>> "$ERR_LOG"
	echo "Moved old RPM DB to \"$OLDDIR\", and reinitialised DB." >> "$LOG" 2>> "$ERR_LOG"
	# echo "You will need to install any special or extra packages."
	echo "" >> "$LOG" 2>> "$ERR_LOG"
fi

# show important message
echo "IMPORTANT: Before running this script the first time (or on a cleaned rpm DB), it is important to run the following \
command to ensure the most recent system packages (and their dependencies) are installed correctly."
echo "Run \"yum install $DB_ONLY_PKGS\"."
echo "It is also a good idea to install the correct email MTA first as well."
echo "You also need to install any special packages not present in the yum repositories."
echo ""
echo "After running the above command and any extra installs, please reboot the system."
echo "Type 'yes' to continue or other to exit."
read TMP
if [ "$TMP" != 'yes' -a "$TMP" != 'YES' ]; then echo "Exiting"; echo -e $USAGE; exit 100; fi

# here we go
echo ""
if [ ! -d "$WORK_DIR" ]; then mkdir -p "$WORK_DIR"; fi

echo "Started $PROG ($VERSION) on $HOSTNAME for system release version $RELEASEVER, basearch $BASEARCH, at $(date '+%Y/%m/%d %H:%M:%S')" >> "$LOG"
echo "Using DateTime stamp $DATETIME" >> "$LOG"

echo "Started $PROG ($VERSION) on $HOSTNAME for system release version $RELEASEVER, basearch $BASEARCH, at $(date '+%Y/%m/%d %H:%M:%S')" >> "$ERR_LOG"
echo "Using DateTime stamp $DATETIME" >> "$ERR_LOG"

echo "Reloading clean yum repo caches."
yum $YUM_OPTS clean all >> "$LOG" 2>> "$ERR_LOG"
yum $YUM_OPTS makecache >> "$LOG" 2>> "$ERR_LOG"

echo "RPM DB backed up to \"$RPM_BD_BAK\"."
tar --exclude=.svn --exclude=temp --exclude=*.lock -czvf "$RPM_BD_BAK" "/var/lib/rpm/" >> "$LOG" 2>> "$ERR_LOG"
echo "RPM DB backed up to \"$RPM_BD_BAK\"." >> "$LOG"

if [ -f "$DRY_RUN_LIST" ]; then rm -f "$DRY_RUN_LIST"; fi	# clear it
if [ $DRY_RUN -ne 0 ]; then
	echo "#!/bin/bash" > "$DRY_RUN_LIST"	# and reset it
	echo "# \"$PROG\" dry run list for $HOSTNAME. Release=$RELEASEVER Basearch=$BASEARCH At $DATETIME" >> "$DRY_RUN_LIST"
	echo "" >> "$DRY_RUN_LIST"
	echo "if [ \"\$HOSTNAME\" != \"$HOSTNAME\" ]; then echo \"Wrong host...\"; exit 200; fi" >> "$DRY_RUN_LIST"
	echo "" >> "$DRY_RUN_LIST"
	echo "echo \"Started $DRY_RUN_LIST ($VERSION) on $HOSTNAME at \$(date '+%Y/%m/%d %H:%M:%S')\" >> \"$LOG\"" >> "$DRY_RUN_LIST"
	echo "echo \"Started $DRY_RUN_LIST ($VERSION) on $HOSTNAME at \$(date '+%Y/%m/%d %H:%M:%S')\" >> \"$ERR_LOG\"" >> "$DRY_RUN_LIST"
	echo "" >> "$DRY_RUN_LIST"
fi

# installed the must installs
for P in $(echo $MUST_INSTALL_PKGS)
do
	TMP=$(rpm -q "$P" | grep -i "not installed")
	if [ ! -z "$TMP" ]; then
		yum $YUM_OPTS install "$P" >> "$LOG" 2>> "$ERR_LOG"
		RET=$?
		if [ "$RET" -ne 0 ]; then
			echo "ERROR: Failed to install \"$P\"."
			exit 5
		fi		
	fi
done

# get a list of all available packages (installed already in rpm db not needed)
echo "Getting list of available packages."	# info
# clean out the yum output columated wrap formatting
if [ "$BASEARCH" == "x86_64" ]; then
	yum $YUM_OPTS list available | sed ':a;N;$!ba;s/\n   /   /g' | egrep '\.x86_64|\.noarch' > "$RPMS_AVAIL_FILE" 2>> "$ERR_LOG"
else
	yum $YUM_OPTS list available | sed ':a;N;$!ba;s/\n   /   /g' | egrep '\.386|\.486|\.586|\.686|\.noarch' > "$RPMS_AVAIL_FILE" 2>> "$ERR_LOG"
fi
# exit	# test

# check $RPMS_AVAIL_FILE for matched in file contents
echo "Scanning available packages."	# info
old_IFS="$IFS"      # save the field separator
IFS=$'\n'     # new field separator, the end of line
for L in $(cat "$RPMS_AVAIL_FILE")
do
	# echo "Package = $L"	# test

	TMP="$(echo "$L" | awk '{ print $1 }' )"	# get name and arch
	NAME="${TMP%.*}"	# extract name
	ARCH="${TMP##*.}"	# extract arch
	VER="$(echo "$L" | awk '{ print $2 }' )"	# get version
	REPO="$(echo "$L" | awk '{ print $3 }' )"	# get repo

	# check for wrap lines in columated $RPMS_AVAIL_FILE
	if [ -z "$REPO" ]; then continue; fi	# wrap line not detected in $RPMS_AVAIL_FILE
	
	LOC="$(repoquery --location "${NAME}.${ARCH}")"	# get package location
	# echo -e "NAME=$NAME, ARCH=$ARCH, VER=$VER, REPO=$REPO,\n	LOC=$LOC\n"	# test

	hasRPMfiles "$NAME" "$LOC" "$ARCH" "$VER" "$REPO"
	RET=$?
	# echo $RET	# test
	if [ $RET -eq 0 ]; then
		# echo -e "NAME=$NAME, ARCH=$ARCH, VER=$VER, REPO=$REPO,\n	LOC=$LOC\n"	# test
		# echo "\"${NAME}.${ARCH}\" has its files present."	# test
		addPkg2DB  "$NAME" "$LOC" "$ARCH" "$VER" "$REPO"
		RET=$?
		if [ $RET -eq 0 ]; then
			(( PKG_CNT += 1 ))
		fi
		# break	# test
	fi
	(( PKG_CHK_CNT += 1 ))
	echo -e -n "Package $PKG_CHK_CNT \r"	# info
	# break	# test
done
IFS=$old_IFS     # restore default field separator

# done
echo "" >> "$LOG"
echo "$PKG_CNT lost packages found."
echo "$PKG_CNT lost packages found." >> "$LOG"
echo -e "$PKG_NAMES" >> "$LOG"
echo "" >> "$LOG"

echo ""
if [ $DO_POST_CHKS -eq 0 -o $DRY_RUN -ne 0 ]; then
	echo "Advise RPM DB and Package Checks by running when additions are finished;-"
	# echo "yum --verbose check dependencies duplicates obsoletes # (take quite a long time)"
	echo "package-cleanup --problems"
	echo "package-cleanup --dupes"
	echo "package-cleanup --leaves	# CAUTION: does not include independantly referenced packages"
	echo "package-cleanup --orphans	# CAUTION: does not include independantly referenced packages"
else
	echo "Running RPM DB and Package Checks"
	echo "RPM DB and Package Checks" >> "$LOG"
	# echo "yum check dependencies duplicates obsoletes # (takes quite a long time)" >> "$LOG"
	# yum --verbose check dependencies duplicates obsoletes >> "$LOG"
	package-cleanup --problems >> "$LOG" 2>> "$ERR_LOG"
	package-cleanup --dupes >> "$LOG" 2>> "$ERR_LOG"
	# package-cleanup --leaves >> "$LOG" 2>> "$ERR_LOG"
	# package-cleanup --orphans >> "$LOG" 2>> "$ERR_LOG"
	echo "" >> "$LOG"
fi
echo ""

echo "Finished $PROG ($VERSION) on $HOSTNAME at $(date '+%Y/%m/%d %H:%M:%S')" >> "$LOG"
echo "" >> "$LOG"
echo "Finished $PROG ($VERSION) on $HOSTNAME at $(date '+%Y/%m/%d %H:%M:%S')" >> "$ERR_LOG"
echo "" >> "$ERR_LOG"

if [ $DRY_RUN -ne 0 ]; then
	echo "" >> "$DRY_RUN_LIST"	# close
	echo "echo \"Finished $DRY_RUN_LIST ($VERSION) on $HOSTNAME at \$(date '+%Y/%m/%d %H:%M:%S')\" >> \"$LOG\"" >> "$DRY_RUN_LIST"
	echo "echo \"Finished $DRY_RUN_LIST ($VERSION) on $HOSTNAME at \$(date '+%Y/%m/%d %H:%M:%S')\" >> \"$ERR_LOG\"" >> "$DRY_RUN_LIST"
	echo "echo \"\" >> \"$LOG\"" >> "$DRY_RUN_LIST"
	echo "echo \"\" >> \"$ERR_LOG\"" >> "$DRY_RUN_LIST"
	echo "#  Check results \"$LOG\" and \"$ERR_LOG\"." >> "$DRY_RUN_LIST"
	echo "" >> "$DRY_RUN_LIST"
	echo "# EOF" >> "$DRY_RUN_LIST"
	echo "" >> "$DRY_RUN_LIST"
fi

echo "Done"
echo "Check \"$LOG\" and \"$ERR_LOG\" for results."

# EOF