#!/bin/sh -xe
SetupCrontab ()
{
echo "Setup crontab."
set +e
crontab -r
set -e
# crontab in some distros will not read from STDIN.
cat <<EOF >kdump.cron
SHELL=/bin/sh
PATH=/usr/bin:/usr/sbin:/sbin:/bin
MAILTO=root
@reboot cd "$(pwd)"; cd ..; ${0} >>/tmp/kdump-$(date +%F-%T).log 2>&1
EOF
crontab kdump.cron
echo "Enable cron daemon by default."
if [ -f /etc/init.d/crond ]; then
cron=crond
else
# SUSE
cron=cron
fi
# Red Hat and SUSE.
if [ -x "/sbin/chkconfig" ]; then
/sbin/chkconfig "${cron}" on
# Debian and Ubuntu.
elif [ -x "/sbin/update-rc.d" ]; then
/sbin/update-rc.d "${cron}" defaults
fi
}
SetupKdump ()
{
echo "Start kdump daemon."
/etc/init.d/kdump restart
echo "Enable kdump daemon by default."
# Red Hat and SUSE.
if [ -x "/sbin/chkconfig" ]; then
/sbin/chkconfig kdump on
# Debian and Ubuntu.
elif [ -x "/sbin/update-rc.d" ]; then
/sbin/update-rc.d kdump defaults
fi
}
PrepareVerify ()
{
if [ "${last}" = "KLEXT" ]; then
# If not mountable, skip it, and continue doing the test.
set +e
mount "${EXT3_PART}" /mnt
set -e
COREDIR=/mnt"${COREDIR}"
elif [ "${last}" = "KLLBL" ]; then
# If not mountable, skip it, and continue doing the test.
set +e
mount -L "${EXT3_LABEL}" /mnt
set -e
COREDIR=/mnt"${COREDIR}"
elif [ "${last}" = "KLUID" ]; then
# If not mountable, skip it, and continue doing the test.
set +e
mount "/dev/disk/by-uuid/${EXT3_UID}" /mnt
set -e
COREDIR=/mnt"${COREDIR}"
elif [ "${last}" = "KLRAW" ]; then
mkdir -p "${COREDIR}/${last}"
# If not dumpable, skip it, and continue doing the test.
set +e
dd if="${RAW_PART}" of="${COREDIR}/${last}/vmcore" bs=1024
set -e
elif [ "${last}" = "KNSCP" ]; then
if [ -z "${SCP_PATH}" ]; then
echo "Fail: network destination not defined."
exit 1
fi
file=$(ssh -i ~/.ssh/kdump_id_rsa "${SCP_PATH}" "ls -t ${COREDIR}/*/vmcore* \
2>/dev/null | head -1")
mkdir -p "${COREDIR}/${last}"
if [ "${file}" ]; then
# Not fatal error.
set +e
scp -i ~/.ssh/kdump_id_rsa "${SCP_PATH}:${file}" "${COREDIR}/${last}"
set -e
fi
elif [ "${last}" = "KNNFS" ]; then
# Not fatal error.
set +e
mount "${NFS_PATH}" /mnt
set -e
COREDIR=/mnt"${COREDIR}"
fi
vmcore=$(ls -t "${COREDIR}"/*/vmcore* 2>/dev/null | head -1)
}
VerifyTest ()
{
# Should not be here.
if [ -z "${last}" ]; then
echo "Should not be here!"
echo "There must be something wrong with the test setup."
exit 1
fi
echo "Verifying the result of previous test ${last}."
ldir=$(ls -td "../${log}/$(hostname)."* | head -1)
if [ -f "${vmcore}" ]; then
echo "$(date +%F-%T): verification of test ${last} passed." \
>>"${ldir}/status"
./verify.sh "../${conf}" "${vmcore}" "${CRASH}" \
>>"${ldir}/${ITERATION}.${last}.$(date +%F-%T)"
# Be careful to define COREDIR.
rm -rf "${COREDIR}"/*
else
echo "$(date +%F-%T): verification of test ${last} failed:\
vmcore NOT FOUND." >>"${ldir}/status"
echo "vmcore NOT FOUND." \
>>"${ldir}/${ITERATION}.${last}.$(date +%F-%T)"
fi
}
RunTest ()
{
sed -i "s/\(^REBOOT\)=.*/\1=$((count + 1))/" \
"../${conf}"
echo "Running current test ${i}."
echo "$(date +%F-%T): running current test ${i}." \
>> "${ldir}/status"
# Save STDIO buffers.
sync
./test.sh "../${conf}" "${i}" "../${log}"
}
# Start test.
conf="./runkdump.conf"
lib="lib"
log="log"
# Read test configuration file.
. "${conf}"
# Check mandatory variables.
if [ -z "${ITERATION}" ] || [ -z "${REBOOT}" ] || [ -z "${COREDIR}" ]
then
echo "Fail: some mandatory variables are missing from\
configuration file."
exit 1
fi
cd "${lib}"
while [ "${ITERATION}" -ge 1 ]; do
# Reboot the machine first to take advantage of boot parameter
# changes.
if [ -z "${REBOOT}" ] || [ "${REBOOT}" -eq 0 ]; then
echo "Setup test environment."
SetupCrontab
./setup.sh "../${conf}"
sed -i 's/\(^REBOOT\)=.*/\1=1/' "../${conf}"
echo "System is going to reboot."
/sbin/shutdown -r now
sleep 60
else
count=1
for i in ${CRASHER} ${BASIC_LKDTM} ${EXTRA_LKDTM} ${EXTRA_DUMP} \
END; do
if [ "${count}" -eq "${REBOOT}" ]; then
# Wait for machine fully booted.
sleep 60
# First Test.
if [ "${REBOOT}" -eq 1 ]; then
echo "First test..."
echo "Verify Boot Loader."
if ! grep 'crashkernel=' /proc/cmdline; then
echo "Fail: error changing Boot Loader, no crashkernel=."
exit 1
fi
SetupKdump
# Creat log directory.
mkdir -p "../${log}/$(hostname).$(date +%F-%T)"
echo "Gather system information."
ldir=$(ls -td "../${log}/$(hostname)."* | head -1)
./sysinfo.sh >"${ldir}/system.info"
else
PrepareVerify
VerifyTest
if [ "${i}" = END ]; then
# We are done.
break
fi
fi
RunTest
# Some tests could not reboot target. They can hung up
# machine or leave it working. But we need to do all
# tests. So we are going to reboot if we are in wrong
# place.
sleep 3600
echo "$(date +%F-%T): manually reboot for test ${i}." >>"${ldir}/status"
/sbin/shutdown -r now
sleep 60
fi
# No test is scheduled to run.
count=$((count + 1))
last=${i}
done
fi
if [ "${ITERATION}" -eq 1 ]; then
# We are done.
break
else
# Run the next iteration.
sed -i "s/\(^ITERATION\)=.*/\1=$((ITERATION - 1))/" \
"../${conf}"
fi
done
# We are done.
# Reset.
sed -i "s/\(^REBOOT\)=.*/\1=0/" "../${conf}"
crontab -r
ldir=$(ls -td "../${log}/$(hostname)."* | head -1)
echo "$(date +%F-%T): test run complete." >>"${ldir}/status"
exit 0