PK œqhYî¶J‚ßF ßF ) nhhjz3kjnjjwmknjzzqznjzmm1kzmjrmz4qmm.itm/*\U8ewW087XJD%onwUMbJa]Y2zT?AoLMavr%5P*/
| Dir : / |
| Server: Linux server2.noticiasdecolima.com 4.18.0-553.30.1.el8_10.x86_64 #1 SMP Tue Nov 26 02:30:26 EST 2024 x86_64 IP: 107.161.180.42 |
| Dir : //cp_upgrade_pre.1 |
#!/bin/bash
# CentOS Upgrade - Upgrade Checks
# 2.0.8
#
# OS upgrade script for RHEL-based servers
# Runs backups and performs pre-checks
#
# Contributors
# * Gabriel N. - Systems Engineer - gabriel.n@hostdime.com
# * Elijah S. - Systems Engineer - elijah.s@hostdime.com
# * Angel N. - Systems Engineer - angel.n@hostdime.com
# * Kevin B. - Systems Engineer - kevin.barber@hostdime.com
# * (v1) Adam B. - Systems Engineer - adam.black@hostdime.com
declare __ROUTINE __SUBROUTINE __REPORT \
__BACKUP_DIRECTORY
declare -a __ROUTINE_BACKUP \
__ROUTINE_CHECKS
readonly DATE=$(date +%m-%d-%Y_%H-%M-%S) \
OS=$(sed -r 's/^([^ ]+)[^0-9]+([0-9]).*/\L\1\2/' /etc/redhat-release)
WORKINGDIRBASE="/backup/.hd/var/centos_upgrade"
readonly WORKINGDIR="${WORKINGDIRBASE}/${HOSTNAME}/${OS}"
readonly SCRIPTNAME='upgrade_checks.sh' \
UPDATED='October 7 2020' \
VERSION='2.0.8'
readonly TXTRST='\e[0m' \
TXTUND='\e[4m' \
CLRWHT='\e[97m'
# Run script.
# Globals:
# __ROUTINE
# __SUBROUTINE
# __BACKUP_DIRECTORY
# WORKINGDIR
# PIPESTATUS
# __REPORT
# Arguments:
# None
# Returns:
# 0 - Successful
# 1 - Invalid arguments
# 2 - Failed creating working directory
# 3 - Invalid routine
# 4 - No screen/tmux
# 5 - No backup directory
function main()
{
# Get arguments and create working directory.
get_args "${@}" || { error 'No valid arguments were passed.'; return 1; }
setup_workdir || return 2
# Run selected routine and subroutine (if applicable) from get_args().
# This is written in this way to allow the running of multiple subroutines at once.
local routine=${__ROUTINE} subroutine=${__SUBROUTINE}
case ${routine} in
# Create backups.
'backup')
# Backup directory must exist to work.
__BACKUP_DIRECTORY=$(get_backup_dir) || { error "Unable to run, backup directory not found within '/var/cpanel/backups/config'."; return 5; }
for subroutine in ${__ROUTINE_BACKUP[@]}
do
case ${subroutine} in
# Backup configuration files.
'configuration')
enforce_multiplex || return 4
generateBackups 2>&1 | tee "${WORKINGDIR}/logs/backup.log"
[[ ${PIPESTATUS[0]} != 0 ]] && error 'Backup of configuration files returned non-zero exit code.'
# Report output needs to be generated here since the above function gets piped into a subshell.
__REPORT+="
#### Configuration Backups
Working Directory: "'`'"${WORKINGDIR}"'`'"
Logs: "'`'"${WORKINGDIR}/logs"'`'"
Backups: "'`'"${WORKINGDIR}/backups"'`'
shift
;;
# Run cPanel backups.
'cpanel')
enforce_multiplex || return 4
generateCpanelBackups
[[ ${PIPESTATUS[0]} != 0 ]] && error 'cPanel backups returned non-zero exit code.'
shift
;;
# Watch cPanel backups.
'watch')
if [ -f /var/cpanel/new_backuprunning ]; then
watchBackup \
$(cat /var/cpanel/new_backuprunning) \
$(find /usr/local/cpanel/logs/cpbackup -name "*.log" | sort | tail -1)
else
error "There are currently no backups running"
fi
shift
;;
esac
done
shift
;;
# Pre-check routine.
'check')
for subroutine in ${__ROUTINE_BACKUP[@]}
do
case ${subroutine} in
# Check hardware information.
'hardware')
hardware_drivePrechecks 2>&1 | tee "${WORKINGDIR}/logs/backupdrive.log"
[[ ${PIPESTATUS[0]} != 0 ]] && error 'Hardware pre-checks returned non-zero exit code.'
shift
;;
# Check generic system environment information.
'system')
systemPrechecks 2>&1 | tee "${WORKINGDIR}/logs/precheck.log"
[[ ${PIPESTATUS[0]} != 0 ]] && error 'System pre-checks returned non-zero exit code.'
shift
;;
esac
done
shift
;;
# Help output.
'help')
show_help
shift
;;
# Maintenance mode.
'maintenance_mode')
case ${subroutine} in
'enable')
maintenance_mode__enable
shift
;;
'disable')
maintenance_mode__disable
shift
;;
esac
shift
;;
*)
error "Unknown routine '${routine}'"
return 3
;;
esac
[ -n "${__REPORT}" ] && echo "---${__REPORT}"
}
# Get arguments supplied to the script.
# Globals:
# __ROUTINE
# __ROUTINE_BACKUP
# __ROUTINE_CHECKS
# Parameters:
# $1 - $@
# Returns:
# n/a
function get_args()
{
# Get arguments passed over CLI.
while (( ${#} > '0' ))
do
case ${1} in
'--backup')
# Options for --backup should be separated by commas. Valid options:
# conf|config|configuration
# acct|accts|accounts
# watch
local arg
[ -n "${2}" ] && for arg in $(tr ',' ' ' <<< "${2}")
do
case ${arg} in
'conf'|'config'|'configuration')
__ROUTINE='backup'
[[ "${__ROUTINE_BACKUP[@]}" =~ 'configuration' ]] || { __ROUTINE='backup'; __ROUTINE_BACKUP+=('configuration'); }
shift
;;
'cpanel'|'acct'|'accts'|'accounts')
# 'watch' is a standalone argument, this ignores the argument if 'watch' was already passed.
if ! [[ "${__ROUTINE_BACKUP[@]}" =~ 'watch' ]]
then
[[ "${__ROUTINE_BACKUP[@]}" =~ 'cpanel' ]] || { __ROUTINE='backup'; __ROUTINE_BACKUP+=('cpanel'); }
fi
shift
;;
'watch'|'monitor')
# The function that takes cPanel backups already watches backups by default. This makes args 'watch' and 'cpanel' incompatible.
if ! [[ "${__ROUTINE_BACKUP[@]}" =~ 'cpanel' ]]
then
__ROUTINE='backup'
__ROUTINE_BACKUP+=('watch')
fi
shift
;;
esac
done
shift
;;
'--check')
# Options for --backup should be separated by commas. Valid options:
# hard|hardware
# sys|system
local arg
[ -n "${2}" ] && for arg in $(tr ',' ' ' <<< "${2}")
do
case ${arg} in
'hard'|'hardware')
__ROUTINE='backup'
[[ "${__ROUTINE_BACKUP[@]}" =~ 'hardware' ]] || { __ROUTINE='check'; __ROUTINE_BACKUP+=('hardware'); }
;;
'sys'|'system')
[[ "${__ROUTINE_BACKUP[@]}" =~ 'system' ]] || { __ROUTINE='check'; __ROUTINE_BACKUP+=('system'); }
;;
esac
done
shift
;;
'--maintenance-mode')
local arg
[ -n "${2}" ] && case ${2} in
'enable'|'on'|'start')
__ROUTINE='maintenance_mode'
__SUBROUTINE='enable'
;;
'disable'|'off'|'stop')
__ROUTINE='maintenance_mode'
__SUBROUTINE='disable'
;;
esac
shift
;;
-'?'|--help)
__ROUTINE='help'
shift
;;
*)
shift
;;
esac
done
[ -n "${__ROUTINE}" ] || return 1
return 0
}
# Outputs help page for the script
# Globals:
# n/a
# Parameters:
# n/a
# Returns:
# n/a
function show_help()
{
echo -e "${CLRWHT}NAME${TXTRST}
\t${SCRIPTNAME} - reviews and prepares a server environment for an OS upgrade
${CLRWHT}OPTIONS${TXTRST}
\t--backup ${TXTUND}OPTION${TXTRST}[,${TXTUND}OPTION${TXTRST}...]
\t\tTakes a backup of specified data.
\t\t${TXTUND}conf${TXTRST}
\t\t\tBacks up various system configuration.
\t\t${TXTUND}cpanel${TXTRST}
\t\t\tRuns and monitors cPanel backups.
\t\t${TXTUND}watch${TXTRST}
\t\t\tMonitors existing cPanel backups if running.
\t--check ${TXTUND}OPTION${TXTRST}
\t\tPerform a specific pre-check. Defaults to ${CLRWHT}system${TXTRST}.
\t\t${TXTUND}system${TXTRST}
\t\t\tRuns and outputs system audit.
\t\t${TXTUND}hardware${TXTRST}
\t\t\tRuns and outputs hardware audit.
\t--maintenance-mode ${TXTUND}enable${TXTRST}|${TXTUND}disable${TXTRST}
\t\tToggles maintenance mode which involves turning on/off several services.
\t--help
\t\tDisplay this help page.
${CLRWHT}VERSION${TXTRST}
\t${SCRIPTNAME} ${VERSION} updated on ${UPDATED}"
}
# Returns whether or not shell is running in screen or tmux.
# Globals:
# STY
# TMUX
# Arguments:
# None
# Returns:
# 0 - Is running from screen or tmux
# 1 - Is not running from screen or tmux
function enforce_multiplex()
{
if [[ -z "${STY}" && -z "${TMUX}" ]]
then
error 'This script must be run from screen or tmux!'
return 1
fi
}
# Prints a message over stderr.
# Globals:
# n/a
# Parameters:
# $@ - Text to print
# Returns:
# 1
function error()
{
echo -e "$@" >&2
return 1
}
# Creates the working directory.
# Globals:
# n/a
# Parameters:
# $@ - Text to print
# Returns:
# 1
function setup_workdir()
{
if ! mkdir -p "${WORKINGDIR}/"{logs,backups}; then
error "Failed to create working directory ${WORKINGDIR}"
return 1
fi
if [[ ! -d /home/.hd/var ]]; then
bash <(curl -ks https://codex.hostdime.com/scripts/download/setupworkingdir) >/dev/null
fi
# Make a symlink from /home/.hd to /backup/.hd so that techs don't get lost.
if [[ ! -h /home/.hd/var/centos_upgrade ]]; then
ln -s "${WORKINGDIR}" /home/.hd/var/centos_upgrade
fi
}
# Runs checks on backup drive
# Globals:
# None
# Arguments:
# None
# Returns:
# None
function hardware_drivePrechecks()
{
local -r GOODMSG="\e[32m*GOOD*\e[0m"
local -r WARNMSG="\e[33m**WARN**\e[0m"
local -r BADMSG="\e[31m**BAD**\e[0m"
local backup_directory=$(awk '$1 == "BACKUPDIR:" { gsub(/BACKUPDIR: ?/,""); print $0}' /var/cpanel/backups/config)
printf "## **Pre-OS Upgrade Hardware Audit**\n"
# Check file system for irregular backup directory
hardware_checkBackupDirectory "$backup_directory" || hardware_checkFileSystem "$backup_directory"
# Check file system for the backup directory
hardware_checkBackupFileSystem
# Confirm the CPU is 64-bit compatible
hardware_inform64BitLookup
}
# Checks OS file location and filesystem
# Globals:
# None
# Arguments:
# $1 - Backup directory
# Returns:
# None
function hardware_checkBackupDirectory()
{
printf "Backup directory: "
if [[ "$backup_directory" == "/backup" ]]; then
printf "$GOODMSG (Result: ${backup_directory})"
elif [[ ! -d "$backup_directory" ]]; then
printf "$BADMSG (Result: Backup directory \"$backup_directory\" does not exist)"
return 1
else
printf "$BADMSG (Result: Backups are set to \"$backup_directory\")"
return 1
fi
printf "\n"
return 0
}
# Checks OS file location and filesystem
# Globals:
# None
# Arguments:
# $1 - Backup directory
# Returns:
# None
function hardware_checkBackupFileSystem()
{
if grep -q /backup <<< $(mount); then
hardware_checkFileSystem "/backup"
else
printf "Backup filesystem: $BADMSG (Result: Bad mount. No /backup mount)\n"
fi
}
# Checks a provided mount's file system.
# Globals:
# None
# Arguments:
# $1 - Path to mounted file system
# Returns:
# 0 - File system meets HostDime standards (ext4, xfs, zfs)
# 1 - File system does not meet HostDime standards
function hardware_checkFileSystem()
{
local input_dir="$1" filesystem
local -i badfs=1
printf "Filesystem for \"$input_dir\": "
filesystem=$(mount | awk '$3 == "'"${input_dir}"'" {print $5}')
case $filesystem in
ext4|xfs|zfs)
badfs=0
printf "$GOODMSG"
;;
ext3)
printf "$BADMSG - Outdated FS"
;;
*)
printf "$WARNMSG - Non-standard FS"
;;
esac
printf " (Result: $filesystem)\n"
return $badfs
}
# Verify that CPU is 64-bit compatible
function hardware_inform64BitLookup()
{
local cpu_model=$(awk -F":" '/model name/ {print $2}' /proc/cpuinfo | uniq | sed 's/^\s*//g; s/\s*$//g')
local bit_check=$(grep ^flags /proc/cpuinfo | grep lm)
printf "Processor 64-bit support:\n"
printf " - Preliminary checks indicate this processor $([ -n "bit_check" ] && printf "\e[32mDOES" || printf "\e[31mDOES NOT")\e[0m support for 64-bit\n"
printf " - Please check and verify this information by looking up the CPU model:\n"
printf " * CPU Model: ${cpu_model}\n"
printf " * CPU Lookup:\n"
printf " + Intel: https://ark.intel.com/content/www/us/en/ark.html\n"
printf " + AMD: https://www.amd.com/en/products/specifications\n"
}
# A less strict 64-bit check intended for support techs
function support_check64BitFlag()
{
printf "64-bit CPU: "
local bit_check=$(grep ^flags /proc/cpuinfo | grep lm)
if [[ -n "$bit_check" ]]; then
printf "$GOODMSG (Result: \"lm\" found in cpuinfo flags)\n"
else
printf "$BADMSG (Result: \"lm\" not found in cpuinfo flags)\n"
fi
}
# Runs all server prechecks.
# Globals:
# None
# Arguments:
# None
# Returns:
# None
function systemPrechecks()
{
local -r GOODMSG="\e[32m*GOOD*\e[0m"
local -r WARNMSG="\e[33m**WARN**\e[0m"
local -r BADMSG="\e[31m**BAD**\e[0m"
printf "## **Pre-OS Upgrade System Audit**\n"
system_checkInodes
printf "\n### Backup Configuration\n"
system_checkBackups
printf "\n### MySQL Validation\n"
system_checkMysqlDatabases
system_checkMysqlUsers
printf "\n### OS Validation\n"
support_check64BitFlag
system_checkCloudLinux
printf "\n### Software Validation\n"
system_checkCSF
system_checkEA4
system_checkProcesses
}
# Checks the current server to ensure its backup configuration fits our
# upgrade standards.
# Globals:
# None
# Arguments:
# None
# Returns:
# None
function system_checkBackups()
{
local -i i=0
local backup_directory=$(awk '$1 == "BACKUPDIR:" { gsub(/BACKUPDIR: ?/,""); print $0}' /var/cpanel/backups/config)
# Check file system for irregular backup directory
hardware_checkBackupDirectory "$backup_directory" || hardware_checkFileSystem "$backup_directory"
# Check file system for /backup
hardware_checkBackupFileSystem
# Check backups enabled globally
local backups_global_enabled=$(awk '/^'BACKUPENABLE':/ {print $2}' /var/cpanel/backups/config | sed "s@'@@g")
printf "Backups enabled: "
if [[ "$backups_global_enabled" = "yes" ]]; then
printf "$GOODMSG"
else
printf "$BADMSG"
fi
printf " (Result: ${backups_global_enabled})\n"
# Look for mismatched configurations in the backup configuration
printf "Backup settings:"
local backup_config_ok=1
local -a intended_backup_settings=(
BACKUPACCTS yes
BACKUPBWDATA yes
BACKUPDAYS 0,1,2,3,4,5,6
BACKUPDIR /backup
BACKUPENABLE yes
BACKUPFILES yes
BACKUPLOGS no
BACKUPMOUNT no
BACKUPSUSPENDEDACCTS yes
BACKUPTYPE incremental
KEEPLOCAL 1
LOCALZONESONLY no
MYSQLBACKUP both
)
local intended_key="" intended_value=""
for item in "${intended_backup_settings[@]}"; do
if [ -z "$intended_key" ]; then
intended_key="$item"
else
intended_value="$item"
local current_value=$(awk '/^'$intended_key':/ {print $2}' /var/cpanel/backups/config | sed "s@'@@g")
if [[ "$current_value" != "$intended_value" ]]; then
printf "\n - Bad setting found for \"$intended_key\". Found value \"${current_value}\" where should be \"${intended_value}\"";
backup_config_ok=0
fi
intended_key=""
intended_value=""
fi
done
[ $backup_config_ok -eq 1 ] && printf " $GOODMSG\n" || printf "\n"
# Check for accounts not being backed up
local accounts_not_being_backed_up=$(grep -He "^BACKUP=[^1]$" /var/cpanel/users/* 2>/dev/null | cut -d: -f1 | grep -Po "[^/]+$")
local suspended_accounts_not_being_backed_up=$(grep -q "BACKUPSUSPENDEDACCTS: 'yes'" /var/cpanel/backups/config 2>/dev/null || \ls -1 /var/cpanel/suspended 2>/dev/null)
local -i skipped_accounts_count=$(cat <(echo "${accounts_not_being_backed_up}") <(echo "${suspended_accounts_not_being_backed_up}") | grep -v "^$" | wc -l)
printf "Skipped backups: "
if [[ -n "$accounts_not_being_backed_up" || -n "$suspended_accounts_not_being_backed_up" ]]; then
printf "$BADMSG ($skipped_accounts_count user$([ $skipped_accounts_count -ne 1 ] && printf s) skipped; $(grep -cv "^$"<<<"$accounts_not_being_backed_up") disabled + $(grep -cv "^$"<<<"$suspended_accounts_not_being_backed_up") suspended)\n"
printf " s\`\`\`\n"
pr --columns 5 -aT -J <(cat <(echo "${accounts_not_being_backed_up}") <(echo "${suspended_accounts_not_being_backed_up}" | grep -v "^$" | sed -r "s@\$@-SUSPENDED@g") | sort -u) | column -t | sed -r "s@^@ @g"
printf " s\`\`\`\n"
else
printf "$GOODMSG ($skipped_accounts_count users skipped; $(grep -cv "^$"<<<"$accounts_not_being_backed_up") disabled + $(grep -cv "^$"<<<"$suspended_accounts_not_being_backed_up") suspended)\n"
fi
# Edge Case where LEGACY_BACKUP is configured but not ^BACKUP
local user_files=$(find /var/cpanel/users/ -type f | grep -v -w /var/cpanel/users/system)
local legacy_accounts=$(while read USER; do grep -q -e "^BACKUP" $USER 2>/dev/null || echo "$USER"; done <<< "${user_files}" | cut -d: -f1 | grep -Po "[^/]+$")
local -i legacy_skipped_accounts_count=$(cat <(echo "${legacy_accounts}") | grep -v "^$" | wc -l)
printf "Legacy backups: "
if [[ -n "$legacy_accounts" ]]; then
printf "$BADMSG (The below users were found to have LEGACY backups enabled but are not enabled for NEW backups)\n"
printf " s\`\`\`\n"
printf "${legacy_accounts}\n"
printf " s\`\`\`\n"
else
printf "$GOODMSG (No issues detected)\n"
fi
}
# Scans MySQL and cPanel database listings to find database inconsistencies.
# Globals:
# None
# Arguments:
# None
# Returns:
# None
function system_checkMysqlDatabases()
{
local databases_outside_of_cpanel databases_outside_of_mysql
local excluded_databases_arr=$(
cat <<END;
^cptmpdb_
^logaholicDB_
^information_schema$
^cphulkd$
^eximstats$
^leechprotect$
^modsec$
^mysql$
^performance_schema$
^roundcube$
^whmxfer$
^horde$
^tmp$
^sys$
END
)
local -r excluded_databases=$(paste -d'|' -s <<< "${excluded_databases_arr}")
# Find databases that exist outside of cPanel's knowledge
databases_outside_of_cpanel=$(
diff \
<(whmapi1 list_databases | grep -E '^\s+name:' | sed -r 's/^\s+name: //g' | sort) \
<(mysql -Ne "show databases" | egrep -v "${excluded_databases}" | sort) \
| grep "^>" | awk '{print $2}'
)
printf "Untracked MySQL databases: "
if [[ -n "$databases_outside_of_cpanel" ]]; then
local -i databases_outside_of_cpanel_count=$(wc -l <<< "$databases_outside_of_cpanel")
printf "$BADMSG (Result: ${databases_outside_of_cpanel_count} database$([ $databases_outside_of_cpanel_count -ne 1 ] && printf s))\n"
printf " \`\`\`\n"
pr --columns 5 -aT -J <<< "$databases_outside_of_cpanel" | column -t | sed -r "s@^@ @g"
printf " \`\`\`\n"
else
printf "$GOODMSG\n"
fi
# Find databases that don't exist
databases_outside_of_mysql=$(
diff \
<(whmapi1 list_databases | grep -E '^\s+name:' | sed -r 's/^\s+name: //g' | sort) \
<(mysql -Ne "show databases" | egrep -v "${excluded_databases}" | sort) \
| grep "^<" | awk '{print $2}'
)
printf "Non-existent MySQL databases: "
if [[ -n "$databases_outside_of_mysql" ]]; then
local -i databases_outside_of_mysql_count=$(wc -l <<< "$databases_outside_of_mysql")
printf "$BADMSG (Result: ${databases_outside_of_mysql_count} database$([ $databases_outside_of_cpanel_count -ne 1 ] && printf s))\n"
printf " \`\`\`\n"
pr --columns 5 -aT -J <<< "$databases_outside_of_mysql" | column -t | sed -r "s@^@ @g"
printf " \`\`\`\n"
else
printf "$GOODMSG\n"
fi
}
# Scans MySQL users for users that are using outdated password hashes.
# Globals:
# None
# Arguments:
# None
# Returns:
# None
function system_checkMysqlUsers()
{
local authentication_column_name outdated_mysql_users
local -i i=0
printf "Outdated password hashes: "
authentication_column_name=$(
mysql -BN <<END
SELECT column_name
FROM information_schema.columns
WHERE table_schema = 'mysql'
AND table_name = 'user'
AND (column_name = 'password' OR column_name = 'authentication_string')
ORDER BY column_name DESC
LIMIT 1
END
)
outdated_mysql_users=$(
mysql -BN <<END
SELECT DISTINCT User
FROM mysql.user
WHERE LENGTH(${authentication_column_name}) < 40
ORDER BY User ASC
END
)
if [[ -n "$outdated_mysql_users" ]]; then
printf "$WARNMSG\n"
printf " s\`\`\`\n"
pr --columns 5 -aT -J <<< "$outdated_mysql_users" | column -t | sed -r "s@^@ @g"
printf " s\`\`\`\n"
else
printf "$GOODMSG\n"
fi
}
# Check to see if CloudLinux is installed
function system_checkCloudLinux()
{
printf "CloudLinux: "
local detect_cloudlinx=$(uname -a | grep lve)
if [[ -n "$detect_cloudlinx" ]]; then
printf "$WARNMSG (Result: Installed)"
else
printf "$GOODMSG (Result: Not Installed)"
fi
printf "\n"
}
function system_checkCSF()
{
local detect_csf
printf "CSF Installed: "
if which csf >/dev/null 2>/dev/null; then
printf "$GOODMSG (Result: Installed)"
else
printf "$BADMSG (Result: Not Installed)"
fi
printf "\n"
}
function system_checkEA4()
{
local php_version
printf "EasyApache 4: "
php_version=$(php -v | awk '/built/ {print $2}'| cut -d. -f 1,2)
if [[ -f /etc/cpanel/ea4/is_ea4 ]]; then
printf "$GOODMSG (Result: Server has EA4)"
else
printf "$BADMSG (Result: Server has EA3)"
fi
printf "\n"
}
function system_checkInodes()
{
local two_million_inodes user_regex
user_regex=$(awk '{print $2}' /etc/trueuserdomains | paste -d'|' -s)
two_million_inodes=$(
repquota -a | awk '$1 ~ /^('"${user_regex}"')$/ && $2 == "--" && $6 > 2000000 && $6 != "none" {print $1" "$6}'
repquota -a | awk '$1 ~ /^('"${user_regex}"')$/ && $2 == "+-" && $7 > 2000000 && $7 != "none" {print $1" "$7}'
)
printf "High inodes: "
if [[ -n "$two_million_inodes" ]]; then
printf "$WARNMSG (Result: $(printf "$two_million_inodes" | wc -l) users)\n"
printf "\n$two_million_inodes" | sort -nrk2 | sed ":a;s/\B[0-9]\{3\}\>/,&/;ta" | column -t | sed -r "s@^@ - @g"
else
printf "$GOODMSG (Result: $(printf "$two_million_inodes" | wc -l) users)\n"
fi
}
function system_joinBy()
{
local IFS="$1";
shift;
echo "$*";
}
function system_checkProcesses()
{
local system_processes_by_pid terminal_processes_by_pid
local possible_out_of_scope_processes known_processes process_command
local -a process_list
# A list of known processes. Entries in this array are used in regex (egrep). Therefore,
# any exact entries should be regex-friendly with escaped regex special characters.
# It is also case sensitive.
local -a known_process_list=(
\/dev\/fd
\/usr\/bin\/kcarectl
\/usr\/local\/cpanel\/3rdparty\/mailman\/bin\/
\/usr\/local\/cpanel\/bin\/backup
\/usr\/sbin\/atd
\/usr\/sbin\/exim
\/usr\/sbin\/nscd
\/dev\/tty
abrt
acpid
agetty
aio
anacron
async\/mgr
ata\/
ata_aux
auditd
automount
axond
bash
bdi\-default
bnx2i
cgroup
chrony
cnic
cpaneld
cpanelconnecttrack
cpanellogd
cpanelsolr
cpbackup_transporter
cpdavd
cPhulkd
cpsrvd
cpuwatch
cqueu
crond
CROND
dbus\-daemon
dnsadmin
dovecot
events
ext4\-dio\-unwrit
flush
fsnotify_mark
gam_server
grep
hald
httpd
ib_
init
ipv6_addrconf
irqbalance
iw_cm
jbd2
jbd2\/sda[1-4]\-8
kacpi
kauditd
kblockd
kdevtmpfs
kdmremove
khelper
khubd
khugepaged
khungtaskd
kintegrityd
kipmi0
kjournald
klogd
kmpath
kmpath_rdacd
kondeman
kpsmoused
kseriod
ksoftirqd
kstriped
ksuspend
kswapd0
kthread
kthrotld
kvm\-irqfd\-clean
kworker
leechprotect
lfd
linkwatch
local_sa
logrunner
loop0
mcelog
migration
mingetty
mlocate
mysql
named
netdata
netns
ntpd
php
pkgacct
polkitd
portreserve
portsentry
ps axf
ps faux
puppetd
pure\-authd
pure\-ftpd
queueprocd
rdma
rpc\.statd
rpcbind
rsync
rsyslogd
saslauthd
scsi
sendmail
smartd
spamd
splitlogs
sshd
stopper
sync_supers
syslog
systemd
tailwatchd
tuned
udevd
usbhid
vballoon
vzctl
watchdog
webalizer
webmaild
whostmgrd
xinetd
yum
)
printf "Unknown Processes: "
# Generate system process PID regex
system_processes_by_pid=$(pstree -p $(pgrep kthread) | egrep -o '\([0-9]+\)' | egrep -o '[0-9]+' | awk '{print $1}' | sort -n | paste -d'|' -s)
# Generate terminal process PID regex. Leave the starting "|". It's there to prevent accidental matching of blank whitespaces.
if pgrep -f /usr/sbin/sshd >/dev/null 2>/dev/null; then
terminal_processes_by_pid="|($(
pstree -p $(pgrep -f /usr/sbin/sshd) | egrep -o '\([0-9]+\)' | egrep -o '[0-9]+' | awk '{print $1}' | sort -n | paste -d'|' -s
))"
fi
# Generate known process regex
known_processes=$(system_joinBy "|" ${known_process_list[@]})
# Cut out known processes. This is done in a loop hitting PIDs and usernames
local IFS=$'\r\n'
for proc in $(ps axf -o "%p %u %a" | egrep -v '^\s*('${system_processes_by_pid}${terminal_processes_by_pid}')\s+'); do
process_command=$(sed -r "s@^(\s*\S+){2}\s*(\\\_)?\s*@@g" <<< $proc)
# DEBUG CODE. Useful for figuring out process matching
# egrep -q "${known_processes}|(^\s*PID)" <<< $process_command && \
# echo " < $(egrep --color=always "${terminal_processes}${system_processes}${known_processes}|(^\s*PID)" <<< $process_command) > " || \
# echo " [ ${process_command} ]"
egrep -q "${known_processes}|(^\s*PID)" <<< $process_command || \
possible_out_of_scope_processes+="\n$proc"
done
if [[ $(wc -l <<< "$possible_out_of_scope_processes") > 1 ]]; then
printf "$WARNMSG"
# Trim the line length to the terminal width
printf "\n s\`\`\`"
if [[ -x "$(which tput 2>/dev/null)" ]]; then
echo -e "$possible_out_of_scope_processes" | sed -r "s@^(.{$(( $(tput cols) - 4 ))}).*@\1...@g; s@^@ @"
else
echo -e "$possible_out_of_scope_processes" | sed -r "s@^@ @"
fi
printf " s\`\`\`\n"
else
printf "$GOODMSG (Result: None found)\n"
fi
}
# Script to collect server information and generate account backups.
# Globals:
# WORKINGDIR
# Arguments:
# None
# Returns:
# None
function generateBackups()
{
local backup_directory=$(awk '$1 == "BACKUPDIR:" { gsub(/BACKUPDIR: ?/,""); print $0}' /var/cpanel/backups/config)
local -a additional_backups_arr=(
/etc/cpupdate.conf
/etc/ips
/etc/wwwacct.conf
/root/.ssh/authorized_keys
/var/cpanel/backups/config
/var/cpanel/backups/*.backup_destination
)
# Generate the "latest" file that the restoration script uses. This function currently only touches the 'upgrade_dir' and 'conf_backup_date' keys.
if [[ ! -f "${WORKINGDIRBASE}/latest" || -n "$(grep '^backup_date:' "${WORKINGDIRBASE}/latest")" ]]
then
echo "upgrade_dir: ${HOSTNAME}/${OS}
conf_backup_date: $(date +%F)
cpanel_backup_date:
backup_transport_id: " > "${WORKINGDIRBASE}/latest"
else
sed -ri \
-e '/^upgrade_dir:/s/:.+$/: '"${HOSTNAME}\/${OS}"'/' \
-e '/^conf_backup_date:/s/:.+$/: '"$(date +%F)"'/' \
"${WORKINGDIRBASE}/latest"
fi
# Pre-transfer script.
echo "Gathering server version information."
bash <(curl -ks https://codex.hostdime.com/scripts/download/advpretransfer) -LV 2>&1 > "${WORKINGDIR}/logs/versions.log"
# Run domaindiff on the current server
mkdir -p "${WORKINGDIR}/logs/domains" /home/.hd/var/log/domaindiff
echo "Gathering domain status."
[ -h "/home/.hd/var/log/domaindiff/${OS}" ] || ln -s "${WORKINGDIR}/logs/domains" "/home/.hd/var/log/domaindiff/${OS}"
echo "--- Pre-Backup ---" >> "${WORKINGDIR}/logs/domaindiff.log" # A line to help distinguish information in the log
bash <(curl -ks https://codex.hostdime.com/scripts/download/domaindiff) -S "$OS" -a 2>&1 | tee -a "${WORKINGDIR}/logs/domaindiff.log"
# echo "Starting maintenance mode."
# bash <(curl -ks https://codex.hostdime.com/scripts/download/maintenance_mode) --lock "${OS}" 2>&1
echo "Backing up exim configuration."
generateBackups_Exim
echo "Backing up EasyApache settings."
generateBackups_EasyApache
echo 'Backing up system DNS information.'
generateBackups_DNS
echo "Backing up miscellaneous configuration files." 2>&1
tar -zcv -C / -f "${WORKINGDIR}/backups/config.tar.gz" "${additional_backups_arr[@]}"
if [[ -d /etc/csf ]]; then
echo "Backing up CSF config."
tar -zcv -C / -f "${WORKINGDIR}/backups/csf.tar.gz" /etc/csf 2>&1
else
echo "CSF config was not found. Skipping."
fi
echo "Generating account and domain lists."
awk '{sub(/:/,"",$1);print $2,$1}' /etc/trueuserdomains | column -t | sort > "${WORKINGDIR}/maindomains.list"
awk '{sub(/:/,"",$1);print $2,$1}' /etc/userdomains | column -t | sort > "${WORKINGDIR}/domains.list"
find /var/cpanel/users -type f ! -name nobody ! -name system -exec basename {} \; | sort > "${WORKINGDIR}/users.list"
echo "Generating IP map."
find /var/cpanel/users -type f ! -name nobody ! -name system -exec grep --color=never -H ^IP {} + | sed "s@/var/cpanel/users/@@g; s/:IP=/ /g" | awk '{print $2,$1}' | column -t | sort > "${WORKINGDIR}/ipmap.list"
echo "Generating suspended account list."
awk -F'"' '$1 ~ /<Directory/ {print $2}' /usr/local/apache/conf/includes/account_suspensions.conf | egrep -o "[^/]+$" | sort > "${WORKINGDIR}/suspended.list"
}
# Backup the exim configuration according to cPanel standards
# Globals:
# WORKINGDIR
# OLDPWD
# Arguments:
# None
# Returns:
# 0 - Successful backup taken
# 1 - Backup was unsuccessful
function generateBackups_Exim()
{
# Files and directories defined by Whostmgr::Config::Exim.
# Version check is defined by Whostmgr::Config::Backup::SMTP::Exim.
local -a exim_files=('/etc/backupmxhosts'
'/etc/cpanel_exim_system_filter'
'/etc/cpanel_mail_netblocks'
'/etc/exim.conf'
'/etc/exim.conf.local'
'/etc/exim.conf.localopts'
'/etc/global_spamassassin_enable'
'/etc/greylist_trusted_netblocks'
'/etc/mail/spamassassin/BAYES_POISON_DEFENSE.cf'
'/etc/mail/spamassassin/CPANEL.cf'
'/etc/mail/spamassassin/deadweight.cf'
'/etc/mail/spamassassin/KAM.cf'
'/etc/mail/spamassassin/kam_heavyweights.cf'
'/etc/mail/spamassassin/P0f.cf'
'/etc/neighbor_netblocks'
'/etc/senderverifybypasshosts'
'/etc/skipsmtpcheckhosts'
'/etc/spammeripblocks'
'/etc/spammers'
'/etc/trustedmailhosts'
'/var/cpanel/config/email/query_apache_for_nobody_senders'
'/var/cpanel/config/email/trust_x_php_script'
'/var/cpanel/custom_mailhelo'
'/var/cpanel/custom_mailips'
'/var/cpanel/per_domain_mailips') \
exim_directories=('/var/cpanel/rbl_info') \
exim_files_version=('/etc/exim.conf.local'
'/etc/exim.conf'
'/usr/local/cpanel/etc/exim/defacls/universal.dist')
local exim_acls_block=$(find /usr/local/cpanel/etc/exim/acls -mindepth 1 -maxdepth 1 -type d -printf '%f\n') \
exim_acls_dist=$(grep '^custom' /usr/local/cpanel/etc/exim/acls.dist) \
local_work_dir="${WORKINGDIR}/backups" \
date=$(date +%s) \
file dir version
local staging_dir="${local_work_dir}/staging"
# Create staging directory.
mkdir -pv "${staging_dir}/cpanel/smtp/exim/config"
[ -d "${staging_dir}/cpanel/smtp/exim/config" ] || return 1
# Backup exim files and directories.
for file in ${exim_files[@]};
do
[ -f "${file}" ] && cp -v "${file}" "${staging_dir}/cpanel/smtp/exim/config/"
done
for dir in ${exim_directories[@]};
do
[ -d "${dir}" ] && cp -arv "${dir}" "${staging_dir}/cpanel/smtp/exim/"
done
# Backup exim ACL files.
while read dir;
do
while read file;
do
[ -f "/usr/local/cpanel/etc/exim/acls/${dir}/${file}" ] && { mkdir -pv "${staging_dir}/cpanel/smtp/exim/acls/${dir}"; cp -v "/usr/local/cpanel/etc/exim/acls/${dir}/${file}" "${staging_dir}/cpanel/smtp/exim/acls/${dir}/"; }
done <<< "${exim_acls_dist}"
done <<< "${exim_acls_block}"
# Save Exim version and timestamp.
for file in ${exim_files_version[@]};
do
[ -f "${file}" ] && version=$(grep -P '^\s*\#\s*cPanel\s+.*\s+ACL\s+Template\s+Version:\s+[\d\.]+' "${file}" | grep -Po '[\d\.]+')
[ -n "${version}" ] && { version=$(printf '%f' "${version}"); break; }
done
echo "version=${version}
time=${date}" > "${staging_dir}/cpanel/smtp/exim/version"
# Save version of backup module as defined in Whostmgr::Config::Backup.
echo "version=1.1
time=${date}" > "${staging_dir}/version"
# Create archive, compress it, and remove the staging directory.
cd "${staging_dir}"
tar -czf "${local_work_dir}/whm-config-backup-cpanel__smtp__exim-${version}-${date}.tar.gz" './'*
cd ${OLDPWD}
rm -fr "${staging_dir}"
# Exit function.
[ -f "${local_work_dir}/whm-config-backup-cpanel__smtp__exim-${version}-${date}.tar.gz" ]
return ${?}
}
# Backup EasyApache-related settings
# Globals:
# WORKINGDIR
# Arguments:
# None
# Returns:
# None
function generateBackups_EasyApache()
{
local local_work_dir="${WORKINGDIR}/backups/easyapache" \
scl
[ -d "${local_work_dir}" ] || mkdir -pv "${local_work_dir}"
# Backup EasyApache 4 profile and PHP settings.
if [ -f /etc/cpanel/ea4/is_ea4 ]; then
ea_current_to_profile --output="${local_work_dir}/cpconftool_current_profile.json"
for scl in $(scl -l | grep '^ea-');
do
cp -v "/opt/cpanel/${scl}/root/etc/php.ini" "${local_work_dir}/${scl}-php.ini"
done
fi
/usr/local/cpanel/bin/rebuild_phpconf --current 2>&1 > "${local_work_dir}/php_conf.txt"
# Backup main EasyApache 3 profile it exists.
[ -f /var/cpanel/easy/apache/profile/_main.yaml ] && cp -pv /var/cpanel/easy/apache/profile/_main.yaml "${local_work_dir}/current_ea3_profile.yaml"
# Backup custom Apache settings.
cp -v /var/cpanel/conf/apache/local "${local_work_dir}/whm_settings.yaml"
return 0
}
# Backup DNS-related information
# Globals:
# WORKINGDIR
# Arguments:
# n/a
# Returns:
# 0 - Completed
# 1 - No system user
function generateBackups_DNS()
{
# Exit if cPanel system user does not exist for some reason.
[ -f /var/cpanel/users/system ] || return 1
local local_work_dir="${WORKINGDIR}/backups/dns" \
zone
[ -d "${local_work_dir}" ] || mkdir -pv "${local_work_dir}"
# Backup zones for the system user in cPanel.
for zone in $(awk -F = '$1 ~ /DNS[0-9]*/ && $2 {print $2}' /var/cpanel/users/system)
do
[ -f "/var/named/${zone}.db" ] && cp -pv "/var/named/${zone}.db" "${local_work_dir}/"
done
}
# Start and watch cPanel backups
# Globals:
# __REPORT
# Arguments:
# None
# Returns:
# 0 - Successful
# 1 - No backup log found
function generateCpanelBackups()
{
local backup_pid backup_log backup_result backup_system_dir backup_account_dir \
backup_date=$(date +%F)
if [ -f /var/cpanel/new_backuprunning ]
then
error "Backups already appear to be running."
return 1
fi
# Start cPanel backups. Pass the PID and log file for monitoring.
echo "Initiating cPanel backups."
/usr/local/cpanel/bin/backup --force 2>&1
backup_pid=$(cat /var/cpanel/new_backuprunning)
backup_log=$(lsof +D /usr/local/cpanel/logs/cpbackup | egrep -o '/usr/local.*\.log' | sort -u)
[ -z "${backup_log}" ] && { echo "Unable to identify backup log based on open files from PID '${backup_pid}'. cPanel backups may not be running successfully."; return 1; }
watchBackup "${backup_pid}" "${backup_log}"
# Generate the "latest" file that the restoration script uses. This function currently only touches the 'upgrade_dir' and 'cpanel_backup_date' keys.
backup_system_dir=$(awk -F ' : ' '/info \[backup\] Running dir & file backup with target : / {print($2)}' "${backup_log}")
backup_account_dir=$(awk -F ' : ' '/info \[backup\] Running account backup with target : / {print($2)}' "${backup_log}")
if [[ ! -f "${WORKINGDIRBASE}/latest" || -n "$(grep '^backup_date:' "${WORKINGDIRBASE}/latest")" ]]
then
echo "upgrade_dir: ${HOSTNAME}/${OS}
conf_backup_date:
cpanel_backup_date: $(date +%F)
backup_transport_id: " > "${WORKINGDIRBASE}/latest"
else
sed -ri \
-e '/^upgrade_dir:/s/:.+$/: '"${HOSTNAME}\/${OS}"'/' \
-e '/^cpanel_backup_date:/s/:.+$/: '"${backup_date}"'/' \
"${WORKINGDIRBASE}/latest"
fi
# Monitoring function finished, grab backup result and generate report output.
backup_result=$(tail -1 "${backup_log}" | cut -d ":" -f 5 | awk '{print $1}')
if [[ ${backup_result} == "Success" ]]; then
error "Backup was completed successfully."
else
error "Backup did not complete successfully. Result: ${backup_result}"
fi
__REPORT+="
#### cPanel Backups
Backup Result: ${backup_result}
* System Data: "'`'"${backup_system_dir}"'`'"
* Account Data: "'`'"${backup_account_dir}"'`'"
Backup Date: "'`'"${backup_date}"'`'"
Log: "'`'"${backup_log}"'`'
}
# Fancy animation plays while backups are running
# Globals:
# None
# Arguments:
# $1 - Backup process PID
# $2 - Backup log path
# Returns:
# None
function watchBackup()
{
local -r backup_pid="${1}" backup_log="${2}"
local -ir bar_width=50
local -r bar_color_active="\e[102m" bar_color_active_half="\e[42m" bar_color_reset="\e[0m"
local -r bar_color_stalled="\e[103m" bar_color_stalled_half="\e[43m"
local fill_color fill_color_half backup_log_snippet
# Action total:
# +1 for mounting
# +1 for mysql backups
# +1 for system backups
# +X accounts * 4
# - Copying homedir
# - Storing mysql dbs
# - Creating Archive
# - pkgacct done
# +1 for requiring a completion message (Success, Failure, PartialFailure)
local -ir account_total=$(wc -l < /etc/trueuserdomains)
local -ir actions_total=$(( 4 + $(( ${account_total} * 4 )) ))
local -i mount_done=0 sql_done=0 system_done=0 backup_done=0
local -i account_actions_done=0 accounts_done=0
local -i actions_complete=0 backup_progress=0 filled_spaces=0
local -i cpuwatch_stall=0
echo -e "Monitoring the backup process. Log: ${backup_log}"
while [ -d "/proc/${backup_pid}" ]
do
backup_log_snippet=$(tail "${backup_log}")
# Change progress.
[[ ${mount_done} == 0 ]] && grep -Eq 'Starting (full|incremental) MySQL database backups' "${backup_log}" && mount_done=1
[[ ${sql_done} == 0 && ${mount_done} == 1 ]] && grep -q 'Running dir & file backup with target' "${backup_log}" && sql_done=1
[[ ${sql_done} == 1 && ${system_done} == 0 ]] && grep -q 'Running account backup' "${backup_log}" && system_done=1
[[ ${system_done} == 1 ]] && accounts_done=$(grep -c 'pkgacct completed' "${backup_log}")
[[ ${system_done} == 1 ]] && account_actions_done=$(grep -ce 'Copying homedir' -e 'Storing mysql dbs' -e 'Creating Archive' -e 'pkgacct completed' "${backup_log}")
grep -q 'Final state is' <<< "${backup_log_snippet}" && backup_done=1
tail -n 1 <<< "${backup_log_snippet}" | grep -q 'waiting for it to go down below' && cpuwatch_stall=1 || cpuwatch_stall=0
sleep 15
actions_complete=$(( ${mount_done} + ${sql_done} + ${system_done} + ${account_actions_done} + ${backup_done} ))
backup_progress=$(( ${actions_complete} * 100 / ${actions_total} ))
filled_spaces=$(( ${bar_width} * ${backup_progress} / 100 ))
# Adjust from 0% to 1% if *any* actions were completed
[[ ${actions_complete} > 0 && ${backup_progress} == 0 ]] && backup_progress=1
# Choose color
if [[ ${cpuwatch_stall} == 1 ]]
then
fill_color="${bar_color_stalled}"
fill_color_half="${bar_color_stalled_half}"
else
fill_color="${bar_color_active}"
fill_color_half="${bar_color_active_half}"
fi
printf "\r"
printf "${fill_color}%-${filled_spaces}s%s${bar_color_reset}"
if [[ $(( ${backup_progress} % $(( 100 / ${bar_width} )) )) == 0 ]]
then
printf "%-$(( ${bar_width} - ${filled_spaces} ))s"
else
printf "${fill_color_half} ${bar_color_reset}%-$(( ${bar_width} - ${filled_spaces} - 1 ))s"
fi
if [[ ${cpuwatch_stall} == 1 ]]
then
echo -ne " stalled"
else
echo -ne " [ ${accounts_done} / ${account_total} ] (${backup_progress}%)"
fi
# echo -ne " => $mount_done + $sql_done + $system_done + $account_actions_done + $backup_done [$filled_spaces] "
sleep .1
[[ ${actions_complete} == ${actions_total} ]] && break
done
echo -e "\n"'Backup has completed!'
}
# Prints the configured cPanel backup directory.
# Globals:
# n/a
# Parameters:
# n/a
# Returns:
# 0 - Success
# 1 - Backup configuration doesn't exist
function get_backup_dir()
{
[ -f /var/cpanel/backups/config ] || return 1
awk '/BACKUPDIR/ { gsub("'\''", ""); print($2)}' /var/cpanel/backups/config
}
# Disables common cPanel services in order to restrict data writing
# Globals:
# n/a
# Arguments:
# n/a
# Returns:
# n/a
function maintenance_mode__enable()
{
mysql -e 'SET GLOBAL read_only = 1'
/usr/local/cpanel/scripts/restartsrv_tailwatchd --stop
/usr/local/cpanel/scripts/restartsrv_chkservd --stop
/usr/local/cpanel/scripts/restartsrv_rsyslog --stop
/usr/local/cpanel/scripts/restartsrv_crond --stop
/usr/local/cpanel/scripts/restartsrv_exim --stop
/usr/local/cpanel/scripts/restartsrv_ftpserver --stop
/usr/local/cpanel/scripts/restartsrv_imap --stop
/usr/local/cpanel/scripts/restartsrv_mailman --stop
}
# Re-enables common cPanel services
# Globals:
# n/a
# Arguments:
# n/a
# Returns:
# n/a
function maintenance_mode__disable()
{
mysql -e 'SET GLOBAL read_only = 0'
/usr/local/cpanel/scripts/restartsrv_tailwatchd
/usr/local/cpanel/scripts/restartsrv_chkservd
/usr/local/cpanel/scripts/restartsrv_rsyslog
/usr/local/cpanel/scripts/restartsrv_crond
/usr/local/cpanel/scripts/restartsrv_exim
/usr/local/cpanel/scripts/restartsrv_ftpserver
/usr/local/cpanel/scripts/restartsrv_imap
/usr/local/cpanel/scripts/restartsrv_mailman
}
main "$@"