Back

dotCMS-Inspector version 3.x

Description

This is a simple tool to gather information about a running dotcms instance. This only works with dotcms 3.0 plus.

Usage: Must be run as root. 

We gather a lot of system information in order to be sure that the environment is set up properly for dotCMS and to catch any unexpected behavior that may be occurring that our log files wouldn’t capture.  This includes things like what OS version is running, system memory information, number and types of CPUS, disk information including free space, all running procs and their owners, network socket statuses, open files handles, and whether this OS is running in a VM or not.

Information gathered about dotCMS installation and running process is intended to be as broad as is reasonable so that in most instances we will only need to gather information at one point in the process (note that for troubleshooting hung processes or addressing historic incidents this will not be the case).   Gathering information on the java process, including smap, status, stat, and JVM info allows us to have a solid understanding of the parent process condition.  We also gather information on the memory being used specifically by dotCMS to check possible performance issues related to memory.  Index information, config files, and information about assets all are used to assess the condition of the core functions of dotCMS.  Finally we gather java dump and garbage collection information in order to have any debugging and error information that may be useful in conjunction with the dotcms log files.

Code

#!/bin/bash

#################
#tool for dotcms-inspectoring sys, db, and dotcms specific info for analysis
#v 0.1
#Matt Yarbrough
#dotCMS
###################
#TODO: 	ability to turn off certain subscripts
#	push publish config
#	cachestats using setup from Chris McCracken or REST API
#	db threads
# 	db locks
#	use mktemp for setup of log storage space
#	/proc/${PID}/smaps, /proc/${PID}/status, and /proc/${PID}/stat

#TEST COMMANDS
command -v awk >/dev/null 2>&1 || { echo >&2 "awk is not installed.  Ending."; exit 1; }	
command -v grep >/dev/null 2>&1 || { echo >&2 "grep is not installed.  Ending."; exit 1; }	
command -v lsof >/dev/null 2>&1 || skipFileHandles=true	
command -v uname >/dev/null 2>&1 || skipOS=true	
command -v cat >/dev/null 2>&1 || skipOS=true
command -v lscpu >/dev/null 2>&1 || skipCPU=true
command -v netstat >/dev/null 2>&1 || skipNetStat=true 
command -v lspci >/dev/null 2>&1 || skipVM=true
command -v tee >/dev/null 2>&1 || skipTee=true

for install in $(ps aux | grep java | grep dotserver | grep org.apache.catalina.startup.Bootstrap | awk '{ print $2 }'); do

DOTPROCPID=$install
DOTOWNER=$(ps aux | grep -v grep | grep $install | awk '{ print $1}')
echo "DOTOWNER: " $DOTOWNER

if [ ! -d logs-$DOTPROCPID ]; then
    mkdir -p logs-$DOTPROCPID;
fi;
getCatHome1=$(ps ax | grep $install)
getCatHome2=${getCatHome1#*\Dcatalina.home=}
getCatHome3=${getCatHome2%%-Djava.io.tmpdir*}
DOTHOME=$( echo "${getCatHome3}" | sed -e "s/^\ *//g" -e "s/\ *$//g")
#ADMINUSER=$2
#ADMINPASS=$3
HOSTNAME=$(hostname)
TOMCATVERSION=${DOTHOME#*\dotserver\/}
RUNDATE=$(date +"%Y%m%d-%H%M%S")
LOGFOLDER=logs-$DOTPROCPID
SYSLOGFILE=$LOGFOLDER/di-system-$HOSTNAME-$RUNDATE.txt
DOTLOGFILE=$LOGFOLDER/di-dotcms-$HOSTNAME-$RUNDATE.txt
#DB info commented, we aren't using this information right now and I'd rather not be grabbing credentials unnecessarily
#DBX=$(grep username="*" $DOTHOME/webapps/ROOT/META-INF/context.xml | grep -v 'username="{your db user}"' | grep -v 'username="{your user}@{your server}"')
#DBU1=${DBX#*\"}
#DBUSER=${DBU1%\"\ pa*}
#DBP1=${DBX#*word=\"}
#DBPASS=${DBP1%\"\ maxAc*}

if [ skipTee != true ]
	then 
		exec &> >(tee -a "$SYSLOGFILE")
	else
		exec > $SYSLOGFILE 2>&1
fi

echo "dotCMS Inspector Run: " $RUNDATE 
echo "JAVA_HOME: " $JAVA_HOME

#SYSTEM FUNCTIONS
function getDB {
	#what database are we?
	echo -e  "\n\n##### DATABASE INFORMATION #####"
	PG="$(grep '<\!--\ POSTGRESQL -->' $DOTHOME/webapps/ROOT/META-INF/context.xml)"
	MY="$(grep '<\!--\ MYSQL UTF8 -->' $DOTHOME/webapps/ROOT/META-INF/context.xml)"

	cd utils
	if [ "$PG" ]
	  then
	    DB="POSTGRESQL"
		echo "Postgres"
	elif [ "$MY" ]
	  then
	    DB="MYSQL"
		echo "Mysql"
	else
	    DB="OTHER"
		echo "Database not supported in this script"
	fi

}

function getOS {
	# Courtesy of user slm on stackexchange
	echo  -e "\n# OS and Version Information:"  
	OS=`uname -s`
	REV=`uname -r`
	MACH=`uname -m`

	GetVersionFromFile()
	{
	    VERSION=`cat $1 | tr "\n" ' ' | sed s/.*VERSION.*=\ // `
	}

	if [ "${OS}" = "SunOS" ] ; then
	    OS=Solaris
	    ARCH=`uname -p` 
	    OSSTR="${OS} ${REV}(${ARCH} `uname -v`)"
	elif [ "${OS}" = "AIX" ] ; then
	    OSSTR="${OS} `oslevel` (`oslevel -r`)"
	elif [ "${OS}" = "Linux" ] ; then
	    KERNEL=`uname -r`
	    if [ -f /etc/redhat-release ] ; then
	        DIST='RedHat'
	        PSUEDONAME=`cat /etc/redhat-release | sed s/.*\(// | sed s/\)//`
	        REV=`cat /etc/redhat-release | sed s/.*release\ // | sed s/\ .*//`
	    elif [ -f /etc/SuSE-release ] ; then
	        DIST=`cat /etc/SuSE-release | tr "\n" ' '| sed s/VERSION.*//`
	        REV=`cat /etc/SuSE-release | tr "\n" ' ' | sed s/.*=\ //`
	    elif [ -f /etc/mandrake-release ] ; then
	        DIST='Mandrake'
	        PSUEDONAME=`cat /etc/mandrake-release | sed s/.*\(// | sed s/\)//`
	        REV=`cat /etc/mandrake-release | sed s/.*release\ // | sed s/\ .*//`
	    elif [ -f /etc/debian_version ] ; then
	        DIST="Debian `cat /etc/debian_version`"
	        REV=""

	    fi
	    if [ -f /etc/UnitedLinux-release ] ; then
	        DIST="${DIST}[`cat /etc/UnitedLinux-release | tr "\n" ' ' | sed s/VERSION.*//`]"
	    fi

	    OSSTR="${OS} ${DIST} ${REV}(${PSUEDONAME} ${KERNEL} ${MACH})"

	fi

	echo ${OSSTR}
	

}

function getMemory {
	#grep 'Mem|Cache|Swap' /proc/meminfo
	#/proc/${PID}/smaps, /proc/${PID}/status, and /proc/${PID}/stat iterate
	echo -e "\n# Total Memory:"  
	grep MemTotal /proc/meminfo | awk '{print $2}'
	echo -e "\n# Cache:"
	grep Cache /proc/meminfo | awk '{print $2}'
	echo -e "\n# Swap:"
	grep Swap /proc/meminfo | awk '{print $2}'
}

function getCPU {
	echo -e "\n# CPU Info:" 
	lscpu
	top -bn 1 | awk 'BEGIN{FS="[ \t%]+"} NR==3{ print 100-$8 }'
}

function getDiskInfo {
	echo -e "\n# Disk Info:"   
	df -h
}

function getProcs {
	echo -e "\n# PS Output:"   
	ps aux	
}

function getNetstat {
	echo -e "\n# Network Sockets:"   
	netstat -tulpn
}

function getVM {
	echo -e "\n# VM Info:" 
	lspci | grep -i vm
}

function getFileHandles {
	echo -e "\n# Open File Handles:"  
	lsof
	echo -e "\n# Network Open File Handles:"
	lsof -i
}

#EXECUTE SYSTEM FUNCTIONS
echo "\n##### SYSTEM INFORMATION #####" 
if [ skipOS != true ]
	then
		getOS
fi
getMemory
if [ skipCPU != true ]
	then
		getCPU
fi
getDiskInfo
getProcs
if [ skipNetStat != true ]
	then
		getNetStat
fi
if [ skipVM != true ]
	then
		getVM
fi		
if [ skipFileHandles != true ]
	then
	getFileHandles
fi
if [ skipTee != true ]
	then 
		exec &> >(tee -a "$DOTLOGFILE")
	else
		exec > $DOTLOGFILE 2>&1
fi

#DOTCMS FUNCTIONS
function getVersion {
	echo -e "\n# Version:" 
	ls $DOTHOME/webapps/ROOT/WEB-INF/lib/dotcms*jar
	echo -e "\n# Tomcat Version:" 
	echo $TOMCATVERSION
}

function getJVMInfo {
	echo -e "\n# JVM INFORMATION AND MEMORY ALLOCATION:" 
	sudo -u $DOTOWNER jps -v
}

function getDotCMSMem {
	echo -e "\n# Memory Info:" 
	echo $DOTPROCPID  | xargs ps -o rss,sz,vsz
}

function getIndexList {
	echo -e "\n# List Indexes:"
	ls -l $DOTHOME/webapps/ROOT/dotsecure/esdata/dotCMSContentIndex_3x/nodes/*/indices
	echo -e "\n# Index Content Volume:"
	find $DOTHOME/webapps/ROOT/dotsecure/esdata/dotCMSContentIndex_3x/nodes/ | wc -l
}

function getConfigFiles {
	echo -e "\n# Config and log are located in the logs folder" 

	cp $DOTHOME/webapps/ROOT/WEB-INF/classes/dotmarketing-config.properties $LOGFOLDER

	cp $DOTHOME/webapps/ROOT/WEB-INF/classes/portal.properties $LOGFOLDER
	
	cp $DOTHOME/webapps/ROOT/WEB-INF/classes/dotcms-config-cluster.properties  $LOGFOLDER

	cp $DOTHOME/conf/server.xml $LOGFOLDER

	cp $DOTHOME/webapps/ROOT/dotsecure/logs/dotcms.log  $LOGFOLDER
	
	cp $DOTHOME/conf/web.xml $LOGFOLDER
	
}

function getPlugins {
	#find -f on plugins, maybe tarball them, exclude build folder
	echo -e "\n# Static Plugins:" 
	ls  $DOTHOME/webapps/ROOT/WEB-INF/lib
	echo -e "\n# Dynamic Plugins: "
	ls $DOTHOME/webapps/ROOT/WEB-INF/felix/load
}

function getJavaDump {
	#this should loop several times to get an idea of what's really going on
	echo -e "\n#Java thread dump located at logs/javadump.txt" 
	PID=$(pgrep -o -x java)
	sudo -u $DOTOWNER jstack $PID >> $LOGFOLDER/javadump.txt
}

function getPushConfig {
	echo -e "\n# Push Info:"
	#use admin login/pw to get via API
}

function getCacheStats {
	#Version dependent, use jsp version of cache stats page (chris has one), add cmd line arg to dotcms inspector to indicate cachestats or not (default not)
	echo "\n# CacheStats:"
}

function getAssetsInfo {
	#look for asset_real_path, test writing  find ls -type f tree and write to separate file exclude dotGenerated, bundles, .zfs
	echo -e "\n# Assets Permissions:" 
	ls -l $DOTHOME/webapps/ROOT/assets 
	
}

function getGCInfo {
	#java opts lists gc log name
	echo -e "\n# GC Info:"

	if [ -f $DOTHOME/webapps/ROOT/dotsecure/logs/*gc*.log ]; then

	cp $DOTHOME/webapps/ROOT/dotsecure/logs/*gc*.log $LOGFOLDER

	else
		echo "no GC log in $DOTHOME/webapps/ROOT/dotsecure/logs/"
	fi
}

#EXECUTE DOTCMS FUNCTIONS
echo "dotCMS Inspector Run: " $RUNDATE 
echo -e "\n\n##### dotCMS INFORMATION #####" 
getVersion
getJVMInfo
getDotCMSMem
getIndexList
getConfigFiles
getPlugins
getJavaDump
#getPushConfig
#getCacheStats
getAssetsInfo
getGCInfo

tar -czf "di-$HOSTNAME-PROC$DOTPROCPID-$RUNDATE.tgz" $LOGFOLDER

rm -Rf $LOGFOLDER

done
exit

Topics

Have you created code you'd like to share with the dotCMS community?

Share your code Become a Contributor