diff options
Diffstat (limited to 'src/pmie/pmie_daily.sh')
-rw-r--r-- | src/pmie/pmie_daily.sh | 540 |
1 files changed, 540 insertions, 0 deletions
diff --git a/src/pmie/pmie_daily.sh b/src/pmie/pmie_daily.sh new file mode 100644 index 0000000..3e386c0 --- /dev/null +++ b/src/pmie/pmie_daily.sh @@ -0,0 +1,540 @@ +#! /bin/sh +# +# Copyright (c) 2013-2014 Red Hat. +# Copyright (c) 2007 Aconex. All Rights Reserved. +# Copyright (c) 1995-2000,2003 Silicon Graphics, Inc. All Rights Reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# Daily administrative script for pmie logfiles +# + +. $PCP_DIR/etc/pcp.env +. $PCP_SHARE_DIR/lib/rc-proc.sh + +# added to handle problem when /var/log/pcp is a symlink, as first +# reported by Micah_Altman@harvard.edu in Nov 2001 +# +_unsymlink_path() +{ + [ -z "$1" ] && return + __d=`dirname $1` + __real_d=`cd $__d 2>/dev/null && $PWDCMND` + if [ -z "$__real_d" ] + then + echo $1 + else + echo $__real_d/`basename $1` + fi +} + +# error messages should go to stderr, not the GUI notifiers +# +unset PCP_STDERR + +# constant setup +# +tmp=`mktemp -d /tmp/pcp.XXXXXXXXX` || exit 1 +status=0 +echo >$tmp/lock +trap "rm -rf \`[ -f $tmp/lock ] && cat $tmp/lock\` $tmp; exit \$status" 0 1 2 3 15 +prog=`basename $0` + +if is_chkconfig_on pmie +then + PMIE_CTL=on +else + PMIE_CTL=off +fi + +# control file for pmie administration ... edit the entries in this +# file to reflect your local configuration (see also -c option below) +# +CONTROL=$PCP_PMIECONTROL_PATH + +# default number of days to keep pmie logfiles +# +CULLAFTER=14 + +# default compression program and days until starting compression +# +COMPRESS=xz +COMPRESSAFTER="" +COMPRESSREGEX="\.(meta|index|Z|gz|bz2|zip|xz|lzma|lzo|lz4)$" + +# mail addresses to send daily logfile summary to +# +MAILME="" + +# search for your mail agent of choice ... +# +MAIL='' +for try in Mail mail email +do + if which $try >/dev/null 2>&1 + then + MAIL=$try + break + fi +done + +# NB: FQDN cleanup; don't guess a 'real name for localhost', and +# definitely don't truncate it a la `hostname -s`. Instead now +# we use such a string only for the default log subdirectory, ie. +# for substituting LOCALHOSTNAME in the third column of $CONTROL. + +# determine path for pwd command to override shell built-in +# (see BugWorks ID #595416). +PWDCMND=`which pwd 2>/dev/null | $PCP_AWK_PROG ' +BEGIN { i = 0 } +/ not in / { i = 1 } +/ aliased to / { i = 1 } + { if ( i == 0 ) print } +'` +if [ -z "$PWDCMND" ] +then + # Looks like we have no choice here... + # force it to a known IRIX location + PWDCMND=/bin/pwd +fi +eval $PWDCMND -P >/dev/null 2>&1 +[ $? -eq 0 ] && PWDCMND="$PWDCMND -P" +here=`$PWDCMND` + +echo > $tmp/usage +cat >> $tmp/usage <<EOF +Options: + -c=FILE,--control=FILE pmie control file + -k=N,--discard=N remove pmie log files after N days + -m=ADDRs,--mail=ADDRs send daily log files to email addresses + -N,--showme perform a dry run, showing what would be done + -V,--verbose verbose output (multiple times for very verbose) + -x=N,--compress-after=N compress pmie log files after N days + -X=PROGRAM,--compressor=PROGRAM use PROGRAM for pmie log file compression + -Y=REGEX,--regex=REGEX egrep filter when compressing files ["$COMPRESSREGEX"] + --help +EOF + +_usage() +{ + pmgetopt --progname=$prog --config=$tmp/usage --usage + status=1 + exit +} + +# option parsing +# +SHOWME=false +RM=rm +KILL=pmsignal +VERBOSE=false +VERY_VERBOSE=false +MYARGS="" + +ARGS=`pmgetopt --progname=$prog --config=$tmp/usage -- "$@"` +[ $? != 0 ] && exit 1 + +eval set -- "$ARGS" +while [ $# -gt 0 ] +do + case "$1" + in + -c) CONTROL="$2" + shift + ;; + -k) CULLAFTER="$2" + shift + check=`echo "$CULLAFTER" | sed -e 's/[0-9]//g'` + if [ ! -z "$check" -a X"$check" != Xforever ] + then + echo "Error: -k option ($CULLAFTER) must be numeric" + status=1 + exit + fi + ;; + -m) MAILME="$2" + shift + ;; + -N) SHOWME=true + RM="echo + rm" + KILL="echo + kill" + MYARGS="$MYARGS -N" + ;; + -V) if $VERBOSE + then + VERY_VERBOSE=true + else + VERBOSE=true + fi + MYARGS="$MYARGS -V" + ;; + -x) COMPRESSAFTER="$2" + shift + check=`echo "$COMPRESSAFTER" | sed -e 's/[0-9]//g'` + if [ ! -z "$check" ] + then + echo "Error: -x option ($COMPRESSAFTER) must be numeric" + status=1 + exit + fi + ;; + -X) COMPRESS="$2" + shift + ;; + -Y) COMPRESSREGEX="$2" + shift + ;; + --) shift + break + ;; + -\?) _usage + ;; + esac + shift +done + +[ $# -ne 0 ] && _usage + +if [ ! -f "$CONTROL" ] +then + echo "$prog: Error: cannot find control file ($CONTROL)" + status=1 + exit +fi + +SUMMARY_LOGNAME=`pmdate -1d %Y%m%d` + +_error() +{ + _report Error "$1" +} + +_warning() +{ + _report Warning "$1" +} + +_report() +{ + echo "$prog: $1: $2" + echo "[$CONTROL:$line] ... inference engine for host \"$host\" unchanged" + touch $tmp/err +} + +_unlock() +{ + rm -f lock + echo >$tmp/lock +} + +# filter for pmie log files in working directory - +# pass in the number of days to skip over (backwards) from today +# +# pv:821339 too many sed commands for IRIX ... split into groups +# of at most 200 days +# +_date_filter() +{ + # start with all files whose names match the patterns used by + # the PCP pmie log file management scripts ... this list may be + # reduced by the sed filtering later on + # + ls | sed -n >$tmp/in -e '/[-.][12][0-9][0-9][0-9][0-1][0-9][0-3][0-9]/p' + + i=0 + while [ $i -le $1 ] + do + dmax=`expr $i + 200` + [ $dmax -gt $1 ] && dmax=$1 + echo "/[-.][12][0-9][0-9][0-9][0-1][0-9][0-3][0-9]/{" >$tmp/sed1 + while [ $i -le $dmax ] + do + x=`pmdate -${i}d %Y%m%d` + echo "/[-.]$x/d" >>$tmp/sed1 + i=`expr $i + 1` + done + echo "p" >>$tmp/sed1 + echo "}" >>$tmp/sed1 + + # cull file names with matching dates, keep other file names + # + sed -n -f $tmp/sed1 <$tmp/in >$tmp/tmp + mv $tmp/tmp $tmp/in + done + + cat $tmp/in +} + + +rm -f $tmp/err $tmp/mail +line=0 +version='' +cat $CONTROL \ +| sed -e "s;PCP_LOG_DIR;$PCP_LOG_DIR;g" \ +| while read host socks logfile args +do + # start in one place for each iteration (beware relative paths) + cd "$here" + line=`expr $line + 1` + + # NB: FQDN cleanup: substitute the LOCALHOSTNAME marker in the config line + # differently for the directory and the pcp -h HOST arguments. + logfile_hostname=`hostname || echo localhost` + logfile=`echo $logfile | sed -e "s;LOCALHOSTNAME;$logfile_hostname;"` + logfile=`_unsymlink_path $logfile` + [ "x$host" = "xLOCALHOSTNAME" ] && host=local: + + $VERY_VERBOSE && echo "[control:$line] host=\"$host\" socks=\"$socks\" log=\"$logfile\" args=\"$args\"" + + case "$host" + in + \#*|'') # comment or empty + continue + ;; + + \$*) # in-line variable assignment + $SHOWME && echo "# $host $socks $logfile $args" + cmd=`echo "$host $socks $logfile $args" \ + | sed -n \ + -e "/='/s/\(='[^']*'\).*/\1/" \ + -e '/="/s/\(="[^"]*"\).*/\1/' \ + -e '/=[^"'"'"']/s/[;&<>|].*$//' \ + -e '/^\\$[A-Za-z][A-Za-z0-9_]*=/{ +s/^\\$// +s/^\([A-Za-z][A-Za-z0-9_]*\)=/export \1; \1=/p +}'` + if [ -z "$cmd" ] + then + # in-line command, not a variable assignment + _warning "in-line command is not a variable assignment, line ignored" + else + case "$cmd" + in + 'export PATH;'*) + _warning "cannot change \$PATH, line ignored" + ;; + 'export IFS;'*) + _warning "cannot change \$IFS, line ignored" + ;; + *) + $SHOWME && echo "+ $cmd" + eval $cmd + ;; + esac + fi + continue + ;; + esac + + if [ -z "$socks" -o -z "$logfile" -o -z "$args" ] + then + _error "insufficient fields in control file record" + continue + fi + + dir=`dirname $logfile` + $VERY_VERBOSE && echo "Check pmie -h $host ... in $dir ..." + + if [ ! -d "$dir" ] + then + if [ "$PMIE_CTL" = "on" ] + then + _error "logfile directory ($dir) does not exist" + fi + continue + fi + + cd $dir + dir=`$PWDCMND` + $SHOWME && echo "+ cd $dir" + + if $VERBOSE + then + echo + echo "=== daily maintenance of pmie log files for host $host ===" + echo + fi + + if [ ! -w $dir ] + then + echo "$prog: Warning: no write access in $dir, skip lock file processing" + else + # demand mutual exclusion + # + fail=true + rm -f $tmp/stamp + for try in 1 2 3 4 + do + if pmlock -v lock >$tmp/out + then + echo $dir/lock >$tmp/lock + fail=false + break + else + if [ ! -f $tmp/stamp ] + then + touch -t `pmdate -30M %Y%m%d%H%M` $tmp/stamp + fi + if [ ! -z "`find lock -newer $tmp/stamp -print 2>/dev/null`" ] + then + : + else + echo "$prog: Warning: removing lock file older than 30 minutes" + LC_TIME=POSIX ls -l $dir/lock + rm -f lock + fi + fi + sleep 5 + done + + if $fail + then + # failed to gain mutex lock + # + if [ -f lock ] + then + echo "$prog: Warning: is another PCP cron job running concurrently?" + LC_TIME=POSIX ls -l $dir/lock + else + echo "$prog: `cat $tmp/out`" + fi + _warning "failed to acquire exclusive lock ($dir/lock) ..." + continue + fi + fi + + # match $logfile from control file to running pmies + pid="" + $VERY_VERBOSE && echo "Looking for logfile=$logfile" + for file in `ls "$PCP_TMP_DIR/pmie"` + do + p_id=$file + file="$PCP_TMP_DIR/pmie/$file" + p_logfile="" + p_pmcd_host="" + + $VERY_VERBOSE && $PCP_ECHO_PROG $PCP_ECHO_N "Check p_id=$p_id ... ""$PCP_ECHO_C" + if ps -p "$p_id" >/dev/null 2>&1 + then + eval `$PCP_BINADM_DIR/pmiestatus $file | $PCP_AWK_PROG ' +NR == 2 { printf "p_logfile=\"%s\"\n", $0; next } +NR == 3 { printf "p_pmcd_host=\"%s\"\n", $0; next } + { next }'` + $VERY_VERBOSE && $PCP_ECHO_PROG $PCP_ECHO_N "p_pmcd_host=$p_pmcd_host p_logfile=$p_logfile""$PCP_ECHO_C" + p_logfile=`_unsymlink_path $p_logfile` + $VERY_VERBOSE && $PCP_ECHO_PROG $PCP_ECHO_N "->$p_logfile ... ""$PCP_ECHO_C" + if [ "$p_logfile" = $logfile ] + then + pid=$p_id + $VERY_VERBOSE && $PCP_ECHO_PROG match + break + fi + $VERY_VERBOSE && $PCP_ECHO_PROG "no match" + else + # ignore, its not a running process + eval $RM -f $file + $VERY_VERBOSE && $PCP_ECHO_PROG "process has vanished" + fi + done + + if [ -z "$pid" ] + then + if [ "$PMIE_CTL" = "on" ] + then + _error "no pmie instance running for host \"$host\"" + fi + else + # now move current logfile name aside and SIGHUP to "roll the logs" + # creating a new logfile with the old name in the process. + # + $SHOWME && echo "+ mv $logfile ${logfile}.{SUMMARY_LOGNAME}" + if mv $logfile ${logfile}.${SUMMARY_LOGNAME} + then + $VERY_VERBOSE && echo "+ $KILL -s HUP $pid" + eval $KILL -s HUP $pid + echo ${logfile}.${SUMMARY_LOGNAME} >> $tmp/mail + else + _error "problems moving logfile \"$logfile\" for host \"$host\"" + touch $tmp/err + fi + fi + + # and cull old logfiles + # + if [ X"$CULLAFTER" != X"forever" ] + then + _date_filter $CULLAFTER >$tmp/list + if [ -s $tmp/list ] + then + if $VERBOSE + then + echo "Log files older than $CULLAFTER days being removed ..." + fmt <$tmp/list | sed -e 's/^/ /' + fi + if $SHOWME + then + cat $tmp/list | xargs echo + rm -f + else + cat $tmp/list | xargs rm -f + fi + fi + fi + + # finally, compress old log files + # (after cull - don't compress unnecessarily) + # + if [ ! -z "$COMPRESSAFTER" ] + then + _date_filter $COMPRESSAFTER | egrep -v "$COMPRESSREGEX" >$tmp/list + if [ -s $tmp/list ] + then + if $VERBOSE + then + echo "Log files older than $COMPRESSAFTER days being compressed ..." + fmt <$tmp/list | sed -e 's/^/ /' + fi + if $SHOWME + then + cat $tmp/list | xargs echo + $COMPRESS + else + cat $tmp/list | xargs $COMPRESS + fi + fi + fi + + _unlock + +done + +if [ -n "$MAILME" -a -s $tmp/mail ] +then + logs="" + for file in `cat $tmp/mail` + do + [ -f $file ] && logs="$logs $file" + done + egrep -v '( OK | OK$|^$|^Log |^pmie: PID)' $logs > $tmp/logmail + if [ ! -s "$tmp/logmail" ] + then + : + elif [ ! -z "$MAIL" ] + then + egrep -v '( OK | OK$|^$)' $logs | \ + $MAIL -s "PMIE summary for $LOCALHOSTNAME" $MAILME + else + echo "$prog: PMIE summary for $LOCALHOSTNAME ..." + egrep -v '( OK | OK$|^$)' $logs + fi + rm -f $tmp/mail $tmp/logmail +fi + +[ -f $tmp/err ] && status=1 +exit |