Simple Nagios alarm when external hooks are not available

nagios
tips

(Daryl Monge) #1

You can’t always have a nice integration between a Nagios monitor and a PagerDuty service via web hooks. It is pretty simple to set up something using email. This technique uses event handlers. Event handlers are “handy” since you can disable an event handler in a Nagios service, hence stopping the PagerDuty email, without actually disabling the service check.

define service{
	use				Production,graphed-service
	service_description		Service Running
	event_handler			pagerduty_service
    check_interval           	1
	notification_options		w,c,u
	check_command			check_running
	}

and

define command{
	command_name	pagerduty_service
    command_line    $USER1$/pagerduty_service $SERVICESTATE$ $SERVICESTATETYPE$ $SERVICEATTEMPT$ "$HOSTALIAS$" "$SERVICEDESC$"
	}

finally, a shell script for the event handler

case "$1" in
    OK)
    ;;
    WARNING)
    ;;
    UNKNOWN)
    ;;
    CRITICAL)
	# Is this a "soft" or a "hard" state?
	case "$2" in
	    # We're in a "soft" state, meaning that Nagios is in the middle of retrying the
	    # check before it turns into a "hard" state and contacts get notified...
	    SOFT)
	    ;;
	    HARD)
		echo "Nagios Service $4 is $1" | mailx -s"Host $4 is down or unreachable" aservice@yoursiteinformation.pagerduty.com
	    ;;
	esac
	;;
esac
exit 0

(system) #2