How to alert when we STOP receiving logs?

We recently had an issue where we needed to review logs from a router during a P1 outage, but found that LM had stopped receiving logs from the device 2 weeks ago.

We need a way to have a "No Data" type of alert for logs, so that if a device stops sending us logs we can be notified and resolve the issue. Instead of finding out 2 weeks later when the logs are needed during an outage.

We can't use the Log Usage datasource for this because it is based on push metrics and does not have a collection interval.

Matt Whitney

Posted 11 months ago·Last reply 10 months ago

2 comments

Joe Williams

·11 months ago

The built in way would be to create a tracked query and alert on that. That tho isn't really scalable if you have to worry about a lot of random devices. So we ended up writing a DataSource, that uses the LM API to query itself to return how many hours ago it last received something. It isn't the cleanest thing in the world, but it gets the job done. You have to change line 20 to match whatever the ID is for Log Usage. Now this only accounts for Log Usage, not a specific log source. So any logs that are processed.

import groovy.json.JsonSlurper
import com.santaba.agent.util.Settings
import com.santaba.agent.live.LiveHostSet
import org.apache.commons.codec.binary.Hex
import javax.crypto.Mac
import javax.crypto.spec.SecretKeySpec


String apiId   = hostProps.get("lmaccess.id")  ?: hostProps.get("logicmonitor.access.id")
String apiKey  = hostProps.get("lmaccess.key") ?: hostProps.get("logicmonitor.access.key")
String apiBearer = hostProps.get("lmbearer.key")
def portalName = hostProps.get("lmaccount")    ?: Settings.getSetting(Settings.AGENT_COMPANY)

String deviceid = hostProps.get("system.deviceId")

Map proxyInfo  = getProxyInfo()

def fields = 'id,dataSourceId,deviceDataSourceId,name,lastCollectedTime,lastUpdatedTime,deviceDataSourceId'
def apipath = "/device/devices/" + deviceid + "/instances"
def apifilter = 'dataSourceId:14012373'
def deviceinstances = apiGetMany(portalName, apiId, apiKey, apipath, proxyInfo, ['size':1000, 'fields': fields, 'filter': apifilter])

instanceid = deviceinstances[0]['id']
devicedatasourceid = deviceinstances[0]['deviceDataSourceId']

def instancepath = "/device/devices/" + deviceid + "/devicedatasources/" + devicedatasourceid + "/instances/" + instanceid + '/data'
def instancedata = apiGet(portalName, apiId, apiKey, instancepath, proxyInfo, ['period':720])

def now = System.currentTimeMillis()

if (instancedata['time'][0] != null && instancedata['time'][0] > 0) {
	//def diffHours = ((instancedata['time'][0] - now) / (1000 * 60 * 60)).toDouble().round(2)
	def diffHours = ((now - instancedata['time'][0]) / (1000 * 60 * 60)).toDouble().round(2)
	println "hours_since_log=${diffHours}"
}
else {
	def diffHours = "NaN"
	println "hours_since_log=${diffHours}"
}

// If script gets to this point, collector should consider this device alive
// keepAlive(hostProps)

return 0


/* Paginated GET method. Returns a list of objects. */
List apiGetMany(portalName, apiId, apiKey, endPoint, proxyInfo, Map args=[:]) {

    def pageSize = args.get('size', 1000) // Default the page size to 1000 if not specified.
    List items = []

    args['size'] = pageSize

    def pageCount = 0
    while (true) {
        pageCount += 1

        // Updated the args
        args['size'] = pageSize
        args['offset'] = items.size()

        def response = apiGet(portalName, apiId, apiKey, endPoint, proxyInfo, args)
        if (response.get("errmsg", "OK") != "OK") {
            throw new Exception("Santaba returned errormsg: ${response?.errmsg}")
        }
        items.addAll(response.items)

        // If we recieved less than we asked for it means we are done
        if (response.items.size() < pageSize) break
    }
    return items
}


/* Simple GET, returns a parsed json payload. No processing. */
def apiGet(portalName, apiId, apiKey, endPoint, proxyInfo, Map args=[:]) {
    def request = rawGet(portalName, apiId, apiKey, endPoint, proxyInfo, args)
    if (request.getResponseCode() == 200) {
        def payload = new JsonSlurper().parseText(request.content.text)
        return payload
    }
    else {
        throw new Exception("Server return HTTP code ${request.getResponseCode()}")
    }
}


/* Raw GET method. */
def rawGet(portalName, apiId, apiKey, endPoint, proxyInfo, Map args=[:]) {
    // def auth = generateAuth(apiId, apiKey, endPoint)
    String apiBearer = hostProps.get("lmbearer.key")
    def auth = "Bearer " + apiBearer

    def headers = ["Authorization": auth, "Content-Type": "application/json", "X-Version":"3"]
    def url = "https://${portalName}.logicmonitor.com/santaba/rest${endPoint}"

    if (args) {
        def encodedArgs = []
        args.each{ k,v ->
            encodedArgs << "${k}=${java.net.URLEncoder.encode(v.toString(), "UTF-8")}"
        }
        url += "?${encodedArgs.join('&')}"
    }

    def request
    if (proxyInfo.enabled) {
        request = url.toURL().openConnection(proxyInfo.proxy)
    }
    else {
        request = url.toURL().openConnection()
    }
    request.setRequestMethod("GET")
    request.setDoOutput(true)
    headers.each{ k,v ->
        request.addRequestProperty(k, v)
    }

    return request
}


/* Generate auth for API calls. */
static String generateAuth(id, key, path) {
    Long epoch_time = System.currentTimeMillis()
    Mac hmac = Mac.getInstance("HmacSHA256")
    hmac.init(new SecretKeySpec(key.getBytes(), "HmacSHA256"))
    def signature = Hex.encodeHexString(hmac.doFinal("GET${epoch_time}${path}".getBytes())).bytes.encodeBase64()

    return "LMv1 ${id}:${signature}:${epoch_time}"
}


/* Helper method to remind the collector this device is not dead */
def keepAlive(hostProps) {
    // Update the liveHost set so tell the collector we are happy.
    hostId = hostProps.get("system.deviceId").toInteger()
    def liveHostSet =  LiveHostSet.getInstance()
    liveHostSet.flag(hostId)
}


/**
* Get collector proxy settings
* @return Map with proxy settings, empty map if proxy not set.
*/
Map getProxyInfo() {
    // Each property must be evaluated for null to determine whether to use collected value or fallback value
    // Elvis operator does not play nice with booleans
    // default to true in absence of property to use collectorProxy as determinant
    Boolean deviceProxy = hostProps.get("proxy.enable")?.toBoolean()
    deviceProxy = (deviceProxy != null) ? deviceProxy : true  
    // if settings are not present, value should be false
    Boolean collectorProxy = Settings.getSetting("proxy.enable")?.toBoolean()
    collectorProxy = (collectorProxy != null) ? collectorProxy : false  

    Map proxyInfo = [:]
    
    if (deviceProxy && collectorProxy) {
        proxyInfo = [
            enabled : true,
            host : hostProps.get("proxy.host") ?: Settings.getSetting("proxy.host"),
            port : hostProps.get("proxy.port") ?: Settings.getSetting("proxy.port") ?: 3128,
            user : Settings.getSetting("proxy.user"),
            pass : Settings.getSetting("proxy.pass")
        ]
    
        proxyInfo["proxy"] = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyInfo.host, proxyInfo.port.toInteger()))
    }

    return proxyInfo
}

Matt Whitney

OP10 months ago

Thanks Joe, we are going to plan on doing something similar in our portal. Appreciate you providing the script as well!

I also am going to submit a feature request to LM for them to give us an out-of-the-box way to alert when a device stops receiving logs for a certain time period, and a way to easily report on device log usage.

1	SUYASH GAIKWAD
2	Sarah Terry
3	SC Sade Cabe
4	RW Rob Winfield
5	UM Urim Mehmedi
6	KD Kenneth Drost