Watchers can be as simple or complex as the query and aggregations they use. Here are some examples to get started with.
SENTINL: HIT WATCHER EXAMPLE:
{
"_index": "watcher",
"_type": "watch",
"_id": "new",
"_source": {
"trigger": {
"schedule": {
"later": "every 5 minutes"
}
},
"input": {
"search": {
"request": {
"index": [
"<mos-{now/d}>",
"<mos-{now/d-1d}>"
],
"body": {}
}
}
},
"condition": {
"script": {
"script": "payload.hits.total > 100"
}
},
"transform": {},
"actions": {
"email_admin": {
"throttle_period": "15m",
"email": {
"to": "alarm@localhost",
"from": "sentinl@localhost",
"subject": "SENTINL Alarm",
"priority": "high",
"body": "Found {{payload.hits.total}} Events"
}
},
"slack_admin": {
"throttle_period": "15m",
"slack": {
"channel": "#kibi",
"message": "SENTINL Alert! Found {{payload.hits.total}} Events"
}
}
}
}
}
SENTINL: TRANSFORM EXAMPLE (ES 2.x):
{
"_index": "watcher",
"_type": "watch",
"_id": "95th",
"_score": 1,
"_source": {
"trigger": {
"schedule": {
"later": "every 5 minutes"
}
},
"input": {
"search": {
"request": {
"index": [
"<access-{now/d}>",
"<access-{now/d-1d}>"
],
"body": {
"size": 0,
"query": {
"filtered": {
"query": {
"query_string": {
"analyze_wildcard": true,
"query": "*"
}
},
"filter": {
"range": {
"@timestamp": {
"from": "now-5m"
}
}
}
}
},
"aggs": {
"response_time_outlier": {
"percentiles": {
"field": "response_time",
"percents": [
95
]
}
}
}
}
}
}
},
"condition": {
"script": {
"script": "payload.aggregations.response_time_outlier.values['95.0'] > 200"
}
},
"transform": {
"script": {
"script": "payload.myvar = payload.aggregations.response_time_outlier.values['95.0']"
}
},
"actions": {
"email_admin": {
"throttle_period": "15m",
"email": {
"to": "username@mycompany.com",
"from": "sentinl@mycompany.com",
"subject": "SENTINL ALARM {{ payload._id }}",
"priority": "high",
"body": "Series Alarm {{ payload._id}}: {{ payload.myvar }}"
}
}
}
}
}
SENTINL: INSERT BACK TO ELASTIC BULK (via NGINX or Direct)
{
"_index": "watcher",
"_type": "watch",
"_id": "surprise",
"_score": 1,
"_source": {
"trigger": {
"schedule": {
"later": "every 50 seconds"
}
},
"input": {
"search": {
"request": {
"index": "my-requests-*",
"body": {
"query": {
"filtered": {
"query": {
"query_string": {
"query": "*",
"analyze_wildcard": true
}
},
"filter": {
"range": {
"@timestamp": {
"from": "now-5m"
}
}
}
}
},
"size": 0,
"aggs": {
"metrics": {
"terms": {
"field": "first_url_part"
}
}
}
}
}
}
},
"condition": {
"script": {
"script": "payload.hits.total > 1"
}
},
"transform": {},
"actions": {
"ES_bulk_request": {
"throttle_period": "1m",
"webhook": {
"method": "POST",
"host": "elasticsearch.foo.bar",
"port": 80,
"path": ":/_bulk",
"body": "{{#payload.aggregations.metrics.buckets}}{\"index\":{\"_index\":\"aggregated_requests\", \"_type\":\"data\"}}\n{\"url\":\"{{key}}\", \"count\":\"{{doc_count}}\", \"execution_time\":\"tbd\"}\n{{/payload.aggregations.metrics.buckets}}",
"headers": {
"Content-Type": "text/plain; charset=ISO-8859-1"
}
}
}
}
}
}