Table of Contents
Archive: This content is maintained for historical reference. Please note that the specific versions and commands may no longer be compatible with modern systems.
This post serves as a reminder of how to ship logs directly from rsyslog to Elasticsearch, bypassing the need for Logstash.
Special thanks to the rsyslog mailing list community for their help in refining this configuration. The following setup has been working reliably in our environment.
The Configuration
Add the following to your rsyslog configuration file (usually found in /etc/rsyslog.d/):
module(load="imfile")
module(load="mmjsonparse")
module(load="omelasticsearch")
template(name="logstash-index"
type="list") {
constant(value="logstash-")
property(name="timereported" dateFormat="rfc3339" position.from="1" position.to="4")
constant(value=".")
property(name="timereported" dateFormat="rfc3339" position.from="6" position.to="7")
constant(value=".")
property(name="timereported" dateFormat="rfc3339" position.from="9" position.to="10")
}
input(type="imfile" File="/var/log/nginx/access.json"
Tag="nginxulyaoth"
PersistStateInterval="10000"
StateFile="nginxulyaoth"
Severity="info"
MaxSubmitAtOnce="20000"
Facility="user"
Ruleset="nginxrule")
template(name="uly-nginx" type="list") {
constant(value="{")
constant(value="\"@timestamp\":\"") property(name="timereported" dateFormat="rfc3339")
constant(value="\",\"host\":\"") property(name="hostname")
constant(value="\",\"severity\":\"") property(name="syslogseverity-text")
constant(value="\",\"facility\":\"") property(name="syslogfacility-text")
constant(value="\",\"tag\":\"") property(name="syslogtag" format="json")
constant(value="\",\"message\":\"") property(name="msg" format="json")
constant(value="\"}")
}
ruleset(name="nginxrule") {
action(type="mmjsonparse" name="jsonparse")
action(type="omelasticsearch"
server="loghost.ulyaoth.net"
serverport="9200"
template="uly-nginx"
searchIndex="logstash-index"
dynSearchIndex="on"
errorFile="/var/log/rsyslog/ES-error.log")
}A improvement you can make on this is to further splitting the JSON log data into individual variables. This will allow for better filtering and visualization within Kibana.
