ma logging: init server
This commit is contained in:
parent
0fbd5ff563
commit
c11c3544ab
19
makefu/2configs/logging/filter/dnsmasq.conf
Normal file
19
makefu/2configs/logging/filter/dnsmasq.conf
Normal file
@ -0,0 +1,19 @@
|
||||
|
||||
if ( [program] == "dnsmasq") {
|
||||
grok {
|
||||
patterns_dir => ["${./patterns}"]
|
||||
match => {
|
||||
"message" => [
|
||||
"^%{logdate:LOGDATE} dnsmasq\[[\d]+\]\: query\[[\w]+\] %{domain:DOMAIN} from %{clientip:CLIENTIP}"
|
||||
, "^%{logdate:LOGDATE} dnsmasq\[[\d]+\]\: reply %{domain:DOMAIN} is %{ip:IP}"
|
||||
, "^%{logdate:LOGDATE} dnsmasq\[[\d]+\]\: %{blocklist:BLOCKLIST} %{domain:DOMAIN} is %{ip:IP}"
|
||||
]
|
||||
}
|
||||
}
|
||||
date {
|
||||
match => [ "LOGDATE", "MMM dd HH:mm:ss", "MMM d HH:mm:ss", "ISO8601" ]
|
||||
}
|
||||
geoip {
|
||||
source => "IP"
|
||||
}
|
||||
}
|
15
makefu/2configs/logging/patterns/dnsmasq
Normal file
15
makefu/2configs/logging/patterns/dnsmasq
Normal file
@ -0,0 +1,15 @@
|
||||
BLOCKLIST [\/\w\.]+
|
||||
DOMAIN [\w\.\-]+
|
||||
DNSID \d+
|
||||
PORT \d+
|
||||
DNSRESPONSE cached|reply|forwarded|query
|
||||
# TODO: there are some strange responses for certain queries like <CNAME> or ...
|
||||
IPORWORD %{IP}|[<>\.\/\w>]+
|
||||
|
||||
# TODO use public suffix list by mozilla
|
||||
TLD [a-z]{2,63}
|
||||
# matches CCSLD and TLD together (e.g. co.uk )
|
||||
CCSLD_TLD [a-z]+\.uk
|
||||
# actually after a CCTLD this would be the third level domain ...
|
||||
PUBLIC_SUFFIX (xn--)?%{FUNCTIONAL_SLD}\.(%{CCSLD_TLD}|%{TLD})
|
||||
FUNCTIONAL_SLD [a-z0-9-]{1,63}
|
@ -1,18 +1,29 @@
|
||||
{pkgs, config, ...}:
|
||||
|
||||
with import <stockholm/lib>;
|
||||
let
|
||||
es-port = 9200;
|
||||
kibana-port = 5601;
|
||||
primaryName = "log.${config.krebs.build.host.name}";
|
||||
serverAliases = [ "${primaryName}.r" "${primaryName}.lan" ];
|
||||
in {
|
||||
|
||||
services.nginx.virtualHosts.${primaryName} = {
|
||||
inherit serverAliases;
|
||||
locations."/" = {
|
||||
proxyPass = "http://localhost:5601/";
|
||||
extraConfig = ''
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
'';
|
||||
};
|
||||
};
|
||||
services.elasticsearch = {
|
||||
enable = true;
|
||||
listenAddress = "0.0.0.0";
|
||||
port = es-port;
|
||||
};
|
||||
services.kibana = {
|
||||
enable = true;
|
||||
listenAddress = "0.0.0.0";
|
||||
port = kibana-port;
|
||||
};
|
||||
|
||||
@ -20,4 +31,110 @@ in {
|
||||
iptables -A INPUT -i retiolum -p tcp --dport ${toString es-port} -j ACCEPT
|
||||
iptables -A INPUT -i retiolum -p tcp --dport ${toString kibana-port} -j ACCEPT
|
||||
'';
|
||||
|
||||
# send logs directly to elasticsearch
|
||||
services.journalbeat = {
|
||||
enable = true;
|
||||
package = pkgs.journalbeat7;
|
||||
extraConfig = ''
|
||||
logging:
|
||||
to_syslog: true
|
||||
level: info
|
||||
metrics.enabled: false
|
||||
template.enabled: false
|
||||
output.logstash:
|
||||
hosts: [ "127.0.0.1:5044" ]
|
||||
template.enabled: false
|
||||
index: journalbeat
|
||||
journalbeat.inputs:
|
||||
- paths: []
|
||||
seek: cursor
|
||||
'';
|
||||
};
|
||||
|
||||
services.logstash = {
|
||||
enable = true;
|
||||
# package = pkgs.logstash5;
|
||||
# plugins = [ pkgs.logstash-contrib ];
|
||||
inputConfig =
|
||||
''
|
||||
syslog {
|
||||
timezone => "Etc/UTC"
|
||||
}
|
||||
beats {
|
||||
port => 5044
|
||||
}
|
||||
'';
|
||||
filterConfig =
|
||||
''
|
||||
# Assume Beats
|
||||
if [syslog] {
|
||||
mutate {
|
||||
add_field => { "program" => "%{[syslog][identifier]}" }
|
||||
}
|
||||
}
|
||||
'' +
|
||||
''
|
||||
if ![program] {
|
||||
mutate {
|
||||
add_field => { "program" => "unknown" }
|
||||
}
|
||||
}
|
||||
'' +
|
||||
''
|
||||
if ([program] == "logstash") {
|
||||
drop {}
|
||||
}
|
||||
'' +
|
||||
''
|
||||
if ( [program] == "dnsmasq") {
|
||||
grok {
|
||||
patterns_dir => ["${./patterns}"]
|
||||
match => {
|
||||
"message" => [
|
||||
"^%{DNSID:dnsid} %{IP:client}/%{PORT} %{DNSRESPONSE:dnstype}\[[\w]+\] %{DOMAIN:domain} from %{IP}"
|
||||
, "^%{DNSID:dnsid} %{IP:client}/%{PORT} %{DNSRESPONSE:dnstype} %{DOMAIN:domain} is %{IPORWORD:resolved_ip}"
|
||||
, "^%{DNSID:dnsid} %{IP:client}/%{PORT} %{DNSRESPONSE:dnstype} %{DOMAIN:domain} to %{IP:upstream_dns}"
|
||||
]
|
||||
}
|
||||
}
|
||||
if [resolved_ip] {
|
||||
geoip {
|
||||
source => "resolved_ip"
|
||||
}
|
||||
}
|
||||
mutate {
|
||||
rename => { "host" => "syslog_host" }
|
||||
}
|
||||
# Target is to parse the the first and second significant part of the domain
|
||||
grok {
|
||||
patterns_dir => ["${./patterns}"]
|
||||
match => { "domain" => [ "%{PUBLIC_SUFFIX:dns_suffix}$" ] }
|
||||
}
|
||||
if [client] {
|
||||
mutate { copy => { "client" => "clientip" } }
|
||||
dns {
|
||||
reverse => [ "client"]
|
||||
action => "replace"
|
||||
hostsfile => [ "/etc/hosts" ]
|
||||
hit_cache_ttl => 1600
|
||||
failed_cache_ttl => 60
|
||||
}
|
||||
}
|
||||
}
|
||||
'' + ''
|
||||
if ( [program] == "proftpd") {
|
||||
kv {
|
||||
field_split => " "
|
||||
}
|
||||
}
|
||||
'';
|
||||
outputConfig =
|
||||
''
|
||||
#stdout {
|
||||
# codec => rubydebug
|
||||
#}
|
||||
elasticsearch { }
|
||||
'';
|
||||
};
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user