Стек ELK (Elasticsearch, Logstash и Kibana) - что это такое? Применение, установка и настройка

# curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.15.1-x86_64.rpm
# rpm -vi filebeat-7.15.1-x86_64.rpm

# yum install filebeat
# apt install filebeat

# systemctl start filebeat
# systemctl enable filebeat

# rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch

# rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch

"number": "7.15.1",
"puild_flavor" : "default",
"build_type" : "rpm",
"build_hash" :"83c34f456ae29d60e94d886e455e6a3409bba9ed",
"build_date" : "2021-10-07T21:56:19.0316081852",
"build_snapshot" : false,
"lucene_version" : "8.9.0",
"minimum_wire_compatibility_version" : "6.8.0",
"minimum_index_compatibility_version" : "6.0.0-beta1"
},
"tagline" : "You Know, for Search"

}

# yum install --enablerepo=elasticsearch elasticsearch

# systemctl daemon-reload
# systemctl enable elasticsearch.service
# systemctl start elasticsearch.service

# systemctl status elasticsearch.service

# curl 127.0.0.1:9200
{
"name": "centos8",
"cluster_name" : "elasticsearch",
"cluster_uuid" : "XSsvp-nvTxi68TaEFN_OUg",
"version" : {
"number" : "7.15.1",
"build_flavor" : "default",
"build_type" : "rpm",
"build_hash" :"83c34f456ae29d60e94d886e455e6a3409bba9ed",
"build_date" : "2021-10-07T21:56:19.031608185Z",
"build_snapshot' : false,
"lucene_version" : "8.9.0",
"minimum_wire_compatibility_version" : "6.8.0",
"minimum_index_compatibility_version" : "6.0.0-beta1"

},

"tagline" : "You Know, for Search"

}

# wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -

# apt install apt-transport-https

# echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | tee -a /etc/apt/sources.list.d/elastic-7.x.list

# apt update && apt install elasticsearch

# systemctl daemon-reload
# systemctl enable elasticsearch.service
# systemctl start elasticsearch.service

# systemctl restart elasticsearch.service

# curl 127.0.0.1:9200
{
"name": "centos8",
"cluster_name" : "elasticsearch",
"cluster_uuid" : "XSsvp-nvTxi68TaEFN_OUg",
"version" : {
"number" : "7.15.1",
"build_flavor" : "default",
"build_type" : "rpm",
"build_hash" :"83c34f456ae29d60e94d886e455e6a3409bba9ed",
"build_date" : "2021-10-07T21:56:19.031608185z",
"build_snapshot' : false,
"lucene_version" : "8.9.0",
"minimum_wire_compatibility_version" : "6.8.0",
"minimum_index_compatibility_version" : "6.0.0-beta1"

},

"tagline" : "You Know, for Search"

}

path.data: /var/lib/elasticsearch # директория для хранения данных
network.host: 127.0.0.1 # слушаем только локальный интерфейс

# systemctl status kibana.service

# netstat -tulnp | grep 9200
tcp6   0    0 127.0.0.1:9200     :::*        LISTEN     19788/java

network.host: 0.0.0.0

discovery.seed_hosts: ["127.0.0.1", "[:1]"]

# rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch

# mcedit /etc/yum.repos.d/kibana.repo

 [kibana-7.x]
name=Kibana repository for 7.x packages
baseurl=https://artifacts.elastic.co/packages/7.x/yum
gogcheck=1
gpekey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md

# yum install kibana

# systemctl daemon-reload
# systemctl enable kibana.service
# systemctl start kibana.service

# systemctl restart kibana.service

# netstat -tulnp | grep 5601
tcp     O   0 127.0.0.1:5601      0.0.0.0:*      LISTEN      20746/node

# wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add -

# echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee /etc/apt/sources list.d/elastic-7.x.list

# apt update && apt install kibana

# systemctl daemon-reload
# systemctl enable kibana.service
# systemctl start kibana.service

# systemctl status kibana.service

# netstat -tulnp | grep 5601
tcp   0   0 127.0.0.1:5601    0.0.0.0:*    LISTEN     20746/node

server.host: "10.20.1.23"

# systemctl restart kibana.service

server.publicBaseUrl is missing and should be configured when running in a production environment. Some features may not behave correctly.

server.publicBaseUr|: "http://10.20.1.23:5601/"

xpack.security.enabled: true

# systemctl restart elasticsearch

xpack.security.transport.ssl.enabled: true

{"statusCode":503,"error":"Service Unavailable","message":"License is not available."}

# /usr/share/elasticsearch/bin/elasticsearch-setup-passwords auto

xpack.security.enabled: true
elasticsearch.username: "kibana_system"
elasticsearch.password: "JArsJZj10YC8LIPIQr4u"

output {
     elasticsearch {
       user => "logstash_nginx_rw"
       password => "gdhsgtadsfsdfgsfdget45t"
       hosts => "localhost:9200"
       index => "nginx-%{+YYYY.MM.dd}"
     }
}

server {
listen 443:

server_name kibana.site.ru;
ss|_certificate /etc/letsencrypt/live/kibana.site.ru/fullchain.pem;
ss|_certificate_key /etc/letsencrypt/live/kibana.site.ru/privkey.pem;

location / {

auth_basic "Restricted Access";
auth_basic_user_file /etc/nginx/htpasswd.kibana;
proxy_pass http://localhost:5601;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection 'upgrade';
proxy_set_header Host $host;
proxy_cache_bypass $http_upgrade;
 }
}

# htpasswd -c /etc/nginx/htpasswd.kibana kibanauser

# yum install httpd-tools

# yum install logstash

# apt install logstash

# systemctl enable logstash.service

$ sudo nano /etc/logstash/conf.d/01-input.cont
input {
 beats {
  port => 5044
  }
}

input {
file {
path => "/var/log/nginx/*access*"
}

}

$ /usr/share/logstash/bin/logstash-plugin list
$ /usr/share/logstash/bin/logstash-plugin list --group output

$ /usr/share/logstash/bin/logstash-plugin update
$ /usr/share/logstash/bin/logstash-plugin install logstash-output-geoip --no-verify

$ sudo nano /etc/logstash/conf.d/O2-output.conf

output {
elasticsearch {
hosts => ["localhost:9200"]

  }
}

$ sudo nano /etc/logstash/conf.d/03-nginx-filter.cont
filter {
if [type] == "nginx-access" {
grok {
match => { "message" => "%{NGINXACCESS}" }
overwrite => [ "message" ]
}
date {
locale => "en"
match => [ "timestamp" , ""dd/MMM/YYYY:HH:mmiss Z" ]

    }
  }
}

patterns_dir => ["/patterns"]

$ mkdir /etc/logstash/patterns
$ cat /etc/logstash/patterns/nginx

NGUSERNAME [a-zA-Z.@-+_%]+
NGUSER %{NGUSERNAME}
NGINXACCESS %{IPORHOST:http_host} %{IPORHOST:clientip} [% {HTTPDATE:timestamp}] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})? | %{DATA:rawrequest})" %{NUMBER:response}
(?:%{NUMBER:bytes} | -) %{QS:referrer} %{QS:agent} %{NUMBER:request_time:float} ( %{UUID:request_id})

$ sudo service logstash restart

$ tail -f /var/log/logstash/logstash-plain.log
[INFO J[logstash.inputs.beats ] Beats inputs: Starting input listener {:address=>"localhost:5044"}
[INFO J[logstash.pipeline ] Pipeline main started
[INFO J[logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}