Elastic Stack (ELK Stack)

yum源

[elasticsearch-6.x]
name=Elasticsearch repository for 6.x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md

下载安装

yum install java-1.8.0-openjdk-devel
服务器安装(端口)：Elasticsearch（9200），Kibana（5601），Logstash（5044）
https://www.elastic.co/downloads
systemctl enable elasticsearch
systemctl enable logstash
systemctl enable kibana

vi /etc/elasticsearch/elasticsearch.yml
node.name: wl-es01
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
network.host: 0.0.0.0
http.port: 9200
script.painless.regex.enabled: true
md5-8fac7b9624e43c19a693fbc18a87121a
vi /etc/kibana/kibana.yml
server.host: "0.0.0.0"
elasticsearch.url: "http://10.200.78.67:9200"
elasticsearch.requestTimeout: 120000
#i18n.defaultLocale: "cn"
tilemap.url: 'http://webrd02.is.autonavi.com/appmaptile?lang=zh_cn&size=1&scale=1&style=7&x={x}&y={y}&z={z}'
md5-8fac7b9624e43c19a693fbc18a87121a
es-jvm.options
#-Xms1g
#-Xmx1g
-Xms48g
-Xmx48g
## GC configuration
#-XX:+UseConcMarkSweepGC
#-XX:CMSInitiatingOccupancyFraction=75
#-XX:+UseCMSInitiatingOccupancyOnly
-XX:+UseG1GC
-XX:MaxGCPauseMillis=200
md5-f229ab4abd44f51691ec9926a30ccf3c
/usr/share/logstash/bin/logstash-plugin update logstash-filter-grok
/usr/share/logstash/bin/logstash-plugin update logstash-filter-geoip
/usr/share/logstash/bin/logstash-plugin update logstash-filter-useragent
/usr/share/logstash/bin/logstash-plugin update logstash-filter-date
/usr/share/logstash/bin/logstash-plugin update logstash-filter-mutate
md5-e790f003fd58ea28fa64b630bc9f6979
vi /etc/logstash/conf.d/gameclient.conf
input {
  beats {
    port => 5044
  }
}
output{
  stdout{
    codec => rubydebug
  }
}
/usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/gameclient.conf
md5-9bc009891305a5173f8e307f71f55f74
./logstash-plugin install logstash-input-mongodb
./logstash-plugin install logstash-filter-xml
vi /etc/logstash/conf.d/mongodb
input {
  mongodb {
    uri => 'mongodb://test:123456@127.0.0.1:27017/test'
    placeholder_db_dir => '/opt/logstash-mongodb/'
    placeholder_db_name =>'test.db'
    collection => 'msgCont'
    batch_size => 1000
    generateId => true
  }
}
filter {
  mutate {
        rename => ["_id", "uid"]
    }
  xml {
       source => "msg"
       store_xml => false
       xpath => {"/message/*/*/text()" => "msg-content"}
   }
}
output {
  elasticsearch {
    hosts => ["http://localhost:9200"]
    index => "im_message"
  }
}
md5-76a989a4b20019c8f5a31026928ff6a3
vi /etc/logstash/conf.d/beats-nginx.conf
input {
  beats {
    port => 5044
  }
}
filter {
  grok {
    match => { "message" => "%{IPORHOST:remote_addr} - (%{USERNAME:remote_user}|-) \\[%{HTTPDATE:time_local}\\] (%{IPORHOST:http_host}|-) \\"%{WORD:method} %{DATA:request_url} HTTP/%{NUMBER:http_version}\\" %{NUMBER:response_code} %{NUMBER:body_sent} \\"%{DATA:referrer}\\" \\"%{DATA:user_agent}\\" \\"%{IPORHOST:x_forwarded_for}\\" “%{NUMBER:request_time}\\"" }
    remove_field => "message"
  }
  date {
    match => [ "time_local", "dd/MMM/YYYY:HH:mm:ss Z" ]
    target => "@timestamp"
    timezone => "-04:00"
    }
  useragent {
    regexes => "/etc/logstash/regexes.yaml"
    target => "ua"
    source => "user_agent"
  }
  mutate {
    convert => { "response_code" => "integer" }
    convert => { "body_sent" => "integer" }
    convert => { "request_time" => "float" }
  }
  if [x_forwarded_for] !~ "^127\\.|^192\\.168\\.|^172\\.1[6-9]\\.|^172\\.2[0-9]\\.|^172\\.3[01]\\.|^10\\.|^100.64\\." {
      geoip {
          source => "x_forwarded_for"
          target => "geoip"
          fields => ["city_name","region_name","country_name","location"]
      }
  if ! [geoip][region_name] and ! [geoip][city_name] {
      mutate {
          add_field => { "client_addr" => "%{[geoip][country_name]}" }
      }
  }
  else if ! [geoip][city_name] {
      mutate {
          add_field => { "client_addr" => "%{[geoip][country_name]},%{[geoip][region_name]}" }
      }
  }
  else if ! [geoip][region_name] {
      mutate {
          add_field => { "client_addr" => "%{[geoip][country_name]},%{[geoip][city_name]}" }
      }
  }
  else {
      mutate {
          add_field => { "client_addr" => "%{[geoip][country_name]},%{[geoip][region_name]},%{[geoip][city_name]}" }
      }
  }
  mutate {
      remove_field => ["[geoip][country_name]","[geoip][region_name]","[geoip][city_name]"]
}
}
}
output {
  elasticsearch {
    hosts => "localhost:9200"
    manage_template => false
    index => "nginx-%{+YYYY.MM.dd}"
#    index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
  }
}
md5-f6ff8f7d4faa65f70e31572a3953d9e5
vi /etc/logstash/regexes.yaml
https://github.com/ua-parser/uap-core/blob/master/regexes.yaml
  #######################
  - regex: 'iPhone'
    device_replacement: 'iPhone'
    brand_replacement: 'Apple'
    model_replacement: 'iPhone'
  - regex: 'Xiaomi_'
    device_replacement: 'XiaoMi'
    brand_replacement: 'XiaoMi'
    model_replacement: 'XiaoMi'
  #######################
  #######################
  - regex: 'Windows NT'
    device_replacement: 'PC'
    brand_replacement: 'PC'
    model_replacement: 'PC'
  - regex: 'Macintosh\\;'
    device_replacement: 'MAC'
    brand_replacement: 'Apple'
    model_replacement: 'MAC'
  ######################
md5-129525e9f05966d666be65ad32732a66
vi /etc/logstash/conf.d/syslog.conf
input {
  tcp {
    port => 10514
    type => "Juniper"
  }
  udp {
    port => 10514
    type => "Juniper"
  }
}
filter {
    grok {
    match => { "message" => "reason=(?<reason>([\\s\\S]*))" }
    }
    kv {
        source => "message"
    include_keys => [ "start_time", "src", "src_port", "dst", "dst_port", "sent", "rcvd", "duration", "session_id", "service" ]
#    remove_field => "message"
    }
    date {  
        match => [ "start_time", "yyyy-MM-dd HH:mm:ss" ]  
        target => "@timestamp"
      timezone => "+08:00"  
    }
    mutate {
        convert => { "src_port" => "integer" }
        convert => { "dst_port" => "integer" }
        convert => { "sent" => "integer" }
        convert => { "rcvd" => "integer" }
        convert => { "duration" => "integer" }
    convert => { "session_id" => "integer" }
    }
    if [dst] !~ "^127\\.|^192\\.168\\.|^172\\.1[6-9]\\.|^172\\.2[0-9]\\.|^172\\.3[01]\\.|^10\\.|^100.64\\." {
        geoip {
        source => "dst"
        target => "dstgeoip"
        fields => ["city_name","region_name","country_name"]
        }
    if ! [dstgeoip][region_name] and ! [dstgeoip][city_name] {
    mutate {
        add_field => { "dstname" => "%{[dstgeoip][country_name]}" }
    }
    }
    else if ! [dstgeoip][city_name] {
        mutate {
            add_field => { "dstname" => "%{[dstgeoip][country_name]},%{[dstgeoip][region_name]}" }
        }
    }
    else if ! [dstgeoip][region_name] {
        mutate {
            add_field => { "dstname" => "%{[dstgeoip][country_name]},%{[dstgeoip][city_name]}" }
        }
    }
    else {
        mutate {
            add_field => { "dstname" => "%{[dstgeoip][country_name]},%{[dstgeoip][region_name]},%{[dstgeoip][city_name]}" }
    }
    }
    mutate {
    remove_field => "dstgeoip"
   }
   }
}
output{
    elasticsearch {
        hosts => "localhost:9200"
        document_type => "Juniper"
        index => "juniper-%{+YYYY.MM.dd}"
    }
#    stdout{
#        codec => rubydebug
#    }
}
md5-a06cdcb7a71379d8b27b86f4e912aacc
input {
  beats {
    port => 5044
    type => "gameclient"
  }
}
filter {
     if [type] == "gameclient" {
          grok {
            match => { "message" => "%{TIMESTAMP_ISO8601:date} \\[%{DATA:level}\\] \\<%{DATA}\\>@%{DATA}\\:%{DATA}\\:%{NUMBER} (?<info>([\\s\\S]*))\\,other\\:(?<other>([\\s\\S]*))" }
          }
          kv {
            source => "info"
            prefix => "info_"
            field_split => ","
            value_split => ":"
          }
          kv {
            source => "other"
            prefix => "other_"
            remove_char_key => "\\"|\\{"
            remove_char_value => "\\"|\\}"
            field_split => ","
            value_split => ":"
          }
          date {
            match => [ "date", "yyyy-MM-dd HH:mm:ss.SSS" ]
            target => "@timestamp"
            timezone => "+08:00"
          }
     }
}
output{
     if [type] == "gameclient" {
        #  stdout{
        #    codec => rubydebug
        #  }
          elasticsearch {
            hosts => "localhost:9200"
            document_type => "gameclient"
            index => "gameclient-%{+YYYY.MM.dd}"
          }
     }
}
md5-d177fd8da50f7a0d0654ff7be5a6f6ad
systemctl enable filebeat
vi /etc/filebeat/filebeat.yml
path.home: /usr/share/filebeat
path.config: /etc/filebeat
path.data: /var/lib/filebeat
path.logs: /var/log/filebeat
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
#output.logstash:
#  hosts: ["10.100.77.60:5044"]
output.elasticsearch:
  hosts: ["10.100.77.60:9200"]
setup.kibana:
  host: "10.100.77.60:5601"
md5-8fac7b9624e43c19a693fbc18a87121a
cd /etc/filebeat
/usr/share/filebeat/bin/filebeat setup --template
/usr/share/filebeat/bin/filebeat setup --dashboards
/usr/share/filebeat/bin/filebeat modules enable nginx
md5-8fac7b9624e43c19a693fbc18a87121a
vi nginx.yml
- module: nginx
  # Access logs
  access:
    enabled: true
    var.paths: ["/home/wwwlogs/static.log"]
  # Error logs
  error:
    enabled: false
    #var.paths:
vi /etc/filebeat/filebeat.yml
path.home: /usr/share/filebeat
path.config: /etc/filebeat
path.data: /var/lib/filebeat
path.logs: /var/log/filebeat
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
output.logstash:
  hosts: ["10.100.77.60:5044"]
#output.elasticsearch:
#  hosts: ["10.100.77.60:9200"]
setup.kibana:
  host: "10.100.77.60:5601"
md5-8fac7b9624e43c19a693fbc18a87121a
#实例
cat /etc/filebeat/filebeat.yml    
path.home: /usr/share/filebeat
path.config: /etc/filebeat
path.data: /var/lib/filebeat
path.logs: /var/log/filebeat
#filebeat.config.modules:
#  path: ${path.config}/modules.d/*.yml
filebeat.prospectors:
- type: log
  enabled: true
  paths:
    - /home/huangliang/19090/log/info*
  fields_under_root: true
  fields:
    type: sit
- type: log
  enabled: true
  paths:
    - /home/huangliang/19091/log/info*
#  tags: ["pro"]
  fields_under_root: true
  fields:
    type: pro
#    logs_env: PRO
output.logstash:
  hosts: ["127.0.0.1:5044"]
setup.kibana:
  host: "localhost:5601"
md5-65b421824d53fc8081becb1bc6c26108
vi /tmp/elasticsearch.template.nginx.json
{
  "index_patterns" : ["nginx*"],
  "mappings" : {
    "doc" : {
      "properties" : {
        "geoip" : {
          "properties" : {
            "location" : {
              "type" : "geo_point"
            }
          }
        }
      }
    }
  }
}
curl -XPUT -H 'Content-Type: application/json' 'http://10.100.77.60:9200/_template/nginx?pretty' -d@/tmp/elasticsearch.template.nginx.json
md5-6fe5404ab4895913dde0fec4ed72ef0b
编辑kibana配置文件kibana.yml，最后面添加:
tilemap.url: 'http://webrd02.is.autonavi.com/appmaptile?lang=zh_cn&size=1&scale=1&style=7&x={x}&y={y}&z={z}'
重启kibana
md5-a897e5949d9af7dbb35b1a9096b23e58
curl localhost:9200/_cat/indices?v
curl 10.100.77.60:9200/_cat/templates
curl localhost:9200/filebeat-6.0.0-2017.12.01?pretty
curl 'http://localhost:9200/_template/nginx?pretty'
curl -XDELETE 'http://localhost:9200/filebeat-*'
curl localhost:9200/gameclient-2018.03.08/?pretty
curl http://10.100.77.60:9200/nginx-2017.12.01/_search?pretty
md5-105f0b472a714ae49183ff4102580fd7
curl -H "Content-Type: application/json" -XGET localhost:9200/gameclient-2018.06.10/doc/_search -d '{"query":{"match":{"gameName":{"query":"2277"}}}}'
md5-406eed5e2f40e3fcd02669c2f0414e61
curl -H "Content-Type: application/json" -XPOST localhost:9200/gameclient-2018.06.10/doc/_update_by_query -d '{"query":{"match":{"gameName":{"query":"2277"}}},"script":{"inline":"ctx._source.gameName='6666'","lang":"painless"}}'
md5-814fcd93b990ae09fc0ea5c5fbb25dd3
curl -H "Content-Type: application/json" -XPOST localhost:9200/gameclient-2018.06.10/doc/_update_by_query -d '{"query":{"match":{"gameName":{"query":"2277"}}},"script":{"inline":"ctx._source.gameName = params.last","params": {"last": "金球争霸"},"lang":"painless"}}'
md5-8fac7b9624e43c19a693fbc18a87121a
cat ch.sh
#!/bin/bash
set -x
cat list.txt|while read line
do
  id=echo $line|awk '{print $1}'
  name=echo $line|awk '{print $2}'
  cat gamedate.txt|while read line2
  do
    curl -H "Content-Type: application/json" -XPOST 10.200.77.45:9200/$line2/doc/_update_by_query -d "{\\"query\\":{\\"match\\":{\\"gameName\\":{\\"query\\": \\"$id\\"}}},\\"script\\":{\\"inline\\":\\"ctx._source.gameName = params.last\\",\\"params\\": {\\"last\\": \\"$name\\"},\\"lang\\":\\"painless\\"}}"
  echo ""
  done
done
md5-3f50f7881963e12ca038c3a737b51824
#!/usr/local/bin/python3
# -*- coding:utf-8 -*-
import time
from datetime import datetime
import sys
from elasticsearch import Elasticsearch
from elasticsearch.helpers import bulk
def set_mapping(es, index_name = "pointlogs", doc_type_name = "point"):
    my_mapping = {
        "mappings":{
            "point": {
                "properties": {
                "@timestamp" : {
                "type" : "date"
            },
                    "x": {
                        "type": "integer"
                    },
                    "y": {
                        "type": "float"
                    },
            "z": {
                        "type": "text"
                    }
                }
            }
        }
    }
    put_my_mapping = {
          "properties": {
                    "@timestamp" : {
                "type" : "date"
                },
            "x": {
                        "type": "integer"
                    },
                    "y": {
                        "type": "float"
                    },
                    "z": {
                        "type": "text"
                    }
                }
    }
    create_index = es.indices.create(index = index_name,body = my_mapping)
    mapping_index = es.indices.put_mapping(index = index_name, doc_type = doc_type_name, body = put_my_mapping,ignore=400)
#    if create_index["acknowledged"] != True or mapping_index["acknowledged"] != True:
#        print ("Index creation failed...")
def set_data(es, input_file, index_name = "pointlogs", doc_type_name="point"):
    i = 0
    count = 0
    ACTIONS = []
    with open(input_file,'r') as fd:
        for num,line in enumerate(fd):
            y = float(line)
            action = {
                "_index": index_name,
                "_type": doc_type_name,
                "_source": {
              "@timestamp" : datetime.now().strftime( "%Y-%m-%dT%H:%M:%S.%f+0800"),
                      "x": num,
                      "y": y,
              "z": "10w.txt",
                }
            }
            i += 1
            ACTIONS.append(action)
            if (i == 100000):
                success, _ = bulk(es, ACTIONS, index = index_name, raise_on_error = True)
                count += success
                print("insert %s lines" % count)
                i = 0
                ACTIONS = []
        success, _ = bulk(es, ACTIONS, index = index_name, raise_on_error=True)
        count += success
        print("ALL insert %s lines" % count)
if __name__ == '__main__':
    es = Elasticsearch(hosts=["127.0.0.1:9200"], timeout=5000)
    set_mapping(es)
    set_data(es,sys.argv[1])
md5-41fc0bdb2e5e37823663c78c9753e0cd
vi /etc/elasticsearch/elasticsearch.yml
script.painless.regex.enabled: true
md5-8fac7b9624e43c19a693fbc18a87121a
def m = /^(\\w+)\\ .*$/.matcher(doc['ua.device.keyword'].value);
if ( m.matches() ) {
   return m.group(1)
} else {
   return "null"
}
md5-3f2044d8e2d2cf8b7826d805a92ae27a
{
  "query": {
    "regexp": {
      "userID": "[0-9].+"
    }
  }
}
---
{
  "query": {
    "prefix": {
      "userID": "demo"
    }
  }
}
md5-0834f530e829501e3f49d4bc639fc9cd
cd /root/node_modules/elasticdump/bin/
./elasticdump --input=http://localhost:9200/.kibana --output=kibana_mapping.json --type=mapping
./elasticdump --input=http://localhost:9200/.kibana --output=kibana.json --type=data
md5-cb2799a30eeceee375a7a5fd86231b90
cd /root/node_modules/elasticdump/bin/
./elasticdump --input=kibana_mapping.json --output=http://localhost:9200/.kibana --type=mapping
./elasticdump --input=kibana.json --output=http://localhost:9200/.kibana --type=data
md5-54a16e0605279c65ca292cd294674e63
./elasticdump --input=http://localhost:9200/gameclient-2018.06.30 --output=gameclient-2018.06.30-2.json --type=data --limit 10000