yum源
[elasticsearch-6.x]
name=Elasticsearch repository for 6.x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md
下载安装
yum install java-1.8.0-openjdk-devel
服务器安装(端口)：Elasticsearch（9200），Kibana（5601），Logstash（5044）
https://www.elastic.co/downloads
systemctl enable elasticsearch
systemctl enable logstash
systemctl enable kibana
vi /etc/elasticsearch/elasticsearch.yml
node.name: wl-es01
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
network.host: 0.0.0.0
http.port: 9200
script.painless.regex.enabled: true
md5-8fac7b9624e43c19a693fbc18a87121a
vi /etc/kibana/kibana.yml
server.host: "0.0.0.0"
elasticsearch.url: "http://10.200.78.67:9200"
elasticsearch.requestTimeout: 120000
#i18n.defaultLocale: "cn"
tilemap.url: 'http://webrd02.is.autonavi.com/appmaptile?lang=zh_cn&size=1&scale=1&style=7&x={x}&y={y}&z={z}'
md5-8fac7b9624e43c19a693fbc18a87121a
es-jvm.options
#-Xms1g
#-Xmx1g
-Xms48g
-Xmx48g
## GC configuration
#-XX:+UseConcMarkSweepGC
#-XX:CMSInitiatingOccupancyFraction=75
#-XX:+UseCMSInitiatingOccupancyOnly
-XX:+UseG1GC
-XX:MaxGCPauseMillis=200
md5-f229ab4abd44f51691ec9926a30ccf3c
/usr/share/logstash/bin/logstash-plugin update logstash-filter-grok
/usr/share/logstash/bin/logstash-plugin update logstash-filter-geoip
/usr/share/logstash/bin/logstash-plugin update logstash-filter-useragent
/usr/share/logstash/bin/logstash-plugin update logstash-filter-date
/usr/share/logstash/bin/logstash-plugin update logstash-filter-mutate
md5-e790f003fd58ea28fa64b630bc9f6979
vi /etc/logstash/conf.d/gameclient.conf
input {
beats {
port => 5044
}
}
output{
stdout{
codec => rubydebug
}
}
/usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/gameclient.conf
md5-9bc009891305a5173f8e307f71f55f74
./logstash-plugin install logstash-input-mongodb
./logstash-plugin install logstash-filter-xml
vi /etc/logstash/conf.d/mongodb
input {
mongodb {
uri => 'mongodb://test:123456@127.0.0.1:27017/test'
placeholder_db_dir => '/opt/logstash-mongodb/'
placeholder_db_name =>'test.db'
collection => 'msgCont'
batch_size => 1000
generateId => true
}
}
filter {
mutate {
rename => ["_id", "uid"]
}
xml {
source => "msg"
store_xml => false
xpath => {"/message/*/*/text()" => "msg-content"}
}
}
output {
elasticsearch {
hosts => ["http://localhost:9200"]
index => "im_message"
}
}
md5-76a989a4b20019c8f5a31026928ff6a3
vi /etc/logstash/conf.d/beats-nginx.conf
input {
beats {
port => 5044
}
}
filter {
grok {
match => { "message" => "%{IPORHOST:remote_addr} - (%{USERNAME:remote_user}|-) \\[%{HTTPDATE:time_local}\\] (%{IPORHOST:http_host}|-) \\"%{WORD:method} %{DATA:request_url} HTTP/%{NUMBER:http_version}\\" %{NUMBER:response_code} %{NUMBER:body_sent} \\"%{DATA:referrer}\\" \\"%{DATA:user_agent}\\" \\"%{IPORHOST:x_forwarded_for}\\" “%{NUMBER:request_time}\\"" }
remove_field => "message"
}
date {
match => [ "time_local", "dd/MMM/YYYY:HH:mm:ss Z" ]
target => "@timestamp"
timezone => "-04:00"
}
useragent {
regexes => "/etc/logstash/regexes.yaml"
target => "ua"
source => "user_agent"
}
mutate {
convert => { "response_code" => "integer" }
convert => { "body_sent" => "integer" }
convert => { "request_time" => "float" }
}
if [x_forwarded_for] !~ "^127\\.|^192\\.168\\.|^172\\.1[6-9]\\.|^172\\.2[0-9]\\.|^172\\.3[01]\\.|^10\\.|^100.64\\." {
geoip {
source => "x_forwarded_for"
target => "geoip"
fields => ["city_name","region_name","country_name","location"]
}
if ! [geoip][region_name] and ! [geoip][city_name] {
mutate {
add_field => { "client_addr" => "%{[geoip][country_name]}" }
}
}
else if ! [geoip][city_name] {
mutate {
add_field => { "client_addr" => "%{[geoip][country_name]},%{[geoip][region_name]}" }
}
}
else if ! [geoip][region_name] {
mutate {
add_field => { "client_addr" => "%{[geoip][country_name]},%{[geoip][city_name]}" }
}
}
else {
mutate {
add_field => { "client_addr" => "%{[geoip][country_name]},%{[geoip][region_name]},%{[geoip][city_name]}" }
}
}
mutate {
remove_field => ["[geoip][country_name]","[geoip][region_name]","[geoip][city_name]"]
}
}
}
output {
elasticsearch {
hosts => "localhost:9200"
manage_template => false
index => "nginx-%{+YYYY.MM.dd}"
# index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
}
}
md5-f6ff8f7d4faa65f70e31572a3953d9e5
vi /etc/logstash/regexes.yaml
https://github.com/ua-parser/uap-core/blob/master/regexes.yaml
#######################
- regex: 'iPhone'
device_replacement: 'iPhone'
brand_replacement: 'Apple'
model_replacement: 'iPhone'
- regex: 'Xiaomi_'
device_replacement: 'XiaoMi'
brand_replacement: 'XiaoMi'
model_replacement: 'XiaoMi'
#######################
#######################
- regex: 'Windows NT'
device_replacement: 'PC'
brand_replacement: 'PC'
model_replacement: 'PC'
- regex: 'Macintosh\\;'
device_replacement: 'MAC'
brand_replacement: 'Apple'
model_replacement: 'MAC'
######################
md5-129525e9f05966d666be65ad32732a66
vi /etc/logstash/conf.d/syslog.conf
input {
tcp {
port => 10514
type => "Juniper"
}
udp {
port => 10514
type => "Juniper"
}
}
filter {
grok {
match => { "message" => "reason=(?<reason>([\\s\\S]*))" }
}
kv {
source => "message"
include_keys => [ "start_time", "src", "src_port", "dst", "dst_port", "sent", "rcvd", "duration", "session_id", "service" ]
# remove_field => "message"
}
date {
match => [ "start_time", "yyyy-MM-dd HH:mm:ss" ]
target => "@timestamp"
timezone => "+08:00"
}
mutate {
convert => { "src_port" => "integer" }
convert => { "dst_port" => "integer" }
convert => { "sent" => "integer" }
convert => { "rcvd" => "integer" }
convert => { "duration" => "integer" }
convert => { "session_id" => "integer" }
}
if [dst] !~ "^127\\.|^192\\.168\\.|^172\\.1[6-9]\\.|^172\\.2[0-9]\\.|^172\\.3[01]\\.|^10\\.|^100.64\\." {
geoip {
source => "dst"
target => "dstgeoip"
fields => ["city_name","region_name","country_name"]
}
if ! [dstgeoip][region_name] and ! [dstgeoip][city_name] {
mutate {
add_field => { "dstname" => "%{[dstgeoip][country_name]}" }
}
}
else if ! [dstgeoip][city_name] {
mutate {
add_field => { "dstname" => "%{[dstgeoip][country_name]},%{[dstgeoip][region_name]}" }
}
}
else if ! [dstgeoip][region_name] {
mutate {
add_field => { "dstname" => "%{[dstgeoip][country_name]},%{[dstgeoip][city_name]}" }
}
}
else {
mutate {
add_field => { "dstname" => "%{[dstgeoip][country_name]},%{[dstgeoip][region_name]},%{[dstgeoip][city_name]}" }
}
}
mutate {
remove_field => "dstgeoip"
}
}
}
output{
elasticsearch {
hosts => "localhost:9200"
document_type => "Juniper"
index => "juniper-%{+YYYY.MM.dd}"
}
# stdout{
# codec => rubydebug
# }
}
md5-a06cdcb7a71379d8b27b86f4e912aacc
input {
beats {
port => 5044
type => "gameclient"
}
}
filter {
if [type] == "gameclient" {
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:date} \\[%{DATA:level}\\] \\<%{DATA}\\>@%{DATA}\\:%{DATA}\\:%{NUMBER} (?<info>([\\s\\S]*))\\,other\\:(?<other>([\\s\\S]*))" }
}
kv {
source => "info"
prefix => "info_"
field_split => ","
value_split => ":"
}
kv {
source => "other"
prefix => "other_"
remove_char_key => "\\"|\\{"
remove_char_value => "\\"|\\}"
field_split => ","
value_split => ":"
}
date {
match => [ "date", "yyyy-MM-dd HH:mm:ss.SSS" ]
target => "@timestamp"
timezone => "+08:00"
}
}
}
output{
if [type] == "gameclient" {
# stdout{
# codec => rubydebug
# }
elasticsearch {
hosts => "localhost:9200"
document_type => "gameclient"
index => "gameclient-%{+YYYY.MM.dd}"
}
}
}
md5-d177fd8da50f7a0d0654ff7be5a6f6ad
systemctl enable filebeat
vi /etc/filebeat/filebeat.yml
path.home: /usr/share/filebeat
path.config: /etc/filebeat
path.data: /var/lib/filebeat
path.logs: /var/log/filebeat
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
#output.logstash:
# hosts: ["10.100.77.60:5044"]
output.elasticsearch:
hosts: ["10.100.77.60:9200"]
setup.kibana:
host: "10.100.77.60:5601"
md5-8fac7b9624e43c19a693fbc18a87121a
cd /etc/filebeat
/usr/share/filebeat/bin/filebeat setup --template
/usr/share/filebeat/bin/filebeat setup --dashboards
/usr/share/filebeat/bin/filebeat modules enable nginx
md5-8fac7b9624e43c19a693fbc18a87121a
vi nginx.yml
- module: nginx
# Access logs
access:
enabled: true
var.paths: ["/home/wwwlogs/static.log"]
# Error logs
error:
enabled: false
#var.paths:
vi /etc/filebeat/filebeat.yml
path.home: /usr/share/filebeat
path.config: /etc/filebeat
path.data: /var/lib/filebeat
path.logs: /var/log/filebeat
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
output.logstash:
hosts: ["10.100.77.60:5044"]
#output.elasticsearch:
# hosts: ["10.100.77.60:9200"]
setup.kibana:
host: "10.100.77.60:5601"
md5-8fac7b9624e43c19a693fbc18a87121a
#实例
cat /etc/filebeat/filebeat.yml
path.home: /usr/share/filebeat
path.config: /etc/filebeat
path.data: /var/lib/filebeat
path.logs: /var/log/filebeat
#filebeat.config.modules:
# path: ${path.config}/modules.d/*.yml
filebeat.prospectors:
- type: log
enabled: true
paths:
- /home/huangliang/19090/log/info*
fields_under_root: true
fields:
type: sit
- type: log
enabled: true
paths:
- /home/huangliang/19091/log/info*
# tags: ["pro"]
fields_under_root: true
fields:
type: pro
# logs_env: PRO
output.logstash:
hosts: ["127.0.0.1:5044"]
setup.kibana:
host: "localhost:5601"
md5-65b421824d53fc8081becb1bc6c26108
vi /tmp/elasticsearch.template.nginx.json
{
"index_patterns" : ["nginx*"],
"mappings" : {
"doc" : {
"properties" : {
"geoip" : {
"properties" : {
"location" : {
"type" : "geo_point"
}
}
}
}
}
}
}
curl -XPUT -H 'Content-Type: application/json' 'http://10.100.77.60:9200/_template/nginx?pretty' -d@/tmp/elasticsearch.template.nginx.json
md5-6fe5404ab4895913dde0fec4ed72ef0b
编辑kibana配置文件kibana.yml，最后面添加:
tilemap.url: 'http://webrd02.is.autonavi.com/appmaptile?lang=zh_cn&size=1&scale=1&style=7&x={x}&y={y}&z={z}'
重启kibana
md5-a897e5949d9af7dbb35b1a9096b23e58
curl localhost:9200/_cat/indices?v
curl 10.100.77.60:9200/_cat/templates
curl localhost:9200/filebeat-6.0.0-2017.12.01?pretty
curl 'http://localhost:9200/_template/nginx?pretty'
curl -XDELETE 'http://localhost:9200/filebeat-*'
curl localhost:9200/gameclient-2018.03.08/?pretty
curl http://10.100.77.60:9200/nginx-2017.12.01/_search?pretty
md5-105f0b472a714ae49183ff4102580fd7
curl -H "Content-Type: application/json" -XGET localhost:9200/gameclient-2018.06.10/doc/_search -d '{"query":{"match":{"gameName":{"query":"2277"}}}}'
md5-406eed5e2f40e3fcd02669c2f0414e61
curl -H "Content-Type: application/json" -XPOST localhost:9200/gameclient-2018.06.10/doc/_update_by_query -d '{"query":{"match":{"gameName":{"query":"2277"}}},"script":{"inline":"ctx._source.gameName='6666'","lang":"painless"}}'
md5-814fcd93b990ae09fc0ea5c5fbb25dd3
curl -H "Content-Type: application/json" -XPOST localhost:9200/gameclient-2018.06.10/doc/_update_by_query -d '{"query":{"match":{"gameName":{"query":"2277"}}},"script":{"inline":"ctx._source.gameName = params.last","params": {"last": "金球争霸"},"lang":"painless"}}'
md5-8fac7b9624e43c19a693fbc18a87121a
cat ch.sh
#!/bin/bash
set -x
cat list.txt|while read line
do
id=`echo $line|awk '{print $1}'`
name=`echo $line|awk '{print $2}'`
cat gamedate.txt|while read line2
do
curl -H "Content-Type: application/json" -XPOST 10.200.77.45:9200/$line2/doc/_update_by_query -d "{\\"query\\":{\\"match\\":{\\"gameName\\":{\\"query\\": \\"$id\\"}}},\\"script\\":{\\"inline\\":\\"ctx._source.gameName = params.last\\",\\"params\\": {\\"last\\": \\"$name\\"},\\"lang\\":\\"painless\\"}}"
echo ""
done
done
md5-3f50f7881963e12ca038c3a737b51824
#!/usr/local/bin/python3
# -*- coding:utf-8 -*-
import time
from datetime import datetime
import sys
from elasticsearch import Elasticsearch
from elasticsearch.helpers import bulk
def set_mapping(es, index_name = "pointlogs", doc_type_name = "point"):
my_mapping = {
"mappings":{
"point": {
"properties": {
"@timestamp" : {
"type" : "date"
},
"x": {
"type": "integer"
},
"y": {
"type": "float"
},
"z": {
"type": "text"
}
}
}
}
}
put_my_mapping = {
"properties": {
"@timestamp" : {
"type" : "date"
},
"x": {
"type": "integer"
},
"y": {
"type": "float"
},
"z": {
"type": "text"
}
}
}
create_index = es.indices.create(index = index_name,body = my_mapping)
mapping_index = es.indices.put_mapping(index = index_name, doc_type = doc_type_name, body = put_my_mapping,ignore=400)
# if create_index["acknowledged"] != True or mapping_index["acknowledged"] != True:
# print ("Index creation failed...")
def set_data(es, input_file, index_name = "pointlogs", doc_type_name="point"):
i = 0
count = 0
ACTIONS = []
with open(input_file,'r') as fd:
for num,line in enumerate(fd):
y = float(line)
action = {
"_index": index_name,
"_type": doc_type_name,
"_source": {
"@timestamp" : datetime.now().strftime( "%Y-%m-%dT%H:%M:%S.%f+0800"),
"x": num,
"y": y,
"z": "10w.txt",
}
}
i += 1
ACTIONS.append(action)
if (i == 100000):
success, _ = bulk(es, ACTIONS, index = index_name, raise_on_error = True)
count += success
print("insert %s lines" % count)
i = 0
ACTIONS = []
success, _ = bulk(es, ACTIONS, index = index_name, raise_on_error=True)
count += success
print("ALL insert %s lines" % count)
if __name__ == '__main__':
es = Elasticsearch(hosts=["127.0.0.1:9200"], timeout=5000)
set_mapping(es)
set_data(es,sys.argv[1])
md5-41fc0bdb2e5e37823663c78c9753e0cd
vi /etc/elasticsearch/elasticsearch.yml
script.painless.regex.enabled: true
md5-8fac7b9624e43c19a693fbc18a87121a
def m = /^(\\w+)\\ .*$/.matcher(doc['ua.device.keyword'].value);
if ( m.matches() ) {
return m.group(1)
} else {
return "null"
}
md5-3f2044d8e2d2cf8b7826d805a92ae27a
{
"query": {
"regexp": {
"userID": "[0-9].+"
}
}
}
---
{
"query": {
"prefix": {
"userID": "demo"
}
}
}
md5-0834f530e829501e3f49d4bc639fc9cd
cd /root/node_modules/elasticdump/bin/
./elasticdump --input=http://localhost:9200/.kibana --output=kibana_mapping.json --type=mapping
./elasticdump --input=http://localhost:9200/.kibana --output=kibana.json --type=data
md5-cb2799a30eeceee375a7a5fd86231b90
cd /root/node_modules/elasticdump/bin/
./elasticdump --input=kibana_mapping.json --output=http://localhost:9200/.kibana --type=mapping
./elasticdump --input=kibana.json --output=http://localhost:9200/.kibana --type=data
md5-54a16e0605279c65ca292cd294674e63
./elasticdump --input=http://localhost:9200/gameclient-2018.06.30 --output=gameclient-2018.06.30-2.json --type=data --limit 10000
