单机部署ELK+kafka收集nginx日志

1. docker-compos.yml

version: '3'

services:
  es_cluster1:
    image: elasticsearch:6.6.2
    restart: always
    container_name: es_cluster1
    hostname: es_cluster1
    ports:
    - "9200:9200"

    volumes:
      - /srv/es1/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:rw
      - /srv/es1/data:/usr/share/elasticsearch/data:rw
      - /srv/es1/logs:/var/log/elasticsearch:rw

    environment:
      - "ES_JAVA_OPTS=-Xms1024m -Xmx1024m"

    ulimits:
      memlock:
        soft: -1
        hard: -1

    networks:
      default:
        ipv4_address: 172.22.0.11

  kibana:
    image: kibana:6.6.2
    container_name: kibana
    ports:
      - 5601:5601
    volumes:
#      - /srv/kibana/localtime:/etc/localtime
      - /srv/kibana/kibana.yml:/usr/share/kibana/config/kibana.yml:rw
    depends_on:
      - es_cluster1
    networks:
      default:
        ipv4_address: 172.22.0.14

  zoo1:
    image: zookeeper # 镜像
    restart: always # 重启
    container_name: zoo1
    hostname: zoo1
    ports:
    - "2181:2181"

    volumes:
    - "./zoo1/zoo.cfg:/conf/zoo.cfg"
    - "./zoo1/data:/data"
    - "./zoo1/datalog:/datalog"

    networks:
      default:
        ipv4_address: 172.22.0.15

  kafka1:
    image: wurstmeister/kafka # 镜像
    restart: always
    container_name: kafka1
    hostname: kafka1
    privileged: true
    ports:
    - 9092:9092
    environment:
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.116.118:9092 # 暴露在外的地址
      KAFKA_ADVERTISED_HOST_NAME: 172.23.0.16 # 
      KAFKA_HOST_NAME: kafka1
      KAFKA_ZOOKEEPER_CONNECT: 172.22.0.15:2181
      KAFKA_ADVERTISED_PORT: 9092 # 暴露在外的端口
      KAFKA_BROKER_ID: 0 # 
      KAFKA_LISTENERS: PLAINTEXT://172.22.0.16:9092
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
    volumes:
    - "./kafka1/kafka:/kafka"
    - "./kafka1/logs:/opt/kafka/logs"
    links:
    - zoo1

    networks:
      default:
        ipv4_address: 172.22.0.16


networks:
  default:
    external:
      name: es_network

2. ES配置文件

[root@localhost srv]# cat /srv/es1/elasticsearch.yml
cluster.name: es-cluster
node.name: es_cluster1
path.logs: /var/log/elasticsearch/
#bootstrap.mlockall: true
network.host: 0.0.0.0
http.port: 9200
bootstrap.memory_lock: true
transport.tcp.compress: true

3. kibana配置文件

[root@localhost srv]# cat /srv/kibana/kibana.yml 
server.name: kibana
server.port: 5601
server.host: "0"
elasticsearch.hosts: [ "http://172.22.0.11:9200" ]
i18n.locale: zh-CN

4. zk配置文件

[root@localhost srv]# cat /srv/zoo1/zoo.cfg 
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/data
dataLogDir=/datalog
clientPort=2181
server.1=172.23.0.11:2888:3888;2181

5. 启动容器

[root@localhost srv]# docker-compose up -d
[root@localhost srv]# docker-compose ps
   Name                  Command               State                          Ports                        
-----------------------------------------------------------------------------------------------------------
es_cluster1   /usr/local/bin/docker-entr ...   Up      0.0.0.0:9200->9200/tcp, 9300/tcp                    
kafka1        start-kafka.sh                   Up      0.0.0.0:9092->9092/tcp                              
kibana        /usr/local/bin/kibana-docker     Up      0.0.0.0:5601->5601/tcp                              
zoo1          /docker-entrypoint.sh zkSe ...   Up      0.0.0.0:2181->2181/tcp, 2888/tcp, 3888/tcp, 8080/tcp

6. 安装Logstash

[root@localhost srv]# wget https://artifacts.elastic.co/downloads/logstash/[root@localhost srv]# logstash-6.4.0.tar.gz
tar -xzf logstash-6.4.0.tar.gz

7. 配置Logstash->kafka->es

[root@localhost srv]# cat /etc/logstash/conf.d/nginx.conf 
input {

    file {
        type => "access_nginx"
        path => "/var/log/nginx/access_json.log"
        start_position => "beginning"
        codec => "json"
        }
}
output {
    kafka {
      bootstrap_servers => "192.168.116.118:9092"
      topic_id => "access_nginx"
      compression_type => "snappy"
    }
    stdout{
         codec => rubydebug
     }
}
[root@localhost srv]# cat /etc/logstash/conf.d/kafka_to_es.conf 
input {
    kafka {
      codec => json
      #zk_connect => "192.168.116.118:2181"
      bootstrap_servers => "192.168.116.118:9092"
      topics => "access_nginx"
      consumer_threads => 5
      decorate_events => true
      type => "access_nginx"
  }
}
output {
  if [type] == "access_nginx"{
    elasticsearch {
      hosts => "192.168.116.118:9200"
      index => "access_nginx-%{+YYY.MM.dd}"
      manage_template => true
    }
  }
}

8. 启动Logstash

[root@localhost bin]#  nohup ./logstash -f /etc/logstash/conf.d/ &

9. 查看ES状态

10. 配置kibana索引
    

11. 查看视图
    

• es状态为yellow处理办法

[root@localhost bin]# curl -XPUT "http://localhost:9200/_settings" -H 'Content-Type: application/json' -d'
{
"index" : {
"number_of_replicas" : 0
}
}'