From fe7b5bd14f76f9eb52e2d45c92109ca6219663fd Mon Sep 17 00:00:00 2001 From: BryantHe Date: Wed, 24 May 2023 18:42:40 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=20elk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 +- docker-compose-logging.yml | 43 +++++++++++++++++++ ...-compose.yml => docker-compose-metrics.yml | 6 +-- fluentd/Dockerfile | 12 ++++++ fluentd/conf/fluent.conf | 31 +++++++++++++ 5 files changed, 91 insertions(+), 5 deletions(-) create mode 100644 docker-compose-logging.yml rename docker-compose.yml => docker-compose-metrics.yml (86%) create mode 100644 fluentd/Dockerfile create mode 100644 fluentd/conf/fluent.conf diff --git a/README.md b/README.md index 173db57..b7ec419 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,12 @@ chassis 微服务基座依赖的服务 - Metrics - Statsd - Statsd-exporter +- Logging + - 整合 ELK TODO: - Metrics - 整合 Prometheus - 整合 Grafana -- Logging - - 整合 ELK - Error Reporting - 整合 Sentry \ No newline at end of file diff --git a/docker-compose-logging.yml b/docker-compose-logging.yml new file mode 100644 index 0000000..4aca4db --- /dev/null +++ b/docker-compose-logging.yml @@ -0,0 +1,43 @@ +version: "3" + +services: + kibana: + image: kibana:8.7.1 + ports: + - "5610:5601" + networks: + chassis_agent_logging: + links: + - "elasticsearch" + elasticsearch: + image: elasticsearch:8.7.1 + hostname: "elasticsearch" + environment: + - "discovery.type=single-node" #单机模式启动 + - "TZ=Asia/Shanghai" # 设置时区 + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" # 设置jvm内存大小 + - "xpack.security.enabled=false" + ports: + - "9200:9200" + expose: + - 9200 + networks: + chassis_agent_logging: + fluentd: + build: + context: ./fluentd + volumes: + - ./fluentd/conf/fluent.conf:/opt/bitnami/fluentd/conf/fluentd.conf + links: + - "elasticsearch" + ports: + - "24224:24224" + - "24224:24224/udp" + networks: + chassis_agent_logging: + +networks: + chassis_agent_logging: + driver: bridge + + diff --git a/docker-compose.yml b/docker-compose-metrics.yml similarity index 86% rename from docker-compose.yml rename to docker-compose-metrics.yml index 15ae45b..1c1fd4d 100644 --- a/docker-compose.yml +++ b/docker-compose-metrics.yml @@ -10,7 +10,7 @@ services: volumes: - "./statsd_config.js:/usr/src/app/config.js" networks: - chassis_agent: + chassis_agent_metrics: statsd-exporter: image: prom/statsd-exporter:latest hostname: "statsd-exporter" @@ -21,10 +21,10 @@ services: volumes: - "./statsd_mapping.yml:/tmp/statsd_mapping.yml" networks: - chassis_agent: + chassis_agent_metrics: networks: - chassis_agent: + chassis_agent_metrics: driver: bridge diff --git a/fluentd/Dockerfile b/fluentd/Dockerfile new file mode 100644 index 0000000..ccf7102 --- /dev/null +++ b/fluentd/Dockerfile @@ -0,0 +1,12 @@ +# fluentd/Dockerfile +FROM bitnami/fluentd:1.16.1-debian-11-r13 +USER root +RUN ["gem", "sources", "--remove", "https://rubygems.org/"] +RUN ["gem", "sources", "--add", "https://gems.ruby-china.com/"] +RUN ["gem", "sources", "-u"] +RUN ["gem", "install", "fluent-plugin-elasticsearch", "--version", "5.3.0"] +RUN ["gem", "install", "fluent-plugin-typecast", "--version", "0.2.0"] + + + + diff --git a/fluentd/conf/fluent.conf b/fluentd/conf/fluent.conf new file mode 100644 index 0000000..97267bb --- /dev/null +++ b/fluentd/conf/fluent.conf @@ -0,0 +1,31 @@ +# fluentd/conf/fluent.conf + + @type forward # source表示日志源来自转发，forward与http是默认安装的组件，不需要额外的插件支持。 + port 24224 # 在24224端口启动forward接收转发 + bind 0.0.0.0 # 允许接收任意IP的转发 + + + + @type copy + + @type elasticsearch # 转发日志到es，这需要安装es的插件 + host elasticsearch # es的配置信息 + port 9200 + index_name fluentd + logstash_format true # 启动转发 + logstash_prefix fluentd-${tag} # 转发到es的索引名称 + logstash_dateformat %Y.%m # 按月来划分索引，可以删除无用数据 + include_tag_key true + type_name application_log + tag_key @log_name + reconnect_on_error true + reload_on_failure true + reload_connections false + + flush_interval 2s # 每2s刷新一次 + + + + @type stdout # 搜索到数据后立即转发到es + +