Kafka消费积压Lag监控工具Burrow的使用

#基础选项
[general]
pidfile="burrow.pid"
stdout-logfile="logs/burrow.out"
access-control-allow-origin="mysite.example.com"
#日志选项
[logging]
filename="logs/burrow.log"
level="info"
maxsize=100
maxbackups=30
maxage=10
use-localtime=false
use-compression=true
#zookeeper选项，这个zookeeper使用burrow自己用的
[zookeeper]
servers=["10.128.0.2:2181", ]
timeout=6
root-path="/burrow"
#burrow作为客户端的配置
[client-profile.burrowclient]
client-id="burrowclient"
kafka-version="0.10.0"
#HTTP监听配置，可以使用HTTPS
[httpserver.default]
address=":8000"
#存储选项
[storage.default]
class-name="inmemory"
workers=20
intervals=15
expire-group=604800
min-distance=1
#报警选项
[notifier.default]
class-name="email"
interval=30
threshold=2
group-whitelist="^important-group-prefix.*$"
group-blacklist="^not-this-group$"
template-open="config/default-email.tmpl"
server="127.0.0.1"
port=25
from="root@localhost.com"
to="hello@ipcpu.com"
#
# kafka cluster config here
#接下来是Kafka相关的内容
#
[cluster.bu-agent-kafka]
class-name="kafka"
servers=[ "10.128.0.65:9092", "10.128.0.66:9092", "10.128.0.67:9092" ]
client-profile="burrowclient"
topic-refresh=300
offset-refresh=60
[consumer.bu-agent-kafka]
class-name="kafka"
cluster="bill-kafka"
servers=[ "10.128.0.65:9092", "10.128.0.66:9092", "10.128.0.67:9092" ]
client-profile="burrowclient"
offsets-topic="__consumer_offsets"
start-latest=true
group-whitelist=".*"
group-blacklist="^not-this-group$"
#@注意这里使用了cluster.kafkaname和consumer.kafkaname两个配置组，
#@第一个用来获取topic和最新offset信息，
#@第二个用来获取消费组和消费组offset及Lag。

nohup /data/apps/burrow/burrow -config-dir=/data/apps/burrow/config/ &

[root@mt ~]##@列出所有监控的Kafka集群
[root@mt ~]# curl -s http://10.128.0.2:8000/v3/kafka |jq
{
  "error": false,
  "message": "cluster list returned",
  "clusters": [
    "nginxlog",
    "bill-kafka"
  ],
  "request": {
    "url": "/v3/kafka",
    "host": "zabbix"
  }
}
[root@mt ~]#
[root@mt ~]# #@列出所有消费组
[root@mt ~]# curl -s http://10.128.0.2:8000/v3/kafka/bill-kafka/consumer |jq
{
  "error": false,
  "message": "consumer list returned",
  "consumers": [
    "group1",
    "group_bill",
    "report_gid"
  ],
  "request": {
    "url": "/v3/kafka/bill-kafka/consumer",
    "host": "zabbix"
  }
}
[root@mt ~]# #@查看消费组健康状态
[root@mt ~]# curl -s http://10.128.0.2:8000/v3/kafka/bill-kafka/consumer/group1/status |jq
{
  "error": false,
  "message": "consumer status returned",
  "status": {
    "cluster": "bill-kafka",
    "group": "group1",
    "status": "OK",
    "complete": 1,
    "partitions": [],
    "partition_count": 18,
    "maxlag": {
      "topic": "shellActivity",
      "partition": 0,
      "owner": "",
      "status": "OK",
      "start": {
        "offset": 101333573,
        "timestamp": 1526997873289,
        "lag": 0
      },
      "end": {
        "offset": 101333604,
        "timestamp": 1526997891689,
        "lag": 0
      },
      "current_lag": 0,
      "complete": 1
    },
    "totallag": 0
  },
  "request": {
    "url": "/v3/kafka/bill-kafka/consumer/group1/status",
    "host": "zabbix"
  }
}