高并发架构实践指南——资源部署(分布式协调服务ZooKeeper)

在zk-node1, zk-node2, zk-node3节点上分别安装ZooKeeper。

安装ZooKeeper

xuej@db1:~/Downloads$ wget https://archive.apache.org/dist/zookeeper/zookeeper-3.6.0/apache-zookeeper-3.6.0-bin.tar.gz
--2022-07-31 17:11:05--  https://archive.apache.org/dist/zookeeper/zookeeper-3.6.0/apache-zookeeper-3.6.0-bin.tar.gz
Resolving archive.apache.org (archive.apache.org)... 138.201.131.134, 2a01:4f8:172:2ec5::2
Connecting to archive.apache.org (archive.apache.org)|138.201.131.134|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 12392394 (12M) [application/x-gzip]
Saving to: ‘apache-zookeeper-3.6.0-bin.tar.gz’

apache-zookeeper-3.6.0-bin.tar.gz                 100%[==========================================================================================================>]  11.82M  20.0KB/s    in 8m 51s  

2022-07-31 17:19:57 (22.8 KB/s) - ‘apache-zookeeper-3.6.0-bin.tar.gz’ saved [12392394/12392394]

xuej@db1:~/Downloads$ tar -zxvf apache-zookeeper-3.6.0-bin.tar.gz

配置ZooKeeper Replicated Cluster

xuej@db1:~/Downloads/apache-zookeeper-3.6.0-bin$ vi /etc/hosts
127.0.0.1       localhost
127.0.1.1       xuej-VirtualBox
192.168.246.43 db1
192.168.246.44 db1-repl0
192.168.246.45 db1-repl1
192.168.246.43 zk1
192.168.246.44 zk2
192.168.246.45 zk3
# The following lines are desirable for IPv6 capable hosts
::1     ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
xuej@db1:~/Downloads$ tar -zxvf apache-zookeeper-3.6.0-bin.tar.gz
xuej@db1:~/Downloads/apache-zookeeper-3.6.0-bin$ vi bin/zkEnv.sh
#!/usr/bin/env bash
JAVA_HOME=/usr/java/jdk-11.0.15+10
ZOOBINDIR="${ZOOBINDIR:-/usr/bin}"
ZOOKEEPER_PREFIX="${ZOOBINDIR}/.."

xuej@db1:~/Downloads/apache-zookeeper-3.6.0-bin$ vi conf/zoo.cfg
# The number of milliseconds of each tick
tickTime=2000
# The number of ticks that the initial 
# synchronization phase can take
initLimit=10
# The number of ticks that can pass between 
# sending a request and getting an acknowledgement
syncLimit=5
# the directory where the snapshot is stored.
# do not use /tmp for storage, /tmp here is just 
# example sakes.
# dataDir=/tmp/zookeeper
dataDir=/var/lib/zookeeper
# the port at which the clients will connect
clientPort=2181
# the maximum number of client connections.
# increase this if you need to handle more clients
#maxClientCnxns=60
#
# Be sure to read the maintenance section of the 
# administrator guide before turning on autopurge.
#
# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
#
# The number of snapshots to retain in dataDir
#autopurge.snapRetainCount=3
# Purge task interval in hours
# Set to "0" to disable auto purge feature
#autopurge.purgeInterval=1

## Metrics Providers
#
# https://prometheus.io Metrics Exporter
#metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider
#metricsProvider.httpPort=7000
#metricsProvider.exportJvmInfo=true
admin.serverPort=9091
server.1=zk1:2888:3888
server.2=zk2:2888:3888
server.3=zk3:2888:3888
xuej@db1:~/Downloads/apache-zookeeper-3.6.0-bin$ sudo ./bin/zkServer.sh start
ZooKeeper JMX enabled by default
Using config: /home/xuej/Downloads/apache-zookeeper-3.6.0-bin/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED
xuej@db1:~/Downloads/apache-zookeeper-3.6.0-bin$ sudo ./bin/zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /home/xuej/Downloads/apache-zookeeper-3.6.0-bin/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost.
Mode: follower
xuej@db1:~/Downloads/apache-zookeeper-3.6.0-bin$ curl http://localhost:9091/commands/stat && echo
{
  "version" : "3.6.0--b4c89dc7f6083829e18fae6e446907ae0b1f22d7, built on 02/25/2020 14:38 GMT",
  "read_only" : false,
  "server_stats" : {
    "packets_sent" : 1,
    "packets_received" : 1,
    "fsync_threshold_exceed_count" : 0,
    "client_response_stats" : {
      "last_buffer_size" : -1,
      "min_buffer_size" : -1,
      "max_buffer_size" : -1
    },
    "provider_null" : false,
    "uptime" : 40552,
    "server_state" : "follower",
    "outstanding_requests" : 0,
    "min_latency" : 0,
    "avg_latency" : 0.0,
    "max_latency" : 0,
    "data_dir_size" : 134250357,
    "log_dir_size" : 134250357,
    "last_processed_zxid" : 64424509957,
    "num_alive_client_connections" : 0
  },
  "client_response" : {
    "last_buffer_size" : -1,
    "min_buffer_size" : -1,
    "max_buffer_size" : -1
  },
  "node_count" : 91,
  "connections" : [ ],
  "secure_connections" : [ ],
  "command" : "stats",
  "error" : null
}
xuej@db1:~/Downloads/apache-zookeeper-3.6.0-bin$ curl http://localhost:9091/commands/srvr && echo
{
  "version" : "3.6.0--b4c89dc7f6083829e18fae6e446907ae0b1f22d7, built on 02/25/2020 14:38 GMT",
  "read_only" : false,
  "server_stats" : {
    "packets_sent" : 1,
    "packets_received" : 1,
    "fsync_threshold_exceed_count" : 0,
    "client_response_stats" : {
      "last_buffer_size" : -1,
      "min_buffer_size" : -1,
      "max_buffer_size" : -1
    },
    "provider_null" : false,
    "uptime" : 67217,
    "server_state" : "follower",
    "outstanding_requests" : 0,
    "min_latency" : 0,
    "avg_latency" : 0.0,
    "max_latency" : 0,
    "data_dir_size" : 134250357,
    "log_dir_size" : 134250357,
    "last_processed_zxid" : 64424509957,
    "num_alive_client_connections" : 0
  },
  "client_response" : {
    "last_buffer_size" : -1,
    "min_buffer_size" : -1,
    "max_buffer_size" : -1
  },
  "node_count" : 91,
  "command" : "server_stats",
  "error" : null
}
xuej@db1:~/Downloads/apache-zookeeper-3.6.0-bin$ curl http://localhost:9091/commands/monitor && echo
{
  "version" : "3.6.0--b4c89dc7f6083829e18fae6e446907ae0b1f22d7, built on 02/25/2020 14:38 GMT",
  "server_state" : "follower",
  "peer_state" : "following - broadcast",
  "ephemerals_count" : 0,
  "min_latency" : 0,
  "avg_latency" : 0.0,
  "num_alive_connections" : 0,
  "max_file_descriptor_count" : 4096,
  "outstanding_requests" : 0,
  "approximate_data_size" : 24681,
  "znode_count" : 91,
  "open_file_descriptor_count" : 62,
  "global_sessions" : 0,
  "local_sessions" : 0,
  "uptime" : 76582,
  "last_client_response_size" : -1,
  "max_latency" : 0,
  "packets_sent" : 1,
  "outstanding_tls_handshake" : 0,
  "packets_received" : 1,
  "max_client_response_size" : -1,
  "connection_drop_probability" : 0.0,
  "watch_count" : 0,
  "min_client_response_size" : -1,
  "quorum_size" : 3,
  "proposal_count" : 0,
  "outstanding_changes_removed" : 0,
  "stale_requests_dropped" : 0,
  "large_requests_rejected" : 0,
  "connection_rejected" : 0,
  "sessionless_connections_expired" : 0,
  "looking_count" : 1,
  "dead_watchers_queued" : 0,
  "stale_requests" : 0,
  "connection_drop_count" : 0,
  "learner_proposal_received_count" : 0,
  "digest_mismatches_count" : 0,
  "dead_watchers_cleared" : 0,
  "response_packet_cache_hits" : 0,
  "bytes_received_count" : 4,
  "add_dead_watcher_stall_time" : 0,
  "request_throttle_wait_count" : 0,
  "response_packet_cache_misses" : 0,
  "ensemble_auth_success" : 0,
  "prep_processor_request_queued" : 0,
  "learner_commit_received_count" : 0,
  "stale_replies" : 0,
  "connection_request_count" : 0,
  "ensemble_auth_fail" : 0,
  "diff_count" : 0,
  "response_packet_get_children_cache_misses" : 0,
  "connection_revalidate_count" : 0,
  "quit_leading_due_to_disloyal_voter" : 0,
  "snap_count" : 0,
  "unrecoverable_error_count" : 0,
  "commit_count" : 0,
  "stale_sessions_expired" : 0,
  "response_packet_get_children_cache_hits" : 0,
  "sync_processor_request_queued" : 0,
  "outstanding_changes_queued" : 0,
  "request_commit_queued" : 0,
  "ensemble_auth_skip" : 0,
  "tls_handshake_exceeded" : 0,
  "revalidate_count" : 0,
  "avg_node_created_watch_count" : 0.0,
  "min_node_created_watch_count" : 0,
  "max_node_created_watch_count" : 0,
  "cnt_node_created_watch_count" : 0,
  "sum_node_created_watch_count" : 0,
  "avg_session_queues_drained" : 0.0,
  "min_session_queues_drained" : 0,
  "max_session_queues_drained" : 0,
  "cnt_session_queues_drained" : 0,
  "sum_session_queues_drained" : 0,
  "avg_write_commit_proc_req_queued" : 0.0,
  "min_write_commit_proc_req_queued" : 0,
  "max_write_commit_proc_req_queued" : 0,
  "cnt_write_commit_proc_req_queued" : 0,
  "sum_write_commit_proc_req_queued" : 0,
  "avg_connection_token_deficit" : 0.0,
  "min_connection_token_deficit" : 0,
  "max_connection_token_deficit" : 0,
  "cnt_connection_token_deficit" : 0,
  "sum_connection_token_deficit" : 0,
  "avg_read_commit_proc_req_queued" : 0.0,
  "min_read_commit_proc_req_queued" : 0,
  "max_read_commit_proc_req_queued" : 0,
  "cnt_read_commit_proc_req_queued" : 0,
  "sum_read_commit_proc_req_queued" : 0,
  "avg_node_deleted_watch_count" : 0.0,
  "min_node_deleted_watch_count" : 0,
  "max_node_deleted_watch_count" : 0,
  "cnt_node_deleted_watch_count" : 0,
  "sum_node_deleted_watch_count" : 0,
  "avg_startup_txns_load_time" : 853.0,
  "min_startup_txns_load_time" : 853,
  "max_startup_txns_load_time" : 853,
  "cnt_startup_txns_load_time" : 1,
  "sum_startup_txns_load_time" : 853,
  "avg_sync_processor_queue_size" : 0.0,
  "min_sync_processor_queue_size" : 0,
  "max_sync_processor_queue_size" : 0,
  "cnt_sync_processor_queue_size" : 1,
  "sum_sync_processor_queue_size" : 0,
  "avg_follower_sync_time" : 32.0,
  "min_follower_sync_time" : 32,
  "max_follower_sync_time" : 32,
  "cnt_follower_sync_time" : 1,
  "sum_follower_sync_time" : 32,
  "avg_prep_processor_queue_size" : 0.0,
  "min_prep_processor_queue_size" : 0,
  "max_prep_processor_queue_size" : 0,
  "cnt_prep_processor_queue_size" : 0,
  "sum_prep_processor_queue_size" : 0,
  "avg_fsynctime" : 0.0,
  "min_fsynctime" : 0,
  "max_fsynctime" : 0,
  "cnt_fsynctime" : 0,
  "sum_fsynctime" : 0,
  "avg_reads_issued_from_session_queue" : 0.0,
  "min_reads_issued_from_session_queue" : 0,
  "max_reads_issued_from_session_queue" : 0,
  "cnt_reads_issued_from_session_queue" : 0,
  "sum_reads_issued_from_session_queue" : 0,
  "avg_snapshottime" : 0.0,
  "min_snapshottime" : 0,
  "max_snapshottime" : 0,
  "cnt_snapshottime" : 0,
  "sum_snapshottime" : 0,
  "avg_startup_txns_loaded" : 40960.0,
  "min_startup_txns_loaded" : 40960,
  "max_startup_txns_loaded" : 40960,
  "cnt_startup_txns_loaded" : 1,
  "sum_startup_txns_loaded" : 40960,
  "avg_reads_after_write_in_session_queue" : 0.0,
  "min_reads_after_write_in_session_queue" : 0,
  "max_reads_after_write_in_session_queue" : 0,
  "cnt_reads_after_write_in_session_queue" : 0,
  "sum_reads_after_write_in_session_queue" : 0,
  "avg_requests_in_session_queue" : 0.0,
  "min_requests_in_session_queue" : 0,
  "max_requests_in_session_queue" : 0,
  "cnt_requests_in_session_queue" : 0,
  "sum_requests_in_session_queue" : 0,
  "avg_write_commit_proc_issued" : 0.0,
  "min_write_commit_proc_issued" : 0,
  "max_write_commit_proc_issued" : 0,
  "cnt_write_commit_proc_issued" : 0,
  "sum_write_commit_proc_issued" : 0,
  "avg_prep_process_time" : 0.0,
  "min_prep_process_time" : 0,
  "max_prep_process_time" : 0,
  "cnt_prep_process_time" : 0,
  "sum_prep_process_time" : 0,
  "avg_pending_session_queue_size" : 0.0,
  "min_pending_session_queue_size" : 0,
  "max_pending_session_queue_size" : 0,
  "cnt_pending_session_queue_size" : 0,
  "sum_pending_session_queue_size" : 0,
  "avg_time_waiting_empty_pool_in_commit_processor_read_ms" : 0.0,
  "min_time_waiting_empty_pool_in_commit_processor_read_ms" : 0,
  "max_time_waiting_empty_pool_in_commit_processor_read_ms" : 0,
  "cnt_time_waiting_empty_pool_in_commit_processor_read_ms" : 0,
  "sum_time_waiting_empty_pool_in_commit_processor_read_ms" : 0,
  "avg_commit_process_time" : 0.0,
  "min_commit_process_time" : 0,
  "max_commit_process_time" : 0,
  "cnt_commit_process_time" : 0,
  "sum_commit_process_time" : 0,
  "avg_dbinittime" : 884.0,
  "min_dbinittime" : 884,
  "max_dbinittime" : 884,
  "cnt_dbinittime" : 1,
  "sum_dbinittime" : 884,
  "avg_netty_queued_buffer_capacity" : 0.0,
  "min_netty_queued_buffer_capacity" : 0,
  "max_netty_queued_buffer_capacity" : 0,
  "cnt_netty_queued_buffer_capacity" : 0,
  "sum_netty_queued_buffer_capacity" : 0,
  "avg_election_time" : 45.0,
  "min_election_time" : 45,
  "max_election_time" : 45,
  "cnt_election_time" : 1,
  "sum_election_time" : 45,
  "avg_commit_commit_proc_req_queued" : 0.0,
  "min_commit_commit_proc_req_queued" : 0,
  "max_commit_commit_proc_req_queued" : 0,
  "cnt_commit_commit_proc_req_queued" : 0,
  "sum_commit_commit_proc_req_queued" : 0,
  "avg_sync_processor_batch_size" : 0.0,
  "min_sync_processor_batch_size" : 0,
  "max_sync_processor_batch_size" : 0,
  "cnt_sync_processor_batch_size" : 0,
  "sum_sync_processor_batch_size" : 0,
  "avg_node_children_watch_count" : 0.0,
  "min_node_children_watch_count" : 0,
  "max_node_children_watch_count" : 0,
  "cnt_node_children_watch_count" : 0,
  "sum_node_children_watch_count" : 0,
  "avg_write_batch_time_in_commit_processor" : 0.0,
  "min_write_batch_time_in_commit_processor" : 0,
  "max_write_batch_time_in_commit_processor" : 0,
  "cnt_write_batch_time_in_commit_processor" : 0,
  "sum_write_batch_time_in_commit_processor" : 0,
  "avg_read_commit_proc_issued" : 0.0,
  "min_read_commit_proc_issued" : 0,
  "max_read_commit_proc_issued" : 0,
  "cnt_read_commit_proc_issued" : 0,
  "sum_read_commit_proc_issued" : 0,
  "avg_concurrent_request_processing_in_commit_processor" : 0.0,
  "min_concurrent_request_processing_in_commit_processor" : 0,
  "max_concurrent_request_processing_in_commit_processor" : 0,
  "cnt_concurrent_request_processing_in_commit_processor" : 0,
  "sum_concurrent_request_processing_in_commit_processor" : 0,
  "avg_node_changed_watch_count" : 0.0,
  "min_node_changed_watch_count" : 0,
  "max_node_changed_watch_count" : 0,
  "cnt_node_changed_watch_count" : 0,
  "sum_node_changed_watch_count" : 0,
  "avg_sync_process_time" : 0.0,
  "min_sync_process_time" : 0,
  "max_sync_process_time" : 0,
  "cnt_sync_process_time" : 0,
  "sum_sync_process_time" : 0,
  "avg_startup_snap_load_time" : 23.0,
  "min_startup_snap_load_time" : 23,
  "max_startup_snap_load_time" : 23,
  "cnt_startup_snap_load_time" : 1,
  "sum_startup_snap_load_time" : 23,
  "avg_prep_processor_queue_time_ms" : 0.0,
  "min_prep_processor_queue_time_ms" : 0,
  "max_prep_processor_queue_time_ms" : 0,
  "cnt_prep_processor_queue_time_ms" : 0,
  "sum_prep_processor_queue_time_ms" : 0,
  "p50_prep_processor_queue_time_ms" : 0,
  "p95_prep_processor_queue_time_ms" : 0,
  "p99_prep_processor_queue_time_ms" : 0,
  "p999_prep_processor_queue_time_ms" : 0,
  "avg_close_session_prep_time" : 0.0,
  "min_close_session_prep_time" : 0,
  "max_close_session_prep_time" : 0,
  "cnt_close_session_prep_time" : 0,
  "sum_close_session_prep_time" : 0,
  "p50_close_session_prep_time" : 0,
  "p95_close_session_prep_time" : 0,
  "p99_close_session_prep_time" : 0,
  "p999_close_session_prep_time" : 0,
  "avg_read_commitproc_time_ms" : 0.0,
  "min_read_commitproc_time_ms" : 0,
  "max_read_commitproc_time_ms" : 0,
  "cnt_read_commitproc_time_ms" : 0,
  "sum_read_commitproc_time_ms" : 0,
  "p50_read_commitproc_time_ms" : 0,
  "p95_read_commitproc_time_ms" : 0,
  "p99_read_commitproc_time_ms" : 0,
  "p999_read_commitproc_time_ms" : 0,
  "avg_updatelatency" : 0.0,
  "min_updatelatency" : 0,
  "max_updatelatency" : 0,
  "cnt_updatelatency" : 0,
  "sum_updatelatency" : 0,
  "p50_updatelatency" : 0,
  "p95_updatelatency" : 0,
  "p99_updatelatency" : 0,
  "p999_updatelatency" : 0,
  "avg_local_write_committed_time_ms" : 0.0,
  "min_local_write_committed_time_ms" : 0,
  "max_local_write_committed_time_ms" : 0,
  "cnt_local_write_committed_time_ms" : 0,
  "sum_local_write_committed_time_ms" : 0,
  "p50_local_write_committed_time_ms" : 0,
  "p95_local_write_committed_time_ms" : 0,
  "p99_local_write_committed_time_ms" : 0,
  "p999_local_write_committed_time_ms" : 0,
  "avg_readlatency" : 0.0,
  "min_readlatency" : 0,
  "max_readlatency" : 0,
  "cnt_readlatency" : 0,
  "sum_readlatency" : 0,
  "p50_readlatency" : 0,
  "p95_readlatency" : 0,
  "p99_readlatency" : 0,
  "p999_readlatency" : 0,
  "avg_quorum_ack_latency" : 0.0,
  "min_quorum_ack_latency" : 0,
  "max_quorum_ack_latency" : 0,
  "cnt_quorum_ack_latency" : 0,
  "sum_quorum_ack_latency" : 0,
  "p50_quorum_ack_latency" : 0,
  "p95_quorum_ack_latency" : 0,
  "p99_quorum_ack_latency" : 0,
  "p999_quorum_ack_latency" : 0,
  "avg_om_commit_process_time_ms" : 0.0,
  "min_om_commit_process_time_ms" : 0,
  "max_om_commit_process_time_ms" : 0,
  "cnt_om_commit_process_time_ms" : 0,
  "sum_om_commit_process_time_ms" : 0,
  "p50_om_commit_process_time_ms" : 0,
  "p95_om_commit_process_time_ms" : 0,
  "p99_om_commit_process_time_ms" : 0,
  "p999_om_commit_process_time_ms" : 0,
  "avg_read_final_proc_time_ms" : 0.0,
  "min_read_final_proc_time_ms" : 0,
  "max_read_final_proc_time_ms" : 0,
  "cnt_read_final_proc_time_ms" : 0,
  "sum_read_final_proc_time_ms" : 0,
  "p50_read_final_proc_time_ms" : 0,
  "p95_read_final_proc_time_ms" : 0,
  "p99_read_final_proc_time_ms" : 0,
  "p999_read_final_proc_time_ms" : 0,
  "avg_commit_propagation_latency" : 0.0,
  "min_commit_propagation_latency" : 0,
  "max_commit_propagation_latency" : 0,
  "cnt_commit_propagation_latency" : 0,
  "sum_commit_propagation_latency" : 0,
  "p50_commit_propagation_latency" : 0,
  "p95_commit_propagation_latency" : 0,
  "p99_commit_propagation_latency" : 0,
  "p999_commit_propagation_latency" : 0,
  "avg_dead_watchers_cleaner_latency" : 0.0,
  "min_dead_watchers_cleaner_latency" : 0,
  "max_dead_watchers_cleaner_latency" : 0,
  "cnt_dead_watchers_cleaner_latency" : 0,
  "sum_dead_watchers_cleaner_latency" : 0,
  "p50_dead_watchers_cleaner_latency" : 0,
  "p95_dead_watchers_cleaner_latency" : 0,
  "p99_dead_watchers_cleaner_latency" : 0,
  "p999_dead_watchers_cleaner_latency" : 0,
  "avg_write_final_proc_time_ms" : 0.0,
  "min_write_final_proc_time_ms" : 0,
  "max_write_final_proc_time_ms" : 0,
  "cnt_write_final_proc_time_ms" : 0,
  "sum_write_final_proc_time_ms" : 0,
  "p50_write_final_proc_time_ms" : 0,
  "p95_write_final_proc_time_ms" : 0,
  "p99_write_final_proc_time_ms" : 0,
  "p999_write_final_proc_time_ms" : 0,
  "avg_proposal_ack_creation_latency" : 0.0,
  "min_proposal_ack_creation_latency" : 0,
  "max_proposal_ack_creation_latency" : 0,
  "cnt_proposal_ack_creation_latency" : 0,
  "sum_proposal_ack_creation_latency" : 0,
  "p50_proposal_ack_creation_latency" : 0,
  "p95_proposal_ack_creation_latency" : 0,
  "p99_proposal_ack_creation_latency" : 0,
  "p999_proposal_ack_creation_latency" : 0,
  "avg_proposal_latency" : 0.0,
  "min_proposal_latency" : 0,
  "max_proposal_latency" : 0,
  "cnt_proposal_latency" : 0,
  "sum_proposal_latency" : 0,
  "p50_proposal_latency" : 0,
  "p95_proposal_latency" : 0,
  "p99_proposal_latency" : 0,
  "p999_proposal_latency" : 0,
  "avg_om_proposal_process_time_ms" : 0.0,
  "min_om_proposal_process_time_ms" : 0,
  "max_om_proposal_process_time_ms" : 0,
  "cnt_om_proposal_process_time_ms" : 0,
  "sum_om_proposal_process_time_ms" : 0,
  "p50_om_proposal_process_time_ms" : 0,
  "p95_om_proposal_process_time_ms" : 0,
  "p99_om_proposal_process_time_ms" : 0,
  "p999_om_proposal_process_time_ms" : 0,
  "avg_sync_processor_queue_and_flush_time_ms" : 0.0,
  "min_sync_processor_queue_and_flush_time_ms" : 0,
  "max_sync_processor_queue_and_flush_time_ms" : 0,
  "cnt_sync_processor_queue_and_flush_time_ms" : 0,
  "sum_sync_processor_queue_and_flush_time_ms" : 0,
  "p50_sync_processor_queue_and_flush_time_ms" : 0,
  "p95_sync_processor_queue_and_flush_time_ms" : 0,
  "p99_sync_processor_queue_and_flush_time_ms" : 0,
  "p999_sync_processor_queue_and_flush_time_ms" : 0,
  "avg_propagation_latency" : 0.0,
  "min_propagation_latency" : 0,
  "max_propagation_latency" : 0,
  "cnt_propagation_latency" : 0,
  "sum_propagation_latency" : 0,
  "p50_propagation_latency" : 0,
  "p95_propagation_latency" : 0,
  "p99_propagation_latency" : 0,
  "p999_propagation_latency" : 0,
  "avg_server_write_committed_time_ms" : 0.0,
  "min_server_write_committed_time_ms" : 0,
  "max_server_write_committed_time_ms" : 0,
  "cnt_server_write_committed_time_ms" : 0,
  "sum_server_write_committed_time_ms" : 0,
  "p50_server_write_committed_time_ms" : 0,
  "p95_server_write_committed_time_ms" : 0,
  "p99_server_write_committed_time_ms" : 0,
  "p999_server_write_committed_time_ms" : 0,
  "avg_sync_processor_queue_time_ms" : 0.0,
  "min_sync_processor_queue_time_ms" : 0,
  "max_sync_processor_queue_time_ms" : 0,
  "cnt_sync_processor_queue_time_ms" : 0,
  "sum_sync_processor_queue_time_ms" : 0,
  "p50_sync_processor_queue_time_ms" : 0,
  "p95_sync_processor_queue_time_ms" : 0,
  "p99_sync_processor_queue_time_ms" : 0,
  "p999_sync_processor_queue_time_ms" : 0,
  "avg_sync_processor_queue_flush_time_ms" : 0.0,
  "min_sync_processor_queue_flush_time_ms" : 0,
  "max_sync_processor_queue_flush_time_ms" : 0,
  "cnt_sync_processor_queue_flush_time_ms" : 0,
  "sum_sync_processor_queue_flush_time_ms" : 0,
  "p50_sync_processor_queue_flush_time_ms" : 0,
  "p95_sync_processor_queue_flush_time_ms" : 0,
  "p99_sync_processor_queue_flush_time_ms" : 0,
  "p999_sync_processor_queue_flush_time_ms" : 0,
  "avg_write_commitproc_time_ms" : 0.0,
  "min_write_commitproc_time_ms" : 0,
  "max_write_commitproc_time_ms" : 0,
  "cnt_write_commitproc_time_ms" : 0,
  "sum_write_commitproc_time_ms" : 0,
  "p50_write_commitproc_time_ms" : 0,
  "p95_write_commitproc_time_ms" : 0,
  "p99_write_commitproc_time_ms" : 0,
  "p999_write_commitproc_time_ms" : 0,
  "avg_zookeeper_write_per_namespace" : 131.0,
  "min_zookeeper_write_per_namespace" : 131,
  "max_zookeeper_write_per_namespace" : 131,
  "cnt_zookeeper_write_per_namespace" : 2,
  "sum_zookeeper_write_per_namespace" : 262,
  "avg_demo_spring_boot_readwrite_ds_write_per_namespace" : 85.5671,
  "min_demo_spring_boot_readwrite_ds_write_per_namespace" : 39,
  "max_demo_spring_boot_readwrite_ds_write_per_namespace" : 3128,
  "cnt_demo_spring_boot_readwrite_ds_write_per_namespace" : 40958,
  "sum_demo_spring_boot_readwrite_ds_write_per_namespace" : 3504658,
  "command" : "monitor",
  "error" : null
}