K8s 中运行Zookeeper

发表于 | 分类于
字数统计: 1.4k | 阅读时长 ≈ 7

1、zk 的yaml资源文件

这里使用statefulSet的方式来部署 Zookeeper，它会创建一个ConfigMap，一个Service，一个Headless Service，一个PodDisruptionBudget，一个StatefulSet

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
# 创建ConfigMap
apiVersion: v1
kind: ConfigMap
metadata:
  name: zk-scripts
  namespace: default
  labels:
    app.kubernetes.io/name: zookeeper
    app.kubernetes.io/component: zookeeper
data:
  setup.sh: |-
    #!/bin/bash
    if [[ -f "/bitnami/zookeeper/data/myid" ]]; then
        export ZOO_SERVER_ID="$(cat /bitnami/zookeeper/data/myid)"
    else
        HOSTNAME="$(hostname -s)"
        if [[ $HOSTNAME =~ (.*)-([0-9]+)$ ]]; then
            ORD=${BASH_REMATCH[2]}
            export ZOO_SERVER_ID="$((ORD + 1 ))"
        else
            echo "Failed to get index from hostname $HOST"
            exit 1
        fi
    fi
    exec /entrypoint.sh /run.sh
---
# 创建headless服务
apiVersion: v1
kind: Service
metadata:
  name: zk-headless
  namespace: default
  labels:
    app.kubernetes.io/name: zookeeper
    app.kubernetes.io/component: zookeeper
spec:
  type: ClusterIP
  clusterIP: None
  publishNotReadyAddresses: true
  ports:
    - name: tcp-client
      port: 2181
      targetPort: client
    - name: tcp-follower
      port: 2888
      targetPort: follower
    - name: tcp-election
      port: 3888
      targetPort: election
  selector:
    app.kubernetes.io/name: zookeeper
    app.kubernetes.io/component: zookeeper
---
# 创建service服务
apiVersion: v1
kind: Service
metadata:
  name: zk
  namespace: default
  labels:
    app.kubernetes.io/name: zookeeper
    app.kubernetes.io/component: zookeeper
spec:
  type: ClusterIP
  sessionAffinity: None
  ports:
    - name: tcp-client
      port: 2181
      targetPort: client
      nodePort: null
    - name: tcp-follower
      port: 2888
      targetPort: follower
    - name: tcp-election
      port: 3888
      targetPort: election
  selector:
    app.kubernetes.io/name: zookeeper
    app.kubernetes.io/component: zookeeper
---
# 创建PodDisruptionBudget 控制器
# Pod Disruption Budget (pod 中断 预算) 简称PDB，含义其实是终止pod前通过 labelSelector 机制获取正常运行的pod数目的限制，目的是对自愿中断的保护措施
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
  name: zookeeper-pod-disruption-budget
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: zookeeper
      app.kubernetes.io/component: zookeeper
  maxUnavailable: 1
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: zk
  namespace: default
  labels:
    app.kubernetes.io/name: zookeeper
    app.kubernetes.io/component: zookeeper
    role: zookeeper
spec:
  replicas: 3
  podManagementPolicy: Parallel
  selector:
    matchLabels:
      app.kubernetes.io/name: zookeeper
      app.kubernetes.io/component: zookeeper
  serviceName: zk-headless
  updateStrategy:
    rollingUpdate: {}
    type: RollingUpdate
  template:
    metadata:
      annotations:
      labels:
        app.kubernetes.io/name: zookeeper
        app.kubernetes.io/component: zookeeper
    spec:
      serviceAccountName: default
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - podAffinityTerm:
                labelSelector:
                  matchLabels:
                    app.kubernetes.io/name: zookeeper
                    app.kubernetes.io/component: zookeeper
                namespaces:
                  - "default"
                topologyKey: kubernetes.io/hostname
              weight: 1
      securityContext:
        fsGroup: 1001
      initContainers:
      containers:
        - name: zookeeper
          image: bitnami/zookeeper:3.8.0
          imagePullPolicy: "IfNotPresent"
          securityContext:
            runAsNonRoot: true
            runAsUser: 1001
          command:
            - /scripts/setup.sh
          resources:
            limits:
              cpu: 1
              memory: 1Gi
            requests:
              cpu: 0.1
              memory: 128Mi
          env:
            - name: BITNAMI_DEBUG
              value: "false"
            - name: ZOO_DATA_LOG_DIR
              value: ""
            - name: ZOO_PORT_NUMBER
              value: "2181"
            - name: ZOO_TICK_TIME
              value: "2000"
            - name: ZOO_INIT_LIMIT
              value: "10"
            - name: ZOO_SYNC_LIMIT
              value: "5"
            - name: ZOO_PRE_ALLOC_SIZE
              value: "65536"
            - name: ZOO_SNAPCOUNT
              value: "100000"
            - name: ZOO_MAX_CLIENT_CNXNS
              value: "60"
            - name: ZOO_4LW_COMMANDS_WHITELIST
              value: "srvr, mntr, ruok"
            - name: ZOO_LISTEN_ALLIPS_ENABLED
              value: "no"
            - name: ZOO_AUTOPURGE_INTERVAL
              value: "0"
            - name: ZOO_AUTOPURGE_RETAIN_COUNT
              value: "3"
            - name: ZOO_MAX_SESSION_TIMEOUT
              value: "40000"
            - name: ZOO_SERVERS
              value: zk-0.zk-headless.default.svc.cluster.local:2888:3888::1 zk-1.zk-headless.default.svc.cluster.local:2888:3888::2 zk-2.zk-headless.default.svc.cluster.local:2888:3888::3
            - name: ZOO_ENABLE_AUTH
              value: "no"
            - name: ZOO_HEAP_SIZE
              value: "1024"
            - name: ZOO_LOG_LEVEL
              value: "ERROR"
            - name: ALLOW_ANONYMOUS_LOGIN
              value: "yes"
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.name
          ports:
            - name: client
              containerPort: 2181
            - name: follower
              containerPort: 2888
            - name: election
              containerPort: 3888
          livenessProbe:
            failureThreshold: 6
            initialDelaySeconds: 30
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 5
            exec:
              command: ['/bin/bash', '-c', 'echo "ruok" | timeout 2 nc -w 2 localhost 2181 | grep imok']
          readinessProbe:
            failureThreshold: 6
            initialDelaySeconds: 5
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 5
            exec:
              command: ['/bin/bash', '-c', 'echo "ruok" | timeout 2 nc -w 2 localhost 2181 | grep imok']
          volumeMounts:
            - name: scripts
              mountPath: /scripts/setup.sh
              subPath: setup.sh
            - name: zookeeper-data
              mountPath: /bitnami/zookeeper
      volumes:
        - name: scripts
          configMap:
            name: zk-scripts
            defaultMode: 0755
  volumeClaimTemplates:
  - metadata:
      name: zookeeper-data
    spec:
      storageClassName: nfs
      accessModes:
      - ReadWriteOnce
      resources:
        requests:
          storage: 2Gi

因为这里使用的镜像bitnami/zookeeper是别人经过调整的，介意的话可以使用k8s 官网的zk yaml部署 https://kubernetes.io/docs/tutorials/stateful-application/zookeeper/

2、部署 zk

1
[root@001-new ~]$ kubectl apply -f zookeeper.yaml

3、查看 zk 服务状态

查看资源清单

1
2
3
4
5
6
7
8
9
[root@001-new ~]$ kubectl get po,svc,pdb,pvc|grep zk
pod/zk-0                                      1/1     Running   0          72m
pod/zk-1                                      1/1     Running   0          72m
pod/zk-2                                      1/1     Running   1          72m
service/zk                      ClusterIP   172.21.6.1      <none>        2181/TCP,2888/TCP,3888/TCP   72m
service/zk-headless             ClusterIP   None            <none>        2181/TCP,2888/TCP,3888/TCP   72m
persistentvolumeclaim/zookeeper-data-zk-0   Bound    pvc-beb9b6e9-f6b0-11ec-a9e9-d27996f947d5   2Gi        RWO            nfs   72m
persistentvolumeclaim/zookeeper-data-zk-1   Bound    pvc-bebb5f58-f6b0-11ec-a9e9-d27996f947d5   2Gi        RWO            nfs   72m
persistentvolumeclaim/zookeeper-data-zk-2   Bound    pvc-bebd09ee-f6b0-11ec-a9e9-d27996f947d5   2Gi        RWO            nfs   72m

查看zk节点状态

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
[root@001-new ~]$ for i in 0 1 2;do kubectl exec zk-$i -- /opt/bitnami/zookeeper/bin/zkServer.sh status;done
/opt/bitnami/java/bin/java
ZooKeeper JMX enabled by default
Using config: /opt/bitnami/zookeeper/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost. Client SSL: false.
Mode: follower
/opt/bitnami/java/bin/java
ZooKeeper JMX enabled by default
Using config: /opt/bitnami/zookeeper/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost. Client SSL: false.
Mode: leader
/opt/bitnami/java/bin/java
ZooKeeper JMX enabled by default
Using config: /opt/bitnami/zookeeper/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost. Client SSL: false.
Mode: follower

让zk节点重新选举

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
# 模拟当前的leader节点意外下线
[root@001-new ~]$ kubectl exec -it zk-1 bash
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
I have no name!@zk-1:/$ /opt/bitnami/zookeeper/bin/zkServer.sh status
/opt/bitnami/java/bin/java
ZooKeeper JMX enabled by default
Using config: /opt/bitnami/zookeeper/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost. Client SSL: false.
Mode: leader
I have no name!@zk-1:/$ ps -ef
UID        PID  PPID  C STIME TTY          TIME CMD
1001         1     0  0 07:06 ?        00:00:06 java -Dzookeeper.log.dir=/opt/bitnami/zookeeper/logs -Dzookeeper.log.file=/opt/bitnami
1001      8877     0  0 08:24 pts/0    00:00:00 bash
1001      8944  8877  0 08:25 pts/0    00:00:00 ps -ef
I have no name!@zk-1:/$ kill 1
I have no name!@zk-1:/$ command terminated with exit code 137
[root@001-new ~]$
[root@001-new ~]$ kubectl get po |grep zk
zk-0                                      1/1     Running   0          79m
zk-1                                      1/1     Running   1          79m
zk-2                                      1/1     Running   1          79m

# 再次查看zk节点状态
[root@001-new ~]$ for i in 0 1 2;do kubectl exec zk-$i -- /opt/bitnami/zookeeper/bin/zkServer.sh status;done
/opt/bitnami/java/bin/java
ZooKeeper JMX enabled by default
Using config: /opt/bitnami/zookeeper/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost. Client SSL: false.
Mode: follower
/opt/bitnami/java/bin/java
ZooKeeper JMX enabled by default
Using config: /opt/bitnami/zookeeper/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost. Client SSL: false.
Mode: follower
ZooKeeper JMX enabled by default
Using config: /opt/bitnami/zookeeper/bin/../conf/zoo.cfg
/opt/bitnami/java/bin/java
Client port found: 2181. Client address: localhost. Client SSL: false.
Mode: leader

获取zk StatefulSet 中每个 Pod 的FQDN域名

1
2
3
4
[root@001-new ~]$ for i in 0 1 2; do kubectl exec zk-$i -- hostname -f; done
zk-0.zk-headless.default.svc.cluster.local
zk-1.zk-headless.default.svc.cluster.local
zk-2.zk-headless.default.svc.cluster.local
-------------本文结束感谢您的阅读-------------
分享到： 收藏夹 复制网址 邮件 微信 QQ空间 腾讯微博 豆瓣 一键分享 更多