kubeadm部署k8s集群

k8s集群环境规划

主机名	IP地址	节点作用	k8s 相关组件
k8s-master	192.168.223.140	k8s-master	kubeadmin，kube-apiserver，kube-controller-manager，kube-scheduler，etcd，kube-proxy，kubelet
k8s-node1	192.168.223.142	k8s-node1	kubeadmin，kube-proxy，kubelet

一、系统初始化配置

系统初始化配置在所有服务器上操作

设置主机名

[root@k8s-master ~]# hostnamectl set-hostname k8s-master
[root@k8s-node1 ~]# hostnamectl set-hostname k8s-node1

## 设置master 节点hosts
[root@k8s-master ~]# vim /etc/hosts
192.168.223.140 k8s-master

## node节点除了配置master hosts还要配置自己的hosts
[root@k8s-node1 ~]# vim /etc/hosts
192.168.223.140 k8s-master
192.168.223.142 k8s-node1

安装依赖包

1	# yum -y install conntrack ntpdate ntp ipvsadm ipset jq iptables curl sysstat libseccomp wget git vim net-tools

禁用firewalld

1	# systemctl stop firewalld && systemctl disable firewalld

关闭swap分区和SELINUX

如果不关闭，默认配置下kubelet将无法启动。使用free -m确认swap已经关闭。

1 2	# swapoff -a && sed -ri 's/.swap./#&/' /etc/fstab # setenforce 0 && sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config

调整时区并关闭系统不需要的服务

# timedatectl set-timezone Asia/Shanghai && timedatectl set-local-rtc 0
# systemctl start ntpd && systemctl enable ntpd
# systemctl restart rsyslog && systemctl restart crond
# systemctl stop postfix && systemctl disable postfix

调整k8s内核参数

# cat >  /etc/sysctl.d/kubernetes.conf<<EOF
#开启网桥模式【重要】
net.bridge.bridge-nf-call-iptables=1
#开启网桥模式【重要】
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
net.ipv4.tcp_tw_recycle=0
#禁止使用swap空间，只有当系统OOM时才允许使用它
vm.swappiness=0
#不检查物理内存是否够用
vm.overcommit_memory=1
#开启OOM
vm.panic_on_oom=0
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
#关闭ipv6【重要】
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720
EOF

# sysctl --system

升级Linux内核为4.44版本

# rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
# rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-2.el7.elrepo.noarch.rpm
# yum --disablerepo="*" --enablerepo="elrepo-kernel" list available
# yum --enablerepo=elrepo-kernel install kernel-ml
# cp /etc/default/grub  /etc/default/grub_bak
# grub2-mkconfig -o /boot/grub2/grub.cfg
##查看系统上的所有可用内核：
# awk -F\' '$1=="menuentry " {print i++ " : " $2}' /etc/grub2.cfg
0 : CentOS Linux (6.1.2-1.el7.elrepo.x86_64) 7 (Core)
1 : CentOS Linux (3.10.0-327.el7.x86_64) 7 (Core)
2 : CentOS Linux (0-rescue-5643801f4d944842ae274438c9c18c90) 7 (Core)
# grub2-set-default 0
# reboot
# uname -a

kube-proxy开启ipvs的前置条件

# modprobe br_netfilter
# cat >/etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
# chmod 755 /etc/sysconfig/modules/ipvs.modules
# bash /etc/sysconfig/modules/ipvs.modules && lsmod |grep -e ip_vs -e nf_conntrack_ipv4

如果使用了较为高版本的内核这里可能会报错：modprobe: FATAL: Module nf_conntrack_ipv4 not found. 这是因为在高版本内核已经把 nf_conntrack_ipv4 替换为 nf_conntrack 了，改为 nf_conntrack 即可。

二、安装docker

一定要安装18.09版以上的docker，否则在初始化k8s master的时候会提示：[WARNING SystemVerification]: this Docker version is not on the list of validated versions: 19.03.3. Latest validated version: 18.09

所有服务器安装docker

# yum install -y yum-utils
# yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
# yum makecache fast
# yum install -y docker-ce-19.03.0 docker-ce-cli-19.03.0 containerd.io
# systemctl start docker && systemctl enable docker

修改docker cgroup driver为systemd

# cat >/etc/docker/daemon.json <<EOF
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file"
}
EOF

# systemctl restart docker
# docker info | grep Cgroup
 Cgroup Driver: systemd
 Cgroup Version: 1

三、kubeadm安装k8s集群

配置k8s yum源

# cat > /etc/yum.repos.d/kubernetes.repo <<EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

2.1 安装kubeadm，kubelet和kubectl

1 2	# yum install -y kubelet-1.20.0 kubeadm-1.20.0 kubectl-1.20.0 # systemctl enable kubelet

##查看初始化需要的镜像
# kubeadm config images list
I0103 16:22:35.211779   23442 version.go:251] remote version is much newer: v1.26.0; falling back to: stable-1.20
k8s.gcr.io/kube-apiserver:v1.20.15
k8s.gcr.io/kube-controller-manager:v1.20.15
k8s.gcr.io/kube-scheduler:v1.20.15
k8s.gcr.io/kube-proxy:v1.20.15
k8s.gcr.io/pause:3.2
k8s.gcr.io/etcd:3.4.13-0
k8s.gcr.io/coredns:1.7.0

默认使用的镜像在国外仓库，后面我们可以改下载的仓库地址

2.2 初始化master节点

将默认的 init 初始化文件存放到指定位置

1	[root@k8s-master ~]# kubeadm config print init-defaults > kubeadm-config.yml

编辑kubeadm-config.yaml，修改本机服务器IP（advertiseAddress）、版本号（kubernetesVersion）、添加国内镜像仓库（imageRepository）、pod网段（podSubnet）

[root@k8s-master ~]# cat kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 192.168.223.140
  bindPort: 6443
nodeRegistration:
  criSocket: /var/run/dockershim.sock
  name: k8s-master
  taints:
  - effect: NoSchedule
    key: node-role.kubernetes.io/master
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns:
  type: CoreDNS
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: k8s.gcr.io
kind: ClusterConfiguration
kubernetesVersion: v1.20.0
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers
networking:
  dnsDomain: cluster.local
  podSubnet: "10.244.0.0/16"
  serviceSubnet: 10.96.0.0/12
scheduler: {}

下载k8s-master节点镜像

1	[root@k8s-master ~]# kubeadm config images pull --config kubeadm-config.yml

初始化集群

建议将集群初始化日志保存起来

[root@k8s-master ~]# kubeadm init --config kubeadm-config.yml
执行成功的结果……
[kubelet-finalize] Updating "/etc/kubernetes/kubelet.conf" to point to a rotatable kubelet client certificate and key
[addons] Applied essential addon: CoreDNS
[addons] Applied essential addon: kube-proxy

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 192.168.223.140:6443 --token riz1na.4fhur5ubekjf4m0l \
    --discovery-token-ca-cert-hash sha256:1f06697abdb74cdfc9601d32a45fcf636084bc7ba3de7180dacf97f1403a98c2

添加 kubectl 集群认证文件

1
2
3

[root@k8s-master ~]# mkdir -p $HOME/.kube
[root@k8s-master ~]# cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
[root@k8s-master ~]# chown $(id -u):$(id -g) $HOME/.kube/config

查看node节点（状态显示notready是因为还未部署网络插件）

1
2
3

[root@k8s-master ~]# kubectl get no
NAME         STATUS     ROLES                  AGE   VERSION
k8s-master   NotReady   control-plane,master   5m   v1.20.0

2.3 部署网络插件

网络插件flannel 和 calico 二选一即可

2.3.1 部署网络插件 flannel

1 2	[root@k8s-master ~]# wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml [root@k8s-master ~]# kubectl apply -f kube-flannel.yml

创建完成后查看pod的状态如下

[root@k8s-master ~]# kubectl get po -nkube-system
NAME                                 READY   STATUS    RESTARTS   AGE
coredns-7f89b7bc75-6qjmf             1/1     Running   0          12m
coredns-7f89b7bc75-d5xv2             1/1     Running   0          12m
etcd-k8s-master                      1/1     Running   0          12m
kube-apiserver-k8s-master            1/1     Running   0          12m
kube-controller-manager-k8s-master   1/1     Running   0          12m
kube-proxy-fpf68                     1/1     Running   0          12m
kube-scheduler-k8s-master            1/1     Running   0          12m

等两分钟之后pod状态变成Running

再查看node节点状态已经变成 ready

1
2
3

[root@k8s-master ~]# kubectl get no
NAME         STATUS   ROLES                  AGE   VERSION
k8s-master   Ready    control-plane,master   14m   v1.20.0

开启ipvs转发模式

[root@k8s-master ~]# kubectl edit configmaps -n kube-system kube-proxy
kind: KubeProxyConfiguration
    metricsBindAddress: ""
    mode: "ipvs"  # 填入`ipvs`
    nodePortAddresses: null

最后再加入node节点，通过ipvsadm -Ln查看

2.3.2 部署网络插件 calico

前往官网查看 calico和k8s 版本对应关系，安装对应版本的calico

1 2	[root@k8s-master ~]# wget https://docs.projectcalico.org/v3.20/manifests/calico.yaml [root@k8s-master ~]# kubectl apply -f calico.yaml

等两分钟之后pod状态变成Running

再查看node节点状态已经变成 ready

1
2
3

[root@k8s-master ~]# kubectl get no
NAME         STATUS   ROLES                  AGE   VERSION
k8s-master   Ready    control-plane,master   58m   v1.20.0

开启ipvs转发模式

[root@k8s-master ~]# kubectl edit configmaps -n kube-system kube-proxy
kind: KubeProxyConfiguration
    metricsBindAddress: ""
    mode: "ipvs"  # 填入`ipvs`
    nodePortAddresses: null

最后再加入node节点，通过ipvsadm -Ln查看

2.4 node 节点加入集群

使用初始化集群时生成的token，在node节点执行命令加入集群，默认token有效期为24小时，如果token过期了重新生成即可

[root@k8s-node1 ~]# kubeadm join 192.168.223.140:6443 --token hpgvxz.q1iso7sfyoipqyp7     --discovery-token-ca-cert-hash sha256:1f06697abdb74cdfc9601d32a45fcf636084bc7ba3de7180dacf97f1403a98c2
执行成功的结果……
[preflight] Running pre-flight checks
[preflight] Reading configuration from the cluster...
[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Starting the kubelet
[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...

This node has joined the cluster:
* Certificate signing request was sent to apiserver and a response was received.
* The Kubelet was informed of the new secure connection details.

Run 'kubectl get nodes' on the control-plane to see this node join the cluster.

去master节点查看是否已加入集群

[root@k8s-master ~]# kubectl get no
NAME         STATUS   ROLES                  AGE   VERSION
k8s-master   Ready    control-plane,master   33m   v1.20.0
k8s-node1    Ready    <none>                 13s   v1.20.0

2.5 部署 nginx 测试

[root@k8s-master ~]# kubectl create deploy nginx --image=nginx
[root@k8s-master ~]# kubectl get po -owide
NAME                     READY   STATUS    RESTARTS   AGE   IP           NODE        NOMINATED NODE   READINESS GATES
nginx-6799fc88d8-vngm5   1/1     Running   0          34s   10.244.1.2   k8s-node1   <none>           <none>
[root@k8s-master ~]# curl -Ik 10.244.1.2
HTTP/1.1 200 OK
Server: nginx/1.21.5
Date: Wed, 04 Jan 2023 06:12:48 GMT
Content-Type: text/html
Content-Length: 615
Last-Modified: Tue, 28 Dec 2021 15:28:38 GMT
Connection: keep-alive
ETag: "61cb2d26-267"
Accept-Ranges: bytes

[root@k8s-master ~]# kubectl delete deploy nginx

2.6 组件 Unhealthy 问题

Kubeadm部署完 k8s集群后，scheduler和controller-manager为Unhealthy

[root@k8s-master ~]# kubectl get cs
NAME                 STATUS      MESSAGE      ERROR
controller-manager   Unhealthy   Get "http://127.0.0.1:10252/healthz": dial tcp 127.0.0.1:10252: connect: connection refused   
scheduler            Unhealthy   Get "http://127.0.0.1:10251/healthz": dial tcp 127.0.0.1:10251: connect: connection refused   
etcd-0               Healthy     {"health":"true"}

出现这种情况，是/etc/kubernetes/manifests/下的kube-controller-manager.yaml和kube-scheduler.yaml设置的默认端口是0导致的，解决方式是注释掉对应的port即可，操作如下：

修改kube-controller-manager.yaml配置文件，将port=0注释

[root@k8s-master ~]# vim /etc/kubernetes/manifests/kube-controller-manager.yaml
  1 apiVersion: v1
  2 kind: Pod
  3 metadata:
  4   creationTimestamp: null
  5   labels:
  6     component: kube-controller-manager
  7     tier: control-plane
  8   name: kube-controller-manager
  9   namespace: kube-system
 10 spec:
 11   containers:
 12   - command:
 13     - kube-controller-manager
 14     - --allocate-node-cidrs=true
 15     - --authentication-kubeconfig=/etc/kubernetes/controller-manager.conf
 16     - --authorization-kubeconfig=/etc/kubernetes/controller-manager.conf
 17     - --bind-address=127.0.0.1
 18     - --client-ca-file=/etc/kubernetes/pki/ca.crt
 19     - --cluster-cidr=10.244.0.0/16
 20     - --cluster-name=kubernetes
 21     - --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt
 22     - --cluster-signing-key-file=/etc/kubernetes/pki/ca.key
 23     - --controllers=*,bootstrapsigner,tokencleaner
 24     - --kubeconfig=/etc/kubernetes/controller-manager.conf
 25     - --leader-elect=true
 26     #- --port=0

修改kube-scheduler.yaml配置文件，将port=0注释掉

[root@k8s-master ~]# vim /etc/kubernetes/manifests/kube-scheduler.yaml
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  labels:
    component: kube-scheduler
    tier: control-plane
  name: kube-scheduler
  namespace: kube-system
spec:
  containers:
  - command:
    - kube-scheduler
    - --authentication-kubeconfig=/etc/kubernetes/scheduler.conf
    - --authorization-kubeconfig=/etc/kubernetes/scheduler.conf
    - --bind-address=127.0.0.1
    - --kubeconfig=/etc/kubernetes/scheduler.conf
    - --leader-elect=true
    #- --port=0

在master节点上重启kubelet

[root@k8s-master ~]# systemctl restart kubelet
[root@k8s-master ~]# kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS    MESSAGE             ERROR
controller-manager   Healthy   ok                  
scheduler            Healthy   ok                  
etcd-0               Healthy   {"health":"true"}

2.7 其它问题

问题：如果node节点kubelet报错：Container runtime network not ready" networkReady="NetworkReady=false reason:NetworkPluginNotReady
解决：将master节点的容器网络配置拷贝到node节点即可 /etc/cni/net.d/*

问题：如果calico node节点活跃pod为0/1是因为calico-node容器获取到了错误的node节点IP

解决：修改calico.yaml在calico-node中添加以下参数

1 2	- name: IP_AUTODETECTION_METHOD value: "interface=ens33"

四、k8s高可用集群部署

k8s高可用集群环境规划，需要准备一个给高可用服务的虚拟IP

主机名	IP地址	节点作用	k8s 相关组件	高可用组件
k8s-master1	192.168.223.140	k8s-master	kubeadmin，kube-apiserver，kube-controller-manager，kube-scheduler，etcd，kube-proxy，kubelet	Haproxy，Keepalived
k8s-master2	192.168.223.144	k8s-master	kubeadmin，kube-apiserver，kube-controller-manager，kube-scheduler，etcd，kube-proxy，kubelet	Haproxy，Keepalived
k8s-master3	192.168.223.145	k8s-master	kubeadmin，kube-apiserver，kube-controller-manager，kube-scheduler，etcd，kube-proxy，kubelet	Haproxy，Keepalived
k8s-node1	192.168.223.142	k8s-node1	kubeadmin，kube-proxy，kubelet	-

系统初始化配置，按照一、系统初始化配置 进行配置
安装docker，按照二、安装docker 进行配置

4.1 安装kubeadm，kubelet和kubectl

3个master节点分别安装kubeadm，kubelet和kubectl

1 2	# yum install -y kubelet-1.20.0 kubeadm-1.20.0 kubectl-1.20.0 # systemctl enable kubelet

4.2 部署etcd集群

cfssl下载 (选一台master 执行)

1
2
3

[root@k8s-master1 ~]# wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -O /usr/bin/cfssl
[root@k8s-master1 ~]# wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -O    /usr/bin/cfssljson
[root@k8s-master1 ~]# chmod +x /usr/bin/cfssl /usr/bin/cfssljson

生成etcd CA、etcd证书 （证书目录在 /etc/etcd/ssl 下）（注意替换 -hostname= 里面IP地址为所有masterIP地址）

[root@k8s-master1 ~]# mkdir -p /etc/etcd/ssl && cd /etc/etcd/ssl
[root@k8s-master1 ssl]# cat > ca-config.json <<EOF
{"signing":{"default":{"expiry":"87600h"},"profiles":{"kubernetes":{"usages":["signing","key encipherment","server auth","client auth"],"expiry":"87600h"}}}}
EOF

[root@k8s-master1 ssl]# cat > etcd-ca-csr.json <<EOF 
{"CN":"etcd","key":{"algo":"rsa","size":2048},"names":[{"C":"CN","ST":"BeiJing","L":"BeiJing","O":"etcd","OU":"etcd"}]}
EOF

[root@k8s-master1 ssl]# cfssl gencert -initca etcd-ca-csr.json | cfssljson -bare etcd-ca
[root@k8s-master1 ssl]# cfssl gencert -ca=etcd-ca.pem -ca-key=etcd-ca-key.pem -config=ca-config.json -hostname=127.0.0.1,192.168.223.140,192.168.223.144,192.168.223.145 -profile=kubernetes etcd-ca-csr.json | cfssljson -bare etcd

[root@k8s-master1 ssl]# rm -rf *.json *.csr

将/etc/etcd/ssl 目录下所有文件拷贝到剩余master 节点相同目录下

1 2	[root@k8s-master1 ssl]# scp /etc/etcd/ssl/* k8s-master2:/etc/etcd/ssl/ [root@k8s-master1 ssl]# scp /etc/etcd/ssl/* k8s-master3:/etc/etcd/ssl/

ETCD下载 (所有master节点执行)

1
2
3

# wget https://github.com/etcd-io/etcd/releases/download/v3.3.12/etcd-v3.3.12-linux-amd64.tar.gz
# tar zxvf etcd-v3.3.12-linux-amd64.tar.gz && cd etcd-v3.3.12-linux-amd64
# cp etcd* /usr/bin/

添加etcd服务 (所有master节点执行) (注意替换 IP地址 192.168 为本机IP，ETCD_NAME 改名)

# cat > /etc/etcd/config << EOF
#[Member]
ETCD_NAME="etcd01"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.223.140:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.223.140:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.223.140:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.223.140:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://192.168.223.140:2380,etcd02=https://192.168.223.144:2380,etcd03=https://192.168.223.145:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
EOF

# vim /usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=neCNork.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=/etc/etcd/config
ExecStart=/usr/bin/etcd \
--name=${ETCD_NAME} \
--data-dir=${ETCD_DATA_DIR} \
--listen-peer-urls=${ETCD_LISTEN_PEER_URLS} \
--listen-client-urls=${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 \
--advertise-client-urls=${ETCD_ADVERTISE_CLIENT_URLS} \
--initial-advertise-peer-urls=${ETCD_INITIAL_ADVERTISE_PEER_URLS} \
--initial-cluster=${ETCD_INITIAL_CLUSTER} \
--initial-cluster-token=${ETCD_INITIAL_CLUSTER_TOKEN} \
--initial-cluster-state=new \
--cert-file=/etc/etcd/ssl/etcd.pem \
--key-file=/etc/etcd/ssl/etcd-key.pem \
--peer-cert-file=/etc/etcd/ssl/etcd.pem \
--peer-key-file=/etc/etcd/ssl/etcd-key.pem \
--trusted-ca-file=/etc/etcd/ssl/etcd-ca.pem \
--peer-trusted-ca-file=/etc/etcd/ssl/etcd-ca.pem
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target

启动3台master上的etcd，查看etcd集群状态

##启动ETCD 
# systemctl enable --now etcd

##检查etcd集群状态
#  ETCDCTL_API=3 etcdctl --cacert=/etc/etcd/ssl/etcd-ca.pem --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem --endpoints="https://192.168.223.140:2379,https://192.168.223.144:2379,https://192.168.223.145:2379" member list
+------------------+---------+--------+------------------------------+------------------------------+
|        ID        | STATUS  |  NAME  |          PEER ADDRS          |         CLIENT ADDRS         |
+------------------+---------+--------+------------------------------+------------------------------+
|  181017bd7f7e566 | started | etcd02 | https://192.168.223.144:2380 | https://192.168.223.144:2379 |
|  f67525124b345fa | started | etcd03 | https://192.168.223.145:2380 | https://192.168.223.145:2379 |
| 3e9b0ead5c1d30ac | started | etcd01 | https://192.168.223.140:2380 | https://192.168.223.140:2379 |
+------------------+---------+--------+------------------------------+------------------------------+


# ETCDCTL_API=3 etcdctl --cacert=/etc/etcd/ssl/etcd-ca.pem --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem --endpoints="https://192.168.223.140:2379,https://192.168.223.144:2379,https://192.168.223.145:2379" --write-out=table endpoint status
+------------------------------+------------------+---------+---------+-----------+-----------+------------+
|           ENDPOINT           |        ID        | VERSION | DB SIZE | IS LEADER | RAFT TERM | RAFT INDEX |
+------------------------------+------------------+---------+---------+-----------+-----------+------------+
| https://192.168.223.140:2379 | 3e9b0ead5c1d30ac |  3.3.12 |   20 kB |      true |        22 |         18 |
| https://192.168.223.144:2379 |  181017bd7f7e566 |  3.3.12 |   20 kB |     false |        22 |         18 |
| https://192.168.223.145:2379 |  f67525124b345fa |  3.3.12 |   20 kB |     false |        22 |         18 |
+------------------------------+------------------+---------+---------+-----------+-----------+------------+

到此部署完了etcd集群，使用kubeadm部署etcd集群可参考官网

4.3 配置apiserver 高可用VIP

高可用我们采用官方推荐的HAproxy + Keepalived，HAproxy和Keepalived以守护进程的方式在所有Master节点部署。

# yum install -y keepalived haproxy

##如果要提前测试VIP能否正常使用，可以使用命令
# ip addr add 192.168.223.150/24 dev eno16777736
##测试完后记得卸载IP
# ip addr del 192.168.223.150 dev eno16777736

配置 Haproxy 服务

所有master节点的haproxy配置相同，haproxy的配置文件是/etc/haproxy/haproxy.cfg。master1节点配置完成之后再分发给master2、master3两个节点。

[root@k8s-master1 ~]# cat /etc/haproxy/haproxy.cfg
global
  maxconn  2000
  ulimit-n  16384
  log  127.0.0.1 local0 err
  stats timeout 30s

defaults
  log global
  mode  http
  option  httplog
  timeout connect 5000
  timeout client  50000
  timeout server  50000
  timeout http-request 15s
  timeout http-keep-alive 15s

frontend monitor-in
  bind *:33305
  mode http
  option httplog
  monitor-uri /monitor

listen stats
  bind    *:8006
  mode    http
  stats   enable
  stats   hide-version
  stats   uri       /stats
  stats   refresh   30s
  stats   realm     Haproxy\ Statistics
  stats   auth      admin:admin

frontend k8s-master
  bind 0.0.0.0:8443
  bind 127.0.0.1:8443
  mode tcp
  option tcplog
  tcp-request inspect-delay 5s
  default_backend k8s-master

backend k8s-master
  mode tcp
  option tcplog
  option tcp-check
  balance roundrobin
  default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
  server master1 k8s-master1:6443  check inter 2000 fall 2 rise 2 weight 100
  server master2 k8s-master2:6443  check inter 2000 fall 2 rise 2 weight 100
  server master3 k8s-master3:6443  check inter 2000 fall 2 rise 2 weight 100

注意修改：server master1 k8s-master1:6443，中的k8s-master1，换成master节点的IP或配置的hosts解析名称

配置Keepalived服务
keepalived中使用track_script机制来配置脚本进行探测kubernetes的master节点是否宕机，并以此切换节点实现高可用。

master1节点的keepalived配置文件如下所示，配置文件所在的位置/etc/keepalived/keepalived.cfg

[root@k8s-master1 ~]# cat /etc/keepalived/keepalived.conf 
! Configuration File for keepalived
global_defs {
    router_id LVS_DEVEL
}
vrrp_script chk_kubernetes {
    script "/etc/keepalived/check_kubernetes.sh"
    interval 2
    weight -5
    fall 3  
    rise 2
}
vrrp_instance VI_1 {
    state MASTER
    interface eno16777736
    mcast_src_ip 192.168.223.140
    virtual_router_id 51
    priority 100
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    virtual_ipaddress {
        192.168.223.150
    }
#    track_script {  ##这里注释是因为haproxy代理的6443端口还没启起来，后面部署了kube-apiserver再启用这个配置
#       chk_kubernetes
#    }
}

注：需要注意几点（记得修改）：

interface：网卡名称
mcast_src_ip：配置多播源地址，此地址是当前主机的ip地址。
priority：keepalived根据此项参数的大小仲裁master节点。我们这里让master节点为kubernetes提供服务，其他两个节点暂时为备用节点。因此master1节点设置为100，master2节点设置为99，master3节点设置为98。
state：我们将master1节点的state字段设置为MASTER，其他两个节点字段修改为BACKUP。
上面的集群检查功能是关闭的，等到集群建立完成后再开启。
virtual_ipaddress：高可用虚拟IP

注：根据上面的注意事项配置master2、master3节点的keepalived服务。

配置健康检测脚本

这里将健康检测脚本放置在/etc/keepalived目录下，check_kubernetes.sh检测脚本如下

[root@master1 keepalived]# cat check_kubernetes.sh 
#!/bin/bash
function check_kubernetes() {
 for ((i=0;i<5;i++));do
  apiserver_pid_id=$(pgrep kube-apiserver)
  if [[ ! -z $apiserver_pid_id ]];then
   return
  else
   sleep 2
  fi
  apiserver_pid_id=0
 done
}

# 1:running  0:stopped
check_kubernetes
if [[ $apiserver_pid_id -eq 0 ]];then
 /usr/bin/systemctl stop keepalived
 exit 1
else
 exit 0
fi

启动Keeplived和Haproxy服务

[root@k8s-master1 ~]# systemctl enable --now keepalived haproxy

##检查一下服务状态
[root@k8s-master1 ~]# systemctl status keepalived haproxy

4.4 初始化k8s 集群

初始化kubeadm 使用的文件，修改以下几项参数

[root@k8s-master1 ~]# kubeadm config print init-defaults > kubeadm-config.yaml
[root@k8s-master1 ~]# vim kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 192.168.223.150  #VIP地址
  bindPort: 6443
nodeRegistration:
  criSocket: /var/run/dockershim.sock
  name: k8s-master1
  taints:
  - effect: NoSchedule
    key: node-role.kubernetes.io/master
---
apiServer:
  certSANs:
  - "192.168.223.150"         #VIP地址
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns:
  type: CoreDNS
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers  #阿里云的镜像站点
controlPlaneEndpoint: "192.168.223.150:8443"   #VIP的地址和端口(haproxy监控端口)
kind: ClusterConfiguration
kubernetesVersion: v1.20.0      #k8s版本号
networking:
  dnsDomain: cluster.local
  serviceSubnet: 10.96.0.0/12   #选择默认即可，当然也可以自定义CIDR
  podSubnet: 172.0.0.0/16       #添加pod网段
scheduler: {}

注意：上面的advertiseAddress字段的值，这个值并非当前主机的网卡地址，而是高可用集群的VIP的地址。
注意：上面的controlPlaneEndpoint这里填写的是VIP的地址，而端口则是haproxy服务的8443端口，也就是我们在haproxy里面配置的这段信息

frontend k8s-master
  bind 0.0.0.0:8443
  bind 127.0.0.1:8443
  mode tcp

这一段里面的8443端，如果你自定义了其他端口，这里请记得修改controlPlaneEndpoint里面的端口。

提前拉取镜像

如果直接采用kubeadm init来初始化，中间会有系统自动拉取镜像的这一步骤，这是比较慢的，我建议分开来做，所以这里就先提前拉取镜像。其他两个master节点在初始化之前也尽量先把镜像拉取下来，这样子减少初始化时间（别忘了将初始化文件拷贝到其余master节点）。

1
2
3

[root@k8s-master1 ~]# kubeadm config images pull --config kubeadm-config.yaml
[root@k8s-master2 ~]# kubeadm config images pull --config kubeadm-config.yaml
[root@k8s-master3 ~]# kubeadm config images pull --config kubeadm-config.yaml

初始化kubenetes的master1节点

[root@k8s-master1 ~]# kubeadm init --config=kubeadm-config.yaml --upload-certs
执行成功结果……
[addons] Applied essential addon: kube-proxy

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of the control-plane node running the following command on each as root:

  kubeadm join 192.168.223.150:8443 --token abcdef.0123456789abcdef \
    --discovery-token-ca-cert-hash sha256:c88fcaa53c0dfb1940a1fde4854be3a76466a7de7375a13894571ca361b2b1a8 \
    --control-plane --certificate-key 0a345539c4a91353017c0aa826dd8b3477f20bd158e052915dda4718eeeb159f

Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 192.168.223.150:8443 --token abcdef.0123456789abcdef \
    --discovery-token-ca-cert-hash sha256:c88fcaa53c0dfb1940a1fde4854be3a76466a7de7375a13894571ca361b2b1a8

添加kubectl config

[root@k8s-master1 ~]# mkdir -p $HOME/.kube
[root@k8s-master1 ~]# sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
[root@k8s-master1 ~]# sudo chown $(id -u):$(id -g) $HOME/.kube/config

[root@k8s-master1 ~]# kubectl get no
NAME          STATUS     ROLES                  AGE   VERSION
k8s-master1   NotReady   control-plane,master   97s   v1.20.0

4.5 启用keepalived 检测功能

k8s集群初始化后kube-apiserver服务已运行，现在可以将3个master 节点keepalived配置文件的 track_script 字段取消注释，并重启keepalived 服务

4.6 Master 节点加入集群

根据提示添加其他master 节点 （master2、master3 上执行）

[root@k8s-master2 ~]# kubeadm join 192.168.223.150:8443 --token abcdef.0123456789abcdef \
    --discovery-token-ca-cert-hash sha256:c88fcaa53c0dfb1940a1fde4854be3a76466a7de7375a13894571ca361b2b1a8 \
    --control-plane --certificate-key 0a345539c4a91353017c0aa826dd8b3477f20bd158e052915dda4718eeeb159f
    
[root@k8s-master3 ~]# kubeadm join 192.168.223.150:8443 --token abcdef.0123456789abcdef \
    --discovery-token-ca-cert-hash sha256:c88fcaa53c0dfb1940a1fde4854be3a76466a7de7375a13894571ca361b2b1a8 \
    --control-plane --certificate-key 0a345539c4a91353017c0aa826dd8b3477f20bd158e052915dda4718eeeb159f

如果节点在加入集群时报错[ERROR Port-10250]: Port 10250 is in use 在报错的节点执行kubeadm reset

4.8 node 节点加入集群

根据提示添加 work 节点

[root@k8s-node1 ~]# kubeadm join 192.168.223.150:8443 --token abcdef.0123456789abcdef \
    --discovery-token-ca-cert-hash sha256:c88fcaa53c0dfb1940a1fde4854be3a76466a7de7375a13894571ca361b2b1a8

[root@k8s-master1 ~]# kubectl get no
NAME          STATUS     ROLES                  AGE   VERSION
k8s-master1   NotReady   control-plane,master   39m   v1.20.0
k8s-master2   NotReady   control-plane,master   10m   v1.20.0
k8s-master3   NotReady   control-plane,master   10m   v1.20.0
k8s-node1     NotReady   <none>                 11s   v1.20.0

到此高可用k8s集群部署基本完成，后续部署网络插件根据 2.3 部署网络插件 进行操作即可

五、集群 token 过期处理

默认token有效期为24小时，当过期之后，该token就不可用了。这时就需要重新创建token，可以直接使用命令快捷生成

1
2

[root@k8s-master ~]# kubeadm token create --print-join-command
kubeadm join 192.168.223.140:6443 --token hpgvxz.q1iso7sfyoipqyp7     --discovery-token-ca-cert-hash sha256:1f06697abdb74cdfc9601d32a45fcf636084bc7ba3de7180dacf97f1403a98c2

如果已运行的集群出现token过期，kubelet、kube-apiserver，kube-controller-manager等服务都报token过期相关的错误，重新生成token后重启k8s相关服务即可恢复集群

六、卸载 flannel 网络插件

master节点删除flannel容器资源

1	[root@k8s-master ~]# kubectl delete -f kube-flannel.yml

在所有节点清理flannel网络插件留下的文件（如果没有创建pod那么cni0网桥只有master存在，意味着只要服务器上有cni0和flannel.l就需要执行下面命令）

ifconfig cni0 down
ip link delete cni0
ifconfig flannel.1 down
ip link delete flannel.1
rm -rf /var/lib/cni/
rm -f /etc/cni/net.d/*

注意：如果有vteh设备的也要清理掉

1 2	# ifconfig vethdc52a97e down # ip link delete vethdc52a97e

清理完成后要重启kubelet

1	# systemctl restart kubelet

七、卸载 k8s 集群

1	# kubeadm reset -f

卸载k8s服务

1	# modprobe -r ipip

#Linux modprobe命令用于自动处理可载入模块。

#-r或–remove 　模块闲置不用时，即自动卸载模块。

# lsmod

#lsmod 命令：用来显示文件、proc/modules的信息，也就是显示当前内核模块装载的模块。

sudo systemctl stop kubelet kube-proxy flanneld kube-apiserver kube-controller-manager kube-scheduler

sudo rm -rf ~/.kube/
sudo rm -rf /etc/kubernetes/
sudo rm -rf /usr/bin/kube*
sudo rm -rf /etc/cni
sudo rm -rf /opt/cni
sudo rm -rf /var/etcd
sudo rm -rf /var/lib/etcd
sudo rm -rf /var/lib/kubelet
sudo rm -rf /var/run/Kubernetes
sudo rm -rf /var/run/flannel/
sudo rm -rf /etc/systemd/system/{etcd,kubelet,kube-apiserver,kube-controller-manager,kube-scheduler,flanneld}.service

mount | grep '/var/lib/kubelet'| awk '{print $3}'|xargs sudo umount
sudo rm -rf /root/local/bin/{etcd,kubelet,kube-apiserver,kube-controller-manager,kube-scheduler,flanneld,mk-docker-opts.sh}
sudo yum clean all
yum remove -y kubelet kubeadm kubectl