openssl req -newkey rsa:2048 -nodes -keyout tls.key -x509 -days 365 -out tls.crt

journalctl -xeu kubelet

#清理docker

docker system prune -f

vi /etc/sysconfig/kubelet

KUBELET_EXTRA_ARGS="--container-runtime=remote --cgroup-driver=systemd --container-runtime-endpoint='unix:///var/run/crio/crio.sock' --runtime-request-timeout=5m"

部署基于cri-o
关闭swap ，关闭selinux,关闭firewalld
swapoff -a
 setenforce 0
 systemctl stop firewalld
 systemctl disable firewalld
2 ) 在 Github 中下载 CRI-O 的二进制压缩包：https://storage.googleapis.com/k8s-conform-cri-o/artifacts/crio-v1.19.0.tar.gz

3） cri-o安装部署：

tar -xf crio-v1.19.0.tar.gz
mkdir -p  /opt/cni/bin
mkdir  -p /usr/local/share/oci-umount/oci-umount.d
mkdir /etc/crio
mkdir -p /usr/local/lib/systemd/system
yum install make -y
make install
 4 ) cri-o的 镜像源配置：
修改：/etc/crio/crio.conf
设置为
pause_image = "registry.aliyuncs.com/google_containers/pause:3.2"
设置为：
registries = ['4v2510z7.mirror.aliyuncs.com:443/library']
5）设置启动服务：
systemctl daemon-reload
systemctl enable --now crio
systemctl start --now crio
systemctl status crio

6 ) cri-o的卸载方法,在解压目录下面执行：
make uninstall

k8s相关的配置搭建：
配置k8s的yum源：（master和node都得执行）
cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
   http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF 

2 ) 安装 kubeadm kubelet kubectl组件，版本和cri-o的版本保持一致（master和node都得执行）
yum install kubectl-1.19.0-0.x86_64 -y 
yum install -y kubelet-1.19.0-0.x86_64 -y
yum install -y kubeadm-1.19.0-0.x86_64 -y
3）配置文件配置：（master和node都得执行）
systemctl enable kubelet

修改/etc/sysconfig/kubelet的参数配置，指定kubelet通过cri-o来进行启动，非常重要（master和node都得执行）
KUBELET_EXTRA_ARGS="--container-runtime=remote --cgroup-driver=systemd --container-runtime-endpoint='unix:///var/run/crio/crio.sock' --runtime-request-timeout=5m"
加载内核模块
modprobe br_netfilter

在/etc/sysctl.conf文件中进行配置
net.ipv4.ip_forward = 1
vm.swappiness = 0
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1

执行sysctl -p，让配置文件生效
 4） 在master上生成配置文件：
kubeadm config print init-defaults > kubeadm-config.yaml
配置文件修改后内容如下：

apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 10.0.2.120
  bindPort: 6443
nodeRegistration:
 # criSocket: /var/run/dockershim.sock
  criSocket: /var/run/crio/crio.sock
  name: cri-2.120
  taints:
  - effect: NoSchedule
    key: node-role.kubernetes.io/master
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns:
  type: CoreDNS
etcd:
  local:
    dataDir: /var/lib/etcd
#imageRepository: k8s.gcr.io
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: v1.19.0
networking:
  dnsDomain: cluster.local
  podSubnet: 10.85.0.0/16
  serviceSubnet: 10.96.0.0/12
scheduler: {}
 
5） 初始化k8s集群

查看所需镜像列表
kubeadm config images list --config kubeadm.yml
拉取镜像
kubeadm config images pull --config kubeadm.yml
根据配置文件启动kubeadm拉起k8s
--v=6 查看日志级别，一个节点可以忽略该参数 --upload-certs
kubeadm init --config=./kubeadm.yml --upload-certs --v=6
 执行完毕后的打印内容：
————————————————
Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 10.0.2.120:6443 --token abcdef.0123456789abcdef \
    --discovery-token-ca-cert-hash sha256:3db371d75d6029e5527233b9ec8400cdc6826a4cb88d626216432f0943232eba
————————————————
 6 ) 在master执行如下命令，使kubectl命令可用：
mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config
kubectl get nodes
NAME        STATUS   ROLES    AGE     VERSION
cri-2.120   Ready    master   9m59s   v1.19.0

查看kubectl get cs状态(k8s的19版本存在问题，修改配置文件，重启kubelet进行恢复)
[root@cri-2 crio-v1.19.0]# kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS      MESSAGE                                                                                       ERROR
scheduler            Unhealthy   Get "http://127.0.0.1:10251/healthz": dial tcp 127.0.0.1:10251: connect: connection refused   
controller-manager   Unhealthy   Get "http://127.0.0.1:10252/healthz": dial tcp 127.0.0.1:10252: connect: connection refused   
etcd-0               Healthy     {"health":"true"}  
————————————————
 原因是kube-controller-manager.yaml和kube-scheduler.yaml设置的默认端口是0，只需在文件中注释掉即可。
在每个主节点上执行
vim /etc/kubernetes/manifests/kube-scheduler.yaml 
# and then comment this line
# - --port=0 
重启kubelet
————————————————
 执行完毕后，查看状态：
[root@cri-2 crio-v1.19.0]#  kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS    MESSAGE             ERROR
scheduler            Healthy   ok                  
controller-manager   Healthy   ok                  
etcd-0               Healthy   {"health":"true"}   

————————————————
 8）将node加入到集群，执行：
kubeadm join 10.0.2.120:6443 --token abcdef.0123456789abcdef \
    --discovery-token-ca-cert-hash sha256:3db371d75d6029e5527233b9ec8400cdc6826a4cb88d626216432f0943232eba 
————————————————
 9）部署flannel网络插件：
 kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
10）部署完毕后的情况查看如下：
[root@cri-2-120 mwt]# kubectl get nodes -o wide
NAME        STATUS   ROLES    AGE     VERSION   INTERNAL-IP   EXTERNAL-IP   OS-IMAGE                KERNEL-VERSION                CONTAINER-RUNTIME
cri-2-121   Ready    <none>   4h42m   v1.19.0   10.0.2.121    <none>        CentOS Linux 7 (Core)   3.10.0-1160.45.1.el7.x86_64   cri-o://1.19.0
cri-2.120   Ready    master   4h43m   v1.19.0   10.0.2.120    <none>        CentOS Linux 7 (Core)   3.10.0-1160.45.1.el7.x86_64   cri-o://1.19.0

[root@cri-2-120 mwt]# kubectl get pods -n kube-system -o wide
————————————————
 11） flannel 插件报错原因：大概的意思就是说，我pod的ip未进行配置，但是我在部署的时候已经在yml文件指定pod的ip地址，为啥还是说没有地址
[root@cri-2-120 mwt]# kubectl logs  kube-flannel-ds-jj9n7  -n kube-system
I1123 13:23:19.362621       1 main.go:520] Determining IP address of default interface
I1123 13:23:19.457117       1 main.go:533] Using interface with name ens192 and address 10.0.2.121
I1123 13:23:19.457155       1 main.go:550] Defaulting external address to interface address (10.0.2.121)
W1123 13:23:19.457188       1 client_config.go:608] Neither --kubeconfig nor --master was specified.  Using the inClusterConfig.  This might not work.
I1123 13:23:19.559039       1 kube.go:116] Waiting 10m0s for node controller to sync
I1123 13:23:19.559097       1 kube.go:299] Starting kube subnet manager
I1123 13:23:20.559212       1 kube.go:123] Node controller sync successful
I1123 13:23:20.559239       1 main.go:254] Created subnet manager: Kubernetes Subnet Manager - cri-2-121
I1123 13:23:20.559264       1 main.go:257] Installing signal handlers
I1123 13:23:20.559400       1 main.go:392] Found network config - Backend type: vxlan
I1123 13:23:20.559490       1 vxlan.go:123] VXLAN config: VNI=1 Port=0 GBP=false Learning=false DirectRouting=false
E1123 13:23:20.559858       1 main.go:293] Error registering network: failed to acquire lease: node "cri-2-121" pod cidr not assigned
I1123 13:23:20.559984       1 main.go:372] Stopping shutdownHandler...

/etc/cni/net.d 这里面的配置文件是指定要使用什么的网络插件来启动网卡，分配哪个网段的ip地址，pod的路由等等信息。
/opt/cni/bin/ 这个存放的是网络插件，比如brige，flannel等
当启动kublet时，会读取/etc/cni/net.d的文件，调用/opt/cni/bin/ 的插件创建相关的网络，并且为启动的pod分配ip地址来进行通信。
我配置flannel失败的原因为：
1. /etc/cni/net.d 中有多个文件有干扰。最后移除所有只留 10-flannel.conflist
2. /opt/cni/bin/目录中，不存在flannel二进制文件，从docker的环境下拷贝过来。
3. 配置的kubeadm-config文件  podSubnet: 10.85.0.0/16 字段名称写错了。
————————————————
 搭建环境查看的文章链接：
https://xujiyou.work/%E4%BA%91%E5%8E%9F%E7%94%9F/CRI-O/%E4%BD%BF%E7%94%A8CRI-O%E5%92%8CKubeadm%E6%90%AD%E5%BB%BA%E9%AB%98%E5%8F%AF%E7%94%A8%20Kubernetes%20%E9%9B%86%E7%BE%A4.html （主要参照，但是存在一些问题）
https://blog.csdn.net/u014230612/article/details/112647016 （重点参照）
https://stdworkflow.com/695/get-http-127-0-0-1-10252-healthz-dial-tcp-127-0-0-1-10252… （问题解决参照）
————————————————
 1 检查master节点的flannel服务
 kubectl get pods -n kube-system | grep flannel
如果master节点没有找到对应的flannel服务，执行kubeadm reset,并重新生成flannel服务

kubeadm reset
rm -rf /etc/kubernetes/admin.conf 
rm -rf $HOME/.kube/config
重置flannel网络

（1）删除node节点（master操作）
kubectl delete node xxx
（2）node 节点删除cni和flannel网卡（node操作）
kubeadm reset
 
ifconfig cni0 down
ifconfig flannel.1 down
ifconfig del flannel.1
ifconfig del cni0
 
ip link del flannel.1
ip link del cni0
 
# 命令执行过程中可能会有报错，有的网卡不存在则忽视
# 后面重新加入后会生成的
（3）加入节点准备工作(master操作)
# 通过 kubedam 重新生成 token
[root@master ~]# kubeadm token create --print-join-command
~~
kubeadm join 192.168.247.136:6443 --token x5phh9.9lpb629032p7dseb     --discovery-token-ca-cert-hash sha256:bd23534d635b46f5316f0d388bd88853a6ddb47b1c04129bf25ea31cdbbfba4a 
 
# 将文件传输至node
[root@master ~]# scp /etc/kubernetes/admin.conf 
usage: scp [-12346BCpqrv] [-c cipher] [-F ssh_config] [-i identity_file]
           [-l limit] [-o ssh_option] [-P port] [-S program]
           [[user@]host1:]file1 ... [[user@]host2:]file2
[root@master ~]# scp /etc/kubernetes/admin.conf  root@192.168.247.137:/etc/kubernetes/admin.conf
root@192.168.247.137's password: 
admin.conf                                                                    100% 5451     3.1MB/s   00:00 
（4）node加入集群
[root@slave1 ~]# kubeadm join 192.168.247.136:6443 --token x5phh9.9lpb629032p7dseb     --discovery-token-ca-cert-hash sha256:bd23534d635b46f5316f0d388bd88853a6ddb47b1c04129bf25ea31cdbbfba4a
~~~
 
This node has joined the cluster:
* Certificate signing request was sent to apiserver and a response was received.
* The Kubelet was informed of the new secure connection details.
 
Run 'kubectl get nodes' on the control-plane to see this node join the cluster.