如何使用二进制方式搭建K8S高可用集群

共计 13608 个字符，预计需要花费 35 分钟才能阅读完成。

这篇文章主要介绍如何使用二进制方式搭建 K8S 高可用集群，文中介绍的非常详细，具有一定的参考价值，感兴趣的小伙伴们一定要看完！

1、系统概述

操作系统版本：CentOS7.5

k8s 版本：1.12

系统要求：关闭 swap、selinux、iptables

具体信息：

拓扑图：

二进制包下载地址

etcd:

https://github.com/coreos/etcd/releases/tag/v3.2.12

flannel:

https://github.com/coreos/flannel/releases/download/v0.10.0/flannel-v0.10.0-linux-amd64.tar.gz

k8s:

https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.12.md

2、自签 Etcd SSL 证书

master01 操作：

# cat cfssl.sh
#!/bin/bash
wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
chmod +x cfssl_linux-amd64 cfssljson_linux-amd64 cfssl-certinfo_linux-amd64
mv cfssl_linux-amd64 /usr/local/bin/cfssl
mv cfssljson_linux-amd64 /usr/local/bin/cfssljson
mv cfssl-certinfo_linux-amd64 /usr/bin/cfssl-certinfo

自签 Etcd SSL 证书

# cat cert-etcd.sh
cat   ca-config.json  EOF
  signing : {
  default : {
  expiry :  87600h 
 },
  profiles : {
  www : {
  expiry :  87600h ,
  usages : [
  signing ,
  key encipherment ,
  server auth ,
  client auth 
 ]
 }
 }
 }
cat   ca-csr.json  EOF
  CN :  etcd CA ,
  key : {
  algo :  rsa ,
  size : 2048
 },
  names : [
 {
  C :  CN ,
  L :  Beijing ,
  ST :  Beijing 
 }
 ]
cat   server-csr.json  EOF
  CN :  etcd ,
  hosts : [
  192.168.247.161 ,
  192.168.247.162 ,
  192.168.247.163 
 ],
  key : {
  algo :  rsa ,
  size : 2048
 },
  names : [
 {
  C :  CN ,
  L :  BeiJing ,
  ST :  BeiJing 
 }
 ]
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server

# ll *.pem
-rw------- 1 root root 1675 Jan 11 15:50 ca-key.pem
-rw-r--r-- 1 root root 1265 Jan 11 15:50 ca.pem
-rw------- 1 root root 1679 Jan 11 15:50 server-key.pem
-rw-r--r-- 1 root root 1338 Jan 11 15:50 server.pem

3、Etcd 数据库集群部署

master01 02 03 操作:

# mkdir -pv /opt/etcd/{bin,cfg,ssl}
# tar zxvf etcd-v3.2.12-linux-amd64.tar.gz
# mv etcd-v3.2.12-linux-amd64/{etcd,etcdctl} /opt/etcd/bin/

master01 操作:

# cd cert-etcd/
[root@master01 cert-etcd]# ll
total 40
-rw-r--r-- 1 root root 287 Jan 11 15:50 ca-config.json
-rw-r--r-- 1 root root 956 Jan 11 15:50 ca.csr
-rw-r--r-- 1 root root 209 Jan 11 15:50 ca-csr.json
-rw------- 1 root root 1675 Jan 11 15:50 ca-key.pem
-rw-r--r-- 1 root root 1265 Jan 11 15:50 ca.pem
-rw-r--r-- 1 root root 1013 Jan 11 15:50 server.csr
-rw-r--r-- 1 root root 296 Jan 11 15:50 server-csr.json
-rw------- 1 root root 1679 Jan 11 15:50 server-key.pem
-rw-r--r-- 1 root root 1338 Jan 11 15:50 server.pem
-rwxr-xr-x 1 root root 1076 Jan 11 15:50 ssl-etcd.sh
[root@master01 cert-etcd]# cp *.pem /opt/etcd/ssl/

# scp -r /opt/etcd master02:/opt/
# scp -r /opt/etcd master03:/opt/

分别在 master01 02 03 操作:

# cat etcd.sh 
#!/bin/bash
# example: ./etcd.sh etcd01 192.168.1.10 etcd02=https://192.168.1.11:2380,etcd03=https://192.168.1.12:2380
ETCD_NAME=$1
ETCD_IP=$2
ETCD_CLUSTER=$3
WORK_DIR=/opt/etcd
cat  EOF  $WORK_DIR/cfg/etcd
#[Member]
ETCD_NAME= ${ETCD_NAME} 
ETCD_DATA_DIR= /var/lib/etcd/default.etcd 
ETCD_LISTEN_PEER_URLS= https://${ETCD_IP}:2380 
ETCD_LISTEN_CLIENT_URLS= https://${ETCD_IP}:2379 
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS= https://${ETCD_IP}:2380 
ETCD_ADVERTISE_CLIENT_URLS= https://${ETCD_IP}:2379 
ETCD_INITIAL_CLUSTER= ${ETCD_NAME}=https://${ETCD_IP}:2380,${ETCD_CLUSTER} 
ETCD_INITIAL_CLUSTER_TOKEN= etcd-cluster 
ETCD_INITIAL_CLUSTER_STATE= new 

--listen-peer-urls=\${ETCD_LISTEN_PEER_URLS} \
--listen-client-urls=\${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 \
--advertise-client-urls=\${ETCD_ADVERTISE_CLIENT_URLS} \
--initial-advertise-peer-urls=\${ETCD_INITIAL_ADVERTISE_PEER_URLS} \
--initial-cluster=\${ETCD_INITIAL_CLUSTER} \
--initial-cluster-token=\${ETCD_INITIAL_CLUSTER_TOKEN} \
--initial-cluster-state=new \
--cert-file=${WORK_DIR}/ssl/server.pem \
--key-file=${WORK_DIR}/ssl/server-key.pem \
--peer-cert-file=${WORK_DIR}/ssl/server.pem \
--peer-key-file=${WORK_DIR}/ssl/server-key.pem \
--trusted-ca-file=${WORK_DIR}/ssl/ca.pem \
--peer-trusted-ca-file=${WORK_DIR}/ssl/ca.pem
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
systemctl daemon-reload
systemctl start etcd
systemctl enable etcd

# ./etcd.sh etcd01 192.168.247.161 etcd02=https://192.168.247.162:2380,etcd03=https://192.168.247.163:2380

# scp etcd.sh master02:/root/
# scp etcd.sh master03:/root/

[root@master02 ~]# ./etcd.sh etcd02 192.168.247.162 etcd01=https://192.168.247.161:2380,etcd03=https://192.168.247.163:2380
[root@master03 ~]# ./etcd.sh etcd03 192.168.247.163 etcd01=https://192.168.247.161:2380,etcd02=https://192.168.247.162:2380

[root@master01 ~]# systemctl restart etcd
# cd /opt/etcd/ssl
# /opt/etcd/bin/etcdctl \
--ca-file=ca.pem --cert-file=server.pem --key-file=server-key.pem \
--endpoints= https://192.168.247.161:2379,https://192.168.247.162:2379,https://192.168.247.163:2379  \
cluster-health
member 1afd7ff8f95cf93 is healthy: got healthy result from https://192.168.247.161:2379
member 8f4e6ce663f0d49a is healthy: got healthy result from https://192.168.247.162:2379
member b6230d9c6f20feeb is healthy: got healthy result from https://192.168.247.163:2379
cluster is healthy

如有报错，查看 /var/log/message 日志

4、node 节点安装 docker

可以放到脚本内执行

# cat docker.sh
yum remove -y docker docker-common docker-selinux docker-engine 
yum install -y yum-utils device-mapper-persistent-data lvm2
wget -O /etc/yum.repos.d/docker-ce.repo https://download.docker.com/linux/centos/docker-ce.repo
sed -i  s+download.docker.com+mirrors.tuna.tsinghua.edu.cn/docker-ce+  /etc/yum.repos.d/docker-ce.repo
yum makecache fast
yum install -y docker-ce 
systemctl enable docker
systemctl start docker
docker version

如果拉取镜像较慢，可以配置 daocloud 提供的 docker 加速器

5、Flannel 网络部署

master01 执行：

# pwd
/opt/etcd/ssl
# /opt/etcd/bin/etcdctl \
--ca-file=ca.pem --cert-file=server.pem --key-file=server-key.pem \
--endpoints= https://192.168.247.161:2379,https://192.168.247.162:2379,https://192.168.247.163:2379  \
set /coreos.com/network/config  {  Network :  172.17.0.0/16 ,  Backend : { Type :  vxlan}}

node01 执行:

# wget https://github.com/coreos/flannel/releases/download/v0.10.0/flannel-v0.10.0-linux-amd64.tar.gz
# tar zxvf flannel-v0.10.0-linux-amd64.tar.gz
# mkdir -pv /opt/kubernetes/{bin,cfg,ssl}
# mv flanneld mk-docker-opts.sh /opt/kubernetes/bin/

# cat /opt/kubernetes/cfg/flanneld
FLANNEL_OPTIONS= --etcd-endpoints=https://192.168.247.161:2379,https://192.168.247.162:2379,https://192.168.247.163:2379 -etcd-cafile=/opt/etcd/ssl/ca.pem -etcd-certfile=/opt/etcd/ssl/server.pem -etcd-keyfile=/opt/etcd/ssl/server-key.pem

将 master 节点的 /opt/etcd/ssl/* 拷贝到 node 节点

[root@master01 ~]# scp -r /opt/etcd/ssl node01:/opt/etcd/

# cat /usr/lib/systemd/system/flanneld.service
[Unit]
Description=Flanneld overlay address etcd agent
After=network-online.target network.target
Before=docker.service
[Service]
Type=notify
EnvironmentFile=/opt/kubernetes/cfg/flanneld
ExecStart=/opt/kubernetes/bin/flanneld --ip-masq $FLANNEL_OPTIONS
ExecStartPost=/opt/kubernetes/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/subnet.env
Restart=on-failure
[Install]
WantedBy=multi-user.target

# cat /usr/lib/systemd/system/docker.service 
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target firewalld.service
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=/run/flannel/subnet.env
ExecStart=/usr/bin/dockerd $DOCKER_NETWORK_OPTIONS
ExecReload=/bin/kill -s HUP $MAINPID
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TimeoutStartSec=0
Delegate=yes
KillMode=process
Restart=on-failure
StartLimitBurst=3
StartLimitInterval=60s
[Install]
WantedBy=multi-user.target

重启 flannel 和 docker：

# systemctl daemon-reload
# systemctl start flanneld
# systemctl enable flanneld
# systemctl restart docker
# systemctl enable docker

# cat /run/flannel/subnet.env 
DOCKER_OPT_BIP= --bip=172.17.12.1/24 
DOCKER_OPT_IPMASQ= --ip-masq=false 
DOCKER_OPT_MTU= --mtu=1450 
DOCKER_NETWORK_OPTIONS=  --bip=172.17.12.1/24 --ip-masq=false --mtu=1450

# ip a
5: docker0:  NO-CARRIER,BROADCAST,MULTICAST,UP  mtu 1500 qdisc noqueue state DOWN group default 
 link/ether 02:42:f0:62:07:73 brd ff:ff:ff:ff:ff:ff
 inet 172.17.12.1/24 brd 172.17.12.255 scope global docker0
 valid_lft forever preferred_lft forever
6: flannel.1:  BROADCAST,MULTICAST,UP,LOWER_UP  mtu 1450 qdisc noqueue state UNKNOWN group default 
 link/ether ca:e9:e0:d4:05:be brd ff:ff:ff:ff:ff:ff
 inet 172.17.12.0/32 scope global flannel.1
 valid_lft forever preferred_lft forever
 inet6 fe80::c8e9:e0ff:fed4:5be/64 scope link 
 valid_lft forever preferred_lft forever

  将介质及配置文件拷贝至 node02 节点

# scp -r /opt/kubernetes node02:/opt/
# cd /usr/lib/systemd/system/
# scp flanneld.service docker.service node02:/usr/lib/systemd/system/
# scp -r /opt/etcd/ssl/ node02:/opt/etcd/

node02 执行：

# mkdir /opt/etcd

# systemctl daemon-reload
# systemctl start flanneld
# systemctl enable flanneld
# systemctl restart docker

# ip a
5: docker0:  NO-CARRIER,BROADCAST,MULTICAST,UP  mtu 1500 qdisc noqueue state DOWN group default 
 link/ether 02:42:ca:2c:48:df brd ff:ff:ff:ff:ff:ff
 inet 172.17.16.1/24 brd 172.17.16.255 scope global docker0
 valid_lft forever preferred_lft forever
6: flannel.1:  BROADCAST,MULTICAST,UP,LOWER_UP  mtu 1450 qdisc noqueue state UNKNOWN group default 
 link/ether ee:73:b2:e8:46:c1 brd ff:ff:ff:ff:ff:ff
 inet 172.17.16.0/32 scope global flannel.1
 valid_lft forever preferred_lft forever
 inet6 fe80::ec73:b2ff:fee8:46c1/64 scope link 
 valid_lft forever preferred_lft forever

网络测试：

[root@node02 opt]# ping 172.17.12.1
PING 172.17.12.1 (172.17.12.1) 56(84) bytes of data.
64 bytes from 172.17.12.1: icmp_seq=1 ttl=64 time=1.07 ms
64 bytes from 172.17.12.1: icmp_seq=2 ttl=64 time=0.300 ms

[root@node01 system]# ping 172.17.16.1
PING 172.17.16.1 (172.17.16.1) 56(84) bytes of data.
64 bytes from 172.17.16.1: icmp_seq=1 ttl=64 time=1.13 ms

6、自签 APIServer SSL 证书

在 master01 执行：

# cat cert-k8s.sh
#创建 ca 证书
cat   ca-config.json  EOF 
  signing : {
  default : {
  expiry :  87600h 
 },
  profiles : {
  kubernetes : {
  expiry :  87600h ,
  usages : [
  signing ,
  key encipherment ,
  server auth ,
  client auth 
 ]
 }
 }
 }
cat   ca-csr.json  EOF 
  CN :  kubernetes ,
  key : {
  algo :  rsa ,
  size : 2048
 },
  names : [
 {
  C :  CN ,
  L :  Beijing ,
  ST :  Beijing ,
  O :  k8s ,
  OU :  System 
 }
 ]
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes server-csr.json | cfssljson -bare server
#生成 kube-proxy 证书：cat   kube-proxy-csr.json  EOF
  CN :  system:kube-proxy ,
  hosts : [],
  key : {
  algo :  rsa ,
  size : 2048
 },
  names : [
 {
  C :  CN ,
  L :  BeiJing ,
  ST :  BeiJing ,
  O :  k8s ,
  OU :  System 
 }
 ]
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin

# ll *.pem
-rw------- 1 root root 1679 Jan 11 22:06 admin-key.pem
-rw-r--r-- 1 root root 1399 Jan 11 22:06 admin.pem
-rw------- 1 root root 1679 Jan 11 22:06 ca-key.pem
-rw-r--r-- 1 root root 1359 Jan 11 22:06 ca.pem
-rw------- 1 root root 1675 Jan 11 22:06 kube-proxy-key.pem
-rw-r--r-- 1 root root 1403 Jan 11 22:06 kube-proxy.pem
-rw------- 1 root root 1679 Jan 11 22:06 server-key.pem
-rw-r--r-- 1 root root 1651 Jan 11 22:06 server.pem

7、部署 Master 组件

master01、02、03 执行：

# mkdir -pv /opt/kubernetes/{bin,cfg,ssl}
# tar zxvf kubernetes-server-linux-amd64.tar.gz
# cd kubernetes/server/bin
# cp kube-apiserver kube-scheduler kube-controller-manager kubectl /opt/kubernetes/bin/
# pwd
/root/cert-k8s
# cp *.pem /opt/kubernetes/ssl/
# head -c 16 /dev/urandom |od -An -t x |tr -d    
1c96cf8a12d4555a52e89bf3925a5c87
# cat /opt/kubernetes/cfg/token.csv
1c96cf8a12d4555a52e89bf3925a5c87,kubelet-bootstrap,10001, system:kubelet-bootstrap

1)、api-server:

# cat api-server.sh 
#!/bin/bash
# example: ./api-server.sh 192.168.247.161 https://192.168.247.161:2379,https://192.168.247.162:2379,https://192.168.247.163:2379
MASTER_IP=$1
ETCD_SERVERS=$2
cat  EOF   /opt/kubernetes/cfg/kube-apiserver 
KUBE_APISERVER_OPTS= --logtostderr=true \\
--v=4 \\
--etcd-servers=${ETCD_SERVERS} \\
--bind-address=${MASTER_IP} \\
--secure-port=6443 \\
--advertise-address=${MASTER_IP} \\
--allow-privileged=true \\
--service-cluster-ip-range=10.0.0.0/24 \\
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota,NodeRestriction \\
--authorization-mode=RBAC,Node \\
--enable-bootstrap-token-auth \\
--token-auth-file=/opt/kubernetes/cfg/token.csv \\
--service-node-port-range=30000-50000 \\
--tls-cert-file=/opt/kubernetes/ssl/server.pem \\
--tls-private-key-file=/opt/kubernetes/ssl/server-key.pem \\
--client-ca-file=/opt/kubernetes/ssl/ca.pem \\
--service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \\
--etcd-cafile=/opt/etcd/ssl/ca.pem \\
--etcd-certfile=/opt/etcd/ssl/server.pem \\
--etcd-keyfile=/opt/etcd/ssl/server-key.pem 
cat  EOF  /usr/lib/systemd/system/kube-apiserver.service 
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
[Service]
EnvironmentFile=-/opt/kubernetes/cfg/kube-apiserver
ExecStart=/opt/kubernetes/bin/kube-apiserver \$KUBE_APISERVER_OPTS
Restart=on-failure
[Install]
WantedBy=multi-user.target
systemctl daemon-reload
systemctl enable kube-apiserver
systemctl restart kube-apiserver

# ./api-server.sh 192.168.247.161 https://192.168.247.161:2379,https://192.168.247.162:2379,https://192.168.247.163:2379

2)、scheduler 组件

# cat scheduler.sh

cat  EOF  /opt/kubernetes/cfg/kube-scheduler 

KUBE_SCHEDULER_OPTS= --logtostderr=true \\

--v=4 \\

--master=127.0.0.1:8080 \\

--leader-elect

cat  EOF  /usr/lib/systemd/system/kube-scheduler.service 

[Unit]

Description=Kubernetes Scheduler

Documentation=https://github.com/kubernetes/kubernetes

[Service]

EnvironmentFile=-/opt/kubernetes/cfg/kube-scheduler

ExecStart=/opt/kubernetes/bin/kube-scheduler \$KUBE_SCHEDULER_OPTS

Restart=on-failure

[Install]

WantedBy=multi-user.target

systemctl daemon-reload

systemctl enable kube-scheduler

systemctl restart kube-scheduler

# ./scheduler.sh