轴承网站建设,网页开发基础知识,dede电影网站源码,类似天书奇谈的网页游戏文章目录 安装前准备安装前设置分发脚本 开始安装k8s集群cfssl 安装部署kubectl命令行工具创建admin证书和私钥创建kubeconfig文件部署ETCD集群部署Flannel网络kube-apiserver 高可用KeepLived 部署部署master节点部署高可用kube-controller-manager集群kube-controller-manage… 文章目录 安装前准备安装前设置分发脚本 开始安装k8s集群cfssl 安装部署kubectl命令行工具创建admin证书和私钥创建kubeconfig文件部署ETCD集群部署Flannel网络kube-apiserver 高可用KeepLived 部署部署master节点部署高可用kube-controller-manager集群kube-controller-manager 创建权限部署高可用kube-schedulerwork节点安装部署Docker组件 部署kubelet组件部署kube-proxy组件 验证集群功能CoreDNS安装 参考文章 安装前准备
192.168.100.26 k8s-01
192.168.100.27 k8s-02
192.168.100.28 k8s-03#node节点
192.168.100.29 k8s-04 #node节点只运行node但是设置证书的时候要添加这个ip
本文档的所有etcd集群、master集群、worker节点均使用以上三台机器并且初始化步骤需要在所有机器上执行命令。如果没有特殊命令所有操作均在192.168.100.26上进行操作
hostnamectl set-hostname k8s-01 #所有机器按照要求修改
hostnamectl set-hostname k8s-02
hostnamectl set-hostname k8s-03
hostnamectl set-hostname k8s-04bash #刷新主机名
# 设置/etc/hosts
cat /etc/hosts EOF
192.168.100.26 k8s-01
192.168.100.27 k8s-02
192.168.100.28 k8s-03
192.168.100.29 k8s-04
EOF# 只要在k8s-01 设置免密主机密码 123456,各位自己改成自己的
wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
yum install -y expectsed -i sPermitRootLogin noPermitRootLogin yesg /etc/ssh/sshd_config
sed -i sPasswordAuthentication noPasswordAuthentication yesg /etc/ssh/sshd_config
systemctl restart sshd
#分发公钥
ssh-keygen -t rsa -P -f /root/.ssh/id_rsa
for i in k8s-01 k8s-02 k8s-03 k8s-04;do
expect -c
spawn ssh-copy-id -i /root/.ssh/id_rsa.pub root$iexpect {\*yes/no*\ {send \yes\r\; exp_continue}\*password*\ {send \123456\r\; exp_continue}\*Password*\ {send \123456\r\;}}
done # 更新path 本次路径全部在/opt 目录下
echo PATH/opt/k8s/bin:$PATH /etc/profile
source /etc/profile
env|grep PATH# 内核以及基础配置更新,内核升级步骤可以自选
yum -y install gcc gcc-c make autoconf libtool-ltdl-devel gd-devel freetype-devel libxml2-devel libjpeg-devel libpng-devel openssh-clients openssl-devel curl-devel bison patch libmcrypt-devel libmhash-devel ncurses-devel binutils compat-libstdc-33 elfutils-libelf elfutils-libelf-devel glibc glibc-common glibc-devel libgcj libtiff pam-devel libicu libicu-devel gettext-devel libaio-devel libaio libgcc libstdc libstdc-devel unixODBC unixODBC-devel numactl-devel glibc-headers sudo bzip2 mlocate flex lrzsz sysstat lsof setuptool system-config-network-tui system-config-firewall-tui ntsysv ntp pv lz4 dos2unix unix2dos rsync dstat iotop innotop mytop telnet iftop expect cmake nc gnuplot screen xorg-x11-utils xorg-x11-xinit rdate bc expat-devel compat-expat1 tcpdump sysstat man nmap curl lrzsz elinks finger bind-utils traceroute mtr ntpdate zip unzip vim wget net-tools
systemctl stop firewalld
systemctl disable firewalld
iptables -F iptables -X iptables -F -t nat iptables -X -t nat
iptables -P FORWARD ACCEPT
swapoff -a
sed -i / swap / s/^\(.*\)$/#\1/g /etc/fstab
setenforce 0
sed -i s/^SELINUX.*/SELINUXdisabled/ /etc/selinux/config# reboot 后yum update -y 执行下# 加载内核模块调优
find /lib/modules/uname -r/ -name ip_vs_rr*
find /lib/modules/uname -r/ -name br_netfilter*
cat /etc/rc.local EOF
modprobe ip_vs_rr
modprobe br_netfilter
EOF
cat /etc/modules-load.d/ipvs.conf EOFip_vs_rrbr_netfilter
EOF
systemctl enable --now systemd-modules-load.service
#验证是否成功
lsmod |egrep ip_vs_rr|br_netfiltercat kubernetes.conf EOF
net.bridge.bridge-nf-call-iptables1
net.bridge.bridge-nf-call-ip6tables1
net.ipv4.ip_forward1
net.ipv4.tcp_tw_recycle0
vm.swappiness0 # 禁止使用 swap 空间只有当系统 OOM 时才允许使用它
vm.overcommit_memory1 # 不检查物理内存是否够用
vm.panic_on_oom0 # 开启 OOM
fs.inotify.max_user_instances8192
fs.inotify.max_user_watches1048576
fs.file-max52706963
fs.nr_open52706963
net.ipv6.conf.all.disable_ipv61
net.netfilter.nf_conntrack_max2310720
EOF
cp kubernetes.conf /etc/sysctl.d/kubernetes.conf
sysctl -p /etc/sysctl.d/kubernetes.conf
需要关闭tcp_tw_recycle否则和NAT冲突会导致服务不通
关闭IPV6防止触发Docker BUGtimedatectl set-timezone Asia/Shanghai#将当前的 UTC 时间写入硬件时钟
timedatectl set-local-rtc 0#重启依赖于系统时间的服务
systemctl restart rsyslog
systemctl restart crond
mkdir -p /opt/k8s/{bin,work} /etc/{kubernetes,etcd}/cert
#在所有节点上执行因为flanneld是在所有节点运行的安装前设置分发脚本
后续所有的使用环境变量都定义在environment.sh中需要根据个人机器及网络环境修改。并且需要拷贝到所有节点的/opt/k8s/bin目录下
cat environment.sh
#!/usr/bin/bash
# 生成 EncryptionConfig 所需的加密 key
export ENCRYPTION_KEY$(head -c 32 /dev/urandom | base64)
# 集群各机器 IP 数组
export NODE_IPS( 192.168.100.26 192.168.100.27 192.168.100.28 192.168.100.29 )
# 集群各 IP 对应的主机名数组
export NODE_NAMES(k8s-01 k8s-02 k8s-03 k8s-04)
# 集群MASTER机器 IP 数组
export MASTER_IPS(192.168.100.26 192.168.100.27 192.168.100.28 )
# 集群所有的master Ip对应的主机
export MASTER_NAMES(k8s-01 k8s-02 k8s-03)
# etcd 集群服务地址列表
export ETCD_ENDPOINTShttps://192.168.100.26:2379,https://192.168.100.27:2379,https://192.168.100.28:2379
# etcd 集群间通信的 IP 和端口
export ETCD_NODESk8s-01https://192.168.100.26:2380,k8s-02https://192.168.100.27:2380,k8s-03https://192.168.100.28:2380
# etcd 集群所有node ip
export ETCD_IPS(192.168.100.26 192.168.100.27 192.168.100.28 192.168.100.29 )
# kube-apiserver 的反向代理(kube-nginx)地址端口
export KUBE_APISERVERhttps://192.168.100.199:8443
# 节点间互联网络接口名称
export IFACEeth0
# etcd 数据目录
export ETCD_DATA_DIR/data/k8s/etcd/data
# etcd WAL 目录建议是 SSD 磁盘分区或者和 ETCD_DATA_DIR 不同的磁盘分区
export ETCD_WAL_DIR/data/k8s/etcd/wal
# k8s 各组件数据目录
export K8S_DIR/data/k8s/k8s
# docker 数据目录
#export DOCKER_DIR/data/k8s/docker
## 以下参数一般不需要修改
# TLS Bootstrapping 使用的 Token可以使用命令 head -c 16 /dev/urandom | od -An -t x | tr -d 生成
#BOOTSTRAP_TOKEN41f7e4ba8b7be874fcff18bf5cf41a7c
# 最好使用 当前未用的网段 来定义服务网段和 Pod 网段
# 服务网段部署前路由不可达部署后集群内路由可达(kube-proxy 保证)
SERVICE_CIDR10.254.0.0/16
# Pod 网段建议 /16 段地址部署前路由不可达部署后集群内路由可达(flanneld 保证)
CLUSTER_CIDR172.30.0.0/16
# 服务端口范围 (NodePort Range)
export NODE_PORT_RANGE1024-32767
# flanneld 网络配置前缀
export FLANNEL_ETCD_PREFIX/kubernetes/network
# kubernetes 服务 IP (一般是 SERVICE_CIDR 中第一个IP)
export CLUSTER_KUBERNETES_SVC_IP10.254.0.1
# 集群 DNS 服务 IP (从 SERVICE_CIDR 中预分配)
export CLUSTER_DNS_SVC_IP10.254.0.2
# 集群 DNS 域名末尾不带点号
export CLUSTER_DNS_DOMAINcluster.local
# 将二进制目录 /opt/k8s/bin 加到 PATH 中
export PATH/opt/k8s/bin:$PATH#分发环境变量
source environment.sh
for node_ip in ${NODE_IPS[]}doecho ${node_ip}scp environment.sh root${node_ip}:/opt/k8s/bin/ssh root${node_ip} chmod x /opt/k8s/bin/*
done开始安装k8s集群
cfssl 安装
为确保安全kubernetes各个组件需要使用x509证书对通信进行加密和认证
CA(Certificate Authority)是自签名的根证书用来签名后续创建的其他证书。本文章使用CloudFlare的PKI工具cfssl创建所有证书。注意: 如果没有特殊指明本文档的所有操作均在k8s-01节点执行远程分发到其他节点
#安装cfssl
mkdir -p /opt/k8s/cert cd /opt/k8s
wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
mv cfssl_linux-amd64 /opt/k8s/bin/cfssl
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
mv cfssljson_linux-amd64 /opt/k8s/bin/cfssljson
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
mv cfssl-certinfo_linux-amd64 /opt/k8s/bin/cfssl-certinfo
chmod x /opt/k8s/bin/*
export PATH/opt/k8s/bin:$PATHcd /opt/k8s/work
cat ca-config.json EOF
{signing: {default: {expiry: 87600h},profiles: {kubernetes: {usages: [signing,key encipherment,server auth,client auth],expiry: 87600h}}}
}
EOF######################
signing 表示该证书可用于签名其它证书生成的ca.pem证书找中CATRUE
server auth 表示client可以用该证书对server提供的证书进行验证
client auth 表示server可以用该证书对client提供的证书进行验证cd /opt/k8s/work
cat ca-csr.json EOF
{CN: kubernetes,key: {algo: rsa,size: 2048},names: [{C: CN,ST: BeiJing,L: BeiJing,O: k8s,OU: 4Paradigm}],ca: {expiry: 876000h}
}
EOF#######################
CN CommonName,kube-apiserver从证书中提取该字段作为请求的用户名(User Name)浏览器使用该字段验证网站是否合法
O Organization,kube-apiserver 从证书中提取该字段作为请求用户和所属组(Group)
kube-apiserver将提取的User、Group作为RBAC授权的用户和标识cd /opt/k8s/work
cfssl gencert -initca ca-csr.json | cfssljson -bare ca
ls ca*#分发证书
#将生成的CA证书、秘钥文件、配置文件拷贝到所有节点的/etc/kubernetes/cert目录下cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[]}doecho ${node_ip}ssh root${node_ip} mkdir -p /etc/kubernetes/certscp ca*.pem ca-config.json root${node_ip}:/etc/kubernetes/certdone
部署kubectl命令行工具
cd /opt/k8s/work
wget http://down.i4t.com/k8s1.14/kubernetes-client-linux-amd64.tar.gz
tar -xzvf kubernetes-client-linux-amd64.tar.gz
cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[]}doecho ${node_ip}scp kubernetes/client/bin/kubectl root${node_ip}:/opt/k8s/bin/ssh root${node_ip} chmod x /opt/k8s/bin/*done
创建admin证书和私钥
kubectl与apiserver https通信apiserver对提供的证书进行认证和授权。kubectl作为集群的管理工具需要被授予最高权限这里创建具有最高权限的admin证书创建证书签名请求
cd /opt/k8s/work
cat admin-csr.json EOF
{CN: admin,hosts: [],key: {algo: rsa,size: 2048},names: [{C: CN,ST: BeiJing,L: BeiJing,O: system:masters,OU: 4Paradigm}]
}
EOF###################
● O 为system:masterskube-apiserver收到该证书后将请求的Group设置为system:masters
● 预定的ClusterRoleBinding cluster-admin将Group system:masters与Role cluster-admin绑定该Role授予API的权限
● 该证书只有被kubectl当做client证书使用所以hosts字段为空
cd /opt/k8s/work
cfssl gencert -ca/opt/k8s/work/ca.pem \-ca-key/opt/k8s/work/ca-key.pem \-config/opt/k8s/work/ca-config.json \-profilekubernetes admin-csr.json | cfssljson -bare admin
ls admin*创建kubeconfig文件
kubeconfig为kubectl的配置文件包含访问apiserver的所有信息如apiserver地址、CA证书和自身使用的证书
cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
# 设置集群参数
kubectl config set-cluster kubernetes \--certificate-authority/opt/k8s/work/ca.pem \--embed-certstrue \--server${KUBE_APISERVER} \--kubeconfigkubectl.kubeconfig
#设置客户端认证参数
kubectl config set-credentials admin \--client-certificate/opt/k8s/work/admin.pem \--client-key/opt/k8s/work/admin-key.pem \--embed-certstrue \--kubeconfigkubectl.kubeconfig
# 设置上下文参数
kubectl config set-context kubernetes \--clusterkubernetes \--useradmin \--kubeconfigkubectl.kubeconfig
# 设置默认上下文
kubectl config use-context kubernetes --kubeconfigkubectl.kubeconfig################
--certificate-authority 验证kube-apiserver证书的根证书
--client-certificate、--client-key 刚生成的admin证书和私钥连接kube-apiserver时使用
--embed-certstrue 将ca.pem和admin.pem证书嵌入到生成的kubectl.kubeconfig文件中 (如果不加入写入的是证书文件路径后续拷贝kubeconfig到其它机器时还需要单独拷贝证书)cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[]}doecho ${node_ip}ssh root${node_ip} mkdir -p ~/.kubescp kubectl.kubeconfig root${node_ip}:~/.kube/configdone
部署ETCD集群 cd /opt/k8s/work
wget http://down.i4t.com/k8s1.14/etcd-v3.3.13-linux-amd64.tar.gz
tar -xvf etcd-v3.3.13-linux-amd64.tar.gz
cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${ETCD_IPS[]}doecho ${node_ip}scp etcd-v3.3.13-linux-amd64/etcd* root${node_ip}:/opt/k8s/binssh root${node_ip} chmod x /opt/k8s/bin/*done# 创建etcd 证书密钥cd /opt/k8s/work
cat etcd-csr.json EOF
{CN: etcd,hosts: [127.0.0.1,192.168.100.26,192.168.100.27,192.168.100.28],key: {algo: rsa,size: 2048},names: [{C: CN,ST: BeiJing,L: BeiJing,O: k8s,OU: 4Paradigm}]
}
EOF#host字段指定授权使用该证书的etcd节点IP或域名列表需要将etcd集群的3个节点都添加其中cd /opt/k8s/work
cfssl gencert -ca/opt/k8s/work/ca.pem \-ca-key/opt/k8s/work/ca-key.pem \-config/opt/k8s/work/ca-config.json \-profilekubernetes etcd-csr.json | cfssljson -bare etcd
ls etcd*pemcd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${ETCD_IPS[]}doecho ${node_ip}ssh root${node_ip} mkdir -p /etc/etcd/certscp etcd*.pem root${node_ip}:/etc/etcd/cert/done# 创建启动文件cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
cat etcd.service.template EOF
[Unit]
DescriptionEtcd Server
Afternetwork.target
Afternetwork-online.target
Wantsnetwork-online.target
Documentationhttps://github.com/coreos
[Service]
Typenotify
WorkingDirectory${ETCD_DATA_DIR}
ExecStart/opt/k8s/bin/etcd \\--data-dir${ETCD_DATA_DIR} \\--wal-dir${ETCD_WAL_DIR} \\--name##NODE_NAME## \\--cert-file/etc/etcd/cert/etcd.pem \\--key-file/etc/etcd/cert/etcd-key.pem \\--trusted-ca-file/etc/kubernetes/cert/ca.pem \\--peer-cert-file/etc/etcd/cert/etcd.pem \\--peer-key-file/etc/etcd/cert/etcd-key.pem \\--peer-trusted-ca-file/etc/kubernetes/cert/ca.pem \\--peer-client-cert-auth \\--client-cert-auth \\--listen-peer-urlshttps://##NODE_IP##:2380 \\--initial-advertise-peer-urlshttps://##NODE_IP##:2380 \\--listen-client-urlshttps://##NODE_IP##:2379,http://127.0.0.1:2379 \\--advertise-client-urlshttps://##NODE_IP##:2379 \\--initial-cluster-tokenetcd-cluster-0 \\--initial-cluster${ETCD_NODES} \\--initial-cluster-statenew \\--auto-compaction-modeperiodic \\--auto-compaction-retention1 \\--max-request-bytes33554432 \\--quota-backend-bytes6442450944 \\--heartbeat-interval250 \\--election-timeout2000
Restarton-failure
RestartSec5
LimitNOFILE65536
[Install]
WantedBymulti-user.target
EOF配置说明 (此处不需要修改任何配置)WorkDirectory、--data-dir 指定etcd工作目录和数据存储为${ETCD_DATA_DIR},需要在启动前创建这个目录 (后面跟着我操作就可以会有创建步骤)
--wal-dir 指定wal目录为了提高性能一般使用SSD和--data-dir不同的盘
--name 指定节点名称当--initial-cluster-state值为new时--name的参数值必须位于--initial-cluster列表中
--cert-file、--key-file ETCD server与client通信时使用的证书和私钥
--trusted-ca-file 签名client证书的CA证书用于验证client证书
--peer-cert-file、--peer-key-file ETCD与peer通信使用的证书和私钥
--peer-trusted-ca-file 签名peer证书的CA证书用于验证peer证书#分发会将配置文件中的#替换成ip
cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for (( i0; i 3; i ))dosed -e s/##NODE_NAME##/${MASTER_NAMES[i]}/ -e s/##NODE_IP##/${ETCD_IPS[i]}/ etcd.service.template etcd-${ETCD_IPS[i]}.service done
ls *.service#NODE_NAMES 和 NODE_IPS 为相同长度的 bash 数组分别为节点名称和对应的 IPcd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[]}doecho ${node_ip}scp etcd-${node_ip}.service root${node_ip}:/etc/systemd/system/etcd.servicedone重命名etcd启动文件并启动etcd服务etcd首次进程启动会等待其他节点加入etcd集群执行启动命令会卡顿一会为正常现象cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[]}doecho ${node_ip}ssh root${node_ip} mkdir -p ${ETCD_DATA_DIR} ${ETCD_WAL_DIR}ssh root${node_ip} systemctl daemon-reload systemctl enable etcd systemctl restart etcd done#这里我们创建了etcd的工作目录#检查启动结果
cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[]}doecho ${node_ip}ssh root${node_ip} systemctl status etcd|grep Activedone正常状态下为这种情况
# for node_ip in ${MASTER_IPS[]}doecho ${node_ip}ssh root${node_ip} systemctl status etcd|grep Activedone192.168.100.26Active: active (running) since Mon 2023-08-21 16:10:15 CST; 52s ago192.168.100.27Active: active (running) since Mon 2023-08-21 16:10:15 CST; 52s ago192.168.100.28Active: active (running) since Mon 2023-08-21 16:10:15 CST; 52s ago可以用这个命令查看journalctl -fu etcd验证etcd是否正常
cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[]}doecho ${node_ip}ETCDCTL_API3 /opt/k8s/bin/etcdctl \--endpointshttps://${node_ip}:2379 \--cacert/etc/kubernetes/cert/ca.pem \--cert/etc/etcd/cert/etcd.pem \--key/etc/etcd/cert/etcd-key.pem endpoint healthdone正常为如下操作
# for node_ip in ${MASTER_IPS[]}doecho ${node_ip}ETCDCTL_API3 /opt/k8s/bin/etcdctl \--endpointshttps://${node_ip}:2379 \--cacert/etc/kubernetes/cert/ca.pem \--cert/etc/etcd/cert/etcd.pem \--key/etc/etcd/cert/etcd-key.pem endpoint healthdone192.168.100.26
https://192.168.100.26:2379 is healthy: successfully committed proposal: took 2.247021ms192.168.100.27
https://192.168.100.27:2379 is healthy: successfully committed proposal: took 2.080702ms192.168.100.28
https://192.168.100.28:2379 is healthy: successfully committed proposal: took 2.262168ms通过etcd 查看集群的leader
source /opt/k8s/bin/environment.sh
ETCDCTL_API3 /opt/k8s/bin/etcdctl \-w table --cacert/etc/kubernetes/cert/ca.pem \--cert/etc/etcd/cert/etcd.pem \--key/etc/etcd/cert/etcd-key.pem \--endpoints${ETCD_ENDPOINTS} endpoint status
# ETCDCTL_API3 /opt/k8s/bin/etcdctl \-w table --cacert/etc/kubernetes/cert/ca.pem \--cert/etc/etcd/cert/etcd.pem \--key/etc/etcd/cert/etcd-key.pem \--endpoints${ETCD_ENDPOINTS} endpoint status
---------------------------------------------------------------------------------------------------
| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | RAFT TERM | RAFT INDEX |
---------------------------------------------------------------------------------------------------
| https://192.168.100.26:2379 | 4fb7383fdc28cdcb | 3.3.13 | 20 kB | true | 2 | 8 |
| https://192.168.100.27:2379 | e69d641fa8a08905 | 3.3.13 | 20 kB | false | 2 | 8 |
| https://192.168.100.28:2379 | 1f54d9cf6d187788 | 3.3.13 | 20 kB | false | 2 | 8 |
---------------------------------------------------------------------------------------------------
部署Flannel网络
Kubernetes要求集群内各个节点(包括master)能通过Pod网段互联互通Flannel使用vxlan技术为各个节点创建一个互通的Pod网络使用的端口为8472.第一次启动时从etcd获取配置的Pod网络为本节点分配一个未使用的地址段然后创建flannel.1网络接口(也可能是其它名称)flannel将分配给自己的Pod网段信息写入/run/flannel/docker文件docker后续使用这个文件中的环境变量设置Docker0网桥从而从这个地址段为本节点的所有Pod容器分配IP下载分发flanneld二进制文件 (本次flanneld不使用Pod运行)
cd /opt/k8s/work
mkdir flannel
wget http://down.i4t.com/k8s1.14/flannel-v0.11.0-linux-amd64.tar.gz
tar -xzvf flannel-v0.11.0-linux-amd64.tar.gz -C flannelflanneld从etcd集群存取网段分配信息而etcd集群开启了双向x509证书认证所以需要为flannel生成证书和私钥创建证书签名请求
cd /opt/k8s/work
cat flanneld-csr.json EOF
{CN: flanneld,hosts: [],key: {algo: rsa,size: 2048},names: [{C: CN,ST: BeiJing,L: BeiJing,O: k8s,OU: 4Paradigm}]
}
EOFcfssl gencert -ca/opt/k8s/work/ca.pem \-ca-key/opt/k8s/work/ca-key.pem \-config/opt/k8s/work/ca-config.json \-profilekubernetes flanneld-csr.json | cfssljson -bare flanneld
ls flanneld*pemcd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[]}doecho ${node_ip}ssh root${node_ip} mkdir -p /etc/flanneld/certscp flanneld*.pem root${node_ip}:/etc/flanneld/certdone
cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
etcdctl \--endpoints${ETCD_ENDPOINTS} \--ca-file/opt/k8s/work/ca.pem \--cert-file/opt/k8s/work/flanneld.pem \--key-file/opt/k8s/work/flanneld-key.pem \mk ${FLANNEL_ETCD_PREFIX}/config {Network:${CLUSTER_CIDR}, SubnetLen: 21, Backend: {Type: vxlan}}
注意:flanneld当前版本v0.11.0不支持etcd v3故使用etcd v2 API写入配置Key和网段数据写入的Pod网段${CLUSTER_CIDR}地址段(如/16)必须小于SubnetLen必须与kube-controller-manager的--cluster-cidr参数一致#创建flanneld的启动文件
cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
cat flanneld.service EOF
[Unit]
DescriptionFlanneld overlay address etcd agent
Afternetwork.target
Afternetwork-online.target
Wantsnetwork-online.target
Afteretcd.service
Beforedocker.service
[Service]
Typenotify
ExecStart/opt/k8s/bin/flanneld \\-etcd-cafile/etc/kubernetes/cert/ca.pem \\-etcd-certfile/etc/flanneld/cert/flanneld.pem \\-etcd-keyfile/etc/flanneld/cert/flanneld-key.pem \\-etcd-endpoints${ETCD_ENDPOINTS} \\-etcd-prefix${FLANNEL_ETCD_PREFIX} \\-iface${IFACE} \\-ip-masq
ExecStartPost/opt/k8s/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker
Restartalways
RestartSec5
StartLimitInterval0
[Install]
WantedBymulti-user.target
RequiredBydocker.service
EOF
mk-docker-opts.sh 脚本将分配给 flanneld 的 Pod 子网段信息写入 /run/flannel/docker 文件后续 docker 启动时使用这个文件中的环境变量配置 docker0 网桥
flanneld 使用系统缺省路由所在的接口与其它节点通信对于有多个网络接口如内网和公网的节点可以用 -iface 参数指定通信接口;
flanneld 运行时需要 root 权限
-ip-masq: flanneld 为访问 Pod 网络外的流量设置 SNAT 规则同时将传递给 Docker 的变量 --ip-masq/run/flannel/docker 文件中设置为 false这样 Docker 将不再创建 SNAT 规则 Docker 的 --ip-masq 为 true 时创建的 SNAT 规则比较“暴力”将所有本节点 Pod 发起的、访问非 docker0 接口的请求做 SNAT这样访问其他节点 Pod 的请求来源 IP 会被设置为 flannel.1 接口的 IP导致目的 Pod 看不到真实的来源 Pod IP。 flanneld 创建的 SNAT 规则比较温和只对访问非 Pod 网段的请求做 SNAT。
cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[]}doecho ${node_ip}scp flanneld.service root${node_ip}:/etc/systemd/system/done
source /opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[]}doecho ${node_ip}ssh root${node_ip} systemctl daemon-reload systemctl enable flanneld systemctl restart flannelddone
source /opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[]}doecho ${node_ip}ssh root${node_ip} systemctl status flanneld|grep Activedone# 检查分配给flanneld的Pod网段信息
source /opt/k8s/bin/environment.sh
etcdctl \--endpoints${ETCD_ENDPOINTS} \--ca-file/etc/kubernetes/cert/ca.pem \--cert-file/etc/flanneld/cert/flanneld.pem \--key-file/etc/flanneld/cert/flanneld-key.pem \get ${FLANNEL_ETCD_PREFIX}/config#查看已分配的Pod子网网段列表source /opt/k8s/bin/environment.sh
etcdctl \--endpoints${ETCD_ENDPOINTS} \--ca-file/etc/kubernetes/cert/ca.pem \--cert-file/etc/flanneld/cert/flanneld.pem \--key-file/etc/flanneld/cert/flanneld-key.pem \ls ${FLANNEL_ETCD_PREFIX}/subnets
#查看某Pod网段对应节点IP和flannel接口地址
source /opt/k8s/bin/environment.sh
etcdctl \--endpoints${ETCD_ENDPOINTS} \--ca-file/etc/kubernetes/cert/ca.pem \--cert-file/etc/flanneld/cert/flanneld.pem \--key-file/etc/flanneld/cert/flanneld-key.pem \get ${FLANNEL_ETCD_PREFIX}/subnets/172.30.104.0-21#后面节点IP需要根据我们查出来的地址进行修改
# etcdctl \--endpoints${ETCD_ENDPOINTS} \--ca-file/etc/kubernetes/cert/ca.pem \--cert-file/etc/flanneld/cert/flanneld.pem \--key-file/etc/flanneld/cert/flanneld-key.pem \ls ${FLANNEL_ETCD_PREFIX}/subnets
/kubernetes/network/subnets/172.30.104.0-21
/kubernetes/network/subnets/172.30.224.0-21
/kubernetes/network/subnets/172.30.24.0-21
/kubernetes/network/subnets/172.30.64.0-21
[rootk8s-01 work]# source /opt/k8s/bin/environment.sh
[rootk8s-01 work]# source /opt/k8s/bin/environment.sh
[rootk8s-01 work]# etcdctl \--endpoints${ETCD_ENDPOINTS} \--ca-file/etc/kubernetes/cert/ca.pem \--cert-file/etc/flanneld/cert/flanneld.pem \--key-file/etc/flanneld/cert/flanneld-key.pem \get ${FLANNEL_ETCD_PREFIX}/subnets/172.30.104.0-21
{PublicIP:192.168.100.28,BackendType:vxlan,BackendData:{VtepMAC:9a:aa:f4:f0:2c:e8}}# 查看节点flannel网络信息
ip addr show
flannel.1网卡的地址为分配的pod自网段的第一个个IP (.0)且是/32的地址
ip addr show|grep flannel.1
到其它节点 Pod 网段请求都被转发到 flannel.1 网卡flanneld 根据 etcd 中子网段的信息如 ${FLANNEL_ETCD_PREFIX}/subnets/172.30.80.0-21来决定进请求发送给哪个节点的互联 IP验证各节点能通过 Pod 网段互通在各节点上部署 flannel 后检查是否创建了 flannel 接口(名称可能为 flannel0、flannel.0、flannel.1 等)
source /opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[]}doecho ${node_ip}ssh ${node_ip} /usr/sbin/ip addr show flannel.1|grep -w inetdonekube-apiserver 高可用
使用Nginx 4层透明代理功能实现k8s节点(master节点和worker节点)高可用访问kube-apiserver的步骤
控制节点的kube-controller-manager、kube-scheduler是多实例部署所以只要一个实例正常就可以保证集群高可用
集群内的Pod使用k8s服务域名kubernetes访问kube-apiserverkube-dns会自动解析多个kube-apiserver节点的IP所以也是高可用的
在每个Nginx进程后端对接多个apiserver实例Nginx对他们做健康检查和负载均衡
kubelet、kube-proxy、controller-manager、schedule通过本地nginx (监听我们vip 192.158.0.54)访问kube-apiserver从而实现kube-apiserver高可用
下载编译nginx (k8s-01安装就可以后面有拷贝步骤)
cd /opt/k8s/work
wget http://down.i4t.com/k8s1.14/nginx-1.15.3.tar.gz
tar -xzvf nginx-1.15.3.tar.gz#编译
cd /opt/k8s/work/nginx-1.15.3
mkdir nginx-prefix
./configure --with-stream --without-http --prefix$(pwd)/nginx-prefix --without-http_uwsgi_module
make make install#############
--without-http_scgi_module --without-http_fastcgi_module
--with-stream开启 4 层透明转发(TCP Proxy)功能
--without-xxx关闭所有其他功能这样生成的动态链接二进制程序依赖最小查看安装后的动态库
ldd ./nginx-prefix/sbin/nginx
由于只开启了 4 层透明转发功能所以除了依赖 libc 等操作系统核心 lib 库外没有对其它 lib 的依赖(如 libz、libssl 等)这样可以方便部署到各版本操作系统中cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[]}doecho ${node_ip}mkdir -p /opt/k8s/kube-nginx/{conf,logs,sbin}done#拷贝二进制到其他文件一次执行不成功就多执行一次
cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[]}doecho ${node_ip}scp /opt/k8s/work/nginx-1.15.3/nginx-prefix/sbin/nginx root${node_ip}:/opt/k8s/kube-nginx/sbin/kube-nginxssh root${node_ip} chmod ax /opt/k8s/kube-nginx/sbin/*ssh root${node_ip} mkdir -p /opt/k8s/kube-nginx/{conf,logs,sbin}sleep 3done# 配置Nginx文件开启4层透明转发cd /opt/k8s/work
cat kube-nginx.conf EOF
worker_processes 1;
events {worker_connections 1024;
}
stream {upstream backend {hash $remote_addr consistent;server 192.168.100.26:6443 max_fails3 fail_timeout30s;server 192.168.100.27:6443 max_fails3 fail_timeout30s;server 192.168.100.28:6443 max_fails3 fail_timeout30s;}server {listen *:8443;proxy_connect_timeout 1s;proxy_pass backend;}
}
EOF#这里需要将server替换我们自己的地址cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[]}doecho ${node_ip}scp kube-nginx.conf root${node_ip}:/opt/k8s/kube-nginx/conf/kube-nginx.confdone#配置启动文件
cd /opt/k8s/work
cat kube-nginx.service EOF
[Unit]
Descriptionkube-apiserver nginx proxy
Afternetwork.target
Afternetwork-online.target
Wantsnetwork-online.target
[Service]
Typeforking
ExecStartPre/opt/k8s/kube-nginx/sbin/kube-nginx -c /opt/k8s/kube-nginx/conf/kube-nginx.conf -p /opt/k8s/kube-nginx -t
ExecStart/opt/k8s/kube-nginx/sbin/kube-nginx -c /opt/k8s/kube-nginx/conf/kube-nginx.conf -p /opt/k8s/kube-nginx
ExecReload/opt/k8s/kube-nginx/sbin/kube-nginx -c /opt/k8s/kube-nginx/conf/kube-nginx.conf -p /opt/k8s/kube-nginx -s reload
PrivateTmptrue
Restartalways
RestartSec5
StartLimitInterval0
LimitNOFILE65536
[Install]
WantedBymulti-user.target
EOFcd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[]}doecho ${node_ip}scp kube-nginx.service root${node_ip}:/etc/systemd/system/donecd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[]}doecho ${node_ip}ssh root${node_ip} systemctl daemon-reload systemctl enable kube-nginx systemctl start kube-nginxdonecd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[]}doecho ${node_ip}ssh root${node_ip} systemctl status kube-nginx |grep Active:doneKeepLived 部署
需要一个vip
在所有master节点安装keeplived
yum install -y keepalived在192.168.100.26 上操作
cat /etc/keepalived/keepalived.conf EOF
! Configuration File for keepalived
global_defs {router_id 192.168.100.26
}
vrrp_script chk_nginx {script /etc/keepalived/check_port.sh 8443interval 2weight -20
}
vrrp_instance VI_1 {state MASTERinterface eth0virtual_router_id 251priority 100advert_int 1mcast_src_ip 192.168.100.26nopreemptauthentication {auth_type PASSauth_pass 11111111}track_script {chk_nginx}virtual_ipaddress {192.168.100.199}
}
EOF## 192.168.100.26 为节点IP192.168.100.199位VIP#将配置拷贝到其他节点并替换相关IP
for node_ip in 192.168.100.26 192.168.100.27 192.168.100.28doecho ${node_ip}scp /etc/keepalived/keepalived.conf $node_ip:/etc/keepalived/keepalived.confdone#替换IP
ssh root192.168.100.27 sed -i s#192.168.100.26#192.168.100.27#g /etc/keepalived/keepalived.conf
ssh root192.168.100.28 sed -i s#192.168.100.26#192.168.100.28#g /etc/keepalived/keepalived.conf#192.168.100.26不替换是因为已经修改好了# 创建健康检查脚本
vim /opt/check_port.sh
CHK_PORT$1if [ -n $CHK_PORT ];thenPORT_PROCESSss -lt|grep $CHK_PORT|wc -lif [ $PORT_PROCESS -eq 0 ];thenecho Port $CHK_PORT Is Not Used,End.exit 1fielseecho Check Port Cant Be Empty!fifor NODE in k8s-01 k8s-02 k8s-03; doecho --- $NODE ---scp -r /opt/check_port.sh $NODE:/etc/keepalived/ssh $NODE systemctl enable --now keepalived
done#没有启动成功请执行下面命令从新启动。启动成功vip肯定就通了
systemctl start keepalived
部署master节点
kubernetes master节点运行组件如下:kube-apiserver、kube-scheduler、kube-controller-manager、kube-nginxkube-apiserver、kube-scheduler、kube-controller-manager均以多实例模式运行
kube-scheduler和kube-controller-manager会自动选举一个leader实例其他实例处于阻塞模式当leader挂了后重新选举产生的leader从而保证服务可用性
kube-apiserver是无状态的需要通过kube-nginx进行代理访问从而保证服务可用性
以下操作都在K8s-01操作下载kubernetes二进制包并分发到所有master节点cd /opt/k8s/work
wget http://down.i4t.com/k8s1.14/kubernetes-server-linux-amd64.tar.gz
tar -xzvf kubernetes-server-linux-amd64.tar.gz
cd kubernetes
tar -xzvf kubernetes-src.tar.gzcd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[]}doecho ${node_ip}scp kubernetes/server/bin/kube-apiserver root${node_ip}:/opt/k8s/bin/scp kubernetes/server/bin/{apiextensions-apiserver,cloud-controller-manager,kube-controller-manager,kube-proxy,kube-scheduler,kubeadm,kubectl,kubelet,mounter} root${node_ip}:/opt/k8s/bin/ssh root${node_ip} chmod x /opt/k8s/bin/*done#同时将kubelet kube-proxy拷贝到所有节点cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[]}doecho ${node_ip}scp kubernetes/server/bin/{kubelet,kube-proxy} root${node_ip}:/opt/k8s/bin/ssh root${node_ip} chmod x /opt/k8s/bin/*done#添加密钥
cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
cat kubernetes-csr.json EOF
{CN: kubernetes,hosts: [127.0.0.1,192.168.100.26,192.168.100.27,192.168.100.28,192.168.100.199,10.254.0.1,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster,kubernetes.default.svc.cluster.local.],key: {algo: rsa,size: 2048},names: [{C: CN,ST: BeiJing,L: BeiJing,O: k8s,OU: 4Paradigm}]
}
EOF#需要将集群的所有IP及VIP添加进去
#如果要添加注意最后的逗号不要忘记添加否则下一步报错
hosts 字段指定授权使用该证书的IP和域名列表这里列出了master节点IP、kubernetes服务的IP和域名kubernetes serviceIP是apiserver自动创建的一般是--service-cluster-ip-range参数指定的网段的第一个IP
$ kubectl get svc kubernetes
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.254.0.1 443/TCP 31d#目前我们是看不到cfssl gencert -ca/opt/k8s/work/ca.pem \-ca-key/opt/k8s/work/ca-key.pem \-config/opt/k8s/work/ca-config.json \-profilekubernetes kubernetes-csr.json | cfssljson -bare kubernetesls kubernetes*pemcd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[]}doecho ${node_ip}ssh root${node_ip} mkdir -p /etc/kubernetes/certscp kubernetes*.pem root${node_ip}:/etc/kubernetes/cert/donecd /opt/k8s/work
source /opt/k8s/bin/environment.sh
cat encryption-config.yaml EOF
kind: EncryptionConfig
apiVersion: v1
resources:- resources:- secretsproviders:- aescbc:keys:- name: key1secret: ${ENCRYPTION_KEY}- identity: {}
EOFcd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[]}doecho ${node_ip}scp encryption-config.yaml root${node_ip}:/etc/kubernetes/done#创建审核文件
cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
cat audit-policy.yaml EOF
apiVersion: audit.k8s.io/v1beta1
kind: Policy
rules:# The following requests were manually identified as high-volume and low-risk, so drop them.- level: Noneresources:- group: resources:- endpoints- services- services/statususers:- system:kube-proxyverbs:- watch- level: Noneresources:- group: resources:- nodes- nodes/statususerGroups:- system:nodesverbs:- get- level: Nonenamespaces:- kube-systemresources:- group: resources:- endpointsusers:- system:kube-controller-manager- system:kube-scheduler- system:serviceaccount:kube-system:endpoint-controllerverbs:- get- update- level: Noneresources:- group: resources:- namespaces- namespaces/status- namespaces/finalizeusers:- system:apiserververbs:- get# Dont log HPA fetching metrics.- level: Noneresources:- group: metrics.k8s.iousers:- system:kube-controller-managerverbs:- get- list# Dont log these read-only URLs.- level: NonenonResourceURLs:- /healthz*- /version- /swagger*# Dont log events requests.- level: Noneresources:- group: resources:- events# node and pod status calls from nodes are high-volume and can be large, dont log responses for expected updates from nodes- level: RequestomitStages:- RequestReceivedresources:- group: resources:- nodes/status- pods/statususers:- kubelet- system:node-problem-detector- system:serviceaccount:kube-system:node-problem-detectorverbs:- update- patch- level: RequestomitStages:- RequestReceivedresources:- group: resources:- nodes/status- pods/statususerGroups:- system:nodesverbs:- update- patch# deletecollection calls can be large, dont log responses for expected namespace deletions- level: RequestomitStages:- RequestReceivedusers:- system:serviceaccount:kube-system:namespace-controllerverbs:- deletecollection# Secrets, ConfigMaps, and TokenReviews can contain sensitive binary data,# so only log at the Metadata level.- level: MetadataomitStages:- RequestReceivedresources:- group: resources:- secrets- configmaps- group: authentication.k8s.ioresources:- tokenreviews# Get repsonses can be large; skip them.- level: RequestomitStages:- RequestReceivedresources:- group: - group: admissionregistration.k8s.io- group: apiextensions.k8s.io- group: apiregistration.k8s.io- group: apps- group: authentication.k8s.io- group: authorization.k8s.io- group: autoscaling- group: batch- group: certificates.k8s.io- group: extensions- group: metrics.k8s.io- group: networking.k8s.io- group: policy- group: rbac.authorization.k8s.io- group: scheduling.k8s.io- group: settings.k8s.io- group: storage.k8s.ioverbs:- get- list- watch# Default level for known APIs- level: RequestResponseomitStages:- RequestReceivedresources:- group: - group: admissionregistration.k8s.io- group: apiextensions.k8s.io- group: apiregistration.k8s.io- group: apps- group: authentication.k8s.io- group: authorization.k8s.io- group: autoscaling- group: batch- group: certificates.k8s.io- group: extensions- group: metrics.k8s.io- group: networking.k8s.io- group: policy- group: rbac.authorization.k8s.io- group: scheduling.k8s.io- group: settings.k8s.io- group: storage.k8s.io# Default level for all other requests.- level: MetadataomitStages:- RequestReceived
EOFcd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[]}doecho ${node_ip}scp audit-policy.yaml root${node_ip}:/etc/kubernetes/audit-policy.yamldone
cat proxy-client-csr.json EOF
{CN: aggregator,hosts: [],key: {algo: rsa,size: 2048},names: [{C: CN,ST: BeiJing,L: BeiJing,O: k8s,OU: 4Paradigm}]
}
EOFCN名称需要位于kube-apiserver的--requestherader-allowed-names参数中否则后续访问metrics时会提示权限不足
生成证书和私钥source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[]}doecho ${node_ip}scp proxy-client*.pem root${node_ip}:/etc/kubernetes/cert/donecd /opt/k8s/work
source /opt/k8s/bin/environment.sh
cat kube-apiserver.service.template EOF
[Unit]
DescriptionKubernetes API Server
Documentationhttps://github.com/GoogleCloudPlatform/kubernetes
Afternetwork.target
[Service]
WorkingDirectory${K8S_DIR}/kube-apiserver
ExecStart/opt/k8s/bin/kube-apiserver \\--advertise-address##NODE_IP## \\--default-not-ready-toleration-seconds360 \\--default-unreachable-toleration-seconds360 \\--feature-gatesDynamicAuditingtrue \\--max-mutating-requests-inflight2000 \\--max-requests-inflight4000 \\--default-watch-cache-size200 \\--delete-collection-workers2 \\--encryption-provider-config/etc/kubernetes/encryption-config.yaml \\--etcd-cafile/etc/kubernetes/cert/ca.pem \\--etcd-certfile/etc/kubernetes/cert/kubernetes.pem \\--etcd-keyfile/etc/kubernetes/cert/kubernetes-key.pem \\--etcd-servers${ETCD_ENDPOINTS} \\--bind-address##NODE_IP## \\--secure-port6443 \\--tls-cert-file/etc/kubernetes/cert/kubernetes.pem \\--tls-private-key-file/etc/kubernetes/cert/kubernetes-key.pem \\--insecure-port0 \\--audit-dynamic-configuration \\--audit-log-maxage15 \\--audit-log-maxbackup3 \\--audit-log-maxsize100 \\--audit-log-truncate-enabled \\--audit-log-path${K8S_DIR}/kube-apiserver/audit.log \\--audit-policy-file/etc/kubernetes/audit-policy.yaml \\--profiling \\--anonymous-authfalse \\--client-ca-file/etc/kubernetes/cert/ca.pem \\--enable-bootstrap-token-auth \\--requestheader-allowed-namesaggregator \\--requestheader-client-ca-file/etc/kubernetes/cert/ca.pem \\--requestheader-extra-headers-prefixX-Remote-Extra- \\--requestheader-group-headersX-Remote-Group \\--requestheader-username-headersX-Remote-User \\--service-account-key-file/etc/kubernetes/cert/ca.pem \\--authorization-modeNode,RBAC \\--runtime-configapi/alltrue \\--enable-admission-pluginsNodeRestriction \\--allow-privilegedtrue \\--apiserver-count3 \\--event-ttl168h \\--kubelet-certificate-authority/etc/kubernetes/cert/ca.pem \\--kubelet-client-certificate/etc/kubernetes/cert/kubernetes.pem \\--kubelet-client-key/etc/kubernetes/cert/kubernetes-key.pem \\--kubelet-httpstrue \\--kubelet-timeout10s \\--proxy-client-cert-file/etc/kubernetes/cert/proxy-client.pem \\--proxy-client-key-file/etc/kubernetes/cert/proxy-client-key.pem \\--service-cluster-ip-range${SERVICE_CIDR} \\--service-node-port-range${NODE_PORT_RANGE} \\--logtostderrtrue \\--v2
Restarton-failure
RestartSec10
Typenotify
LimitNOFILE65536
[Install]
WantedBymulti-user.target
EOF--advertise-addressapiserver 对外通告的 IPkubernetes 服务后端节点 IP
--default-*-toleration-seconds设置节点异常相关的阈值
--max-*-requests-inflight请求相关的最大阈值
--etcd-*访问 etcd 的证书和 etcd 服务器地址
--experimental-encryption-provider-config指定用于加密 etcd 中 secret 的配置
--bind-address https 监听的 IP不能为 127.0.0.1否则外界不能访问它的安全端口 6443
--secret-porthttps 监听端口
--insecure-port0关闭监听 http 非安全端口(8080)
--tls-*-file指定 apiserver 使用的证书、私钥和 CA 文件
--audit-*配置审计策略和审计日志文件相关的参数
--client-ca-file验证 client (kue-controller-manager、kube-scheduler、kubelet、kube-proxy 等)请求所带的证书
--enable-bootstrap-token-auth启用 kubelet bootstrap 的 token 认证
--requestheader-*kube-apiserver 的 aggregator layer 相关的配置参数proxy-client HPA 需要使用
--requestheader-client-ca-file用于签名 --proxy-client-cert-file 和 --proxy-client-key-file 指定的证书在启用了 metric aggregator 时使用
--requestheader-allowed-names不能为空值为逗号分割的 --proxy-client-cert-file 证书的 CN 名称这里设置为 aggregator
--service-account-key-file签名 ServiceAccount Token 的公钥文件kube-controller-manager 的 --service-account-private-key-file 定私钥文件两者配对使用
--runtime-configapi/alltrue 启用所有版本的 APIs如 autoscaling/v2alpha1
--authorization-modeNode,RBAC、--anonymous-authfalse 开启 Node 和 RBAC 授权模式拒绝未授权的请求
--enable-admission-plugins启用一些默认关闭的 plugins
--allow-privileged运行执行 privileged 权限的容器
--apiserver-count3指定 apiserver 实例的数量
--event-ttl指定 events 的保存时间
--kubelet-如果指定则使用 https 访问 kubelet APIs需要为证书对应的用户(上面 kubernetes.pem 证书的用户为 kubernetes) 用户定义 RBAC 规则否则访问 kubelet API 时提示未授权
--proxy-client-*apiserver 访问 metrics-server 使用的证书
--service-cluster-ip-range 指定 Service Cluster IP 地址段
--service-node-port-range 指定 NodePort 的端口范围
如果 kube-apiserver 机器没有运行 kube-proxy则还需要添加 --enable-aggregator-routingtrue 参数关于 --requestheader-XXX 相关参数参考
https://github.com/kubernetes-incubator/apiserver-builder/blob/master/docs/concepts/auth.md
https://docs.bitnami.com/kubernetes/how-to/configure-autoscaling-custom-metrics/
注意: requestheader-client-ca-file指定的CA证书必须具有client auth and server auth如果--requestheader-allowed-names为空或者--proxy-client-cert-file证书的CN名称不在allowed-names中则后续查看node或者Pods的metrics失败为各个节点创建和分发kube-apiserver启动文件替换模板文件的变量为各个节点生成启动文件
cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for (( i0; i 3; i )) #这里是三个节点所以为3,请根据实际情况修改,后边不在提示,同理dosed -e s/##NODE_NAME##/${MASTER_NAMES[i]}/ -e s/##NODE_IP##/${MASTER_IPS[i]}/ kube-apiserver.service.template kube-apiserver-${MASTER_IPS[i]}.service done
ls kube-apiserver*.servicecd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[]}doecho ${node_ip}scp kube-apiserver-${node_ip}.service root${node_ip}:/etc/systemd/system/kube-apiserver.servicedonesource /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[]}doecho ${node_ip}ssh root${node_ip} mkdir -p ${K8S_DIR}/kube-apiserverssh root${node_ip} systemctl daemon-reload systemctl enable kube-apiserver systemctl restart kube-apiserverdone
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[]}doecho ${node_ip}ssh root${node_ip} systemctl status kube-apiserver |grep Active:done
#如果开启不了查看日志 journalctl -u kube-apiserver#打印kube-apiserver写入etcd数据
source /opt/k8s/bin/environment.sh
ETCDCTL_API3 etcdctl \--endpoints${ETCD_ENDPOINTS} \--cacert/opt/k8s/work/ca.pem \--cert/opt/k8s/work/etcd.pem \--key/opt/k8s/work/etcd-key.pem \get /registry/ --prefix --keys-only#检查集群信息
netstat -lntup|grep kube[rootk8s-01 work]# kubectl cluster-info
Kubernetes master is running at https://192.168.100.199:8443To further debug and diagnose cluster problems, use kubectl cluster-info dump.
[rootk8s-01 work]# kubectl get all --all-namespaces
NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
default service/kubernetes ClusterIP 10.254.0.1 none 443/TCP 2m26s
[rootk8s-01 work]# kubectl get componentstatuses
NAME STATUS MESSAGE ERROR
controller-manager Unhealthy Get http://127.0.0.1:10252/healthz: dial tcp 127.0.0.1:10252: connect: connection refused
scheduler Unhealthy Get http://127.0.0.1:10251/healthz: dial tcp 127.0.0.1:10251: connect: connection refused
etcd-2 Healthy {health:true}
etcd-1 Healthy {health:true}
etcd-0 Healthy {health:true} 如果提示有报错请检查~/.kube/config以及配置证书是否有问题授权kube-apiserver访问kubelet API的权限在执行kubectl命令时apiserver会将请求转发到kubelet的https端口。这里定义的RBAC规则授权apiserver使用的证书(kubernetes.pem)用户名(CN:kubernetes)访问kubelet API的权限kubectl create clusterrolebinding kube-apiserver:kubelet-apis --clusterrolesystem:kubelet-api-admin --user kubernetes
部署高可用kube-controller-manager集群
该集群包含三个节点启动后通过竞争选举机制产生一个leader节点其他节点为阻塞状态。当leader节点不可用时阻塞节点将会在此选举产生新的leader从而保证服务的高可用。为保证通信安全这里采用x509证书和私钥kube-controller-manager在与apiserver的安全端口http 10252通信使用创建kube-controller-manager证书和私钥创建证书签名请求
cd /opt/k8s/work
cat kube-controller-manager-csr.json EOF
{CN: system:kube-controller-manager,key: {algo: rsa,size: 2048},hosts: [127.0.0.1,192.168.100.26,192.168.100.27,192.168.100.28],names: [{C: CN,ST: BeiJing,L: BeiJing,O: system:kube-controller-manager,OU: 4Paradigm}]
}
EOF#这里的IP地址为master iphost列表包含所有的kube-controller-manager节点IP(VIP不需要输入)
CN和O均为system:kube-controller-managerkubernetes内置的ClusterRoleBindings system:kube-controller-manager赋予kube-controller-manager工作所需权限
生成证书和私钥cd /opt/k8s/work
cfssl gencert -ca/opt/k8s/work/ca.pem \-ca-key/opt/k8s/work/ca-key.pem \-config/opt/k8s/work/ca-config.json \-profilekubernetes kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager
ls kube-controller-manager*pem将生成的证书和私钥分发到所有master节点
cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[]}doecho ${node_ip}scp kube-controller-manager*.pem root${node_ip}:/etc/kubernetes/cert/done# 创建和分发kubeconfig文件
#kube-controller-manager使用kubeconfig文件访问apiserver
#该文件提供了apiserver地址、嵌入的CA证书和kube-controller-manager证书cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
kubectl config set-cluster kubernetes \--certificate-authority/opt/k8s/work/ca.pem \--embed-certstrue \--server${KUBE_APISERVER} \--kubeconfigkube-controller-manager.kubeconfig
kubectl config set-credentials system:kube-controller-manager \--client-certificatekube-controller-manager.pem \--client-keykube-controller-manager-key.pem \--embed-certstrue \--kubeconfigkube-controller-manager.kubeconfig
kubectl config set-context system:kube-controller-manager \--clusterkubernetes \--usersystem:kube-controller-manager \--kubeconfigkube-controller-manager.kubeconfig
kubectl config use-context system:kube-controller-manager --kubeconfigkube-controller-manager.kubeconfigcd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[]}doecho ${node_ip}scp kube-controller-manager.kubeconfig root${node_ip}:/etc/kubernetes/done创建kube-controller-manager启动文件cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
cat kube-controller-manager.service.template EOF
[Unit]
DescriptionKubernetes Controller Manager
Documentationhttps://github.com/GoogleCloudPlatform/kubernetes
[Service]
WorkingDirectory${K8S_DIR}/kube-controller-manager
ExecStart/opt/k8s/bin/kube-controller-manager \\--profiling \\--cluster-namekubernetes \\--controllers*,bootstrapsigner,tokencleaner \\--kube-api-qps1000 \\--kube-api-burst2000 \\--leader-elect \\--use-service-account-credentials\\--concurrent-service-syncs2 \\--bind-address0.0.0.0 \\#--secure-port10252 \\--tls-cert-file/etc/kubernetes/cert/kube-controller-manager.pem \\--tls-private-key-file/etc/kubernetes/cert/kube-controller-manager-key.pem \\#--port0 \\--authentication-kubeconfig/etc/kubernetes/kube-controller-manager.kubeconfig \\--client-ca-file/etc/kubernetes/cert/ca.pem \\--requestheader-allowed-names \\--requestheader-client-ca-file/etc/kubernetes/cert/ca.pem \\--requestheader-extra-headers-prefixX-Remote-Extra- \\--requestheader-group-headersX-Remote-Group \\--requestheader-username-headersX-Remote-User \\--authorization-kubeconfig/etc/kubernetes/kube-controller-manager.kubeconfig \\--cluster-signing-cert-file/etc/kubernetes/cert/ca.pem \\--cluster-signing-key-file/etc/kubernetes/cert/ca-key.pem \\--experimental-cluster-signing-duration876000h \\--horizontal-pod-autoscaler-sync-period10s \\--concurrent-deployment-syncs10 \\--concurrent-gc-syncs30 \\--node-cidr-mask-size24 \\--service-cluster-ip-range${SERVICE_CIDR} \\--pod-eviction-timeout6m \\--terminated-pod-gc-threshold10000 \\--root-ca-file/etc/kubernetes/cert/ca.pem \\--service-account-private-key-file/etc/kubernetes/cert/ca-key.pem \\--kubeconfig/etc/kubernetes/kube-controller-manager.kubeconfig \\--logtostderrtrue \\--v2
Restarton-failure
RestartSec5
[Install]
WantedBymulti-user.target
EOF参数解释--port0关闭监听非安全端口http同时 --address 参数无效--bind-address 参数有效
--secure-port10252、--bind-address0.0.0.0: 在所有网络接口监听 10252 端口的 https /metrics 请求
--kubeconfig指定 kubeconfig 文件路径kube-controller-manager 使用它连接和验证 kube-apiserver
--authentication-kubeconfig 和 --authorization-kubeconfigkube-controller-manager 使用它连接 apiserver对 client 的请求进行认证和授权。kube-controller-manager 不再使用 --tls-ca-file 对请求 https metrics 的 Client 证书进行校验。如果没有配置这两个 kubeconfig 参数则 client 连接 kube-controller-manager https 端口的请求会被拒绝(提示权限不足)。
--cluster-signing-*-file签名 TLS Bootstrap 创建的证书
--experimental-cluster-signing-duration指定 TLS Bootstrap 证书的有效期
--root-ca-file放置到容器 ServiceAccount 中的 CA 证书用来对 kube-apiserver 的证书进行校验
--service-account-private-key-file签名 ServiceAccount 中 Token 的私钥文件必须和 kube-apiserver 的 --service-account-key-file 指定的公钥文件配对使用
--service-cluster-ip-range 指定 Service Cluster IP 网段必须和 kube-apiserver 中的同名参数一致
--leader-electtrue集群运行模式启用选举功能被选为 leader 的节点负责处理工作其它节点为阻塞状态
--controllers*,bootstrapsigner,tokencleaner启用的控制器列表tokencleaner 用于自动清理过期的 Bootstrap token
--horizontal-pod-autoscaler-*custom metrics 相关参数支持 autoscaling/v2alpha1
--tls-cert-file、--tls-private-key-file使用 https 输出 metrics 时使用的 Server 证书和秘钥
--use-service-account-credentialstrue: kube-controller-manager 中各 controller 使用 serviceaccount 访问 kube-apiservercd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for (( i0; i 3; i ))dosed -e s/##NODE_NAME##/${MASTER_NAMES[i]}/ -e s/##NODE_IP##/${MASTER_IPS[i]}/ kube-controller-manager.service.template kube-controller-manager-${MASTER_IPS[i]}.service done
ls kube-controller-manager*.servicecd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[]}doecho ${node_ip}scp kube-controller-manager-${node_ip}.service root${node_ip}:/etc/systemd/system/kube-controller-manager.servicedonesource /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[]}doecho ${node_ip}ssh root${node_ip} mkdir -p ${K8S_DIR}/kube-controller-managerssh root${node_ip} systemctl daemon-reload systemctl enable kube-controller-manager systemctl restart kube-controller-managerdone
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[]}doecho ${node_ip}ssh root${node_ip} systemctl status kube-controller-manager|grep Activedone#检查服务状态
[rootk8s-01 work]# netstat -lnpt | grep kube-cont
tcp6 0 0 :::10257 :::* LISTEN 9664/kube-controlle
tcp6 0 0 :::10252 :::* LISTEN 9664/kube-controlle
kube-controller-manager 创建权限
ClusteRole system:kube-controller-manager的权限太小只能创建secret、serviceaccount等资源,将controller的权限分散到ClusterRole system:controller:xxx中[rootk8s-01 work]# kubectl describe clusterrole system:kube-controller-manager
Name: system:kube-controller-manager
Labels: kubernetes.io/bootstrappingrbac-defaults
Annotations: rbac.authorization.kubernetes.io/autoupdate: true
PolicyRule:Resources Non-Resource URLs Resource Names Verbs--------- ----------------- -------------- -----secrets [] [] [create delete get update]endpoints [] [] [create get update]serviceaccounts [] [] [create get update]events [] [] [create patch update]tokenreviews.authentication.k8s.io [] [] [create]subjectaccessreviews.authorization.k8s.io [] [] [create]configmaps [] [] [get]namespaces [] [] [get]*.* [] [] [list watch]
需要在 kube-controller-manager 的启动参数中添加 --use-service-account-credentialstrue 参数这样 main controller 会为各 controller 创建对应的 ServiceAccount XXX-controller。内置的 ClusterRoleBinding system:controller:XXX 将赋予各 XXX-controller ServiceAccount 对应的 ClusterRole system:controller:XXX 权限。
[rootk8s-01 work]# kubectl get clusterrole|grep controller
system:controller:attachdetach-controller 12m
system:controller:certificate-controller 12m
system:controller:clusterrole-aggregation-controller 12m
system:controller:cronjob-controller 12m
system:controller:daemon-set-controller 12m
system:controller:deployment-controller 12m
system:controller:disruption-controller 12m
system:controller:endpoint-controller 12m
system:controller:expand-controller 12m
system:controller:generic-garbage-collector 12m
system:controller:horizontal-pod-autoscaler 12m
system:controller:job-controller 12m
system:controller:namespace-controller 12m
system:controller:node-controller 12m
system:controller:persistent-volume-binder 12m
system:controller:pod-garbage-collector 12m
system:controller:pv-protection-controller 12m
system:controller:pvc-protection-controller 12m
system:controller:replicaset-controller 12m
system:controller:replication-controller 12m
system:controller:resourcequota-controller 12m
system:controller:route-controller 12m
system:controller:service-account-controller 12m
system:controller:service-controller 12m
system:controller:statefulset-controller 12m
system:controller:ttl-controller 12m
system:kube-controller-manager 12m#以 deployment controller 为例
[rootk8s-01 work]# kubectl describe clusterrole system:controller:deployment-controller
Name: system:controller:deployment-controller
Labels: kubernetes.io/bootstrappingrbac-defaults
Annotations: rbac.authorization.kubernetes.io/autoupdate: true
PolicyRule:Resources Non-Resource URLs Resource Names Verbs--------- ----------------- -------------- -----replicasets.apps [] [] [create delete get list patch update watch]replicasets.extensions [] [] [create delete get list patch update watch]events [] [] [create patch update]pods [] [] [get list update watch]deployments.apps [] [] [get list update watch]deployments.extensions [] [] [get list update watch]deployments.apps/finalizers [] [] [update]deployments.apps/status [] [] [update]deployments.extensions/finalizers [] [] [update]deployments.extensions/status [] [] [update]#查看当前的 leader
[rootk8s-01 work]# kubectl get endpoints kube-controller-manager --namespacekube-system -o yaml
apiVersion: v1
kind: Endpoints
metadata:annotations:control-plane.alpha.kubernetes.io/leader: {holderIdentity:k8s-01_ede3a742-4001-11ee-b717-4201c0a8641a,leaseDurationSeconds:15,acquireTime:2023-08-21T09:05:52Z,renewTime:2023-08-21T09:09:23Z,leaderTransitions:0}creationTimestamp: 2023-08-21T09:05:52Zname: kube-controller-managernamespace: kube-systemresourceVersion: 604selfLink: /api/v1/namespaces/kube-system/endpoints/kube-controller-manageruid: ede66bf9-4001-11ee-b6ee-4201c0a8641a部署高可用kube-scheduler
cd /opt/k8s/work
cat kube-scheduler-csr.json EOF
{CN: system:kube-scheduler,hosts: [127.0.0.1,192.168.100.26,192.168.100.27,192.168.100.28],key: {algo: rsa,size: 2048},names: [{C: CN,ST: BeiJing,L: BeiJing,O: system:kube-scheduler,OU: 4Paradigm}]
}
EOF
hosts 列表包含所有 kube-scheduler 节点 IP
CN 和 O 均为 system:kube-schedulerkubernetes 内置的 ClusterRoleBindings system:kube-scheduler 将赋予 kube-scheduler 工作所需的权限
生成证书和私钥cd /opt/k8s/work
cfssl gencert -ca/opt/k8s/work/ca.pem \-ca-key/opt/k8s/work/ca-key.pem \-config/opt/k8s/work/ca-config.json \-profilekubernetes kube-scheduler-csr.json | cfssljson -bare kube-scheduler
ls kube-scheduler*pemcd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[]}doecho ${node_ip}scp kube-scheduler*.pem root${node_ip}:/etc/kubernetes/cert/done
cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
kubectl config set-cluster kubernetes \--certificate-authority/opt/k8s/work/ca.pem \--embed-certstrue \--server${KUBE_APISERVER} \--kubeconfigkube-scheduler.kubeconfig
kubectl config set-credentials system:kube-scheduler \--client-certificatekube-scheduler.pem \--client-keykube-scheduler-key.pem \--embed-certstrue \--kubeconfigkube-scheduler.kubeconfig
kubectl config set-context system:kube-scheduler \--clusterkubernetes \--usersystem:kube-scheduler \--kubeconfigkube-scheduler.kubeconfig
kubectl config use-context system:kube-scheduler --kubeconfigkube-scheduler.kubeconfig
cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[]}doecho ${node_ip}scp kube-scheduler.kubeconfig root${node_ip}:/etc/kubernetes/done#创建 kube-scheduler 配置文件cd /opt/k8s/work
cat kube-scheduler.yaml.template EOF
apiVersion: kubescheduler.config.k8s.io/v1alpha1
kind: KubeSchedulerConfiguration
bindTimeoutSeconds: 600
clientConnection:burst: 200kubeconfig: /etc/kubernetes/kube-scheduler.kubeconfigqps: 100
enableContentionProfiling: false
enableProfiling: true
hardPodAffinitySymmetricWeight: 1
healthzBindAddress: 127.0.0.1:10251
leaderElection:leaderElect: true
metricsBindAddress: ##NODE_IP##:10251
EOF--kubeconfig指定 kubeconfig 文件路径kube-scheduler 使用它连接和验证 kube-apiserver
--leader-electtrue集群运行模式启用选举功能被选为 leader 的节点负责处理工作其它节点为阻塞状态
替换模板文件中的变量cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for (( i0; i 3; i ))dosed -e s/##NODE_NAME##/${NODE_NAMES[i]}/ -e s/##NODE_IP##/${NODE_IPS[i]}/ kube-scheduler.yaml.template kube-scheduler-${NODE_IPS[i]}.yamldone
ls kube-scheduler*.yamlcd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[]}doecho ${node_ip}scp kube-scheduler-${node_ip}.yaml root${node_ip}:/etc/kubernetes/kube-scheduler.yamldonecd /opt/k8s/work
cat kube-scheduler.service.template EOF
[Unit]
DescriptionKubernetes Scheduler
Documentationhttps://github.com/GoogleCloudPlatform/kubernetes
[Service]
WorkingDirectory${K8S_DIR}/kube-scheduler
ExecStart/opt/k8s/bin/kube-scheduler \\--config/etc/kubernetes/kube-scheduler.yaml \\--bind-address##NODE_IP## \\--secure-port10259 \\--port0 \\--tls-cert-file/etc/kubernetes/cert/kube-scheduler.pem \\--tls-private-key-file/etc/kubernetes/cert/kube-scheduler-key.pem \\--authentication-kubeconfig/etc/kubernetes/kube-scheduler.kubeconfig \\--client-ca-file/etc/kubernetes/cert/ca.pem \\--requestheader-allowed-names \\--requestheader-client-ca-file/etc/kubernetes/cert/ca.pem \\--requestheader-extra-headers-prefixX-Remote-Extra- \\--requestheader-group-headersX-Remote-Group \\--requestheader-username-headersX-Remote-User \\--authorization-kubeconfig/etc/kubernetes/kube-scheduler.kubeconfig \\--logtostderrtrue \\--v2
Restartalways
RestartSec5
StartLimitInterval0
[Install]
WantedBymulti-user.target
EOF
cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for (( i0; i 3; i ))dosed -e s/##NODE_NAME##/${NODE_NAMES[i]}/ -e s/##NODE_IP##/${NODE_IPS[i]}/ kube-scheduler.service.template kube-scheduler-${NODE_IPS[i]}.service done
ls kube-scheduler*.servicecd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[]}doecho ${node_ip}scp kube-scheduler-${node_ip}.service root${node_ip}:/etc/systemd/system/kube-scheduler.servicedonesource /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[]}doecho ${node_ip}ssh root${node_ip} mkdir -p ${K8S_DIR}/kube-schedulerssh root${node_ip} systemctl daemon-reload systemctl enable kube-scheduler systemctl restart kube-scheduler
done
#检查服务运行状态
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[]}doecho ${node_ip}ssh root${node_ip} systemctl status kube-scheduler|grep Activedone
查看输出的 metrics注意以下命令在 kube-scheduler 节点上执行。kube-scheduler 监听 10251 和 10251 端口10251接收 http 请求非安全端口不需要认证授权10259接收 https 请求安全端口需要认证授权两个接口都对外提供 /metrics 和 /healthz 的访问。[rootk8s-01 work]# curl -s http://192.168.100.26:10251/metrics|head
# HELP apiserver_audit_event_total Counter of audit events generated and sent to the audit backend.
# TYPE apiserver_audit_event_total counter
apiserver_audit_event_total 0
# HELP apiserver_audit_requests_rejected_total Counter of apiserver requests rejected due to an error in audit logging backend.
# TYPE apiserver_audit_requests_rejected_total counter
apiserver_audit_requests_rejected_total 0
# HELP apiserver_client_certificate_expiration_seconds Distribution of the remaining lifetime on the certificate used to authenticate a request.
# TYPE apiserver_client_certificate_expiration_seconds histogram
apiserver_client_certificate_expiration_seconds_bucket{le0} 0
apiserver_client_certificate_expiration_seconds_bucket{le1800} 0#查看当前leader
kubectl get endpoints kube-scheduler --namespacekube-system -o yaml
work节点安装
kubernetes work节点运行如下组件: docker、kubelet、kube-proxy、flanneld、kube-nginx前面已经安装flanneld这就不在安装了
source /opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[]}doecho ${node_ip}ssh root${node_ip} yum install -y epel-releasessh root${node_ip} yum install -y conntrack ipvsadm ntp ntpdate ipset jq iptables curl sysstat libseccomp modprobe ip_vs done部署Docker组件
yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum makecache fast
yum -y install docker-cemkdir -p /etc/docker/
cat /etc/docker/daemon.json EOF
{exec-opts: [native.cgroupdriversystemd],registry-mirrors: [https://hjvrgh7a.mirror.aliyuncs.com],log-driver: json-file,log-opts: {max-size: 100m},storage-driver: overlay2
}
EOF另一个版本mkdir -p /etc/docker/
cat /etc/docker/daemon.json EOF
{exec-opts: [native.cgroupdriversystemd],registry-mirrors: [https://hub-mirror.c.163.com,https://mirror.baidubce.com],log-driver: json-file,log-opts: {max-size: 100m},storage-driver: overlay2
}
EOF#这里配置当时镜像加速器,可以不进行配置但是建议配置
要添加我们harbor仓库需要在添加下面一行insecure-registries: [harbor.xxx.com],
默认docker hub需要https协议使用上面配置不需要配置https# 以下操作 每一台都要执行
/usr/lib/systemd/system/docker.service
cat /usr/lib/systemd/system/docker.service
[Unit]
DescriptionDocker Application Container Engine
Documentationhttps://docs.docker.com
BindsTocontainerd.service
Afternetwork-online.target firewalld.service containerd.service
Wantsnetwork-online.target
Requiresdocker.socket[Service]
Typenotify
# the default is not to use systemd for cgroups because the delegate issues still
# exists and systemd currently does not support the cgroup feature set required
# for containers run by docker
ExecStart/usr/bin/dockerd $DOCKER_NETWORK_OPTIONS -H fd:// --containerd/run/containerd/containerd.sock
EnvironmentFile-/run/flannel/docker
ExecReload/bin/kill -s HUP $MAINPID
TimeoutSec0
RestartSec2
Restartalways# Note that StartLimit* options were moved from Service to Unit in systemd 229.
# Both the old, and new location are accepted by systemd 229 and up, so using the old location
# to make them work for either version of systemd.
StartLimitBurst3# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230.
# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make
# this option work for either version of systemd.
StartLimitInterval60s# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILEinfinity
LimitNPROCinfinity
LimitCOREinfinity# Comment TasksMax if your systemd version does not support it.
# Only systemd 226 and above support this option.
TasksMaxinfinity# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegateyes# kill only the docker process, not all processes in the cgroup
KillModeprocess[Install]
WantedBymulti-user.targetsource /opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[]}doecho ${node_ip}ssh root${node_ip} systemctl daemon-reload systemctl enable docker systemctl restart dockerdone
source /opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[]}doecho ${node_ip}ssh root${node_ip} systemctl status docker|grep Activedone# 检查docker网桥
source /opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[]}doecho ${node_ip}ssh root${node_ip} /usr/sbin/ip addr show flannel.1 /usr/sbin/ip addr show docker0done部署kubelet组件
kubelet运行在每个worker节点上接收kube-apiserver发送的请求管理Pod容器执行交互命令kubelet启动时自动向kube-apiserver注册节点信息内置的cAdivsor统计和监控节点的资源使用资源情况。为确保安全部署时关闭了kubelet的非安全http端口对请求进行认证和授权拒绝未授权的访问创建kubelet bootstrap kubeconfig文件
cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_name in ${NODE_NAMES[]}doecho ${node_name}# 创建 tokenexport BOOTSTRAP_TOKEN$(kubeadm token create \--description kubelet-bootstrap-token \--groups system:bootstrappers:${node_name} \--kubeconfig ~/.kube/config)# 设置集群参数kubectl config set-cluster kubernetes \--certificate-authority/etc/kubernetes/cert/ca.pem \--embed-certstrue \--server${KUBE_APISERVER} \--kubeconfigkubelet-bootstrap-${node_name}.kubeconfig# 设置客户端认证参数kubectl config set-credentials kubelet-bootstrap \--token${BOOTSTRAP_TOKEN} \--kubeconfigkubelet-bootstrap-${node_name}.kubeconfig# 设置上下文参数kubectl config set-context default \--clusterkubernetes \--userkubelet-bootstrap \--kubeconfigkubelet-bootstrap-${node_name}.kubeconfig# 设置默认上下文kubectl config use-context default --kubeconfigkubelet-bootstrap-${node_name}.kubeconfigdone
向kubeconfig写入的是tokenbootstrap结束后kube-controller-manager为kubelet创建client和server证书
查看kubeadm为各个节点创建的token[rootk8s-01 work]# kubeadm token list --kubeconfig ~/.kube/config
TOKEN TTL EXPIRES USAGES DESCRIPTION EXTRA GROUPS
1r8xjv.oypmp64rmji9r4d1 23h 2023-08-22T17:28:3508:00 authentication,signing kubelet-bootstrap-token system:bootstrappers:k8s-01
7a6n62.yzd0y8rb2vnufqlc 23h 2023-08-22T17:28:3608:00 authentication,signing kubelet-bootstrap-token system:bootstrappers:k8s-04
fyw4n9.3wwgsgwgjyhxxw11 23h 2023-08-22T17:28:3508:00 authentication,signing kubelet-bootstrap-token system:bootstrappers:k8s-02
i3cd24.fbvpytiu9rliclke 23h 2023-08-22T17:28:3608:00 authentication,signing kubelet-bootstrap-token system:bootstrappers:k8s-03token有效期为1天超期后将不能被用来bootstrap kubelet且会被kube-controller-manager的token cleaner清理
kube-apiserver接收kubelet的bootstrap token后将请求的user设置为system:bootstrap; group设置为system:bootstrappers后续将为这个group设置ClusterRoleBinding
#查看各token关联的Secret
[rootk8s-01 work]# kubectl get secrets -n kube-system|grep bootstrap-token
bootstrap-token-1r8xjv bootstrap.kubernetes.io/token 7 73s
bootstrap-token-7a6n62 bootstrap.kubernetes.io/token 7 72s
bootstrap-token-fyw4n9 bootstrap.kubernetes.io/token 7 73s
bootstrap-token-i3cd24 bootstrap.kubernetes.io/token 7 72s#分发 bootstrap kubeconfig 文件到所有 worker 节点
cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_name in ${NODE_NAMES[]}doecho ${node_name}scp kubelet-bootstrap-${node_name}.kubeconfig root${node_name}:/etc/kubernetes/kubelet-bootstrap.kubeconfigdone#创建和分发kubelet
cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
cat kubelet-config.yaml.template EOF
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: ##NODE_IP##
staticPodPath:
syncFrequency: 1m
fileCheckFrequency: 20s
httpCheckFrequency: 20s
staticPodURL:
port: 10250
readOnlyPort: 0
rotateCertificates: true
serverTLSBootstrap: true
authentication:anonymous:enabled: falsewebhook:enabled: truex509:clientCAFile: /etc/kubernetes/cert/ca.pem
authorization:mode: Webhook
registryPullQPS: 0
registryBurst: 20
eventRecordQPS: 0
eventBurst: 20
enableDebuggingHandlers: true
enableContentionProfiling: true
healthzPort: 10248
healthzBindAddress: ##NODE_IP##
clusterDomain: ${CLUSTER_DNS_DOMAIN}
clusterDNS:- ${CLUSTER_DNS_SVC_IP}
nodeStatusUpdateFrequency: 10s
nodeStatusReportFrequency: 1m
imageMinimumGCAge: 2m
imageGCHighThresholdPercent: 85
imageGCLowThresholdPercent: 80
volumeStatsAggPeriod: 1m
kubeletCgroups:
systemCgroups:
cgroupRoot:
cgroupsPerQOS: true
cgroupDriver: systemd
runtimeRequestTimeout: 10m
hairpinMode: promiscuous-bridge
maxPods: 220
podCIDR: ${CLUSTER_CIDR}
podPidsLimit: -1
resolvConf: /etc/resolv.conf
maxOpenFiles: 1000000
kubeAPIQPS: 1000
kubeAPIBurst: 2000
serializeImagePulls: false
evictionHard:memory.available: 100Mi
nodefs.available: 10%
nodefs.inodesFree: 5%
imagefs.available: 15%
evictionSoft: {}
enableControllerAttachDetach: true
failSwapOn: true
containerLogMaxSize: 20Mi
containerLogMaxFiles: 10
systemReserved: {}
kubeReserved: {}
systemReservedCgroup:
kubeReservedCgroup:
enforceNodeAllocatable: [pods]
EOFaddresskubelet 安全端口https10250监听的地址不能为 127.0.0.1否则 kube-apiserver、heapster 等不能调用 kubelet 的 API
readOnlyPort0关闭只读端口(默认 10255)等效为未指定
authentication.anonymous.enabled设置为 false不允许匿名访问 10250 端口
authentication.x509.clientCAFile指定签名客户端证书的 CA 证书开启 HTTP 证书认证
authentication.webhook.enabledtrue开启 HTTPs bearer token 认证
对于未通过 x509 证书和 webhook 认证的请求(kube-apiserver 或其他客户端)将被拒绝提示 Unauthorized
authroization.modeWebhookkubelet 使用 SubjectAccessReview API 查询 kube-apiserver 某 user、group 是否具有操作资源的权限(RBAC)
featureGates.RotateKubeletClientCertificate、featureGates.RotateKubeletServerCertificate自动 rotate 证书证书的有效期取决于 kube-controller-manager 的 --experimental-cluster-signing-duration 参数
需要 root 账户运行
为各个节点创建和分发kubelet配置文件cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[]}do echo ${node_ip}sed -e s/##NODE_IP##/${node_ip}/ kubelet-config.yaml.template kubelet-config-${node_ip}.yaml.templatescp kubelet-config-${node_ip}.yaml.template root${node_ip}:/etc/kubernetes/kubelet-config.yamldonecd /opt/k8s/work
source /opt/k8s/bin/environment.sh
cat kubelet.service.template EOF
[Unit]
DescriptionKubernetes Kubelet
Documentationhttps://github.com/GoogleCloudPlatform/kubernetes
Afterdocker.service
Requiresdocker.service
[Service]
WorkingDirectory${K8S_DIR}/kubelet
ExecStart/opt/k8s/bin/kubelet \\--allow-privilegedtrue \\--bootstrap-kubeconfig/etc/kubernetes/kubelet-bootstrap.kubeconfig \\--cert-dir/etc/kubernetes/cert \\--cni-conf-dir/etc/cni/net.d \\--container-runtimedocker \\--container-runtime-endpointunix:///var/run/dockershim.sock \\--root-dir${K8S_DIR}/kubelet \\--kubeconfig/etc/kubernetes/kubelet.kubeconfig \\--config/etc/kubernetes/kubelet-config.yaml \\--hostname-override##NODE_NAME## \\--pod-infra-container-imageregistry.aliyuncs.com/google_containers/pause:3.2 \\--image-pull-progress-deadline15m \\--volume-plugin-dir${K8S_DIR}/kubelet/kubelet-plugins/volume/exec/ \\--logtostderrtrue \\--v2
Restartalways
RestartSec5
StartLimitInterval0
[Install]
WantedBymulti-user.target
EOF如果设置了 --hostname-override 选项则 kube-proxy 也需要设置该选项否则会出现找不到 Node 的情况
--bootstrap-kubeconfig指向 bootstrap kubeconfig 文件kubelet 使用该文件中的用户名和 token 向 kube-apiserver 发送 TLS Bootstrapping 请求
K8S approve kubelet 的 csr 请求后在 --cert-dir 目录创建证书和私钥文件然后写入 --kubeconfig 文件
--pod-infra-container-image 不使用 redhat 的 pod-infrastructure:latest 镜像它不能回收容器的僵尸
分发启动文件cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_name in ${NODE_NAMES[]}do echo ${node_name}sed -e s/##NODE_NAME##/${node_name}/ kubelet.service.template kubelet-${node_name}.servicescp kubelet-${node_name}.service root${node_name}:/etc/systemd/system/kubelet.servicedoneBootstrap Token Auth 和授予权限 kubelet 启动时查找 --kubeletconfig
参数对应的文件是否存在如果不存在则使用 --bootstrap-kubeconfig 指定的 kubeconfig 文件向
kube-apiserver 发送证书签名请求 (CSR)。 kube-apiserver 收到 CSR 请求后对其中的 Token
进行认证认证通过后将请求的 user 设置为 system:bootstrap:group 设置为
system:bootstrappers这一过程称为 Bootstrap Token Auth。创建user和group的CSR权限不创建kubelet会启动失败
kubectl create clusterrolebinding kubelet-bootstrap --clusterrolesystem:node-bootstrapper --groupsystem:bootstrappers#启动kubelet
source /opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[]}doecho ${node_ip}ssh root${node_ip} mkdir -p ${K8S_DIR}/kubelet/kubelet-plugins/volume/exec/ssh root${node_ip} /usr/sbin/swapoff -assh root${node_ip} systemctl daemon-reload systemctl enable kubelet systemctl restart kubeletdone关闭 swap 分区否则 kubelet 会启动失败
kubelet 启动后使用 --bootstrap-kubeconfig 向 kube-apiserver 发送 CSR 请求当这个CSR 被 approve 后kube-controller-manager 为 kubelet 创建 TLS 客户端证书、私钥和--kubeletconfig 文件。 注意kube-controller-manager 需要配置 --cluster-signing-cert-file 和 --cluster-signing-key-file 参数才会为 TLS Bootstrap 创建证书和私钥。[rootk8s-01 work]# kubectl get csr
NAME AGE REQUESTOR CONDITION
csr-4fhqr 25s system:bootstrap:i3cd24 Pending
csr-85jzl 20s system:bootstrap:fyw4n9 Pending
csr-9lwfj 9s system:bootstrap:fyw4n9 Pending
csr-bzhvt 20s system:bootstrap:i3cd24 Pending
csr-gw4w4 4s system:bootstrap:fyw4n9 Pending
csr-j42zv 15s system:bootstrap:fyw4n9 Pending
csr-jc7mx 5s system:bootstrap:1r8xjv Pending
csr-lns5s 3s system:bootstrap:i3cd24 Pending
csr-pjczh 14s system:bootstrap:i3cd24 Pending
csr-qztj9 27s system:bootstrap:1r8xjv Pending
csr-s9c2x 16s system:bootstrap:1r8xjv Pending
csr-th5pc 9s system:bootstrap:i3cd24 Pending
csr-vvsdv 10s system:bootstrap:1r8xjv Pending
csr-wgj7x 26s system:bootstrap:fyw4n9 Pending
csr-xd8gb 21s system:bootstrap:1r8xjv Pending这里节点均处于pending(等待)状态自动approve CSR请求创建三个ClusterRoleBinding分别用于自动approve client、renew client、renew server证cd /opt/k8s/work
cat csr-crb.yaml EOF# Approve all CSRs for the group system:bootstrapperskind: ClusterRoleBindingapiVersion: rbac.authorization.k8s.io/v1metadata:name: auto-approve-csrs-for-groupsubjects:- kind: Groupname: system:bootstrappersapiGroup: rbac.authorization.k8s.ioroleRef:kind: ClusterRolename: system:certificates.k8s.io:certificatesigningrequests:nodeclientapiGroup: rbac.authorization.k8s.io
---# To let a node of the group system:nodes renew its own credentialskind: ClusterRoleBindingapiVersion: rbac.authorization.k8s.io/v1metadata:name: node-client-cert-renewalsubjects:- kind: Groupname: system:nodesapiGroup: rbac.authorization.k8s.ioroleRef:kind: ClusterRolename: system:certificates.k8s.io:certificatesigningrequests:selfnodeclientapiGroup: rbac.authorization.k8s.io
---
# A ClusterRole which instructs the CSR approver to approve a node requesting a
# serving cert matching its client cert.
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:name: approve-node-server-renewal-csr
rules:
- apiGroups: [certificates.k8s.io]resources: [certificatesigningrequests/selfnodeserver]verbs: [create]
---# To let a node of the group system:nodes renew its own server credentialskind: ClusterRoleBindingapiVersion: rbac.authorization.k8s.io/v1metadata:name: node-server-cert-renewalsubjects:- kind: Groupname: system:nodesapiGroup: rbac.authorization.k8s.ioroleRef:kind: ClusterRolename: approve-node-server-renewal-csrapiGroup: rbac.authorization.k8s.io
EOF
kubectl apply -f csr-crb.yamlauto-approve-csrs-for-group 自动approve node的第一次CSR注意第一次CSR时请求的Group为system:bootstrappers
node-client-cert-renewal 自动approve node后续过期的client证书自动生成的证书Group为system:nodes
node-server-cert-renewal 自动approve node后续过期的server证书自动生成的证书Group查看kubelet等待1-10分钟3个节点的CSR都会自动approvedPending的CSR用于创建kubelet serve证书需要手动approve (后面步骤)目前所有节点均为ready状态NAME STATUS ROLES AGE VERSION
k8s-01 Ready 2m29s v1.14.2
k8s-02 Ready 2m28s v1.14.2
k8s-03 Ready 2m28s v1.14.2
k8s-04 Ready 2m27s v1.14.2kube-controller-manager为各node生成了kubeconfig文件和公钥
[rootk8s-01 work]# ls -l /etc/kubernetes/kubelet.kubeconfig
-rw------- 1 root root 2316 Aug 21 17:34 /etc/kubernetes/kubelet.kubeconfig
[rootk8s-01 work]# ls -l /etc/kubernetes/cert/|grep kubelet
-rw------- 1 root root 1273 Aug 21 17:34 kubelet-client-2023-08-21-17-34-58.pem
lrwxrwxrwx 1 root root 59 Aug 21 17:34 kubelet-client-current.pem - /etc/kubernetes/cert/kubelet-client-2023-08-21-17-34-58.pem手动approve server cert csr基于安全考虑CSR approving controllers不会自动approve kubelet server证书签名请求需要手动approve
kubectl get csr | grep Pending | awk {print $1} | xargs kubectl certificate approvekubelet API接口kubelet启动后监听多个端口用于接受kube-apiserver或其他客户端发送的请求[rootk8s-02 work]# netstat -lntup|grep kubelet
tcp 0 0 127.0.0.1:40605 0.0.0.0:* LISTEN 27600/kubelet
tcp 0 0 192.168.100.27:10248 0.0.0.0:* LISTEN 27600/kubelet
tcp 0 0 192.168.100.27:10250 0.0.0.0:* LISTEN 27600/kubelet 10248: healthz http 服务
10250: https 服务访问该端口时需要认证和授权即使访问 /healthz 也需要
未开启只读端口 10255
从 K8S v1.10 开始去除了 --cadvisor-port 参数默认 4194 端口不支持访问 cAdvisor UI API
bear token认证和授权创建一个ServiceAccount将它和ClusterRole system:kubelet-api-admin绑定从而具有调用kubelet API的权限kubectl create sa kubelet-api-test
kubectl create clusterrolebinding kubelet-api-test --clusterrolesystem:kubelet-api-admin --serviceaccountdefault:kubelet-api-test
SECRET$(kubectl get secrets | grep kubelet-api-test | awk {print $1})
TOKEN$(kubectl describe secret ${SECRET} | grep -E ^token | awk {print $2})
echo ${TOKEN}[rootk8s-01 work]# kubectl create clusterrolebinding kubelet-api-test --clusterrolesystem:kubelet-api-admin --serviceaccountdefault:kubelet-api-test
clusterrolebinding.rbac.authorization.k8s.io/kubelet-api-test created
[rootk8s-01 work]# SECRET$(kubectl get secrets | grep kubelet-api-test | awk {print $1})
[rootk8s-01 work]# TOKEN$(kubectl describe secret ${SECRET} | grep -E ^token | awk {print $2})
[rootk8s-01 work]# echo ${TOKEN}
eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJkZWZhdWx0Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZWNyZXQubmFtZSI6Imt1YmVsZXQtYXBpLXRlc3QtdG9rZW4tMndtbjYiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoia3ViZWxldC1hcGktdGVzdCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6ImI3NWEzMzE4LTQwMGMtMTFlZS1iNmVlLTQyMDFjMGE4NjQxYSIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDpkZWZhdWx0Omt1YmVsZXQtYXBpLXRlc3QifQ.nvB9YjjVclFqrMMmAVF2dMD5hHJ6UzW6ujNO-i1ijFTEoG1cD1FQTAmdYS-hdbY_1heB5O3sFNRO6hF-E-s_yKF3aEcaxtiyJX95WE2Wt52VxA4750WV9Gqyz5HxGpsluEwlkz7eLh8jPwQHJqAE__sO1zo4-68hVGjbuEm8vFh5Bz1iQUfWFPK6RXaIzqcnkMe3T4SlZrtzHvd5ttmA2s4fsIZTvM6T157NUem7POF0MwWMIhtO1F49bMTxf2fe90dGSqZFx0gtWKoe671sbAcf6emcRkWz93GCumBrxZw5bLsm3dX_NWFj8kkVAB9WG-RGGGyK7iRg-oizXrumiA
部署kube-proxy组件
kube-proxy运行在所有worker节点上它监听apiserver中service和endpoint的变化情况创建路由规则提供服务IP和负载均衡功能。这里使用ipvs模式的kube-proxy进行部署在各个节点需要安装ipvsadm和ipset命令加载ip_vs内核模块cd /opt/k8s/work
cat kube-proxy-csr.json EOF
{CN: system:kube-proxy,key: {algo: rsa,size: 2048},names: [{C: CN,ST: BeiJing,L: BeiJing,O: k8s,OU: 4Paradigm}]
}
EOFCN指定该证书的 User 为 system:kube-proxy
预定义的 RoleBinding system:node-proxier 将User system:kube-proxy 与 Role system:node-proxier 绑定该 Role 授予了调用 kube-apiserver Proxy 相关 API 的权限
该证书只会被 kube-proxy 当做 client 证书使用所以 hosts 字段为空
生成证书和私钥cd /opt/k8s/work
cfssl gencert -ca/opt/k8s/work/ca.pem \-ca-key/opt/k8s/work/ca-key.pem \-config/opt/k8s/work/ca-config.json \-profilekubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
ls kube-proxy*cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
kubectl config set-cluster kubernetes \--certificate-authority/opt/k8s/work/ca.pem \--embed-certstrue \--server${KUBE_APISERVER} \--kubeconfigkube-proxy.kubeconfig
kubectl config set-credentials kube-proxy \--client-certificatekube-proxy.pem \--client-keykube-proxy-key.pem \--embed-certstrue \--kubeconfigkube-proxy.kubeconfig
kubectl config set-context default \--clusterkubernetes \--userkube-proxy \--kubeconfigkube-proxy.kubeconfig
kubectl config use-context default --kubeconfigkube-proxy.kubeconfig--embed-certstrue将 ca.pem 和 admin.pem 证书内容嵌入到生成的kubectl-proxy.kubeconfig文件中(不加时写入的是证书文件路径)
分发 kubeconfig 文件cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_name in ${NODE_NAMES[]}doecho ${node_name}scp kube-proxy.kubeconfig root${node_name}:/etc/kubernetes/done创建kube-proxy配置文件从v1.10开始kube-proxy部分参数可以配置在文件中可以使用--write-config-to选项生成该配置文件cd /opt/k8s/work
cat kube-proxy-config.yaml.template EOF
kind: KubeProxyConfiguration
apiVersion: kubeproxy.config.k8s.io/v1alpha1
clientConnection:burst: 200kubeconfig: /etc/kubernetes/kube-proxy.kubeconfigqps: 100
bindAddress: ##NODE_IP##
healthzBindAddress: ##NODE_IP##:10256
metricsBindAddress: ##NODE_IP##:10249
enableProfiling: true
clusterCIDR: ${CLUSTER_CIDR}
hostnameOverride: ##NODE_NAME##
mode: ipvs
portRange:
kubeProxyIPTablesConfiguration:masqueradeAll: false
kubeProxyIPVSConfiguration:scheduler: rrexcludeCIDRs: []
EOFbindAddress: 监听地址
clientConnection.kubeconfig: 连接 apiserver 的 kubeconfig 文件
-clusterCIDR: kube-proxy 根据 --cluster-cidr判断集群内部和外部流量指定 --cluster-cidr 或 --masquerade-all 选项后 kube-proxy 才会对访问 Service IP 的请求做 SNAT
hostnameOverride: 参数值必须与 kubelet 的值一致否则 kube-proxy 启动后会找不到该 Node从而不会创建任何 ipvs 规则
mode: 使用 ipvs 模式
分发和创建kube-proxy配置文件cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for (( i0; i 4; i ))do echo ${NODE_NAMES[i]}sed -e s/##NODE_NAME##/${NODE_NAMES[i]}/ -e s/##NODE_IP##/${NODE_IPS[i]}/ kube-proxy-config.yaml.template kube-proxy-config-${NODE_NAMES[i]}.yaml.templatescp kube-proxy-config-${NODE_NAMES[i]}.yaml.template root${NODE_NAMES[i]}:/etc/kubernetes/kube-proxy-config.yamldone#我这里一共有4个节点要运行所以这里写4这是整个集群的node节点的数量 这里一定要注意修改#创建和分发 kube-proxy systemd unit 文件cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
cat kube-proxy.service EOF
[Unit]
DescriptionKubernetes Kube-Proxy Server
Documentationhttps://github.com/GoogleCloudPlatform/kubernetes
Afternetwork.target
[Service]
WorkingDirectory${K8S_DIR}/kube-proxy
ExecStart/opt/k8s/bin/kube-proxy \\--config/etc/kubernetes/kube-proxy-config.yaml \\--logtostderrtrue \\--v2
Restarton-failure
RestartSec5
LimitNOFILE65536
[Install]
WantedBymulti-user.target
EOFcd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_name in ${NODE_NAMES[]}do echo ${node_name}scp kube-proxy.service root${node_name}:/etc/systemd/system/done启动 kube-proxy 服务cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[]}doecho ${node_ip}ssh root${node_ip} mkdir -p ${K8S_DIR}/kube-proxyssh root${node_ip} modprobe ip_vs_rrssh root${node_ip} systemctl daemon-reload systemctl enable kube-proxy systemctl restart kube-proxydonesource /opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[]}doecho ${node_ip}ssh root${node_ip} systemctl status kube-proxy|grep Activedone[rootk8s-01 work]# netstat -lnpt|grep kube-prox
tcp 0 0 192.168.100.26:10249 0.0.0.0:* LISTEN 32089/kube-proxy
tcp 0 0 192.168.100.26:10256 0.0.0.0:* LISTEN 32089/kube-proxy source /opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[]}doecho ${node_ip}ssh root${node_ip} /usr/sbin/ipvsadm -lndone[rootk8s-01 work]# for node_ip in ${NODE_IPS[]}doecho ${node_ip}ssh root${node_ip} /usr/sbin/ipvsadm -lndone192.168.100.26
IP Virtual Server version 1.2.1 (size4096)
Prot LocalAddress:Port Scheduler Flags- RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 10.254.0.1:443 rr- 192.168.100.26:6443 Masq 1 0 0 - 192.168.100.27:6443 Masq 1 0 0 - 192.168.100.28:6443 Masq 1 0 0 192.168.100.27
IP Virtual Server version 1.2.1 (size4096)
Prot LocalAddress:Port Scheduler Flags- RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 10.254.0.1:443 rr- 192.168.100.26:6443 Masq 1 0 0 - 192.168.100.27:6443 Masq 1 0 0 - 192.168.100.28:6443 Masq 1 0 0 192.168.100.28
IP Virtual Server version 1.2.1 (size4096)
Prot LocalAddress:Port Scheduler Flags- RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 10.254.0.1:443 rr- 192.168.100.26:6443 Masq 1 0 0 - 192.168.100.27:6443 Masq 1 0 0 - 192.168.100.28:6443 Masq 1 0 0 192.168.100.29
IP Virtual Server version 1.2.1 (size4096)
Prot LocalAddress:Port Scheduler Flags- RemoteAddress:Port Forward Weight ActiveConn InActConn可见所有通过 https 访问 K8S SVC kubernetes 的请求都转发到 kube-apiserver 节点的 6443 端口
验证集群功能
cd /opt/k8s/work
cat nginx-ds.yml EOF
apiVersion: v1
kind: Service
metadata:name: nginx-dslabels:app: nginx-ds
spec:type: NodePortselector:app: nginx-dsports:- name: httpport: 80targetPort: 80
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:name: nginx-dslabels:addonmanager.kubernetes.io/mode: Reconcile
spec:template:metadata:labels:app: nginx-dsspec:containers:- name: my-nginximage: nginxports:- containerPort: 80
EOFkubectl create -f nginx-ds.yml# kubectl get pod -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
nginx-ds-29n8p 1/1 Running 0 116s 172.17.0.2 k8s-02 source /opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[]}doecho ${node_ip}ssh ${node_ip} ping -c 1 172.17.0.2ssh ${node_ip} ping -c 1 172.30.96.2ssh ${node_ip} ping -c 1 172.17.0.2done#检查服务IP和端口可达性
kubectl get svc |grep nginx-ds
source /opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[]}doecho ${node_ip}ssh ${node_ip} curl -s 10.254.248.73done
CoreDNS安装
source /opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[]}doecho $node_ipssh $node_ip wget -P /opt/ http://down.i4t.com/coredns_v1.4.tarssh $node_ip docker load -i /opt/coredns_v1.4.tardonewget -P /opt/ http://down.i4t.com/k8s1.14/coredns.yamlkubectl create -f /opt/coredns.yamlkubectl get pod -n kube-system -l k8s-appkube-dns
参考文章
https://i4t.com/4253.html
