You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
17 KiB
17 KiB
利用国内源搭建k8s集群
一、前期准备
1.1 准备环境
# Linux版本
Rocky Linux 9.4 Mini
# 更新系统
dnf clean all -y
dnf update -y
# K8S的三台服务器
10.10.14.200 k8s-master
10.10.14.201 k8s-node1
10.10.14.202 k8s-node2
1.2 系统初始化
设置系统时区为上海
timedatectl set-timezone Asia/Shanghai
clock -w
# 查看时区
ls -l /etc/localtime
关闭防火墙：
systemctl stop firewalld
systemctl disable firewalld
sed -i 's/enforcing/disabled/' /etc/selinux/config
setenforce 0
关闭swap分区：
sed -ri 's/.*swap.*/#&/' /etc/fstab
swapoff -a
在master上执行
hostnamectl set-hostname k8s-master
在node1上执行
hostnamectl set-hostname k8s-node1
在node2上执行
hostnamectl set-hostname k8s-node2
在每个节点添加hosts：
cat >> /etc/hosts << EOF
10.10.14.200 k8s-master
10.10.14.201 k8s-node1
10.10.14.202 k8s-node2
EOF
将桥接的IPv4流量传递到iptables的链：
在每个节点添加如下的命令：
cat > /etc/sysctl.d/k8s.conf << EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
vm.swappiness = 0
EOF
在每个节点加载br\_netfilter模块
modprobe br_netfilter
sysctl --system
查看是否加载
lsmod | grep br_netfilter
在每个节点添加时间同步：
安装ntpdate时间同步插件
dnf install chrony -y
systemctl enable --now chronyd
编辑内容
cat > /etc/chrony.conf << EOF
server 0.pool.ntp.org iburst
server 1.pool.ntp.org iburst
server 2.pool.ntp.org iburst
server 3.pool.ntp.org iburst
EOF
systemctl restart chronyd
手工同步
chronyc makestep
在每个节点安装ipset和ipvsadm：
安装
yum -y install ipset ipvsadm
配置
mkdir -p /etc/sysconfig/modules/
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
EOF
授权、运行、检查是否加载：
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack
二、开始安装
2.1 安装Docker
所有节点安装Docker：
获取镜像源
yum install -y yum-utils
curl -o /etc/yum.repos.d/docker-ce.repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum clean all
yum makecache
安装
# 找出有哪些可用的Docker版本
yum list docker-ce --showduplicates | sort -r
yum list docker-ce-cli --showduplicates | sort -r
yum list containerd.io --showduplicates | sort -r
docker-ce.x86_64 3:20.10.24-3.el9 docker-ce-stable
...
docker-ce-cli.x86_64 1:20.10.24-3.el9 docker-ce-stable
...
yum install -y docker-ce docker-ce-cli containerd.io
设置开机自启动并启动
systemctl enable docker && systemctl start docker
配置加速
mkdir -p /etc/docker
tee /etc/docker/daemon.json <<-'EOF'
{
"exec-opts": ["native.cgroupdriver=systemd"]
}
EOF
systemctl daemon-reload
systemctl restart docker
# 上传 cri-dockerd-0.3.8.amd64.tgz
tar xf cri-dockerd-0.3.8.amd64.tgz
mv cri-dockerd/cri-dockerd /usr/bin/
rm -rf cri-dockerd cri-dockerd-0.3.8.amd64.tgz
# 配置启动项
cat > /etc/systemd/system/cri-docker.service<<EOF
[Unit]
Description=CRI Interface for Docker Application Container Engine
Documentation=https://docs.mirantis.com
After=network-online.target firewalld.service docker.service
Wants=network-online.target
Requires=cri-docker.socket
[Service]
Type=notify
# ExecStart=/usr/bin/cri-dockerd --container-runtime-endpoint fd://
# 指定用作 Pod 的基础容器的容器镜像（“pause 镜像”）
ExecStart=/usr/bin/cri-dockerd --pod-infra-container-image=registry.k8s.io/pause:3.9 --container-runtime-endpoint fd://
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always
StartLimitBurst=3
StartLimitInterval=60s
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TasksMax=infinity
Delegate=yes
KillMode=process
[Install]
WantedBy=multi-user.target
EOF
cat > /etc/systemd/system/cri-docker.socket <<EOF
[Unit]
Description=CRI Docker Socket for the API
PartOf=cri-docker.service
[Socket]
ListenStream=%t/cri-dockerd.sock
SocketMode=0660
SocketUser=root
SocketGroup=docker
[Install]
WantedBy=sockets.target
EOF
systemctl daemon-reload
systemctl enable cri-docker && systemctl start cri-docker && systemctl status cri-docker
2.2 安装 kubelet、kubeadm、kubectl
配置k8s源(所有节点)
cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://pkgs.k8s.io/core:/stable:/v1.29/rpm/
enabled=1
gpgcheck=1
gpgkey=https://pkgs.k8s.io/core:/stable:/v1.29/rpm/repodata/repomd.xml.key
# exclude=kubelet kubeadm kubectl cri-tools kubernetes-cni
EOF
安装kubelet、kubeadm、kubectl (所有节点)
# 安装默认版本
#yum install -y kubelet kubeadm kubectl
# 安装指定版本
yum -y install kubeadm-1.29.0-150500.1.1 kubelet-1.29.0-150500.1.1 kubectl-1.29.0-150500.1.1
配置 cgroup 驱动与docker一致(所有节点)
cp /etc/sysconfig/kubelet{,.bak}
cat > /etc/sysconfig/kubelet <<EOF
KUBELET_EXTRA_ARGS="--cgroup-driver=systemd"
EOF
systemctl enable kubelet
安装自动补全工具
yum install bash-completion -y
source /usr/share/bash-completion/bash_completion
echo "source <(kubectl completion bash)" >> ~/.bashrc
source ~/.bashrc
查看镜像配置
kubeadm config images list --kubernetes-version=v1.29.0
如果网络有限制，请提前下载需要用到的镜像
# 重点，重点，重点
# kubeadm部署集群，需要用到k8s配置镜像和Calico网络配置镜像
# 由于默认拉取镜像地址k8s.gcr.io国内无法访问，国内镜像仓库我也没有找到，所以建议提前下载好，导入镜像。
# 所有需要用到镜像(k8s配置镜像和Calico网络配置镜像)：
# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
calico/kube-controllers v3.27.0 4e87edec0297 12 days ago 75.5MB
calico/cni v3.27.0 8e8d96a874c0 12 days ago 211MB
calico/pod2daemon-flexvol v3.27.0 6506d2e0be2d 12 days ago 15.4MB
calico/node v3.27.0 1843802b91be 13 days ago 340MB
registry.k8s.io/kube-apiserver v1.29.0 1443a367b16d 2 weeks ago 127MB
registry.k8s.io/kube-scheduler v1.29.0 7ace497ddb8e 2 weeks ago 59.5MB
registry.k8s.io/kube-controller-manager v1.29.0 0824682bcdc8 2 weeks ago 122MB
registry.k8s.io/kube-proxy v1.29.0 98262743b26f 2 weeks ago 82.2MB
registry.k8s.io/etcd 3.5.10-0 a0eed15eed44 8 weeks ago 148MB
registry.k8s.io/coredns/coredns v1.11.1 cbb01a7bd410 4 months ago 59.8MB
registry.k8s.io/pause 3.9 e6f181688397 14 months ago 744kB
办法
docker pull calico/kube-controllers:v3.27.0
docker pull calico/cni:v3.27.0
docker pull calico/pod2daemon-flexvol:v3.27.0
docker pull calico/node:v3.27.0
# 国内镜像源
# https://docker.aityp.com/
# 偷梁换柱
docker pull swr.cn-north-4.myhuaweicloud.com/ddn-k8s/registry.k8s.io/kube-apiserver:v1.29.8
docker tag swr.cn-north-4.myhuaweicloud.com/ddn-k8s/registry.k8s.io/kube-apiserver:v1.29.8 registry.k8s.io/kube-apiserver:v1.29.0
docker pull swr.cn-north-4.myhuaweicloud.com/ddn-k8s/registry.k8s.io/kube-scheduler:v1.29.8
docker tag swr.cn-north-4.myhuaweicloud.com/ddn-k8s/registry.k8s.io/kube-scheduler:v1.29.8 registry.k8s.io/kube-scheduler:v1.29.0
docker pull swr.cn-north-4.myhuaweicloud.com/ddn-k8s/registry.k8s.io/kube-controller-manager:v1.29.8
docker tag swr.cn-north-4.myhuaweicloud.com/ddn-k8s/registry.k8s.io/kube-controller-manager:v1.29.8 registry.k8s.io/kube-controller-manager:v1.29.0
docker pull swr.cn-north-4.myhuaweicloud.com/ddn-k8s/registry.k8s.io/kube-proxy:v1.29.8
docker tag swr.cn-north-4.myhuaweicloud.com/ddn-k8s/registry.k8s.io/kube-proxy:v1.29.8 registry.k8s.io/kube-proxy:v1.29.0
docker pull swr.cn-north-4.myhuaweicloud.com/ddn-k8s/registry.k8s.io/etcd:3.5.10-0
docker tag swr.cn-north-4.myhuaweicloud.com/ddn-k8s/registry.k8s.io/etcd:3.5.10-0 registry.k8s.io/etcd:3.5.10-0
docker pull swr.cn-north-4.myhuaweicloud.com/ddn-k8s/registry.k8s.io/coredns/coredns:v1.11.1
docker tag swr.cn-north-4.myhuaweicloud.com/ddn-k8s/registry.k8s.io/coredns/coredns:v1.11.1 registry.k8s.io/coredns/coredns:v1.11.1
docker pull swr.cn-north-4.myhuaweicloud.com/ddn-k8s/registry.k8s.io/pause:3.9
docker tag swr.cn-north-4.myhuaweicloud.com/ddn-k8s/registry.k8s.io/pause:3.9 registry.k8s.io/pause:3.9
集群初始化(master节点运行)
# 初始化集群
kubeadm init --apiserver-advertise-address 10.10.14.200 --kubernetes-version v1.29.0 --pod-network-cidr=10.244.0.0/16 --cri-socket=unix:///var/run/cri-dockerd.sock
# 如果网络有问题，请使用如下命令初始化
kubeadm init \
--apiserver-advertise-address 10.10.14.200 # master节点ip \
--kubernetes-version v1.29.0 \
--pod-network-cidr=10.244.0.0/16 # pod分配的ip \
--cri-socket=unix:///var/run/cri-dockerd.sock --image-repository registry.cn-hangzhou.aliyuncs.com/google_containers
记录下下面的命令：
Then you can join any number of worker nodes by running the following on each as root:
kubeadm join 10.10.14.200:6443 --token 3yk4me.5k595v6hm2qz463s \
--discovery-token-ca-cert-hash sha256:9e83f5ebfaefa83523e16d546d56b9f3803d4083a71d18fe49217f72306a2058
创建配置目录（master）
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
# 创建可永久使用的token
kubeadm token create --ttl 0 --print-join-command
node节点执行如下命令添加节点（node节点运行）
kubeadm join 10.10.14.200:6443 --token 3yk4me.5k595v6hm2qz463s \
--discovery-token-ca-cert-hash sha256:9e83f5ebfaefa83523e16d546d56b9f3803d4083a71d18fe49217f72306a2058 --cri-socket unix:///var/run/cri-dockerd.sock
2.3 集群网络插件 calico 部署
（master节点运行）
建议使用flannel组件
# 将此文件下载直接apply即可
# wget https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml
# 我已经下载好了，直接上传就行了
kubectl apply -f kube-flannel.yml
应用operator资源清单文件
# wget https://raw.githubusercontent.com/projectcalico/calico/v3.26.4/manifests/tigera-operator.yaml
kubectl create -f tigera-operator.yaml
通过自定义方式安装
#wget https://raw.githubusercontent.com/projectcalico/calico/v3.26.4/manifests/custom-resources.yaml
修改文件第13行，修改为使用kubeadm init ----pod-network-cidr对应的IP地址段
vi custom-resources.yaml
11 ipPools:
12 - blockSize: 26
13 cidr: 10.244.0.0/16
14 encapsulation: VXLANCrossSubnet
应用资源清单文件
kubectl apply -f custom-resources.yaml
监视calico-sysem命名空间中pod运行情况
watch kubectl get pods -n calico-system
查看calico是否正常运行
kubectl get pods -n calico-system
NAME READY STATUS RESTARTS AGE
calico-kube-controllers-7bc767bbcb-pxppk 1/1 Running 0 5m46s
calico-node-6cc8l 0/1 Running 0 5m46s
calico-node-vkvjz 1/1 Running 0 5m46s
calico-node-wvk6q 1/1 Running 0 5m46s
calico-typha-74545574b-6jpgq 1/1 Running 0 5m46s
calico-typha-74545574b-vx9kv 1/1 Running 0 5m40s
csi-node-driver-7pxtt 2/2 Running 0 5m46s
csi-node-driver-lflc6 2/2 Running 0 5m46s
csi-node-driver-r5npp 2/2 Running 0 5m46s
查看集群节点是否正常运行
kubectl get nodes
[root@k8s-master ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master Ready control-plane 15m v1.29.0
k8s-node1 Ready <none> 12m v1.29.0
k8s-node2 Ready <none> 12m v1.29.0
查看所有pod是否正常运行
kubectl get pod -A
NAMESPACE NAME READY STATUS RESTARTS AGE
calico-apiserver calico-apiserver-6bb6cf484f-48c57 1/1 Running 0 56s
calico-apiserver calico-apiserver-6bb6cf484f-xlbh4 1/1 Running 0 56s
calico-system calico-kube-controllers-7bc767bbcb-pxppk 1/1 Running 0 6m59s
calico-system calico-node-6cc8l 1/1 Running 0 6m59s
calico-system calico-node-vkvjz 1/1 Running 0 6m59s
calico-system calico-node-wvk6q 1/1 Running 0 6m59s
calico-system calico-typha-74545574b-6jpgq 1/1 Running 0 6m59s
calico-system calico-typha-74545574b-vx9kv 1/1 Running 0 6m53s
calico-system csi-node-driver-7pxtt 2/2 Running 0 6m59s
calico-system csi-node-driver-lflc6 2/2 Running 0 6m59s
calico-system csi-node-driver-r5npp 2/2 Running 0 6m59s
kube-flannel kube-flannel-ds-qf7tg 0/1 CrashLoopBackOff 6 (5m6s ago) 11m
kube-flannel kube-flannel-ds-tlczf 1/1 Running 0 11m
kube-flannel kube-flannel-ds-xn98c 1/1 Running 0 11m
kube-system coredns-76f75df574-q6vps 1/1 Running 0 16m
kube-system coredns-76f75df574-srxnf 1/1 Running 0 16m
kube-system etcd-k8s-master 1/1 Running 0 16m
kube-system kube-apiserver-k8s-master 1/1 Running 0 16m
kube-system kube-controller-manager-k8s-master 1/1 Running 0 16m
kube-system kube-proxy-8t78q 1/1 Running 0 13m
kube-system kube-proxy-glwfx 1/1 Running 0 16m
kube-system kube-proxy-qg4t7 1/1 Running 0 13m
kube-system kube-scheduler-k8s-master 1/1 Running 0 16m
tigera-operator tigera-operator-7f8cd97876-7s58q 1/1 Running 0 9m19s
注：以后所有yaml文件都只在Master节点执行。 安装目录：/etc/kubernetes/ 组件配置文件目录：/etc/kubernetes/manifests/
三、测试
测试集群是否正常运行pod
#创建测试pod nginx
kubectl create deployment web -r 2 --image=nginx
deployment.apps/web created
# 使用nodeport将端口映射出来
kubectl expose deployment web --port=80 --type=NodePort
service/web exposed
查看pod运行状态
kubectl get pod,svc
NAME READY STATUS RESTARTS AGE
pod/web-76fd95c67-ckvcn 0/1 ContainerCreating 0 23s
pod/web-76fd95c67-zl9dz 0/1 ContainerCreating 0 23s
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 17m
service/web NodePort 10.107.0.63 <none> 80:31129/TCP 11s
5.2、命令行测试
curl 10.10.14.201:31129
111-test
curl 10.10.14.201:31129
2222-test
四、资料
KubePi 是一个现代化的 K8s 面板
https://github.com/1Panel-dev/KubePi
kubeadm部署kubernetes1.29