Kubernetes is a great platform for container orchestration and everything else. Recently, Kubernetes has gone far ahead both in terms of functionality and in terms of security and fault tolerance. The Kubernetes architecture allows you to easily survive failures of various kinds and always stay afloat.
Today we will break the cluster, delete certificates, rejoin nodes live, and all this, if possible, without downtime for already running services.
So let's get started. The main control-plane Kubernetes consists of just a few components:
etcd - used as a database
kube-apiserver - API and heart of our cluster
kube-controller-manager - performs operations on Kubernetes resources
kube-scheduler - main scheduler
kubelets - which directly launch containers on hosts
TLS-, , . - Kubernetes, , :
# tree /etc/kubernetes/pki/
/etc/kubernetes/pki/
├── apiserver.crt
├── apiserver-etcd-client.crt
├── apiserver-etcd-client.key
├── apiserver.key
├── apiserver-kubelet-client.crt
├── apiserver-kubelet-client.key
├── ca.crt
├── ca.key
├── CTNCA.pem
├── etcd
│ ├── ca.crt
│ ├── ca.key
│ ├── healthcheck-client.crt
│ ├── healthcheck-client.key
│ ├── peer.crt
│ ├── peer.key
│ ├── server.crt
│ └── server.key
├── front-proxy-ca.crt
├── front-proxy-ca.key
├── front-proxy-client.crt
├── front-proxy-client.key
├── sa.key
└── sa.pub
static pods /etc/kubernetes/manifests/
, .. . . , Kubernetes, - .
:
TLS-, , - kubeadm, kubespray . kubeadm .. Kubernetes, .
, . :
rm -rf /etc/kubernetes/
:
CA etcd (
/etc/kubernetes/pki/etcd
)
CA Kubernetes (
/etc/kubernetes/pki
)
Kubeconfig cluster-admin, kube-controller-manager, kube-scheduler kubelet ( base64 CA-
/etc/kubernetes/*.conf
)
- etcd, kube-apiserver, kube-scheduler kube-controller-manager (
/etc/kubernetes/manifests
)
,
control-plane
, control-plane :
crictl rm `crictl ps -aq`
: kubeadm , .
etcd, (3 -) etcd- .
kubeadm init phase certs etcd-ca
- CA etcd-. , -:
/etc/kubernetes/pki/etcd/ca.{key,crt}
etcd- static- control-plane :
kubeadm init phase certs etcd-healthcheck-client
kubeadm init phase certs etcd-peer
kubeadm init phase certs etcd-server
kubeadm init phase etcd local
etcd-:
# crictl ps
CONTAINER ID IMAGE CREATED STATE NAME ATTEMPT POD ID
ac82b4ed5d83a 0369cf4303ffd 2 seconds ago Running etcd 0 bc8b4d568751b
, Kubernetes, master- :
kubeadm init phase certs all
kubeadm init phase kubeconfig all
kubeadm init phase control-plane all
cp -f /etc/kubernetes/admin.conf ~/.kube/config
SSL- Kubernetes-.
kubeadm , cluster-info kube-public .. CA.
kubeadm init phase bootstrap-token
CA, control-plane , .
/etc/kubernetes/pki/{ca,front-proxy-ca}.{key,crt}
/etc/kubernetes/pki/sa.{key,pub}
, Kubernetes, :
kubeadm init phase upload-certs --upload-certs
Kubernetes 2 , :
kubeadm join phase control-plane-prepare all kubernetes-apiserver:6443 --control-plane --token cs0etm.ua7fbmwuf1jz946l --discovery-token-ca-cert-hash sha256:555f6ececd4721fed0269d27a5c7f1c6d7ef4614157a18e56ed9a1fd031a3ab8 --certificate-key 385655ee0ab98d2441ba8038b4e8d03184df1806733eac131511891d1096be73
kubeadm join phase control-plane-join all
, API Kubernetes , CA front-proxy client, apiserver aggregation layer . kube-apiserver .
:
kubectl get cm -n kube-system extension-apiserver-authentication -o yaml
control-plane.
, NotReady
:
kubectl get node
apiserver, CA. kubeadm, .
CA :
systemctl stop kubelet
rm -rf /var/lib/kubelet/pki/ /etc/kubernetes/kubelet.conf
kubeadm init phase kubeconfig kubelet
kubeadm init phase kubelet-start
:
kubeadm token create --print-join-command
:
systemctl stop kubelet
rm -rf /var/lib/kubelet/pki/ /etc/kubernetes/pki/ /etc/kubernetes/kubelet.conf
kubeadm join phase kubelet-start kubernetes-apiserver:6443 --token cs0etm.ua7fbmwuf1jz946l --discovery-token-ca-cert-hash sha256:555f6ececd4721fed0269d27a5c7f1c6d7ef4614157a18e56ed9a1fd031a3ab8
,
/etc/kubernetes/pki/
, .
kubelet' , . , controller-manager NotReady- .
controller-manager, :
rm /etc/kubernetes/manifests/kube-controller-manager.yaml
crictl rmp `crictl ps --name kube-controller-manager -q`
, controller-manager . static-manifest controller-manager .
:
kubeadm init phase control-plane controller-manager
join token, cluster-info.
kubelet CA ( serverTLSBootstrap: true
), csr kubelet':
kubectl get csr
kubectl certificate approve <csr>
ServiceAccounts
. /etc/kubernetes/pki/sa.key
- jwt- ServiceAccounts, .
, kubernetes.io/service-account-token
:
kubectl get secret --all-namespaces | awk '/kubernetes.io\/service-account-token/ { print "kubectl delete secret -n " $1 " " $2}' | sh -s
kube-controller-manager , .
, , ::
kubectl get pod --field-selector 'spec.serviceAccountName!=default' --no-headers --all-namespaces | awk '{print "kubectl delete pod -n " $1 " " $2}'
serviceAccount. kube-system
, .. kube-proxy CNI-, .
. ! etcd-.