-
[K8s] 초기 설치 후, componentstatuses UnhealthySystem & Dev Story/K8s 2021. 2. 19. 18:21
root@k8s-master:~# kubectl version Client Version: version.Info{Major:"1", Minor:"20", GitVersion:"v1.20.2", GitCommit:"faecb196815e248d3ecfb03c680a4507229c2a56", GitTreeState:"clean", BuildDate:"2021-01-13T13:28:09Z", GoVersion:"go1.15.5", Compiler:"gc", Platform:"linux/amd64"} Server Version: version.Info{Major:"1", Minor:"20", GitVersion:"v1.20.2", GitCommit:"faecb196815e248d3ecfb03c680a4507229c2a56", GitTreeState:"clean", BuildDate:"2021-01-13T13:20:00Z", GoVersion:"go1.15.5", Compiler:"gc", Platform:"linux/amd64"}K8s 1.20.2 버전에서 "kubectl get componentstatuses" 명령어로 상태 확인 시 아래와 같이 스케쥴러와 컨트롤러가 Unhealthy로 나타나는 문제 발생
root@k8s-master:~# kubectl get componentstatuses Warning: v1 ComponentStatus is deprecated in v1.19+ NAME STATUS MESSAGE ERROR scheduler Unhealthy Get "http://127.0.0.1:10251/healthz": dial tcp 127.0.0.1:10251: connect: connection refused controller-manager Unhealthy Get "http://127.0.0.1:10252/healthz": dial tcp 127.0.0.1:10252: connect: connection refused etcd-0 Healthy {"health":"true"}Pod도 잘 떠있고, 네트워크 포트도 정상적으로 Listen 된 상태이지만 포트가 다르게 보이는데... 이게 문제인가?
root@k8s-master:~# kubectl get pods -n kube-system NAME READY STATUS RESTARTS AGE calico-kube-controllers-86bddfcff-ktp5m 1/1 Running 1 9d calico-node-p9p25 1/1 Running 1 9d calico-node-q88cs 1/1 Running 1 9d calico-node-twrxt 1/1 Running 1 9d coredns-74ff55c5b-4z59m 1/1 Running 1 9d coredns-74ff55c5b-tcgcd 1/1 Running 1 9d etcd-k8s-master 1/1 Running 1 9d kube-apiserver-k8s-master 1/1 Running 1 9d kube-controller-manager-k8s-master 1/1 Running 0 17m kube-proxy-r86ft 1/1 Running 1 9d kube-proxy-t4jsw 1/1 Running 1 9d kube-proxy-z8s4g 1/1 Running 1 9d kube-scheduler-k8s-master 1/1 Running 0 17mroot@k8s-master:~# netstat -atnp | grep -i listen tcp 0 0 127.0.0.1:10248 0.0.0.0:* LISTEN 2070337/kubelet tcp 0 0 127.0.0.1:10249 0.0.0.0:* LISTEN 3673/kube-proxy tcp 0 0 127.0.0.1:9099 0.0.0.0:* LISTEN 4279/calico-node tcp 0 0 192.168.0.13:2379 0.0.0.0:* LISTEN 2424/etcd tcp 0 0 127.0.0.1:2379 0.0.0.0:* LISTEN 2424/etcd tcp 0 0 192.168.0.13:2380 0.0.0.0:* LISTEN 2424/etcd tcp 0 0 127.0.0.1:2381 0.0.0.0:* LISTEN 2424/etcd tcp 0 0 127.0.0.1:10257 0.0.0.0:* LISTEN 2070042/kube-contro tcp 0 0 127.0.0.1:10259 0.0.0.0:* LISTEN 2070706/kube-schedu tcp 0 0 0.0.0.0:179 0.0.0.0:* LISTEN 4419/bird tcp 0 0 127.0.0.53:53 0.0.0.0:* LISTEN 651/systemd-resolve tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 725/sshd: /usr/sbin tcp 0 0 127.0.0.1:37591 0.0.0.0:* LISTEN 2070337/kubelet tcp6 0 0 :::10250 :::* LISTEN 2070337/kubelet tcp6 0 0 :::10251 :::* LISTEN 2070706/kube-schedu tcp6 0 0 :::6443 :::* LISTEN 2353/kube-apiserver tcp6 0 0 :::10252 :::* LISTEN 2070042/kube-contro tcp6 0 0 :::10256 :::* LISTEN 3673/kube-proxy tcp6 0 0 :::22 :::* LISTEN 725/sshd: /usr/sbin"/etc/kubernetes/manifests" 경로로 이동
root@k8s-master:~# cd /etc/kubernetes/manifests/ root@k8s-master:/etc/kubernetes/manifests#에디터로 "kube-controller-manager.yaml" 파일을 열고, 24번째 --port=0을 주석 처리
apiVersion: v1 kind: Pod metadata: creationTimestamp: null labels: component: kube-controller-manager tier: control-plane name: kube-controller-manager namespace: kube-system spec: containers: - command: - kube-controller-manager - --authentication-kubeconfig=/etc/kubernetes/controller-manager.conf - --authorization-kubeconfig=/etc/kubernetes/controller-manager.conf - --bind-address=127.0.0.1 - --client-ca-file=/etc/kubernetes/pki/ca.crt - --cluster-name=kubernetes - --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt - --cluster-signing-key-file=/etc/kubernetes/pki/ca.key - --controllers=*,bootstrapsigner,tokencleaner - --kubeconfig=/etc/kubernetes/controller-manager.conf - --leader-elect=true # - --port=0 - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt - --root-ca-file=/etc/kubernetes/pki/ca.crt - --service-account-private-key-file=/etc/kubernetes/pki/sa.key - --use-service-account-credentials=true image: k8s.gcr.io/kube-controller-manager:v1.20.2 imagePullPolicy: IfNotPresent에디터로 "kube-scheduler.yaml" 파일을 열고, 10번째 --port=0을 주석 처리
apiVersion: v1 kind: Pod metadata: creationTimestamp: null labels: component: kube-scheduler tier: control-plane name: kube-scheduler namespace: kube-system spec: containers: - command: - kube-scheduler - --authentication-kubeconfig=/etc/kubernetes/scheduler.conf - --authorization-kubeconfig=/etc/kubernetes/scheduler.conf - --bind-address=127.0.0.1 - --kubeconfig=/etc/kubernetes/scheduler.conf - --leader-elect=true # - --port=0 image: k8s.gcr.io/kube-scheduler:v1.20.2 imagePullPolicy: IfNotPresent그리고 나서, kubelet을 재기동한다.
root@k8s-master:~# systemctl restart kubelet.serviceHealthy 상태로 확인
root@k8s-master:~# kubectl get componentstatuses Warning: v1 ComponentStatus is deprecated in v1.19+ NAME STATUS MESSAGE ERROR scheduler Healthy ok controller-manager Healthy ok etcd-0 Healthy {"health":"true"}