System & Dev Story/K8s
[K8s] 초기 설치 후, componentstatuses Unhealthy
일기™
2021. 2. 19. 18:21
root@k8s-master:~# kubectl version
Client Version: version.Info{Major:"1", Minor:"20", GitVersion:"v1.20.2", GitCommit:"faecb196815e248d3ecfb03c680a4507229c2a56", GitTreeState:"clean", BuildDate:"2021-01-13T13:28:09Z", GoVersion:"go1.15.5", Compiler:"gc", Platform:"linux/amd64"}
Server Version: version.Info{Major:"1", Minor:"20", GitVersion:"v1.20.2", GitCommit:"faecb196815e248d3ecfb03c680a4507229c2a56", GitTreeState:"clean", BuildDate:"2021-01-13T13:20:00Z", GoVersion:"go1.15.5", Compiler:"gc", Platform:"linux/amd64"}
K8s 1.20.2 버전에서 "kubectl get componentstatuses" 명령어로 상태 확인 시 아래와 같이 스케쥴러와 컨트롤러가 Unhealthy로 나타나는 문제 발생
root@k8s-master:~# kubectl get componentstatuses
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME STATUS MESSAGE ERROR
scheduler Unhealthy Get "http://127.0.0.1:10251/healthz": dial tcp 127.0.0.1:10251: connect: connection refused
controller-manager Unhealthy Get "http://127.0.0.1:10252/healthz": dial tcp 127.0.0.1:10252: connect: connection refused
etcd-0 Healthy {"health":"true"}
Pod도 잘 떠있고, 네트워크 포트도 정상적으로 Listen 된 상태이지만 포트가 다르게 보이는데... 이게 문제인가?
root@k8s-master:~# kubectl get pods -n kube-system
NAME READY STATUS RESTARTS AGE
calico-kube-controllers-86bddfcff-ktp5m 1/1 Running 1 9d
calico-node-p9p25 1/1 Running 1 9d
calico-node-q88cs 1/1 Running 1 9d
calico-node-twrxt 1/1 Running 1 9d
coredns-74ff55c5b-4z59m 1/1 Running 1 9d
coredns-74ff55c5b-tcgcd 1/1 Running 1 9d
etcd-k8s-master 1/1 Running 1 9d
kube-apiserver-k8s-master 1/1 Running 1 9d
kube-controller-manager-k8s-master 1/1 Running 0 17m
kube-proxy-r86ft 1/1 Running 1 9d
kube-proxy-t4jsw 1/1 Running 1 9d
kube-proxy-z8s4g 1/1 Running 1 9d
kube-scheduler-k8s-master 1/1 Running 0 17mroot@k8s-master:~# netstat -atnp | grep -i listen
tcp 0 0 127.0.0.1:10248 0.0.0.0:* LISTEN 2070337/kubelet
tcp 0 0 127.0.0.1:10249 0.0.0.0:* LISTEN 3673/kube-proxy
tcp 0 0 127.0.0.1:9099 0.0.0.0:* LISTEN 4279/calico-node
tcp 0 0 192.168.0.13:2379 0.0.0.0:* LISTEN 2424/etcd
tcp 0 0 127.0.0.1:2379 0.0.0.0:* LISTEN 2424/etcd
tcp 0 0 192.168.0.13:2380 0.0.0.0:* LISTEN 2424/etcd
tcp 0 0 127.0.0.1:2381 0.0.0.0:* LISTEN 2424/etcd
tcp 0 0 127.0.0.1:10257 0.0.0.0:* LISTEN 2070042/kube-contro
tcp 0 0 127.0.0.1:10259 0.0.0.0:* LISTEN 2070706/kube-schedu
tcp 0 0 0.0.0.0:179 0.0.0.0:* LISTEN 4419/bird
tcp 0 0 127.0.0.53:53 0.0.0.0:* LISTEN 651/systemd-resolve
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 725/sshd: /usr/sbin
tcp 0 0 127.0.0.1:37591 0.0.0.0:* LISTEN 2070337/kubelet
tcp6 0 0 :::10250 :::* LISTEN 2070337/kubelet
tcp6 0 0 :::10251 :::* LISTEN 2070706/kube-schedu
tcp6 0 0 :::6443 :::* LISTEN 2353/kube-apiserver
tcp6 0 0 :::10252 :::* LISTEN 2070042/kube-contro
tcp6 0 0 :::10256 :::* LISTEN 3673/kube-proxy
tcp6 0 0 :::22 :::* LISTEN 725/sshd: /usr/sbin
"/etc/kubernetes/manifests" 경로로 이동
root@k8s-master:~# cd /etc/kubernetes/manifests/
root@k8s-master:/etc/kubernetes/manifests#
에디터로 "kube-controller-manager.yaml" 파일을 열고, 24번째 --port=0을 주석 처리
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: null
labels:
component: kube-controller-manager
tier: control-plane
name: kube-controller-manager
namespace: kube-system
spec:
containers:
- command:
- kube-controller-manager
- --authentication-kubeconfig=/etc/kubernetes/controller-manager.conf
- --authorization-kubeconfig=/etc/kubernetes/controller-manager.conf
- --bind-address=127.0.0.1
- --client-ca-file=/etc/kubernetes/pki/ca.crt
- --cluster-name=kubernetes
- --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt
- --cluster-signing-key-file=/etc/kubernetes/pki/ca.key
- --controllers=*,bootstrapsigner,tokencleaner
- --kubeconfig=/etc/kubernetes/controller-manager.conf
- --leader-elect=true
# - --port=0
- --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
- --root-ca-file=/etc/kubernetes/pki/ca.crt
- --service-account-private-key-file=/etc/kubernetes/pki/sa.key
- --use-service-account-credentials=true
image: k8s.gcr.io/kube-controller-manager:v1.20.2
imagePullPolicy: IfNotPresent
에디터로 "kube-scheduler.yaml" 파일을 열고, 10번째 --port=0을 주석 처리
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: null
labels:
component: kube-scheduler
tier: control-plane
name: kube-scheduler
namespace: kube-system
spec:
containers:
- command:
- kube-scheduler
- --authentication-kubeconfig=/etc/kubernetes/scheduler.conf
- --authorization-kubeconfig=/etc/kubernetes/scheduler.conf
- --bind-address=127.0.0.1
- --kubeconfig=/etc/kubernetes/scheduler.conf
- --leader-elect=true
# - --port=0
image: k8s.gcr.io/kube-scheduler:v1.20.2
imagePullPolicy: IfNotPresent
그리고 나서, kubelet을 재기동한다.
root@k8s-master:~# systemctl restart kubelet.service
Healthy 상태로 확인
root@k8s-master:~# kubectl get componentstatuses
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME STATUS MESSAGE ERROR
scheduler Healthy ok
controller-manager Healthy ok
etcd-0 Healthy {"health":"true"}