🚨 EC2 클러스터 파드 강제종료현상

ubuntu@ip-172-31-28-195:~$ kubectl get pods -n kube-system
NAME                                       READY   STATUS              RESTARTS         AGE
coredns-5dd5756b68-hzzj7                   0/1     ContainerCreating   0                6m2s
coredns-5dd5756b68-tfcjw                   0/1     ContainerCreating   0                6m2s
etcd-ip-172-31-28-195                      1/1     Running             19 (6m57s ago)   5m11s
kube-apiserver-ip-172-31-28-195            1/1     Running             19 (6m27s ago)   6m24s
kube-controller-manager-ip-172-31-28-195   1/1     Running             21 (6m57s ago)   5m14s
kube-proxy-6rpxj                           1/1     Running             1 (34s ago)      41s
kube-proxy-skrv8                           0/1     CrashLoopBackOff    4 (40s ago)      6m2s
kube-scheduler-ip-172-31-28-195            1/1     Running             19 (6m57s ago)   5m14s
ubuntu@ip-172-31-28-195:~$ kubectl apply -f https://projectcalico.docs.tigera.io/manifests/calico.yaml
The connection to the server 172.31.28.195:6443 was refused - did you specify the right host or port?
ubuntu@ip-172-31-28-195:~$ kubectl get pods -n kube-system
The connection to the server 172.31.28.195:6443 was refused - did you specify the right host or port?
ubuntu@ip-172-31-28-195:~$ kubectl get nodes
The connection to the server 172.31.28.195:6443 was refused - did you specify the right host or port?

https://evalle.github.io/blog/20180517-apiserver.html

kube-apiserver 로그

2023-10-28T18:53:06.414158608Z stderr F I1028 18:53:06.414115       1 cache.go:32] Waiting for caches to sync for AvailableConditionController controller
2023-10-28T18:53:06.41424918Z stderr F I1028 18:53:06.414199       1 tlsconfig.go:240] "Starting DynamicServingCertificateController"
2023-10-28T18:53:06.414416185Z stderr F I1028 18:53:06.414374       1 system_namespaces_controller.go:67] Starting system namespaces controller
2023-10-28T18:53:06.41458004Z stderr F I1028 18:53:06.414482       1 controller.go:80] Starting OpenAPI V3 AggregationController
2023-10-28T18:53:06.414691778Z stderr F I1028 18:53:06.414641       1 handler_discovery.go:412] Starting ResourceDiscoveryManager
2023-10-28T18:53:06.414903873Z stderr F I1028 18:53:06.414866       1 customresource_discovery_controller.go:289] Starting DiscoveryController
2023-10-28T18:53:06.415650496Z stderr F I1028 18:53:06.415593       1 apf_controller.go:372] Starting API Priority and Fairness config controller
2023-10-28T18:53:06.416089312Z stderr F I1028 18:53:06.416038       1 gc_controller.go:78] Starting apiserver lease garbage collector
2023-10-28T18:53:06.416508711Z stderr F I1028 18:53:06.416457       1 dynamic_serving_content.go:132] "Starting controller" name="aggregator-proxy-cert::/etc/kubernetes/pki/front-proxy-client.crt::/etc/kubernetes/pki/front-proxy-client.key"
2023-10-28T18:53:06.417110815Z stderr F I1028 18:53:06.417057       1 gc_controller.go:78] Starting apiserver lease garbage collector
2023-10-28T18:53:06.417435432Z stderr F I1028 18:53:06.417394       1 controller.go:116] Starting legacy_token_tracking_controller
2023-10-28T18:53:06.417443454Z stderr F I1028 18:53:06.417408       1 shared_informer.go:311] Waiting for caches to sync for configmaps
2023-10-28T18:53:06.41754834Z stderr F I1028 18:53:06.417507       1 crdregistration_controller.go:111] Starting crd-autoregister controller
2023-10-28T18:53:06.417553772Z stderr F I1028 18:53:06.417519       1 shared_informer.go:311] Waiting for caches to sync for crd-autoregister
2023-10-28T18:53:06.41765846Z stderr F I1028 18:53:06.417624       1 controller.go:134] Starting OpenAPI controller
2023-10-28T18:53:06.417734357Z stderr F I1028 18:53:06.417703       1 controller.go:85] Starting OpenAPI V3 controller
2023-10-28T18:53:06.417803499Z stderr F I1028 18:53:06.417774       1 naming_controller.go:291] Starting NamingConditionController
2023-10-28T18:53:06.417954554Z stderr F I1028 18:53:06.417841       1 establishing_controller.go:76] Starting EstablishingController
2023-10-28T18:53:06.418015864Z stderr F I1028 18:53:06.417935       1 nonstructuralschema_controller.go:192] Starting NonStructuralSchemaConditionController
2023-10-28T18:53:06.418091351Z stderr F I1028 18:53:06.418059       1 apiapproval_controller.go:186] Starting KubernetesAPIApprovalPolicyConformantConditionController
2023-10-28T18:53:06.418155174Z stderr F I1028 18:53:06.418127       1 crd_finalizer.go:266] Starting CRDFinalizer
2023-10-28T18:53:06.41965546Z stderr F I1028 18:53:06.419592       1 apiservice_controller.go:97] Starting APIServiceRegistrationController
2023-10-28T18:53:06.423373459Z stderr F I1028 18:53:06.423305       1 cache.go:32] Waiting for caches to sync for APIServiceRegistrationController controller
2023-10-28T18:53:06.423546731Z stderr F I1028 18:53:06.423509       1 controller.go:78] Starting OpenAPI AggregationController
2023-10-28T18:53:06.427505412Z stderr F I1028 18:53:06.427433       1 cluster_authentication_trust_controller.go:440] Starting cluster_authentication_trust_controller controller
2023-10-28T18:53:06.427527444Z stderr F I1028 18:53:06.427502       1 shared_informer.go:311] Waiting for caches to sync for cluster_authentication_trust_controller
2023-10-28T18:53:06.42759846Z stderr F I1028 18:53:06.414204       1 aggregator.go:164] waiting for initial CRD sync...
2023-10-28T18:53:06.448699521Z stderr F I1028 18:53:06.447611       1 dynamic_cafile_content.go:157] "Starting controller" name="client-ca-bundle::/etc/kubernetes/pki/ca.crt"
2023-10-28T18:53:06.448712749Z stderr F I1028 18:53:06.447825       1 dynamic_cafile_content.go:157] "Starting controller" name="request-header::/etc/kubernetes/pki/front-proxy-ca.crt"
2023-10-28T18:53:06.514353963Z stderr F I1028 18:53:06.514250       1 cache.go:39] Caches are synced for AvailableConditionController controller
2023-10-28T18:53:06.52889435Z stderr F I1028 18:53:06.528784       1 shared_informer.go:318] Caches are synced for cluster_authentication_trust_controller
2023-10-28T18:53:06.52917583Z stderr F I1028 18:53:06.529107       1 cache.go:39] Caches are synced for APIServiceRegistrationController controller
2023-10-28T18:53:06.571407783Z stderr F I1028 18:53:06.571309       1 apf_controller.go:377] Running API Priority and Fairness config worker
2023-10-28T18:53:06.571434984Z stderr F I1028 18:53:06.571334       1 apf_controller.go:380] Running API Priority and Fairness periodic rebalancing process
2023-10-28T18:53:06.571673434Z stderr F I1028 18:53:06.571615       1 shared_informer.go:318] Caches are synced for configmaps
2023-10-28T18:53:06.571796406Z stderr F I1028 18:53:06.571744       1 shared_informer.go:318] Caches are synced for crd-autoregister
2023-10-28T18:53:06.572621157Z stderr F I1028 18:53:06.572560       1 aggregator.go:166] initial CRD sync complete...
2023-10-28T18:53:06.572632241Z stderr F I1028 18:53:06.572578       1 autoregister_controller.go:141] Starting autoregister controller
2023-10-28T18:53:06.572636368Z stderr F I1028 18:53:06.572586       1 cache.go:32] Waiting for caches to sync for autoregister controller
2023-10-28T18:53:06.573284003Z stderr F I1028 18:53:06.573225       1 cache.go:39] Caches are synced for autoregister controller
2023-10-28T18:53:06.577118543Z stderr F I1028 18:53:06.577031       1 controller.go:624] quota admission added evaluator for: leases.coordination.k8s.io
2023-10-28T18:53:06.583432068Z stderr F I1028 18:53:06.583345       1 shared_informer.go:318] Caches are synced for node_authorizer
2023-10-28T18:53:07.420057171Z stderr F I1028 18:53:07.417743       1 storage_scheduling.go:111] all system priority classes are created successfully or already exist.
2023-10-28T18:53:07.79130305Z stderr F W1028 18:53:07.791097       1 lease.go:263] Resetting endpoints for master service "kubernetes" to [172.31.28.195]
2023-10-28T18:53:07.792564449Z stderr F I1028 18:53:07.792459       1 controller.go:624] quota admission added evaluator for: endpoints
2023-10-28T18:53:07.804236855Z stderr F I1028 18:53:07.803981       1 controller.go:624] quota admission added evaluator for: endpointslices.discovery.k8s.io
2023-10-28T18:53:12.746045468Z stderr F I1028 18:53:12.745863       1 controller.go:128] Shutting down kubernetes service endpoint reconciler
2023-10-28T18:53:12.750661531Z stderr F W1028 18:53:12.750551       1 lease.go:263] Resetting endpoints for master service "kubernetes" to []
2023-10-28T18:53:12.759550951Z stderr F I1028 18:53:12.759448       1 autoregister_controller.go:165] Shutting down autoregister controller
2023-10-28T18:53:12.759575385Z stderr F I1028 18:53:12.759485       1 apiapproval_controller.go:198] Shutting down KubernetesAPIApprovalPolicyConformantConditionController
2023-10-28T18:53:12.759757575Z stderr F I1028 18:53:12.759692       1 establishing_controller.go:87] Shutting down EstablishingController
2023-10-28T18:53:12.759842049Z stderr F I1028 18:53:12.759803       1 naming_controller.go:302] Shutting down NamingConditionController
2023-10-28T18:53:12.759918992Z stderr F I1028 18:53:12.759887       1 controller.go:115] Shutting down OpenAPI V3 controller
2023-10-28T18:53:12.759984482Z stderr F I1028 18:53:12.759949       1 controller.go:162] Shutting down OpenAPI controller
2023-10-28T18:53:12.760057904Z stderr F I1028 18:53:12.759969       1 crdregistration_controller.go:142] Shutting down crd-autoregister controller
2023-10-28T18:53:12.760063785Z stderr F I1028 18:53:12.760024       1 controller.go:129] Ending legacy_token_tracking_controller
2023-10-28T18:53:12.760067472Z stderr F I1028 18:53:12.760030       1 controller.go:130] Shutting down legacy_token_tracking_controller
2023-10-28T18:53:12.760143908Z stderr F I1028 18:53:12.760101       1 gc_controller.go:91] Shutting down apiserver lease garbage collector
2023-10-28T18:53:12.76014964Z stderr F I1028 18:53:12.760119       1 apf_controller.go:384] Shutting down API Priority and Fairness config worker
2023-10-28T18:53:12.760206838Z stderr F I1028 18:53:12.760170       1 storage_flowcontrol.go:187] APF bootstrap ensurer is exiting
2023-10-28T18:53:12.760277785Z stderr F I1028 18:53:12.760243       1 customresource_discovery_controller.go:325] Shutting down DiscoveryController
2023-10-28T18:53:12.760345053Z stderr F I1028 18:53:12.760309       1 system_namespaces_controller.go:77] Shutting down system namespaces controller
2023-10-28T18:53:12.760899427Z stderr F I1028 18:53:12.760834       1 apiservice_controller.go:131] Shutting down APIServiceRegistrationController
2023-10-28T18:53:12.760983052Z stderr F I1028 18:53:12.760942       1 cluster_authentication_trust_controller.go:463] Shutting down cluster_authentication_trust_controller controller
2023-10-28T18:53:12.76106586Z stderr F I1028 18:53:12.761027       1 crd_finalizer.go:278] Shutting down CRDFinalizer
2023-10-28T18:53:12.761131697Z stderr F I1028 18:53:12.761099       1 gc_controller.go:91] Shutting down apiserver lease garbage collector
2023-10-28T18:53:12.761205926Z stderr F I1028 18:53:12.761168       1 available_controller.go:439] Shutting down AvailableConditionController
2023-10-28T18:53:12.761732527Z stderr F I1028 18:53:12.759499       1 nonstructuralschema_controller.go:204] Shutting down NonStructuralSchemaConditionController
2023-10-28T18:53:12.761913685Z stderr F I1028 18:53:12.761867       1 controller.go:84] Shutting down OpenAPI AggregationController
2023-10-28T18:53:12.762020029Z stderr F I1028 18:53:12.761977       1 dynamic_cafile_content.go:171] "Shutting down controller" name="client-ca-bundle::/etc/kubernetes/pki/ca.crt"
2023-10-28T18:53:12.76220127Z stderr F I1028 18:53:12.762153       1 dynamic_serving_content.go:146] "Shutting down controller" name="aggregator-proxy-cert::/etc/kubernetes/pki/front-proxy-client.crt::/etc/kubernetes/pki/front-proxy-client.key"
2023-10-28T18:53:12.762302752Z stderr F I1028 18:53:12.762257       1 object_count_tracker.go:151] "StorageObjectCountTracker pruner is exiting"
2023-10-28T18:53:12.763035514Z stderr F I1028 18:53:12.762356       1 dynamic_cafile_content.go:171] "Shutting down controller" name="client-ca-bundle::/etc/kubernetes/pki/ca.crt"
2023-10-28T18:53:12.775540736Z stderr F I1028 18:53:12.775334       1 dynamic_serving_content.go:146] "Shutting down controller" name="serving-cert::/etc/kubernetes/pki/apiserver.crt::/etc/kubernetes/pki/apiserver.key"
2023-10-28T18:53:12.775566406Z stderr F I1028 18:53:12.762574       1 controller.go:86] Shutting down OpenAPI V3 AggregationController
2023-10-28T18:53:12.775571385Z stderr F I1028 18:53:12.762591       1 controller.go:159] Shutting down quota evaluator
2023-10-28T18:53:12.775575329Z stderr F I1028 18:53:12.775404       1 controller.go:178] quota evaluator worker shutdown
2023-10-28T18:53:12.775579756Z stderr F I1028 18:53:12.762094       1 dynamic_cafile_content.go:171] "Shutting down controller" name="request-header::/etc/kubernetes/pki/front-proxy-ca.crt"
2023-10-28T18:53:12.775592582Z stderr F I1028 18:53:12.775465       1 controller.go:178] quota evaluator worker shutdown
2023-10-28T18:53:12.77559653Z stderr F I1028 18:53:12.775486       1 controller.go:178] quota evaluator worker shutdown
2023-10-28T18:53:12.775599865Z stderr F I1028 18:53:12.775496       1 controller.go:178] quota evaluator worker shutdown
2023-10-28T18:53:12.775603209Z stderr F I1028 18:53:12.775502       1 controller.go:178] quota evaluator worker shutdown
2023-10-28T18:53:12.775606593Z stderr F I1028 18:53:12.762923       1 secure_serving.go:258] Stopped listening on [::]:6443
2023-10-28T18:53:12.775632233Z stderr F I1028 18:53:12.762942       1 tlsconfig.go:255] "Shutting down DynamicServingCertificateController"
2023-10-28T18:53:12.776744288Z stderr F I1028 18:53:12.776651       1 dynamic_cafile_content.go:171] "Shutting down controller" name="request-header::/etc/kubernetes/pki/front-proxy-ca.crt"