yatai 등에서는 ECR에 image를 PUSH하기 위해 docker login을 하는데 credential 정보를 aws cli로 가져와서 사용함. 그러나 이 경우, credential이 12시간 마다 초기화되므로 이후에는 만료되어 yatai-image-builder가 image build 후에 ECR에 push 할 때, permission error가 발생하게 됨.
그러므로 이 credential을 대략 6시간 마다 업데이트해줄 필요가 있는데 이를 k8s에서 cronjob으로 주기적인 업데이트를 수행하도록 만들어보자.
테스트 환경
차트를 생성하고 워킹 디렉토리 등록 및 templates를 아래와 같이 생성 및 삭제하자.
$ helm create ecr-refresher
export WORKDIR=$(pwd)
cd ecr-refresher/templates
tree .
.
├── _helpers.tpl
├── configmap.yaml
├── cronjob.yaml
├── deployment.yaml
├── role.yaml
├── rolebinding.yaml
├── secret.yaml
├── serviceaccount.yaml
└── tests
1 directory, 8 files
template 들에서 사용할 기본적인 변수들을 작성한 _helpers.tpl과 각 template들의 내용을 검토하자.
_helpers.tpl
{{/*
Expand the name of the chart.
*/}}
{{- define "ecr-refresher.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "ecr-refresher.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "ecr-refresher.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Common labels
*/}}
{{- define "ecr-refresher.labels" -}}
helm.sh/chart: {{ include "ecr-refresher.chart" . }}
{{ include "ecr-refresher.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
Selector labels
*/}}
{{- define "ecr-refresher.selectorLabels" -}}
app.kubernetes.io/name: {{ include "ecr-refresher.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
Create the name of workloads
*/}}
{{- define "ecr-refresher.cronJobName" -}}
{{- printf "%s-%s" (include "ecr-refresher.fullname" .) "cronjob" }}
{{- end }}
{{- define "ecr-refresher.deploymentName" -}}
{{- printf "%s-%s" (include "ecr-refresher.fullname" .) "deployment" }}
{{- end }}
{{- define "ecr-refresher.serviceAccountName" -}}
{{- printf "%s-%s" (include "ecr-refresher.fullname" .) "serviceaccount" }}
{{- end }}
{{- define "ecr-refresher.secretName" -}}
{{- printf "%s-%s" (include "ecr-refresher.fullname" .) "env-secret" }}
{{- end }}
{{- define "ecr-refresher.configMapName" -}}
{{- printf "%s-%s" (include "ecr-refresher.fullname" .) "env-configmap" }}
{{- end }}
{{- define "ecr-refresher.roleName" -}}
{{- printf "%s-%s" (include "ecr-refresher.fullname" .) "update-role" }}
{{- end }}
{{- define "ecr-refresher.roleBindingName" -}}
{{- printf "%s-%s" (include "ecr-refresher.fullname" .) "rolebinder" }}
{{- end }}
configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "ecr-refresher.configMapName" . }}
data:
TARGET_SECRET_NAMES: {{ join " " .Values.targetSecretNames }}
TARGET_DEPLOYMENT_NAMES: {{ join " " .Values.targetDeploymentNames }}
binaryData:
run.sh: {{ (.Files.Get "run.sh" | b64enc) }}
secret.yaml
apiVersion: v1
kind: Secret
type: Opaque
metadata:
name: {{ include "ecr-refresher.secretName" . }}
namespace: {{ .Release.Namespace }}
data:
AWS_ACCESS_KEY_ID: {{ .Values.ecrRegistry.awsAccessKeyID | b64enc }}
AWS_SECRET_ACCESS_KEY: {{ .Values.ecrRegistry.awsSecretAccessKey | b64enc }}
AWS_DEFAULT_REGION: {{ .Values.ecrRegistry.awsDefaultRegion | b64enc }}
serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "ecr-refresher.serviceAccountName" . }}
namespace: {{ .Release.Namespace }}
labels:
{{- include "ecr-refresher.labels" . | nindent 4 }}
role.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: {{ include "ecr-refresher.roleName" . }}
namespace: {{ .Release.Namespace }}
rules:
- apiGroups: [""]
resources: ["secrets"]
resourceNames: [{{ join ", " .Values.targetSecretNames }}]
verbs: ["get", "watch", "list", "patch"]
- apiGroups: ["apps"]
resources: ["deployments"]
resourceNames: [{{ join ", " .Values.targetDeploymentNames }}]
verbs: ["patch"]
rolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: {{ include "ecr-refresher.roleBindingName" . }}
namespace: {{ .Release.Namespace }}
subjects:
- kind: ServiceAccount
name: {{ include "ecr-refresher.serviceAccountName" . }}
namespace: {{ .Release.Namespace }}
roleRef:
kind: Role
name: {{ include "ecr-refresher.roleName" . }}
apiGroup: rbac.authorization.k8s.io
deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "ecr-refresher.deploymentName" . }}
labels:
app: {{ include "ecr-refresher.deploymentName" . }}
spec:
replicas: 1
selector:
matchLabels:
app: {{ include "ecr-refresher.deploymentName" . }}
template:
metadata:
labels:
app: {{ include "ecr-refresher.deploymentName" . }}
spec:
containers:
- name: {{ include "ecr-refresher.deploymentName" . }}
image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
workingDir: /run-script
command:
- "sleep"
args:
- "1000000000000000000"
env:
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: {{ include "ecr-refresher.secretName" . }}
key: AWS_ACCESS_KEY_ID
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: {{ include "ecr-refresher.secretName" . }}
key: AWS_SECRET_ACCESS_KEY
- name: TARGET_SECRET_NAMES
valueFrom:
configMapKeyRef:
name: {{ include "ecr-refresher.configMapName" . }}
key: TARGET_SECRET_NAMES
- name: TARGET_DEPLOYMENT_NAMES
valueFrom:
configMapKeyRef:
name: {{ include "ecr-refresher.configMapName" . }}
key: TARGET_DEPLOYMENT_NAMES
volumeMounts:
- name: run-script
mountPath: /run-script/run.sh
subPath: run.sh
volumes:
- name: run-script
configMap:
name: {{ include "ecr-refresher.configMapName" . }}
items:
- key: run.sh
path: run.sh
defaultMode: 0755
serviceAccountName: {{ include "ecr-refresher.serviceAccountName" . }}
cronjob.yaml
apiVersion: batch/v1
kind: CronJob
metadata:
name: {{ include "ecr-refresher.cronJobName" . }}
namespace: {{ .Release.Namespace }}
spec:
schedule: "{{ .Values.cronJob.period }}"
jobTemplate:
spec:
template:
spec:
containers:
- name: {{ include "ecr-refresher.cronJobName" . }}
image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
command:
- "/bin/sh"
args:
- "-c"
- "/run-script/run.sh -s $TARGET_SECRET_NAMES -d $TARGET_DEPLOYMENT_NAMES"
env:
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: {{ include "ecr-refresher.secretName" . }}
key: AWS_ACCESS_KEY_ID
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: {{ include "ecr-refresher.secretName" . }}
key: AWS_SECRET_ACCESS_KEY
- name: TARGET_SECRET_NAMES
valueFrom:
configMapKeyRef:
name: {{ include "ecr-refresher.configMapName" . }}
key: TARGET_SECRET_NAMES
- name: TARGET_DEPLOYMENT_NAMES
valueFrom:
configMapKeyRef:
name: {{ include "ecr-refresher.configMapName" . }}
key: TARGET_DEPLOYMENT_NAMES
volumeMounts:
- name: run-script
mountPath: /run-script/run.sh
subPath: run.sh
volumes:
- name: run-script
configMap:
name: {{ include "ecr-refresher.configMapName" . }}
items:
- key: run.sh
path: run.sh
defaultMode: 0755
restartPolicy: OnFailure # or Never, OnFailure
serviceAccountName: {{ include "ecr-refresher.serviceAccountName" . }}
이제, 이 템플릿들에 유동적으로 값을 변경하며 사용할 변수들이 담긴 values.yaml 파일을 검토하자.
values.yaml
# cronjob's image
image:
repository: amazon/aws-cli
tag: 2.7.6
pullPolicy: Always
pullSecrets: []
# cronjob's rule
cronJob:
failedJobsHistoryLimit: 3
successfullJobsHistroyLimit: 4
period: "30 7-21/4 * * *"
suspend: false
# ecr
ecrRegistry:
registry: 868593138253.dkr.ecr.ap-northeast-2.amazonaws.com
awsDefaultRegion: ap-northeast-2
awsAccessKeyID: YOUR_ECR_ACCESS_KEY_ID
awsSecretAccessKey: YOUR_ECR_SECRET_ACCESS_KEY
# to update role's permission refered to as resourceName and patch secrets with these names and restart deployment
targetSecretNames:
- yatai-image-builder-env
- test-env
targetDeploymentNames:
- yatai-image-builder
# etc
resources:
limits:
cpu: 50m
memory: 16Mi
requests:
cpu: 100m
memory: 32Mi
cronjob을 통해 수행할 쉘 스크립트를 워킹디렉토리에 작성하자.
(아래에 스크립트는 간단히 짠 것이므로 효율적으로 코드로 수정하면 더 좋음.)
$ cd $WORKDIR
touch run.sh
run.sh
#!/bin/bash
############ variables #############
CA_CERTIFICATE='/var/run/secrets/kubernetes.io/serviceaccount/ca.crt'
BEARER_TOKEN=`cat /var/run/secrets/kubernetes.io/serviceaccount/token`
NAMESPACE=`cat /var/run/secrets/kubernetes.io/serviceaccount/namespace`
while getopts "s:d:" option;
do
case $option in
s)
TARGET_SECRET_NAMES=($OPTARG)
;;
d)
TARGET_DEPLOYMENT_NAMES=($OPTARG)
;;
esac
done
echo "[$(date +%Y-%m-%dT%H:%M:%S%z)][INFO][PRINT_VAR][CA_CERTIFICATE] $CA_CERTIFICATE"
echo "[$(date +%Y-%m-%dT%H:%M:%S%z)][INFO][PRINT_VAR][BEARER_TOKEN] $BEARER_TOKEN"
echo "[$(date +%Y-%m-%dT%H:%M:%S%z)][INFO][PRINT_VAR][NAMESPACE] $NAMESPACE"
echo "[$(date +%Y-%m-%dT%H:%M:%S%z)][INFO][PRINT_VAR][TARGET_SECRET_NAMES] ${TARGET_SECRET_NAMES[@]}"
echo "[$(date +%Y-%m-%dT%H:%M:%S%z)][INFO][PRINT_VAR][TARGET_DEPLOYMENT_NAMES] ${TARGET_DEPLOYMENT_NAMES[@]}"
####################################
####### aws-cli-get-ecr-cred #######
ECR_PUSH_TOKEN=$(aws ecr get-login-password --region ap-northeast-2)
echo "[$(date +%Y-%m-%dT%H:%M:%S%z)][INFO][PRINT_VAR][ECR_PUSH_TOKEN] $ECR_PUSH_TOKEN"
####################################
########### patch-secret ###########
for TARGET_SECRET_NAME in "${TARGET_SECRET_NAMES[@]}"
do
echo [$(date +%Y-%m-%dT%H:%M:%S%z)][INFO][PATCH_SECRET][START] $TARGET_SECRET_NAME
curl -v \
--cacert $CA_CERTIFICATE \
-H "Authorization: Bearer $BEARER_TOKEN" \
-H "Content-Type: application/json-patch+json" \
-X PATCH \
-d '[{"op": "replace", "path": "/data/DOCKER_REGISTRY_PASSWORD", "value": "'"$(echo -n $ECR_PUSH_TOKEN | base64 | tr -d '[:space:]')"'"}]' \
https://kubernetes.default.svc/api/v1/namespaces/$NAMESPACE/secrets/$TARGET_SECRET_NAME
echo [$(date +%Y-%m-%dT%H:%M:%S%z)][INFO][PATCH_SECRET][END] $TARGET_SECRET_NAME
done
####################################
######## restart-deployment ########
for TARGET_DEPLOYMENT_NAME in "${TARGET_DEPLOYMENT_NAMES[@]}"
do
echo [$(date +%Y-%m-%dT%H:%M:%S%z)][INFO][RESTART_DEPLOYMENT][START] $TARGET_DEPLOYMENT_NAME
curl -v \
--cacert $CA_CERTIFICATE \
-H "Authorization: Bearer $BEARER_TOKEN" \
-H "Content-Type: application/strategic-merge-patch+json" \
-X PATCH \
-d '{"spec":{"template":{"metadata":{"annotations":{"kubectl.kubernetes.io/restartedAt":"'"$(date +%Y-%m-%dT%H:%M:%S%z)"'"}}}}}' \
https://kubernetes.default.svc/apis/apps/v1/namespaces/$NAMESPACE/deployments/$TARGET_DEPLOYMENT_NAME \
echo [$(date +%Y-%m-%dT%H:%M:%S%z)][INFO][RESTART_DEPLOYMENT][END] $TARGET_DEPLOYMENT_NAME
done
####################################
스크립트 실행 명령어
./run.sh -d $TARGET_DEPLOYMENT_NAMES -s $TARGET_SECRET_NAMES
### 배포
$ helm upgrade --install \
yatai-ecr-refresher . \
-n yatai-image-builder \
--set ecrRegistry.awsAccessKeyID=<자신의 key 넣기> \
--set ecrRegistry.awsSecretAccessKey=<자신의 key 넣기>
### 확인
$ helm ls -l name=yatai-ecr-refresher -n yatai-image-builder
NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION
yatai-ecr-refresher yatai-image-builder 1 2023-03-03 10:52:54.374523 +0900 KST deployed ecr-refresher-0.1.0 1.16.0
$ k get configmap,secret,sa,role,rolebinding,deployment,cronjob,job,pod -n yatai-image-builder -l app.kubernetes.io/instance=yatai-ecr-refresher
NAME DATA AGE
configmap/yatai-ecr-refresher-env-configmap 3 112s
NAME TYPE DATA AGE
secret/yatai-ecr-refresher-env-secret Opaque 3 112s
NAME SECRETS AGE
serviceaccount/yatai-ecr-refresher-serviceaccount 1 112s
NAME CREATED AT
role.rbac.authorization.k8s.io/yatai-ecr-refresher-update-role 2023-03-03T01:52:54Z
NAME ROLE AGE
rolebinding.rbac.authorization.k8s.io/yatai-ecr-refresher-rolebinder Role/yatai-ecr-refresher-update-role 113s
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/yatai-ecr-refresher-deployment 1/1 1 1 113s
NAME SCHEDULE SUSPEND ACTIVE LAST SCHEDULE AGE
cronjob.batch/yatai-ecr-refresher-cronjob 30 7-21/4 * * * False 0 <none> 113s
NAME READY STATUS RESTARTS AGE
pod/yatai-ecr-refresher-deployment-7b9db55856-97944 1/1 Running 0 114s