Pormetheus, Grafana EKS 배포

Prometheus git url

git clone https://prometheus-community.github.io/helm-charts
cd ./helm-charts/charts/prometheus

values.yaml에서 service.type : NodePort변경

배포 후 target group health check "/-/healthy" 으로 변경

kubectl create ns prometheus
kubectl config set-context --current --namespace=monitoring
helm install -n prometheus prometheus-community/prometheus  -f values.yaml

Add Target Prometheus

kubectl edit cm prometheus-server -n monitoring

- job_name : partner-api
	metrics_path: '/metrics'
	static_configs:
	- targets: ['service.mall.svc.cluster.local:80']
- job_name : mall-api
	static_configs:
	- targets: ['service.mall.svc.cluster.local:80']

kubectl rollout restart deployment prometheus-server -n monitoring

Grafana

---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  namespace: monitoring
  name: grafana-pvc
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 1Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: grafana
  name: grafana
  namespace: monitoring
spec:
  selector:
    matchLabels:
      app: grafana
  template:
    metadata:
      labels:
        app: grafana
    spec:
      securityContext:
        fsGroup: 472
        supplementalGroups:
          - 0
      containers:
        - name: grafana
          image: grafana/grafana:latest
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 3000
              name: http-grafana
              protocol: TCP
          readinessProbe:
            failureThreshold: 3
            httpGet:
              path: /robots.txt
              port: 3000
              scheme: HTTP
            initialDelaySeconds: 10
            periodSeconds: 30
            successThreshold: 1
            timeoutSeconds: 2
          livenessProbe:
            failureThreshold: 3
            initialDelaySeconds: 30
            periodSeconds: 10
            successThreshold: 1
            tcpSocket:
              port: 3000
            timeoutSeconds: 1
          resources:
            requests:
              cpu: 250m
              memory: 750Mi
          volumeMounts:
            - mountPath: /var/lib/grafana
              name: grafana-pv
      volumes:
        - name: grafana-pv
          persistentVolumeClaim:
            claimName: grafana-pvc
---
apiVersion: v1
kind: Service
metadata:
  namespace: monitoring
  name: grafana
spec:
  ports:
    - port: 80
      protocol: TCP
      targetPort: 3000
  selector:
    app: grafana
  sessionAffinity: None
  type: ClusterIP

k apply -f grafana.yaml

Deploy Ingress ALB

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  namespace: monitoring
  name: monitoring-ingress
  annotations:
    # Ingress Core Settings    
    alb.ingress.kubernetes.io/scheme: internet-facing
    alb.ingress.kubernetes.io/target-type: ip
    alb.ingress.kubernetes.io/load-balancer-name: monitoring-ingress
    alb.ingress.kubernetes.io/target-group-attributes: stickiness.enabled=true,stickiness.lb_cookie.duration_seconds=60
    # SSL Settings
    alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]'
    alb.ingress.kubernetes.io/certificate-arn: 
    alb.ingress.kubernetes.io/ssl-redirect: '443'

spec:
  ingressClassName: alb
  rules:
  - host: prometheus.example.com
    http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: prometheus-server
                port:
                  number: 80
  - host: grafana.example.com
    http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: grafana
                port:
                  number: 80

grafana admin password reset

kubectl exec --namespace monitoring -it loki-stack-grafana-7fc6b6db4d-p8zx7 -c grafana grafana cli admin reset-admin-password helloworld