0. 실행환경

AWS EC2 t2.xlarge
OS : Ubuntu 22.04
Kafka :
Docker Compose : v2.7.0

1. 실행

1) docker-compose.yml 생성

vi docker-compose.yml

version: '2'
services:
  zookeeper-1:
    image: confluentinc/cp-zookeeper:latest
    environment:
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_INIT_LIMIT: 5
      ZOOKEEPER_SYNC_LIMIT: 2
    ports:
      - "22181:2181"
  
  zookeeper-2:
    image: confluentinc/cp-zookeeper:latest
    environment:
      ZOOKEEPER_SERVER_ID: 2
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_INIT_LIMIT: 5
      ZOOKEEPER_SYNC_LIMIT: 2
    ports:
      - "32181:2181"

  zookeeper-3:
    image: confluentinc/cp-zookeeper:latest
    environment:
      ZOOKEEPER_SERVER_ID: 3
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_INIT_LIMIT: 5
      ZOOKEEPER_SYNC_LIMIT: 2
    ports:
      - "42181:2181"


  
  kafka-1:
    image: confluentinc/cp-kafka:latest
    depends_on:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3
    ports:
      - 29092:29092
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1:9092,PLAINTEXT_HOST://localhost:29092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      
  kafka-2:
    image: confluentinc/cp-kafka:latest
    depends_on:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3
    ports:
      - "39092:39092"
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-2:9092,PLAINTEXT_HOST://localhost:39092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1

  kafka-3:
    image: confluentinc/cp-kafka:latest
    depends_on:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3
    ports:
      - "49092:49092"
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-3:9092,PLAINTEXT_HOST://localhost:49092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1

# docker-compose 설정 변수
depends_on : 서비스들의 우선순위 지정. depends_on에 설정된 서비스가 실행되어야 해당 서비스가 올라간다.
environment: 환경변수 설정

# Kafka 설정 변수
KAFKA_BROKER_ID: 유일값이어야함. 
KAFKA_ZOOKEEPER_CONNECT: zookeeper 지정
KAFKA_ADVERTISED_LISTENERS: 외부에서 접속용 리스너 설정
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 보안을 위한 프로토콜 매핑. 여기 설정 값이 KAFKA_ADVERTISED_LISTENERS 과 함께 key-value로 매핑됨.
KAFKA_INTER_BROKER_LISTENER_NAME: 도커 내부에서 리스너 이름을 지정
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 트랜잭션 상태에서 복제 수
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 트랜잭션 최소 ISR(InSyncReplicas 설정) 수

2) docker compose 실행

docker-compose -f docker-compose.yml up -d

3) 토픽 생성

docker-compose exec kafka-1 kafka-topics --create --topic test-topic --bootstrap-server kafka-1:9092 --replication-factor 3 --partitions 2

=> Created topic test-topic.

--bootstrap-server <service:port> : 클라이언트가 접근하는 토픽 파티션의 메타데이터를 요청하기 위한 설정
--replication-factor : 토픽 복제 수
--partition: 토픽내에 파티션 수

4) 토픽 확인

docker-compose exec kafka-1 kafka-topics --describe --topic test-topic --bootstrap-server kafka-1:9092 

=>
Topic: test-topic	TopicId: zrU8TR3IQu2l24nQkYZ1jA	PartitionCount: 2	ReplicationFactor: 3	Configs:
	Topic: test-topic	Partition: 0	Leader: 3	Replicas: 3,1,2	Isr: 3,1,2
	Topic: test-topic	Partition: 1	Leader: 1	Replicas: 1,2,3	Isr: 1,2,3
        

Leader : 파티션의 리더 브로커
Replicas : 데이터 복제
Isr : In sync replica (동기화된 복제본)

5) 컨슈머 실행

docker-compose exec kafka-1 bash
[appuser@6e847b6b1748 ~]$ kafka-console-consumer --topic test-topic --bootstrap-server kafka-1:9092

6) producer 실행

$ docker-compose exec kafka-1 bash 
[appuser@6e847b6b1748 ~]$ kafka-console-producer --topic test-topic --broker-list kafka-1:9092

producer

consumer

3. Replication 수 변경