1.카프카란
% wget https://downloads.apache.org/kafka/2.8.1/kafka_2.12-2.8.1.tgz
% tar -xvzf kafka_2.12-2.8.1.tgz
% cp -r kafka_2.12-2.8.1 kafka_1
% cp -r kafka_2.12-2.8.1 kafka_2
% cp -r kafka_2.12-2.8.1 kafka_3
% mkdir logs_1
% mkdir logs_2
% mkdir logs_3
% mkdir data
% cd data
% mkdir zookeeper_1
% mkdir zookeeper_2
% mkdir zookeeper_3
% echo 1 > zookeeper_1/myid
% echo 2 > zookeeper_2/myid
% echo 3 > zookeeper_3/myid
~
dataDir=/Users/publicai/kafka/data/zookeeper_1
~
clientPort=2181
~
# admin.serverPort=8080
tickTime=2000
initLimit=5
syncLimit=2
server.1=localhost:2666:3666
server.2=localhost:2667:3667
server.3=localhost:2668:3668
# The id of the broker. This must be set to a unique integer for each broker.
broker.id=1
..
############################# Socket Server Settings #############################
listeners=PLAINTEXT://localhost:9092
...
############################# Log Basics #############################
log.dirs=/Users/publicai/kafka/logs_1
...
############################# Zookeeper #############################
zookeeper.connect=localhost:2181,localhost:2182,localhost:2183
-- 주키퍼 실행
bin/zookeeper-server-start.sh config/zookeeper.properties
-- kafka 실행
bin/kafka-server-start.sh config/server.properties
주피터 터미널 3개
kafka 터미널 3개
% bin/kafka-topics.sh --create --zookeeper localhost:2181, localhost:2182, \
localhost:2183 --replication-factor 2 --partitions 1 --topic dataengineering
아래 메시지가 뜨면 토픽 생성 완료
bin/kafka-topics.sh --list --zookeeper localhost:2181, localhost:2182, localhost:2183
bin/kafka-console-producer.sh --broker-list localhost:9092, localhost:9093, localhost:9094 --topic dataengineering
bin/kafka-console-consumer.sh --bootstrap-server localhost:9092, localhost:9093, localhost:9094 --topic dataengineering --from-beginning
생산자 : 왼쪽, 소비자 : 오른쪽