Docker Compose 로 Kafka KRaft 서버 올리기

3개의 브로커(broker)와 스키마 레지스트리(schema-registry) 서버를 구축하는 docker compose 예제

명령어

• 실행 (https://docs.docker.com/engine/reference/commandline/compose_up/)

docker compose up --build --force-recreate // 필요에 맞는 옵션을 사용한다.

• 중단 (https://docs.docker.com/engine/reference/commandline/compose_down/)

docker-compose down --volumes // 필요에 맞는 옵션을 사용한다.

• CLUSTER_ID 생성 명령어

kafka-storage random-uuid

docker-compose.yml

version: "3"
services:
  broker-1:
    image: confluentinc/cp-kafka:latest
    container_name: broker-1
    hostname: broker-1
    ports:
      - "9092:9092"
    environment:
      KAFKA_NODE_ID: 1
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_LISTENERS: INTERNAL://broker-1:29092,CONTROLLER://broker-1:29093,EXTERNAL://0.0.0.0:9092
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker-1:29092,EXTERNAL://localhost:9092
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@broker-1:29093,2@broker-2:29093,3@broker-3:29093
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
      CLUSTER_ID: RCukC15VQ6SaOHhAvJec6g // kafka-storage random-uuid 를 사용해 만들어준다.
      KAFKA_LOG_DIRS: /tmp/broker-logs
    networks:
      - kafka-network
  broker-2:
    image: confluentinc/cp-kafka:latest
    container_name: broker-2
    hostname: broker-2
    ports:
      - "9093:9093"
    environment:
      KAFKA_NODE_ID: 2
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_LISTENERS: INTERNAL://broker-2:29092,CONTROLLER://broker-2:29093,EXTERNAL://0.0.0.0:9093
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker-2:29092,EXTERNAL://localhost:9093
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@broker-1:29093,2@broker-2:29093,3@broker-3:29093
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
      CLUSTER_ID: RCukC15VQ6SaOHhAvJec6g // kafka-storage random-uuid 를 사용해 만들어준다.
      KAFKA_LOG_DIRS: /tmp/broker-logs
    networks:
      - kafka-network
  broker-3:
    image: confluentinc/cp-kafka:latest
    container_name: broker-3
    hostname: broker-3
    ports:
      - "9094:9094"
    environment:
      KAFKA_NODE_ID: 3
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_LISTENERS: INTERNAL://broker-3:29092,CONTROLLER://broker-3:29093,EXTERNAL://0.0.0.0:9094
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker-3:29092,EXTERNAL://localhost:9094
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@broker-1:29093,2@broker-2:29093,3@broker-3:29093
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
      CLUSTER_ID: RCukC15VQ6SaOHhAvJec6g // kafka-storage random-uuid 를 사용해 만들어준다.
      KAFKA_LOG_DIRS: /tmp/broker-logs
    networks:
      - kafka-network
  schema-registry:
    image: confluentinc/cp-schema-registry:latest
    container_name: schema-registry
    hostname: schema-registry
    ports:
      - "8081:8081"
    environment:
      # Advertised host name, required for intercommunication between multiple instances of Schema Registry.
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      # List of Kafka brokers to connect to.
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: broker-1:29092,broker-2:29092,broker-3:29092
      # Comma-separated list of listeners for API requests supporting HTTP or HTTPS.
      SCHEMA_REGISTRY_LISTENERS: http://schema-registry:8081
    depends_on:
      - broker-1
      - broker-2
      - broker-3
    networks:
      - kafka-network
  rest-proxy:
    image: confluentinc/cp-kafka-rest
    container_name: rest-proxy
    hostname: rest-proxy
    ports:
      - "8082:8082"
    environment:
      KAFKA_REST_LISTENERS: http://rest-proxy:8082
      KAFKA_REST_SCHEMA_REGISTRY_URL: http://schema-registry:8081
      KAFKA_REST_HOST_NAME: rest-proxy
      KAFKA_REST_BOOTSTRAP_SERVERS: broker-1:29092,broker-2:29092,broker-3:29092
    depends_on:
      - broker-1
      - broker-2
      - broker-3
      - schema-registry
    networks:
      - kafka-network 
networks:
  kafka-network: // 원하는 네크워크명
    driver: bridge
volumes:
  kafka-volume: // 원하는 볼륨명