services: kafka-1: image: apache/kafka:latest container_name: kafka-1 hostname: kafka-1 ports: - "9092:9092" # External Client Port environment: KAFKA_NODE_ID: 1 KAFKA_PROCESS_ROLES: broker,controller # --- NETWORKING CONFIGS --- # 1. Listeners: Where the process listens inside the container # CONTROLLER:9093, INTERNAL:29092, EXTERNAL:9092 KAFKA_LISTENERS: CONTROLLER://:9093,INTERNAL://:29092,EXTERNAL://:9092 # 2. Advertised Listeners: What the broker tells others to connect to # Internal -> within docker network, this network is made by docker-compose # see `sudo docker network ls` # External -> for my python script # these are the two ways you can talk to me KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-1:29092,EXTERNAL://localhost:9092 # 3. Security Map: Define the protocol for the new names KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT # 4. Inter-broker communication MUST use the INTERNAL listener KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL # when brokers talk to each other, use INTERNAL KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER # controller uses CONTROLLER listener # Quorum voters use the Internal Docker DNS KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka-1:9093,2@kafka-2:9093,3@kafka-3:9093 KAFKA_LOG_DIRS: /var/lib/kafka/data # --- REPLICATION DEFAULTS --- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2 KAFKA_NUM_PARTITIONS: 3 KAFKA_DEFAULT_REPLICATION_FACTOR: 3 KAFKA_MIN_INSYNC_REPLICAS: 2 volumes: - ./kafka-1-data:/var/lib/kafka/data kafka-2: image: apache/kafka:latest container_name: kafka-2 hostname: kafka-2 ports: - "9094:9092" # Host:9094 -> Container:9092 environment: KAFKA_NODE_ID: 2 KAFKA_PROCESS_ROLES: broker,controller # Listeners KAFKA_LISTENERS: CONTROLLER://:9093,INTERNAL://:29092,EXTERNAL://:9092 # Advertised: Internal uses kafka-2, External uses localhost:9094 KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-2:29092,EXTERNAL://localhost:9094 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka-1:9093,2@kafka-2:9093,3@kafka-3:9093 KAFKA_LOG_DIRS: /var/lib/kafka/data # --- SAME DEFAULTS --- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2 KAFKA_NUM_PARTITIONS: 3 KAFKA_DEFAULT_REPLICATION_FACTOR: 3 KAFKA_MIN_INSYNC_REPLICAS: 2 volumes: - ./kafka-2-data:/var/lib/kafka/data kafka-3: image: apache/kafka:latest container_name: kafka-3 hostname: kafka-3 ports: - "9096:9092" # Host:9096 -> Container:9092 environment: KAFKA_NODE_ID: 3 KAFKA_PROCESS_ROLES: broker,controller # Listeners KAFKA_LISTENERS: CONTROLLER://:9093,INTERNAL://:29092,EXTERNAL://:9092 # Advertised: Internal uses kafka-3, External uses localhost:9096 KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-3:29092,EXTERNAL://localhost:9096 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka-1:9093,2@kafka-2:9093,3@kafka-3:9093 KAFKA_LOG_DIRS: /var/lib/kafka/data # --- SAME DEFAULTS --- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2 KAFKA_NUM_PARTITIONS: 3 KAFKA_DEFAULT_REPLICATION_FACTOR: 3 KAFKA_MIN_INSYNC_REPLICAS: 2 volumes: - ./kafka-3-data:/var/lib/kafka/data