feat: replication example

2026-01-25 04:56:42 +00:00
parent 15c93c71fb
commit 0beb588a70
5 changed files with 178 additions and 0 deletions
@@ -0,0 +1,107 @@
+services:
+    kafka-1:
+        image: apache/kafka:latest
+        container_name: kafka-1
+        hostname: kafka-1
+        ports:
+            - "9092:9092" # External Client Port
+        environment:
+            KAFKA_NODE_ID: 1
+            KAFKA_PROCESS_ROLES: broker,controller
+
+            # --- NETWORKING CONFIGS ---
+            # 1. Listeners: Where the process listens inside the container
+            # CONTROLLER:9093, INTERNAL:29092, EXTERNAL:9092
+            KAFKA_LISTENERS: CONTROLLER://:9093,INTERNAL://:29092,EXTERNAL://:9092
+
+            # 2. Advertised Listeners: What the broker tells others to connect to
+            # Internal -> within docker network, this network is made by docker-compose
+            # see `sudo docker network ls`
+            # External -> for my python script
+
+            # these are the two ways you can talk to me
+            KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-1:29092,EXTERNAL://localhost:9092
+
+            # 3. Security Map: Define the protocol for the new names
+            KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
+
+            # 4. Inter-broker communication MUST use the INTERNAL listener
+            KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL # when brokers talk to each other, use INTERNAL
+            KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER # controller uses CONTROLLER listener
+
+            # Quorum voters use the Internal Docker DNS
+            KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka-1:9093,2@kafka-2:9093,3@kafka-3:9093
+
+            KAFKA_LOG_DIRS: /var/lib/kafka/data
+
+            # --- REPLICATION DEFAULTS ---
+            KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
+            KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
+            KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
+            KAFKA_NUM_PARTITIONS: 3
+            KAFKA_DEFAULT_REPLICATION_FACTOR: 3
+            KAFKA_MIN_INSYNC_REPLICAS: 2
+        volumes:
+            - ./kafka-1-data:/var/lib/kafka/data
+
+    kafka-2:
+        image: apache/kafka:latest
+        container_name: kafka-2
+        hostname: kafka-2
+        ports:
+            - "9094:9092" # Host:9094 -> Container:9092
+        environment:
+            KAFKA_NODE_ID: 2
+            KAFKA_PROCESS_ROLES: broker,controller
+
+            # Listeners
+            KAFKA_LISTENERS: CONTROLLER://:9093,INTERNAL://:29092,EXTERNAL://:9092
+            # Advertised: Internal uses kafka-2, External uses localhost:9094
+            KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-2:29092,EXTERNAL://localhost:9094
+
+            KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
+            KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
+            KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+            KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka-1:9093,2@kafka-2:9093,3@kafka-3:9093
+
+            KAFKA_LOG_DIRS: /var/lib/kafka/data
+            # --- SAME DEFAULTS ---
+            KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
+            KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
+            KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
+            KAFKA_NUM_PARTITIONS: 3
+            KAFKA_DEFAULT_REPLICATION_FACTOR: 3
+            KAFKA_MIN_INSYNC_REPLICAS: 2
+        volumes:
+            - ./kafka-2-data:/var/lib/kafka/data
+
+    kafka-3:
+        image: apache/kafka:latest
+        container_name: kafka-3
+        hostname: kafka-3
+        ports:
+            - "9096:9092" # Host:9096 -> Container:9092
+        environment:
+            KAFKA_NODE_ID: 3
+            KAFKA_PROCESS_ROLES: broker,controller
+
+            # Listeners
+            KAFKA_LISTENERS: CONTROLLER://:9093,INTERNAL://:29092,EXTERNAL://:9092
+            # Advertised: Internal uses kafka-3, External uses localhost:9096
+            KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-3:29092,EXTERNAL://localhost:9096
+
+            KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
+            KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
+            KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+            KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka-1:9093,2@kafka-2:9093,3@kafka-3:9093
+
+            KAFKA_LOG_DIRS: /var/lib/kafka/data
+            # --- SAME DEFAULTS ---
+            KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
+            KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
+            KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
+            KAFKA_NUM_PARTITIONS: 3
+            KAFKA_DEFAULT_REPLICATION_FACTOR: 3
+            KAFKA_MIN_INSYNC_REPLICAS: 2
+        volumes:
+            - ./kafka-3-data:/var/lib/kafka/data