Kafka - yeardream-de-project-team4/k8s_project GitHub Wiki

kafka architecture in k8s

---

producer

• producer.jar

git clone --branch feature/kafka https://github.com/yeardream-de-project-team4/k8s_project.git

• producer.yaml

apiVersion: batch/v1
kind: Job
metadata:
  name: airport-producer-job
spec:
  template:
    spec:
      restartPolicy: Never
      containers:
      - name: my-container
        image: 나의 producer image
        volumeMounts:
        - name: nfs-volume
          mountPath: /app/resources
      volumes:
      - name: nfs-volume
        persistentVolumeClaim:
          claimName: my-nfs-pvc

---

broker

※ zookeeper 및 Kafka의 config는 env 부분에 각자 원하는대로 설정하면 된다. 참고로 아래는 기본적인 설정만 되어있다.

• zookeeper.yaml

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: zookeeper-statefulset
  labels:
    app: zookeeper
spec:
  replicas: 3
  serviceName: zksvc
  selector:
    matchLabels:
      app: zookeeper
  template:
    metadata:
      labels:
        app: zookeeper
    spec:
      containers:
      - name: zookeeper
        image: confluentinc/cp-zookeeper:7.0.1
        ports:
        - containerPort: 2181
        env:
        - name: ZOOKEEPER_CLIENT_PORT
          value: "2181"
        - name: ZOOKEEPER_TICK_TIME
          value: "2000"
  volumeClaimTemplates:
  - metadata:
      name: datadir
    spec:
      accessModes: [ "ReadWriteOnce" ]
      storageClassName: "zoo-pv"
      resources:
        requests:
          storage: 10Gi

---
apiVersion: v1
kind: Service
metadata:
  name: zksvc
spec:
  type: ClusterIP
  clusterIP: None
  selector:
    app: zookeeper
  ports:
    - protocol: TCP
      port: 2181
      targetPort: 2181

• zookeeper_pv.yaml

apiVersion: v1
kind: PersistentVolume
metadata:
  name: zoo-pv-0
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Delete
  storageClassName: 'zoo-pv'
  hostPath:
    path: /data/
    type: DirectoryOrCreate
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: zoo-pv-1
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Delete
  storageClassName: 'zoo-pv'
  hostPath:
    path: /data/
    type: DirectoryOrCreate
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: zoo-pv-2
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Delete
  storageClassName: 'zoo-pv'
  hostPath:
    path: /data/
    type: DirectoryOrCreate

• Kafka.yaml

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: kafka-statefulset
  labels:
    app: kafka
spec:
  replicas: 3
  serviceName: kafka-service
  selector:
    matchLabels:
      app: kafka
  template:
    metadata:
      labels:
        app: kafka
    spec:
      containers:
      - name: broker
        image: confluentinc/cp-kafka:7.0.1
        ports:
        - containerPort: 9092
        env:
        - name: MY_POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: KAFKA_ZOOKEEPER_CONNECT
          value: zookeeper-statefulset-0.zksvc:2181,zookeeper-statefulset-1.zksvc:2181,zookeeper-statefulset-2.zksvc:2181
        - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
          value: PLAINTEXT:PLAINTEXT,PLAINTEXT_INTERNAL:PLAINTEXT
        - name: KAFKA_ADVERTISED_LISTENERS
          value: PLAINTEXT://:29092,PLAINTEXT_INTERNAL://$(MY_POD_NAME).kafka-service:9092
        - name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR
          value: "1"
        - name: KAFKA_TRANSACTION_STATE_LOG_MIN_ISR
          value: "1"
        - name: KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR
          value: "1"
  volumeClaimTemplates:
  - metadata:
      name: kafkadir
    spec:
      accessModes: ["ReadWriteOnce"]
      storageClassName: "kafka"
      resources:
        requests:
          storage: 10Gi

---

apiVersion: v1
kind: Service
metadata:
  name: kafka-service
spec:
  type: ClusterIP
  clusterIP: None
  selector:
    app: kafka
  ports:
    - protocol: TCP
      port: 9092
      targetPort: 9092

• Kafka_pv.yaml

apiVersion: v1
kind: PersistentVolume
metadata:
  name: kafka-0
  namespace: kafka-ns
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Delete
  storageClassName: "kafka"
  hostPath:
    path: /data/
    type: DirectoryOrCreate
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: kafka-1
  namespace: kafka-ns
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Delete
  storageClassName: "kafka"
  hostPath:
    path: /data/
    type: DirectoryOrCreate
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: kafka-2
  namespace: kafka-ns
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Delete
  storageClassName: "kafka"
  hostPath:
    path: /data/
    type: DirectoryOrCreate

---

consumer

• consumer.jar

git clone --branch feature/kafka https://github.com/yeardream-de-project-team4/k8s_project.git

• consumer.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: airport-consumer
spec:
  replicas: 3
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
        - name: airport-consumer
          image: 나의 consumer image
          ports:
            - containerPort: 80

---