#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

## Credits to Franck Cuny : https://github.com/fcuny/distributedlog-on-k8s/blob/master/bookkeeper.statefulset.yaml

# SSDs for bookie journal storage
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: bookie-ssd
provisioner: kubernetes.io/gce-pd
parameters:
  type: pd-ssd
  zones: us-east1-b
---
# HDDs for bookie ledger storage
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: bookie-hdd
provisioner: kubernetes.io/gce-pd
parameters:
  type: pd-standard
  zones: us-east1-b
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: bookie-config
data:
  PULSAR_MEM: "\" -Xms4g -Xmx4g -XX:MaxDirectMemorySize=4g\""
  PULSAR_GC: "\" -XX:+UseG1GC \""
  dbStorage_writeCacheMaxSizeMb: "1024"
  dbStorage_readAheadCacheMaxSizeMb: "1024"
  zkServers: zk-0.zookeeper,zk-1.zookeeper,zk-2.zookeeper
  statsProviderClass: org.apache.bookkeeper.stats.prometheus.PrometheusMetricsProvider
  useHostNameAsBookieID: "true"
---
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
  name: bookkeeper
  labels:
    app: pulsar
    component: bookkeeper
spec:
  serviceName: bookkeeper
  replicas: 3
  template:
    metadata:
      labels:
        app: pulsar
        component: bookkeeper
        cluster: us-east
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "8000"
    spec:
      terminationGracePeriodSeconds: 0
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - weight: 1
              podAffinityTerm:
                topologyKey: "kubernetes.io/hostname"
                labelSelector:
                  matchExpressions:
                    - key: "app"
                      operator: In
                      values:
                        - bookkeeper
      initContainers:
        - name: bookie-metaformat
          image: apachepulsar/pulsar:latest
          command: ["sh", "-c"]
          args:
            - >
              bin/apply-config-from-env.py conf/bookkeeper.conf &&
              bin/bookkeeper shell metaformat --nonInteractive || true;
          envFrom:
            - configMapRef:
                name: bookie-config
      containers:
        - name: bookie
          image: apachepulsar/pulsar:latest
          command: ["sh", "-c"]
          args:
            - >
              bin/apply-config-from-env.py conf/bookkeeper.conf &&
              bin/apply-config-from-env.py conf/pulsar_env.sh &&
              bin/pulsar bookie
          ports:
            - containerPort: 3181
              hostPort: 3181
              name: client
          envFrom:
            - configMapRef:
                name: bookie-config
          volumeMounts:
            - name: journal-disk
              mountPath: /pulsar/data/bookkeeper/journal
            - name: ledgers-disk
              mountPath: /pulsar/data/bookkeeper/ledgers
      initContainers:
        # The first time, initialize BK metadata in zookeeper
        # Otherwise ignore error if it's already there
        - name: bookie-metaformat
          image: apachepulsar/pulsar:latest
          command: ["sh", "-c"]
          args:
            - >
              bin/apply-config-from-env.py conf/bookkeeper.conf &&
              bin/bookkeeper shell metaformat --nonInteractive || true;
          envFrom:
            - configMapRef:
                name: bookie-config
  volumeClaimTemplates:
    - metadata:
        name: journal-disk
        annotations:
          volume.alpha.kubernetes.io/storage-class: default
        labels:
          component: bookkeeper
      spec:
        accessModes: [ "ReadWriteOnce" ]
        resources:
          requests:
            storage: 10Gi
        storageClassName: bookie-ssd
    - metadata:
        name: ledgers-disk
        annotations:
          volume.alpha.kubernetes.io/storage-class: default
        labels:
          component: bookkeeper
      spec:
        accessModes: [ "ReadWriteOnce" ]
        resources:
          requests:
            storage: 10Gi
        storageClassName: bookie-hdd


---
##
## Bookie headless service.
## This is required for resolving intra-cluser hostnames (via k8s DNS)
## from the brokers.
##
apiVersion: v1
kind: Service
metadata:
    name: bookkeeper
    labels:
        app: pulsar
        component: bookkeeper
spec:
    ports:
      - port: 3181
        name: server
    clusterIP: None
    selector:
        app: pulsar
        component: bookkeeper

---
##
## Run BookKeeper auto-recovery from a different set of containers
## Auto-Recovery makes sure to restore the replication factor when any bookie
## crashes and it's not recovering on its own.
##
apiVersion: apps/v1beta1
kind: Deployment
metadata:
  name: bookie-autorecovery
spec:
  replicas: 2
  template:
    metadata:
      labels:
        app: pulsar
        component: bookkeeper-replication
    spec:
      containers:
        - name: replication-worker
          image: apachepulsar/pulsar:latest
          command: ["sh", "-c"]
          args:
            - >
              bin/apply-config-from-env.py conf/bookkeeper.conf &&
              bin/bookkeeper autorecovery
          envFrom:
            - configMapRef:
                name: bookie-config
          env:
              ## Configure for lower mem usage
            - name: PULSAR_MEM
              value: "\" -Xmx256m \""
            - name: PULSAR_GC
              value: "\"  \""
