#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

## Credits to Franck Cuny : https://github.com/fcuny/distributedlog-on-k8s/blob/master/bookkeeper.statefulset.yaml

# SSDs for bookie journal storage
kind: StorageClass
apiVersion: storage.k8s.io/v1beta1
metadata:
  name: bookie-ssd
provisioner: kubernetes.io/aws-ebs
parameters:
  type: gp2
  zones: us-east-2a
  fsType: xfs
#  To create encrypted ebs volume using kms
#  encrypted: "true"
#  kmsKeyId: <enter the key id here>
reclaimPolicy: Retain
---
# HDDs for bookie ledger storage
kind: StorageClass
apiVersion: storage.k8s.io/v1beta1
metadata:
  name: bookie-hdd
provisioner: kubernetes.io/aws-ebs
parameters:
  type: st1
  zones: us-east-2a
  fsType: xfs
#  To create encrypted ebs volume using kms
#  encrypted: "true"
#  kmsKeyId: <enter the key id here>
reclaimPolicy: Retain
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: bookie-config
data:
  PULSAR_MEM: "\"-Dio.netty.leakDetectionLevel=disabled -Dio.netty.recycler.linkCapacity=1024 -XX:+UseG1GC -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB -XX:+ExitOnOutOfMemoryError -XX:+PerfDisableSharedMem -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintHeapAtGC -verbosegc -XX:G1LogLevel=finest -Xms28g -Xmx28g -XX:MaxDirectMemorySize=28g\""
  dbStorage_writeCacheMaxSizeMb: "2048" # Write cache size (direct memory)
  dbStorage_readAheadCacheMaxSizeMb: "2048" # Read cache size (direct memory)
  dbStorage_rocksDB_blockCacheSize: "4294967296"
  journalMaxSizeMB: "2048"
  zkServers: zk-0.zookeeper,zk-1.zookeeper,zk-2.zookeeper
  statsProviderClass: org.apache.bookkeeper.stats.prometheus.PrometheusMetricsProvider
  useHostNameAsBookieID: "true"
---
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
    name: bookkeeper
    labels:
        app: pulsar
        component: bookkeeper
        role: bookie
spec:
    serviceName: bookkeeper
    replicas: 2
    template:
        metadata:
            labels:
                app: pulsar
                component: bookkeeper
                # Specify cluster to allow aggregation by cluster in
                # the metrics
                cluster: us-east
                role: bookie
            annotations:
                prometheus.io/scrape: "true"
                prometheus.io/port: "8000"

        spec:
            containers:
              - name: bookie
                image: apachepulsar/pulsar:latest
                command: ["sh", "-c"]
                args:
                  - >
                    bin/apply-config-from-env.py conf/bookkeeper.conf &&
                    bin/apply-config-from-env.py conf/pulsar_env.sh &&
                    bin/pulsar bookie
                ports:
                  - containerPort: 3181
                    name: client
                envFrom:
                  - configMapRef:
                        name: bookie-config
                env:

                volumeMounts:
                  - name: journal-disk
                    mountPath: /pulsar/data/bookkeeper/journal
                  - name: ledgers-disk
                    mountPath: /pulsar/data/bookkeeper/ledgers

            initContainers:
                # The first time, initialize BK metadata in zookeeper
                 # Otherwise ignore error if it's already there
              - name: bookie-metaformat
                image: apachepulsar/pulsar:latest
                command: ["sh", "-c"]
                args:
                  - >
                    bin/apply-config-from-env.py conf/bookkeeper.conf &&
                    bin/bookkeeper shell metaformat --nonInteractive || true;
                envFrom:
                  - configMapRef:
                        name: bookie-config

# Uncomment the following if bookie should be deployed on a specific instance type
#            nodeSelector:
#              beta.kubernetes.io/instance-type: m4.2xlarge

    volumeClaimTemplates:
      - metadata:
          name: journal-disk
        spec:
          accessModes: [ "ReadWriteOnce" ]
          resources:
            requests:
              storage: 200Gi
          storageClassName: bookie-ssd
      - metadata:
          name: ledgers-disk
        spec:
          accessModes: [ "ReadWriteOnce" ]
          resources:
            requests:
              storage: 1000Gi
          storageClassName: bookie-hdd

---
##
## Define the Bookie headless service
## In practice, in this case, it is only useful to have a view of
## all the bookie pods that are present
##
apiVersion: v1
kind: Service
metadata:
    name: bookkeeper
    labels:
        app: pulsar
        component: bookkeeper
        role: bookie
spec:
    ports:
      - port: 3181
        name: server
    clusterIP: None
    selector:
        app: pulsar
        component: bookkeeper

---

##
## Run BookKeeper auto-recovery from a different set of containers
## Auto-Recovery makes sure to restore the replication factor when any bookie
## crashes and it's not recovering on its own.
##
apiVersion: apps/v1beta1
kind: Deployment
metadata:
    name: bookie-autorecovery
spec:
    replicas: 2
    template:
        metadata:
            labels:
                app: pulsar
                component: bookkeeper-replication
                role: bookie
        spec:
            containers:
              - name: replication-worker
                image: apachepulsar/pulsar:latest
                command: ["sh", "-c"]
                args:
                  - >
                    bin/apply-config-from-env.py conf/bookkeeper.conf &&
                    bin/bookkeeper autorecovery
                envFrom:
                  - configMapRef:
                        name: bookie-config
                env:
                    ## Configure for lower mem usage
                  - name: PULSAR_MEM
                    value: "\" -Xmx256m \""
                  - name: PULSAR_GC
                    value: "\"  \""
# Uncomment the following if bookie should be deployed on a specific instance type
#            nodeSelector:
#              beta.kubernetes.io/instance-type: m4.2xlarge


---
