Update kubernetes deployment apis k8s post 1.9 (#4700)

**Motivation**

Fixes #4698 and #4699 

**List of changes**

1. Two new folders under pulsar/deployment/kubernetes/generic : 
- original : contains the original scripts (pre Kubernetes 1.9)
- k8s-1-9-and-above : contains the new scripts with APIs ipdated (Kubernetes 1.9 and above)

2. bookie.yaml :
- Migrate DaemonSet api version from extensions/v1beta1 to apps/v1
- Declare the bookie service before the DaemonSet object for bookies

3. broker.yaml
- Migrate Deployment api version from apps/v1beta1 to apps/v1
- Declare the bookie service before the Deployment object for brokers

4. monitoring.yaml :
- Migrate all Deployment api version from apps/v1beta1 to apps/v1
- Declare each service before the Deployment object the service is bound to

5. zookeeper.yaml :
- Migrate StatefulSet api version from apps/v1beta1 to apps/v1
- Declare the service before the StatefulSet object for zookeeper

6. proxy.yaml
- Migrate Deployment api version from apps/v1beta1 to apps/v1
- Declare the bookie service before the Deployment object for proxy.

deployment/kubernetes/generic/k8s-1-9-and-above/bookie.yaml

+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+    name: bookie-config
+data:
+    PULSAR_MEM: "\" -Xms64m -Xmx256m -XX:MaxDirectMemorySize=256m\""
+    dbStorage_writeCacheMaxSizeMb: "32" # Write cache size (direct memory)
+    dbStorage_readAheadCacheMaxSizeMb: "32" # Read cache size (direct memory)
+    zkServers: zookeeper
+    statsProviderClass: org.apache.bookkeeper.stats.prometheus.PrometheusMetricsProvider
+---
+##
+## Define the Bookie headless service
+## In practice, in this case, it is only useful to have a view of
+## all the bookie pods that are present
+##
+apiVersion: v1
+kind: Service
+metadata:
+    name: bookkeeper
+    labels:
+        app: pulsar
+        component: bookkeeper
+spec:
+    ports:
+      - port: 3181
+        name: server
+    clusterIP: None
+    selector:
+        app: pulsar
+        component: bookkeeper
+---
+## BookKeeper servers need to access the local disks and the pods
+## cannot be moved across different nodes.
+## For this reason, we run BK as a daemon set, one for each node in the
+## cluster, unless restricted by label selectors
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+    name: bookie
+    labels:
+        app: pulsar
+        component: bookkeeper
+spec:
+    template:
+        metadata:
+            labels:
+                app: pulsar
+                component: bookkeeper
+                # Specify cluster to allow aggregation by cluster in
+                # the metrics
+                cluster: local
+            annotations:
+                prometheus.io/scrape: "true"
+                prometheus.io/port: "8000"
+
+        spec:
+            containers:
+              - name: bookie
+                image: apachepulsar/pulsar-all:latest
+                command: ["sh", "-c"]
+                args:
+                  - >
+                    bin/apply-config-from-env.py conf/bookkeeper.conf &&
+                    bin/apply-config-from-env.py conf/pulsar_env.sh &&
+                    bin/pulsar bookie
+                ports:
+                  - containerPort: 3181
+                    hostPort: 3181
+                    name: client
+                envFrom:
+                  - configMapRef:
+                        name: bookie-config
+                env:
+                  - name: advertisedAddress
+                    valueFrom:
+                        fieldRef:
+                            fieldPath: status.hostIP
+
+                volumeMounts:
+                  - name: journal-disk
+                    mountPath: /pulsar/data/bookkeeper/journal
+                  - name: ledgers-disk
+                    mountPath: /pulsar/data/bookkeeper/ledgers
+
+                # bin/bookkeeper shell bookiesanity
+
+            initContainers:
+                # The first time, initialize BK metadata in zookeeper
+                 # Otherwise ignore error if it's already there
+              - name: bookie-metaformat
+                image: apachepulsar/pulsar-all:latest
+                command: ["sh", "-c"]
+                args:
+                  - >
+                    bin/apply-config-from-env.py conf/bookkeeper.conf &&
+                    bin/bookkeeper shell metaformat --nonInteractive || true;
+                envFrom:
+                  - configMapRef:
+                        name: bookie-config
+
+            volumes:
+                # Mount local disks
+              - name: journal-disk
+                hostPath:
+                    path: /mnt/disks/ssd0
+              - name: ledgers-disk
+                hostPath:
+                    path: /mnt/disks/ssd1

deployment/kubernetes/generic/k8s-1-9-and-above/broker.yaml

+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+    name: broker-config
+data:
+    # Tune for available memory. Increase the heap up to 24G to have
+    # better GC behavior at high throughput
+    PULSAR_MEM: "\" -Xms64m -Xmx128m -XX:MaxDirectMemorySize=128m\""
+    zookeeperServers: zookeeper
+    configurationStoreServers: zookeeper
+    clusterName: local
+    # change the managed ledger settings if you have more bookies
+    managedLedgerDefaultEnsembleSize: "1"
+    managedLedgerDefaultWriteQuorum: "1"
+    managedLedgerDefaultAckQuorum: "1"
+    # enable pulsar functions
+    functionsWorkerEnabled: "true"
+    PF_pulsarFunctionsCluster: local
+---
+##
+## Define the Broker headless service
+## In practice, in this case, it is only useful to have a view of
+## all the broker pods that are present
+##
+apiVersion: v1
+kind: Service
+metadata:
+    name: broker
+    labels:
+        app: pulsar
+        component: broker
+spec:
+    ports:
+      - port: 8080
+        name: http
+      - port: 6650
+        name: pulsar
+    clusterIP: None
+    selector:
+        app: pulsar
+        component: broker
+---
+##
+## Broker deployment definition
+##
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+    name: broker
+spec:
+    replicas: 3
+    template:
+        metadata:
+            labels:
+                app: pulsar
+                component: broker
+            annotations:
+                prometheus.io/scrape: "true"
+                prometheus.io/port: "8080"
+        spec:
+            containers:
+              - name: broker
+                image: apachepulsar/pulsar-all:latest
+                command: ["sh", "-c"]
+                args:
+                  - >
+                    bin/apply-config-from-env.py conf/broker.conf &&
+                    bin/apply-config-from-env.py conf/pulsar_env.sh &&
+                    bin/gen-yml-from-env.py conf/functions_worker.yml &&
+                    bin/pulsar broker
+                ports:
+                  - containerPort: 8080
+                    # hostPort: 8080
+                  - containerPort: 6650
+                    # hostPort: 6650
+                envFrom:
+                  - configMapRef:
+                        name: broker-config
+                env:
+                 - name: advertisedAddress
+                   valueFrom:
+                       fieldRef:
+                           fieldPath: status.podIP

deployment/kubernetes/generic/k8s-1-9-and-above/monitoring.yaml

+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+    name: prometheus-config
+data:
+    # Include prometheus configuration file, setup to monitor all the
+    # Kubernetes pods with the "scrape=true" annotation.
+    prometheus.yml: |
+        global:
+            scrape_interval: 15s
+        scrape_configs:
+          - job_name: 'prometheus'
+            static_configs:
+              - targets: ['localhost:9090']
+          - job_name: 'kubernetes-pods'
+            kubernetes_sd_configs:
+              - role: pod
+
+            relabel_configs:
+              - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+                action: keep
+                regex: true
+              - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+                action: replace
+                target_label: __metrics_path__
+                regex: (.+)
+              - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
+                action: replace
+                regex: ([^:]+)(?::\d+)?;(\d+)
+                replacement: $1:$2
+                target_label: __address__
+              - action: labelmap
+                regex: __meta_kubernetes_pod_label_(.+)
+              - source_labels: [__meta_kubernetes_namespace]
+                action: replace
+                target_label: kubernetes_namespace
+              - source_labels: [__meta_kubernetes_pod_label_component]
+                action: replace
+                target_label: job
+              - source_labels: [__meta_kubernetes_pod_name]
+                action: replace
+                target_label: kubernetes_pod_name
+---
+## PROMOTHEUS - Service
+apiVersion: v1
+kind: Service
+metadata:
+    name: prometheus
+    labels:
+        app: pulsar
+        component: prometheus
+spec:
+    type: NodePort
+    ports:
+      - name: prometheus
+        nodePort: 30003
+        port: 9090
+        protocol: TCP
+    selector:
+        app: pulsar
+        component: prometheus
+---
+## PROMOTHEUS - Deployment
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+    name: prometheus
+spec:
+    replicas: 1
+    template:
+        metadata:
+            labels:
+                app: pulsar
+                component: prometheus
+        spec:
+            containers:
+              - name: prometheus
+                image: prom/prometheus:v1.6.3
+                volumeMounts:
+                  - name: config-volume
+                    mountPath: /etc/prometheus
+                  - name: data-volume
+                    mountPath: /prometheus
+                ports:
+                  - containerPort: 9090
+            volumes:
+              - name: config-volume
+                configMap:
+                    name: prometheus-config
+              - name: data-volume
+                emptyDir: {}
+---
+## GRAFANA - Service
+apiVersion: v1
+kind: Service
+metadata:
+    name: grafana
+    labels:
+        app: pulsar
+        component: grafana
+spec:
+    type: NodePort
+    ports:
+      - name: grafana
+        nodePort: 30004
+        port: 3000
+        protocol: TCP 
+    selector:
+        app: pulsar
+        component: grafana
+---
+## GRAFANA - Deployment
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+    name: grafana
+spec:
+    replicas: 1
+    template:
+        metadata:
+            labels:
+                app: pulsar
+                component: grafana
+        spec:
+            containers:
+              - name: grafana
+                image: apachepulsar/pulsar-grafana:latest
+                ports:
+                  - containerPort: 3000
+                env:
+                  - name: PROMETHEUS_URL
+                    value: http://prometheus:9090/
+---
+## PULSAR DASHBOARD - Deployment
+apiVersion: v1
+kind: Service
+metadata:
+    name: pulsar-dashboard
+    labels:
+        app: pulsar
+        component: dashboard
+spec:
+    type: NodePort
+    ports:
+      - name: dashboard
+        nodePort: 30005
+        port: 80
+        protocol: TCP
+    selector:
+        app: pulsar
+        component: dashboard
+---
+## PULSAR DASHBOARD - Deployment
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+    name: pulsar-dashboard
+spec:
+    replicas: 1
+    template:
+        metadata:
+            labels:
+                app: pulsar
+                component: dashboard
+        spec:
+            containers:
+              - name: grafana
+                image: apachepulsar/pulsar-dashboard:latest
+                ports:
+                  - containerPort: 80
+                env:
+                  - name: SERVICE_URL
+                    value: http://broker:8080/

deployment/kubernetes/generic/k8s-1-9-and-above/proxy.yaml

+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+    name: proxy-config
+data:
+    PULSAR_MEM: "\" -Xms64m -Xmx128m -XX:MaxDirectMemorySize=128m\""
+    zookeeperServers: zookeeper
+    configurationStoreServers: zookeeper
+    clusterName: local
+---
+##
+## Expose all nodes on port so that you can reach cluster from outside k8
+##
+apiVersion: v1
+kind: Service
+metadata:
+    name: proxy
+    labels:
+        app: pulsar
+        component: proxy
+spec:
+    type: NodePort
+    ports:
+      - name: http
+        nodePort: 30001
+        port: 8080
+        protocol: TCP
+      - name: tcp
+        nodePort: 30002
+        port: 6650
+        protocol: TCP
+    selector:
+        app: pulsar
+        component: proxy
+---
+##
+## Proxy deployment definition
+##
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+    name: proxy
+spec:
+    replicas: 2
+    template:
+        metadata:
+            labels:
+                app: pulsar
+                component: proxy
+            annotations:
+                prometheus.io/scrape: "true"
+                prometheus.io/port: "8080"
+        spec:
+            containers:
+              - name: proxy
+                image: apachepulsar/pulsar-all:latest
+                command: ["sh", "-c"]
+                args:
+                  - >
+                    bin/apply-config-from-env.py conf/proxy.conf &&
+                    bin/apply-config-from-env.py conf/pulsar_env.sh &&
+                    bin/pulsar proxy
+                ports:
+                  - containerPort: 8080
+                envFrom:
+                  - configMapRef:
+                        name: proxy-config

deployment/kubernetes/generic/k8s-1-9-and-above/zookeeper.yaml

+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+    name: zookeeper-config
+data:
+    PULSAR_MEM: "\" -Xms100m -Xmx256m \""
+    PULSAR_GC: "\" -XX:+UseG1GC -XX:MaxGCPauseMillis=10\""
+---
+## Define a disruption budget to ensure there are at least
+## 2 ZK servers running all the time
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+    name: zk-budget
+spec:
+    selector:
+        matchLabels:
+            app: zk
+    minAvailable: 2
+---
+##
+## Define the ZooKeeper headless service
+##
+apiVersion: v1
+kind: Service
+metadata:
+    annotations:
+        service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
+    name: zookeeper
+    labels:
+        app: pulsar
+        component: zookeeper
+spec:
+    ports:
+      - port: 2888
+        name: server
+      - port: 3888
+        name: leader-election
+    clusterIP: None
+    selector:
+        app: pulsar
+        component: zookeeper
+---
+## Define a StatefulSet for ZK servers
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+    name: zk
+    labels:
+        app: pulsar
+        component: zookeeper
+spec:
+    serviceName: zookeeper
+    replicas: 3
+    template:
+        metadata:
+            labels:
+                app: pulsar
+                component: zookeeper
+                cluster: us-central
+            annotations:
+                pod.alpha.kubernetes.io/initialized: "true"
+                prometheus.io/scrape: "true"
+                prometheus.io/port: "8080"
+
+        spec:
+            # Make sure multiple pods of ZK don't get scheduled on the
+            # same node, unless there are no other available nodes
+            affinity:
+                podAntiAffinity:
+                    preferredDuringSchedulingIgnoredDuringExecution:
+                      - weight: 1
+                        podAffinityTerm:
+                            labelSelector:
+                                matchExpressions:
+                                  - key: "app"
+                                    operator: In
+                                    values:
+                                      - zookeeper
+                            topologyKey: "kubernetes.io/hostname"
+            containers:
+              - name: zookeeper
+                image: apachepulsar/pulsar-all:latest
+                command: ["sh", "-c"]
+                args:
+                  - >
+                    bin/apply-config-from-env.py conf/zookeeper.conf &&
+                    bin/apply-config-from-env.py conf/pulsar_env.sh &&
+                    bin/generate-zookeeper-config.sh conf/zookeeper.conf &&
+                    bin/pulsar zookeeper
+                ports:
+                  - containerPort: 2181
+                    name: client
+                  - containerPort: 2888
+                    name: server
+                  - containerPort: 3888
+                    name: leader-election
+                env:
+                  - name: ZOOKEEPER_SERVERS
+                    value: zk-0,zk-1,zk-2
+                envFrom:
+                  - configMapRef:
+                        name: zookeeper-config
+
+                readinessProbe:
+                    exec:
+                        command:
+                          - "bin/pulsar-zookeeper-ruok.sh"
+                    initialDelaySeconds: 5
+                    timeoutSeconds: 5
+
+                livenessProbe:
+                    exec:
+                        command:
+                          - "bin/pulsar-zookeeper-ruok.sh"
+                    initialDelaySeconds: 15
+                    timeoutSeconds: 5
+
+                volumeMounts:
+                  - name: datadir
+                    mountPath: /pulsar/data
+            volumes:
+              - name: datadir
+                emptyDir: {}

deployment/kubernetes/generic/original/admin.yaml

+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+apiVersion: v1
+kind: Pod
+metadata:
+    name: pulsar-admin
+spec:
+    containers:
+      - name: pulsar-admin
+        image: apachepulsar/pulsar:latest
+        command: ["sh", "-c"]
+        args:
+          - >
+            bin/apply-config-from-env.py conf/client.conf &&
+            bin/apply-config-from-env.py conf/pulsar_env.sh &&
+            bin/apply-config-from-env.py conf/pulsar_tools_env.sh &&
+            sleep 10000000000
+        envFrom:
+          - configMapRef:
+                name: broker-config
+        env:
+          - name: webServiceUrl
+            value: "http://proxy:8080/"
+          - name: brokerServiceUrl
+            value: "pulsar://proxy:6650/"
+          - name: PULSAR_MEM
+            value: "\"-Xms64m -Xmx128m\""

deployment/kubernetes/generic/original/cluster-metadata.yaml

+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: pulsar-cluster-metadata-init
+  labels:
+    app: pulsar
+    component: broker
+spec:
+  template:
+    spec:
+      containers:
+        - name: pulsar-cluster-metadata-init-container
+          image: apachepulsar/pulsar:latest
+          command: ["sh", "-c"]
+          args:
+            - >
+              bin/pulsar initialize-cluster-metadata \
+                --cluster local \
+                --zookeeper zookeeper \
+                --configuration-store zookeeper \
+                --web-service-url http://broker.default.svc.cluster.local:8080/ \
+                --broker-service-url pulsar://broker.default.svc.cluster.local:6650/ || true;
+      restartPolicy: Never