Merge pull request #3442 from xyz-li/app

add extra annotations and labels to helm release

build/ks-apiserver/Dockerfile

 # Copyright 2020 The KubeSphere Authors. All rights reserved.
 # Use of this source code is governed by an Apache license
 # that can be found in the LICENSE file.
-FROM alpine/helm:3.4.2 as helm-base
-
 FROM alpine:3.11
 
+ARG HELM_VERSION=v3.5.2
+
 RUN apk add --no-cache ca-certificates
-COPY --from=helm-base /usr/bin/helm /usr/bin/helm
+# install helm
+RUN wget https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz && \
+    tar xvf helm-${HELM_VERSION}-linux-amd64.tar.gz && \
+    rm helm-${HELM_VERSION}-linux-amd64.tar.gz && \
+    mv linux-amd64/helm /usr/bin/ && \
+    rm -rf linux-amd64
 # To speed up building process, we copy binary directly from make
 # result instead of building it again, so make sure you run the 
 # following command first before building docker image

build/ks-controller-manager/Dockerfile

 # Copyright 2020 The KubeSphere Authors. All rights reserved.
 # Use of this source code is governed by an Apache license
 # that can be found in the LICENSE file.
-FROM alpine/helm:3.4.2 as helm-base
-
 FROM alpine:3.11
 
+ARG HELM_VERSION=v3.5.2
+ARG KUSTOMIZE_VERSION=v4.0.5
+
 RUN apk add --no-cache ca-certificates
-COPY --from=helm-base /usr/bin/helm /usr/bin/helm
+# install helm
+RUN wget https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz && \
+    tar xvf helm-${HELM_VERSION}-linux-amd64.tar.gz && \
+    rm helm-${HELM_VERSION}-linux-amd64.tar.gz && \
+    mv linux-amd64/helm /usr/bin/ && \
+    rm -rf linux-amd64
+# install kustomize
+RUN wget https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2F${KUSTOMIZE_VERSION}/kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz && \
+    tar xvf kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz && \
+    rm kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz && \
+    mv kustomize /usr/bin
+    
 COPY  /bin/cmd/controller-manager /usr/local/bin/
 
 EXPOSE 8443 8080

go.mod

@@ -109,6 +109,7 @@ require (
 	sigs.k8s.io/controller-runtime v0.6.4
 	sigs.k8s.io/controller-tools v0.4.0
 	sigs.k8s.io/kubefed v0.4.0
+	sigs.k8s.io/kustomize v2.0.3+incompatible
 	sigs.k8s.io/yaml v1.2.0
 )
 

pkg/controller/openpitrix/helmrelease/helm_release_controller.go

@@ -31,6 +31,7 @@ import (
 	"k8s.io/klog"
 	"kubesphere.io/kubesphere/pkg/apis/application/v1alpha1"
 	"kubesphere.io/kubesphere/pkg/client/informers/externalversions"
+	"kubesphere.io/kubesphere/pkg/constants"
 	"kubesphere.io/kubesphere/pkg/simple/client/openpitrix/helmrepoindex"
 	"kubesphere.io/kubesphere/pkg/simple/client/openpitrix/helmwrapper"
 	"kubesphere.io/kubesphere/pkg/simple/client/s3"
@@ -309,7 +310,10 @@ func (r *ReconcileHelmRelease) createOrUpgradeHelmRelease(rls *v1alpha1.HelmRele
 	}
 
 	// If clusterConfig is empty, this application will be installed in current host.
-	hw := helmwrapper.NewHelmWrapper(clusterConfig, rls.GetRlsNamespace(), rls.Spec.Name, helmwrapper.SetMock(r.helmMock))
+	hw := helmwrapper.NewHelmWrapper(clusterConfig, rls.GetRlsNamespace(), rls.Spec.Name,
+		// We just add kubesphere.io/creator annotation now.
+		helmwrapper.SetAnnotations(map[string]string{constants.CreatorAnnotationKey: rls.GetCreator()}),
+		helmwrapper.SetMock(r.helmMock))
 	var res helmwrapper.HelmRes
 	if upgrade {
 		res, err = hw.Upgrade(rls.Spec.ChartName, string(chartData), string(rls.Spec.Values))

pkg/simple/client/openpitrix/helmwrapper/helm_wrapper.go

@@ -20,20 +20,34 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"gopkg.in/yaml.v3"
 	"k8s.io/klog"
 	kpath "k8s.io/utils/path"
 	"kubesphere.io/kubesphere/pkg/utils/idutils"
 	"os"
 	"os/exec"
-	"path"
 	"path/filepath"
+	"sigs.k8s.io/kustomize/pkg/types"
 	"strings"
 	"time"
 )
 
+const (
+	workspaceBase = "/tmp/helm-operator"
+)
+
 var (
 	UninstallNotFoundFormat = "Error: uninstall: Release not loaded: %s: release: not found"
 	StatusNotFoundFormat    = "Error: release: not found"
+
+	kustomizationFile  = "kustomization.yaml"
+	postRenderExecFile = "helm-post-render.sh"
+	// kustomize cannot read stdio now, so we save helm stdout to file, then kustomize reads that file and build the resources
+	kustomizeBuild = `#!/bin/sh
+# save helm stdout to file, then kustomize read this file
+cat > ./.local-helm-output.yaml
+kustomize build
+`
 )
 
 type HelmRes struct {
@@ -131,9 +145,9 @@ func (c *helmWrapper) Status() (status *releaseStatus, err error) {
 
 func (c *helmWrapper) Workspace() string {
 	if c.workspaceSuffix == "" {
-		return path.Join(c.base, fmt.Sprintf("%s_%s", c.Namespace, c.ReleaseName))
+		return filepath.Join(c.base, fmt.Sprintf("%s_%s", c.Namespace, c.ReleaseName))
 	} else {
-		return path.Join(c.base, fmt.Sprintf("%s_%s_%s", c.Namespace, c.ReleaseName, c.workspaceSuffix))
+		return filepath.Join(c.base, fmt.Sprintf("%s_%s_%s", c.Namespace, c.ReleaseName, c.workspaceSuffix))
 	}
 }
 
@@ -145,6 +159,11 @@ type helmWrapper struct {
 	ReleaseName string
 	ChartName   string
 
+	// add labels to helm chart
+	labels map[string]string
+	// add annotations to helm chart
+	annotations map[string]string
+
 	// helm cmd path
 	cmdPath string
 	// base should be /dev/shm on linux
@@ -158,11 +177,16 @@ func (c *helmWrapper) kubeConfigPath() string {
 	if len(c.Kubeconfig) == 0 {
 		return ""
 	}
-	return path.Join(c.Workspace(), "kube.config")
+	return filepath.Join(c.Workspace(), "kube.config")
+}
+
+// The dir where chart saved
+func (c *helmWrapper) chartDir() string {
+	return filepath.Join(c.Workspace(), "chart")
 }
 
 func (c *helmWrapper) chartPath() string {
-	return filepath.Join(c.Workspace(), fmt.Sprintf("%s.tgz", c.ChartName))
+	return filepath.Join(c.chartDir(), fmt.Sprintf("%s.tgz", c.ChartName))
 }
 
 func (c *helmWrapper) cleanup() {
@@ -185,6 +209,13 @@ func SetDryRun(dryRun bool) Option {
 	}
 }
 
+// extra annotations added to all resources in chart
+func SetAnnotations(annotations map[string]string) Option {
+	return func(wrapper *helmWrapper) {
+		wrapper.annotations = annotations
+	}
+}
+
 func SetMock(mock bool) Option {
 	return func(wrapper *helmWrapper) {
 		wrapper.mock = mock
@@ -208,6 +239,43 @@ func NewHelmWrapper(kubeconfig, ns, rls string, options ...Option) *helmWrapper
 	return c
 }
 
+func (c *helmWrapper) setupPostRenderEnvironment() error {
+	if len(c.labels) == 0 && len(c.annotations) == 0 {
+		return nil
+	}
+
+	// build the executable file
+	postRender, err := os.OpenFile(filepath.Join(c.Workspace(), postRenderExecFile), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755)
+	if err != nil {
+		return err
+	}
+	_, err = postRender.WriteString(kustomizeBuild)
+	if err != nil {
+		return err
+	}
+	postRender.Close()
+
+	// create kustomization.yaml
+	kustomization, err := os.OpenFile(filepath.Join(c.Workspace(), kustomizationFile), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return err
+	}
+
+	kustomizationConfig := types.Kustomization{
+		Resources:         []string{"./.local-helm-output.yaml"},
+		CommonAnnotations: c.annotations, // add extra annotations to output
+		CommonLabels:      c.labels,      // add extra labels to output
+	}
+
+	err = yaml.NewEncoder(kustomization).Encode(kustomizationConfig)
+	if err != nil {
+		return err
+	}
+	kustomization.Close()
+
+	return nil
+}
+
 // ensureWorkspace check whether workspace exists or not.
 // If not exists, create workspace dir.
 func (c *helmWrapper) ensureWorkspace() error {
@@ -222,6 +290,12 @@ func (c *helmWrapper) ensureWorkspace() error {
 		}
 	}
 
+	err := os.MkdirAll(c.chartDir(), os.ModeDir|os.ModePerm)
+	if err != nil {
+		klog.Errorf("mkdir %s failed, error: %s", c.chartDir(), err)
+		return err
+	}
+
 	if len(c.Kubeconfig) > 0 {
 		kubeFile, err := os.OpenFile(c.kubeConfigPath(), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
 		if err != nil {
@@ -243,7 +317,7 @@ func (c *helmWrapper) createChart(chartName, chartData, values string) error {
 	c.ChartName = chartName
 
 	// write chart
-	f, err := os.Create(path.Join(c.chartPath()))
+	f, err := os.Create(c.chartPath())
 
 	if err != nil {
 		return err
@@ -257,7 +331,7 @@ func (c *helmWrapper) createChart(chartName, chartData, values string) error {
 	f.Close()
 
 	// write values
-	f, err = os.Create(path.Join(c.Workspace(), "values.yaml"))
+	f, err = os.Create(filepath.Join(c.Workspace(), "values.yaml"))
 	if err != nil {
 		return err
 	}
@@ -271,6 +345,7 @@ func (c *helmWrapper) createChart(chartName, chartData, values string) error {
 	return nil
 }
 
+// helm uninstall
 func (c *helmWrapper) Uninstall() (res HelmRes, err error) {
 	start := time.Now()
 	defer func() {
@@ -332,6 +407,7 @@ func (c *helmWrapper) Uninstall() (res HelmRes, err error) {
 	return
 }
 
+// helm upgrade
 func (c *helmWrapper) Upgrade(chartName, chartData, values string) (res HelmRes, err error) {
 	// TODO: check release status first
 	if true {
@@ -342,6 +418,7 @@ func (c *helmWrapper) Upgrade(chartName, chartData, values string) (res HelmRes,
 	}
 }
 
+// helm install
 func (c *helmWrapper) Install(chartName, chartData, values string) (res HelmRes, err error) {
 	return c.install(chartName, chartData, values, false)
 }
@@ -359,6 +436,11 @@ func (c *helmWrapper) install(chartName, chartData, values string, upgrade bool)
 	}
 	defer c.cleanup()
 
+	err = c.setupPostRenderEnvironment()
+	if err != nil {
+		return
+	}
+
 	if err = c.createChart(chartName, chartData, values); err != nil {
 		return
 	}
@@ -393,7 +475,7 @@ func (c *helmWrapper) install(chartName, chartData, values string, upgrade bool)
 	cmd.Args = append(cmd.Args, c.ReleaseName, c.chartPath(), "--namespace", c.Namespace)
 
 	if len(values) > 0 {
-		cmd.Args = append(cmd.Args, "--values", path.Join(c.Workspace(), "values.yaml"))
+		cmd.Args = append(cmd.Args, "--values", filepath.Join(c.Workspace(), "values.yaml"))
 	}
 
 	if c.dryRun {
@@ -404,6 +486,11 @@ func (c *helmWrapper) install(chartName, chartData, values string, upgrade bool)
 		cmd.Args = append(cmd.Args, "--kubeconfig", c.kubeConfigPath())
 	}
 
+	// Post render, add annotations or labels to resources
+	if len(c.labels) > 0 || len(c.annotations) > 0 {
+		cmd.Args = append(cmd.Args, "--post-renderer", filepath.Join(c.Workspace(), postRenderExecFile))
+	}
+
 	if klog.V(8) {
 		// output debug info
 		cmd.Args = append(cmd.Args, "--debug")

pkg/simple/client/openpitrix/helmwrapper/helm_wrapper_darwin.go

@@ -17,6 +17,5 @@ limitations under the License.
 package helmwrapper
 
 const (
-	workspaceBase = "/tmp/helm-operator"
-	helmPath      = "/usr/local/bin/helm"
+	helmPath = "/usr/local/bin/helm"
 )

pkg/simple/client/openpitrix/helmwrapper/helm_wrapper_linux.go

@@ -17,6 +17,5 @@ limitations under the License.
 package helmwrapper
 
 const (
-	workspaceBase = "/dev/shm/helm-operator"
-	helmPath      = "/usr/bin/helm"
+	helmPath = "/usr/bin/helm"
 )

pkg/simple/client/openpitrix/helmwrapper/helm_wrapper_test.go

@@ -18,12 +18,15 @@ package helmwrapper
 
 import (
 	"fmt"
+	"kubesphere.io/kubesphere/pkg/constants"
 	"os"
 	"testing"
 )
 
 func TestHelmInstall(t *testing.T) {
-	wr := NewHelmWrapper("", "dummy", "dummy", SetMock(true))
+	wr := NewHelmWrapper("", "dummy", "dummy",
+		SetAnnotations(map[string]string{constants.CreatorAnnotationKey: "1234"}),
+		SetMock(true))
 
 	res, err := wr.Install("dummy-chart", "", "dummy-value")
 	if err != nil {