Merge branch 'main' into 'develop'

refactor: refactor code structure

See merge request platform/CloudNative4AI/cluster-lifecycle/di-orchestrator!49

.gitignore

@@ -29,3 +29,5 @@ bin
 coverage.out.*
 
 config/webhook/manifests.yaml
+
+hello/*
\ No newline at end of file

.gitlab-ci.yml

 variables:
   REGISTRY: registry.sensetime.com/cloudnative4ai
-  VERSION: v0.2.2
+  VERSION: v0.3.0
   PROJECT: di-orchestrator
   # dind config
   DOCKER_HOST: tcp://localhost:2376
@@ -15,14 +15,10 @@ variables:
 before_script:
   # this is necessary.
   - export PATH="/go/bin:/root/go/bin:${PATH}"
-  - git config --global user.name "gitlab-runner"
-  - git config --global user.email "no-exist@sensetime.com"
-  # set proxy
-  - export GOPROXY=https://goproxy.cn
-  - export GOPRIVATE=go-sensephoenix.sensetime.com
   # set image name
   - export PROJECT_IMAGE="${REGISTRY}/${PROJECT}:${VERSION}-${CI_COMMIT_SHORT_SHA}"
   - export PROJECT_IMAGE_RELEASE="${REGISTRY}/${PROJECT}:${VERSION}"
+  - export https_proxy=http://10.54.0.110:8080
 
 stages:
   - lint
@@ -33,7 +29,7 @@ stages:
 
 lint:
   stage: lint
-  image: registry.sensetime.com/cloudnative4ai/ci-deploy-kit:latest
+  image: ${PHOENIX_CI_IMAGE_NEW}
   tags: 
     - cloudnative4ai-group-runner-phoenix
   script:
@@ -42,18 +38,20 @@ lint:
 
 unit-test:
   stage: test
-  image: registry.sensetime.com/cloudnative4ai/ci-deploy-kit:latest
+  image: ${PHOENIX_CI_IMAGE_NEW}
   tags: 
     - cloudnative4ai-group-runner-phoenix
   script:
+    - unset https_proxy
     - curl -L https://download.phoenix.sensetime.com/mirrors/kubebuilder/v2.3.2/kubebuilder_2.3.2_linux_amd64.tar.gz | tar -xz -C /tmp/
     - mv /tmp/kubebuilder_2.3.2_linux_amd64 /usr/local/kubebuilder
     - export PATH=$PATH:/usr/local/kubebuilder/bin
+    - export https_proxy=http://10.54.0.110:8080
     - make test
 
 build-manual:
   stage: build
-  image: registry.sensetime.com/cloudnative4ai/ci-deploy-kit:latest
+  image: ${PHOENIX_CI_IMAGE_NEW}
   tags: 
     - cloudnative4ai-group-runner-phoenix
   allow_failure: false
@@ -67,14 +65,14 @@ build-manual:
     - develop
     - /^release.*$/
   script:
-    - make docker-build
+    - make dev-images
     - make docker-push
   after_script:
     - docker image prune -f
 
 build-release:
   stage: build
-  image: registry.sensetime.com/cloudnative4ai/ci-deploy-kit:latest
+  image: ${PHOENIX_CI_IMAGE_NEW}
   tags: 
     - cloudnative4ai-group-runner-phoenix
   allow_failure: false
@@ -87,14 +85,14 @@ build-release:
     - develop
     - /^release.*$/
   script:
-    - make docker-build
+    - make dev-images
     - make docker-push
   after_script:
     - docker image prune -f
 
 tag:
   stage: release
-  image: registry.sensetime.com/cloudnative4ai/ci-deploy-kit:latest
+  image: ${PHOENIX_CI_IMAGE_NEW}
   tags:
     - cloudnative4ai-group-runner-phoenix
   services:

Dockerfile

-# Build the di-operator binary
+# Build the di-orchestrator binary
 FROM registry.sensetime.com/cloudnative4ai/golang:1.15 as builder
 
 WORKDIR /workspace
@@ -7,48 +7,22 @@ COPY go.mod go.mod
 COPY go.sum go.sum
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
-RUN export http_proxy=http://172.16.1.135:3128 && https_proxy=http://172.16.1.135:3128 \
-    && go env -w GO111MODULE=on \
-    && go env -w GOPROXY=https://goproxy.cn \
-    && go mod download
+RUN go mod download
 
 # Copy the go source
 COPY cmd/ cmd/
-COPY api/ api/
-COPY common/ common/
-COPY controllers/ controllers/
-COPY server/ server/
-COPY utils/ utils/
+COPY pkg/ pkg/
+COPY main.go main.go
 
-# Build operator
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o di-operator cmd/operator/main.go
-# Build webhook
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o di-webhook cmd/webhook/main.go
-# Build server
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o di-server cmd/server/main.go
+# Build orchestrator
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o di-orchestrator ./main.go
 
-# Use distroless as minimal base image to package the di-operator binary
+# Use distroless as minimal base image to package the di-orchestrator binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
-FROM registry.sensetime.com/cloudnative4ai/ubi:v1.0.0 as di-operator
+FROM registry.sensetime.com/cloudnative4ai/ubi:v1.0.0
 LABEL maintainer="opendilab.contact@gmail.com"
 RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
 WORKDIR /
-COPY --from=builder /workspace/di-operator .
+COPY --from=builder /workspace/di-orchestrator .
 
-ENTRYPOINT ["/di-operator"]
-
-FROM registry.sensetime.com/cloudnative4ai/ubi:v1.0.0 as di-webhook
-LABEL maintainer="opendilab.contact@gmail.com"
-RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
-WORKDIR /
-COPY --from=builder /workspace/di-webhook .
-
-ENTRYPOINT ["/di-webhook"]
-
-FROM registry.sensetime.com/cloudnative4ai/ubi:v1.0.0 as di-server
-LABEL maintainer="opendilab.contact@gmail.com"
-RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
-WORKDIR /
-COPY --from=builder /workspace/di-server .
-
-ENTRYPOINT ["/di-server"]
+ENTRYPOINT ["/di-orchestrator"]

Dockerfile.dev

-# Build the di-operator binary
+# Build the di-orchestrator binary
 
-# Use distroless as minimal base image to package the di-operator binary
+# Use distroless as minimal base image to package the di-orchestrator binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
-FROM registry.sensetime.com/cloudnative4ai/ubi:v1.0.0 as di-operator
+FROM registry.sensetime.com/cloudnative4ai/ubi:v1.0.0 as di-orchestrator
 LABEL maintainer="opendilab.contact@gmail.com"
 RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
 WORKDIR /
-COPY ./bin/di-operator  .
+COPY ./bin/di-orchestrator  .
 
-ENTRYPOINT ["/di-operator"]
+ENTRYPOINT ["/di-orchestrator"]
 
-FROM registry.sensetime.com/cloudnative4ai/ubi:v1.0.0 as di-webhook
-LABEL maintainer="opendilab.contact@gmail.com"
-RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
-WORKDIR /
-COPY ./bin/di-webhook .
-
-ENTRYPOINT ["/di-webhook"]
-
-FROM registry.sensetime.com/cloudnative4ai/ubi:v1.0.0 as di-server
-LABEL maintainer="opendilab.contact@gmail.com"
-RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
-WORKDIR /
-COPY ./bin/di-server .
-
-ENTRYPOINT ["/di-server"]

Makefile

 
 # di-operator version
-VERSION ?= v0.2.2
+VERSION ?= v0.3.0
 MASTER_VERSION := $(VERSION)
 
 COMMIT_SHORT_SHA=$(shell git log -n 1 | head -n 1 | sed -e 's/^commit //' | head -c 8)
@@ -16,19 +16,11 @@ VERSION := $(MASTER_VERSION)
 endif
 
 # Image URL to use all building/pushing image targets
-IMG_BASE ?= registry.sensetime.com/cloudnative4ai/di-operator
-SERVER_IMG_BASE ?= registry.sensetime.com/cloudnative4ai/di-server
-WEBHOOK_IMG_BASE ?= registry.sensetime.com/cloudnative4ai/di-webhook
+IMG_BASE ?= registry.sensetime.com/cloudnative4ai/di-orchestrator
 
 IMG ?= ${IMG_BASE}:${VERSION}
 MASTER_IMG ?= ${IMG_BASE}:${MASTER_VERSION}
 
-SERVER_IMG ?= ${SERVER_IMG_BASE}:${VERSION}
-MASTER_SERVER_IMG ?= ${SERVER_IMG_BASE}:${MASTER_VERSION}
-
-WEBHOOK_IMG ?= ${WEBHOOK_IMG_BASE}:${VERSION}
-MASTER_WEBHOOK_IMG ?= ${WEBHOOK_IMG_BASE}:${MASTER_VERSION}
-
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false"
 
@@ -61,7 +53,7 @@ help: ## Display this help.
 
 manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
 	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=di-operator-cluster-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
-	cd config/manager && $(KUSTOMIZE) edit set image ${IMG_BASE}=${MASTER_IMG} ${SERVER_IMG_BASE}=${MASTER_SERVER_IMG} ${WEBHOOK_IMG_BASE}=${MASTER_WEBHOOK_IMG}
+	cd config/manager && $(KUSTOMIZE) edit set image ${IMG_BASE}=${MASTER_IMG}
 	./hack/update-image-tags.sh config/manager ${MASTER_VERSION}
 	./hack/update-image-tags.sh config/v1.15.5 ${MASTER_VERSION}
 	./hack/update-version.sh ${MASTER_VERSION}
@@ -73,7 +65,7 @@ manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and Cust
 # used `make manifests` when commited git
 dev-manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
 	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=di-operator-cluster-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
-	cd config/manager && $(KUSTOMIZE) edit set image ${IMG_BASE}=${IMG} ${SERVER_IMG_BASE}=${SERVER_IMG} ${WEBHOOK_IMG_BASE}=${WEBHOOK_IMG}
+	cd config/manager && $(KUSTOMIZE) edit set image ${IMG_BASE}=${IMG}
 	./hack/update-image-tags.sh config/manager ${VERSION}
 	./hack/update-image-tags.sh config/v1.15.5 ${VERSION}
 	./hack/update-version.sh ${VERSION}
@@ -93,53 +85,30 @@ lint:
 
 .PHONY: test
 test: ginkgo ## Run tests.
-	$(GINKGO) -nodes 4 -v -cover -coverprofile=coverage.out ./api/v1alpha1 ./controllers ./server/http ./common/gpuallocator 
-	go tool cover -func=./api/v1alpha1/coverage.out 
-	go tool cover -func=./controllers/coverage.out 
-	go tool cover -func=./server/http/coverage.out 
-	go tool cover -func=./common/gpuallocator/coverage.out
+	$(GINKGO) -nodes 4 -v -cover -coverprofile=coverage.out ./pkg/... 
+	go tool cover -func=./pkg/api/v1alpha1/coverage.out 
+	go tool cover -func=./pkg/controllers/coverage.out 
+	go tool cover -func=./pkg/server/http/coverage.out 
+	go tool cover -func=./pkg/common/gpuallocator/coverage.out
 
 ##@ Build
 
 build: generate  ## Build di-operator binary.
-	go build -o bin/di-operator cmd/operator/main.go
-	go build -o bin/di-server cmd/server/main.go
-	go build -o bin/di-webhook cmd/webhook/main.go
-
-run: manifests generate fmt vet ## Run a controller from your host.
-	go run ./cmd/operator/main.go
-
-operator-image:
-	docker build -t ${IMG} --target di-operator .
-
-webhook-image:
-	docker build -t ${WEBHOOK_IMG} --target di-webhook .
-
-server-image:
-	docker build -t ${SERVER_IMG} --target di-server .
+	go build -o bin/di-orchestrator ./main.go
 
-docker-build: operator-image webhook-image server-image ## Build docker image with the di-operator.
+docker-build: ## Build docker image with the di-operator.
+	docker build -t ${IMG} -f Dockerfile .
 
 dev-images: build
-	docker build -t ${IMG} -f Dockerfile.dev --target di-operator .
-	docker build -t ${WEBHOOK_IMG} -f Dockerfile.dev --target di-webhook .
-	docker build -t ${SERVER_IMG} -f Dockerfile.dev --target di-server .
+	docker build -t ${IMG} -f Dockerfile.dev .
 
 docker-push: ## Push docker image with the di-operator.
 	docker push ${IMG}
-	docker push ${SERVER_IMG}
-	docker push ${WEBHOOK_IMG}
 
 docker-release: ## Release docker image with the di-operator.
 	docker pull ${IMG}
-	docker pull ${SERVER_IMG}
-	docker pull ${WEBHOOK_IMG}
 	docker tag ${IMG} ${MASTER_IMG}
-	docker tag ${SERVER_IMG} ${MASTER_SERVER_IMG}
-	docker tag ${WEBHOOK_IMG} ${MASTER_WEBHOOK_IMG}
 	docker push ${MASTER_IMG} 
-	docker push ${MASTER_SERVER_IMG}
-	docker push ${MASTER_WEBHOOK_IMG}
 
 ##@ Deployment
 

cmd/common/common.go

+/*
+Copyright 2021 The OpenDILab authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package common
+
+import (
+	goflag "flag"
+
+	"github.com/go-logr/logr"
+	"github.com/spf13/cobra"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+)
+
+type GenericFlags struct {
+}
+
+func NewGenericFlags() *GenericFlags {
+	return &GenericFlags{}
+}
+
+func (f *GenericFlags) AddFlags(cmd *cobra.Command) {}
+
+var (
+	Logger logr.Logger
+)
+
+func init() {
+	opts := zap.Options{
+		Development: true,
+	}
+	opts.BindFlags(goflag.CommandLine)
+	goflag.Parse()
+
+	Logger = zap.New(zap.UseFlagOptions(&opts))
+}

cmd/operator/main.go → cmd/operator/operator.go

@@ -13,30 +13,66 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-
-package main
+package operator
 
 import (
-	"flag"
 	"os"
 
-	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
-	// to ensure that exec-entrypoint and run can make use of them.
-	_ "k8s.io/client-go/plugin/pkg/client/auth"
-
+	"github.com/spf13/cobra"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
-	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
-	div1alpha1 "opendilab.org/di-orchestrator/api/v1alpha1"
-	dicommon "opendilab.org/di-orchestrator/common"
-	"opendilab.org/di-orchestrator/controllers"
-	//+kubebuilder:scaffold:imports
+	cmdcommon "opendilab.org/di-orchestrator/cmd/common"
+	div1alpha1 "opendilab.org/di-orchestrator/pkg/api/v1alpha1"
+	dicommon "opendilab.org/di-orchestrator/pkg/common"
+	"opendilab.org/di-orchestrator/pkg/controllers"
 )
 
+type CreateOptions struct {
+	MetricAddress        string
+	ProbeAddress         string
+	EnableLeaderElection bool
+}
+
+func NewCreateOptions() *CreateOptions {
+	return &CreateOptions{
+		MetricAddress:        ":8443",
+		ProbeAddress:         ":8080",
+		EnableLeaderElection: false,
+	}
+}
+
+func (o *CreateOptions) AddFlags(cmd *cobra.Command) {
+	cmd.Flags().StringVar(&o.MetricAddress, "metric-addr", o.MetricAddress, "The address the metric endpoint binds to.")
+	cmd.Flags().StringVar(&o.ProbeAddress, "probe-addr", o.ProbeAddress, "The address the probe endpoint binds to.")
+	cmd.Flags().BoolVar(&o.EnableLeaderElection, "leader-elect", o.EnableLeaderElection,
+		"Enable leader election for controller manager. "+
+			"Enabling this will ensure there is only one active controller manager.")
+}
+
+func NewCmdOperator() *cobra.Command {
+	o := NewCreateOptions()
+	var operatorCmd = &cobra.Command{
+		Use:   "operator",
+		Short: "Command to run di-operator ",
+		Long: `Run di-operator with specified configuration.
+
+Examples:
+	# Start di-operator with metric address and probe address specified.
+	di-orchestrator operator --metric-addr :8443 --probe-addr :8080
+`,
+		Run: func(cmd *cobra.Command, args []string) {
+			cobra.CheckErr(runCommand(cmd, o))
+		},
+	}
+
+	o.AddFlags(operatorCmd)
+	return operatorCmd
+}
+
 var (
 	scheme   = runtime.NewScheme()
 	setupLog = ctrl.Log.WithName("setup")
@@ -49,38 +85,26 @@ func init() {
 	//+kubebuilder:scaffold:scheme
 }
 
-func main() {
-	var metricsAddr, probeAddr, serverAddr string
-	var enableLeaderElection bool
-	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
-	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
-	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
-		"Enable leader election for controller manager. "+
-			"Enabling this will ensure there is only one active controller manager.")
-	flag.StringVar(&serverAddr, "server-address", dicommon.DefaultServerURL, "The address to access  server.")
-
-	opts := zap.Options{
-		Development: true,
-	}
-	opts.BindFlags(flag.CommandLine)
-	flag.Parse()
-
-	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
+func runCommand(cmd *cobra.Command, options *CreateOptions) error {
+	ctrl.SetLogger(cmdcommon.Logger)
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:                 scheme,
-		MetricsBindAddress:     metricsAddr,
-		HealthProbeBindAddress: probeAddr,
-		LeaderElection:         enableLeaderElection,
+		MetricsBindAddress:     options.MetricAddress,
+		HealthProbeBindAddress: options.ProbeAddress,
+		LeaderElection:         options.EnableLeaderElection,
 		LeaderElectionID:       "12841a5d.opendilab.org",
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")
-		os.Exit(1)
+		return err
 	}
 
 	// set DefaultDIServerURL
-	dicommon.DefaultServerURL = serverAddr
+	serverAddr := os.Getenv(dicommon.ServerURLEnv)
+	if serverAddr != "" {
+		dicommon.DefaultServerURL = serverAddr
+	}
 
 	reconciler := &controllers.DIJobReconciler{
 		Client:   mgr.GetClient(),
@@ -90,23 +114,24 @@ func main() {
 	}
 	if err = reconciler.SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "DIJob")
-		os.Exit(1)
+		return err
 	}
 
 	//+kubebuilder:scaffold:builder
 
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
 		setupLog.Error(err, "unable to set up health check")
-		os.Exit(1)
+		return err
 	}
 	if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
 		setupLog.Error(err, "unable to set up ready check")
-		os.Exit(1)
+		return err
 	}
 
 	setupLog.Info("starting manager")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		setupLog.Error(err, "problem running manager")
-		os.Exit(1)
+		return err
 	}
+	return nil
 }

cmd/root.go

+/*
+Copyright 2021 The OpenDILab authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package cmd
+
+import (
+	goflag "flag"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+
+	"opendilab.org/di-orchestrator/cmd/operator"
+	"opendilab.org/di-orchestrator/cmd/server"
+	"opendilab.org/di-orchestrator/cmd/webhook"
+)
+
+// rootCmd represents the base command when called without any subcommands
+var (
+	rootCmd = &cobra.Command{
+		Use:   "di-orchestrator",
+		Short: "A component responsible for managing DI-engine jobs.",
+		Long: `DI Orchestrator is a component responsible for managing DI-engine jobs in Kubernetes cluster.
+This application allows you to run di-operator, di-server and di-webhook locally with comandline.`,
+		// Uncomment the following line if your bare application
+		// has an action associated with it:
+		Run: func(cmd *cobra.Command, args []string) {
+			cmd.Help()
+		},
+	}
+)
+
+// Execute adds all child commands to the root command and sets flags appropriately.
+// This is called by main.main(). It only needs to happen once to the rootCmd.
+func Execute() {
+	cobra.CheckErr(rootCmd.Execute())
+}
+
+func init() {
+	rootCmd.AddCommand(server.NewCmdServer())
+	rootCmd.AddCommand(operator.NewCmdOperator())
+	rootCmd.AddCommand(webhook.NewCmdWebhook())
+
+	// add all the flags in go flagset into pflagset
+	pflag.CommandLine.AddGoFlagSet(goflag.CommandLine)
+}

cmd/server/main.go

-package main
-
-import (
-	"context"
-	"flag"
-	"fmt"
-	"log"
-	"os"
-	"os/signal"
-	"syscall"
-	"time"
-
-	"github.com/google/uuid"
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/dynamic"
-	"k8s.io/client-go/dynamic/dynamicinformer"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/client-go/tools/leaderelection"
-	"k8s.io/client-go/tools/leaderelection/resourcelock"
-	"sigs.k8s.io/controller-runtime/pkg/log/zap"
-
-	dicommon "opendilab.org/di-orchestrator/common"
-	gpualloc "opendilab.org/di-orchestrator/common/gpuallocator"
-	serverdynamic "opendilab.org/di-orchestrator/server/dynamic"
-	serverhttp "opendilab.org/di-orchestrator/server/http"
-)
-
-var (
-	DefaultLeaseLockNamespace = "di-system"
-	DefaultLeaseLockName      = "di-server"
-
-	DefaultAGConfigNamespace = "di-system"
-	DefaultAGConfigName      = "aggregator-config"
-)
-
-func main() {
-	var kubeconfig, serverBindAddress, leaseLockName, leaseLockNamespace, agconfigNamespace, agconfigName string
-	var gpuAllocPolicy, serverAddr string
-	var enableLeaderElection bool
-	if flag.Lookup("kubeconfig") == nil {
-		flag.StringVar(&kubeconfig, "kubeconfig", "", "The kubeconfig file to access kubernetes cluster. ")
-	}
-
-	flag.StringVar(&serverBindAddress, "server-bind-address", ":8080", "The address for server to bind to.")
-	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
-		"Enable leader election for controller manager. "+
-			"Enabling this will ensure there is only one active controller manager.")
-	flag.StringVar(&leaseLockNamespace, "lease-lock-namespace", DefaultLeaseLockNamespace, "The lease lock resource namespace")
-	flag.StringVar(&leaseLockName, "lease-lock-name", DefaultLeaseLockName, "The lease lock resource name")
-	flag.StringVar(&agconfigNamespace, "agconfig-namespace", DefaultAGConfigNamespace, "The AggregatorConfig namespace to manage actors and learners.")
-	flag.StringVar(&agconfigName, "agconfig-name", DefaultAGConfigName, "The AggregatorConfig name to manage actors and learners.")
-	flag.StringVar(&gpuAllocPolicy, "gpu-alloc-policy", gpualloc.SimpleGPUAllocPolicy, "The policy for server to allocate gpus to pods.")
-	flag.StringVar(&serverAddr, "server-address", dicommon.DefaultServerURL, "The address to connect to  server.")
-	flag.Parse()
-
-	kubeconfig = flag.Lookup("kubeconfig").Value.String()
-
-	cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
-	if err != nil {
-		log.Fatalf("Failed to get kubeconfig: %v", err)
-	}
-
-	kubeClient := kubernetes.NewForConfigOrDie(cfg)
-	dynamicClient := dynamic.NewForConfigOrDie(cfg)
-
-	dif := dynamicinformer.NewFilteredDynamicSharedInformerFactory(dynamicClient, serverdynamic.ResyncPeriod, corev1.NamespaceAll, nil)
-
-	dyi := serverdynamic.NewDynamicInformer(dif)
-
-	// start dynamic informer
-	stopCh := make(chan struct{})
-	go dif.Start(stopCh)
-
-	opts := zap.Options{
-		Development: true,
-	}
-	opts.BindFlags(flag.CommandLine)
-	flag.Parse()
-
-	logger := zap.New(zap.UseFlagOptions(&opts))
-
-	agconfig := fmt.Sprintf("%s/%s", agconfigNamespace, agconfigName)
-	diServer := serverhttp.NewDIServer(kubeClient, dynamicClient, logger, agconfig, dyi, gpuAllocPolicy)
-	dicommon.DefaultServerURL = serverAddr
-
-	if !enableLeaderElection {
-		if err := diServer.Start(serverBindAddress); err != nil {
-			log.Fatalf("Failed to start DIServer: %v", err)
-		}
-		return
-	}
-
-	run := func(ctx context.Context) {
-		if err := diServer.Start(serverBindAddress); err != nil {
-			log.Fatalf("Failed to start DIServer: %v", err)
-		}
-	}
-
-	// use a Go context so we can tell the leaderelection code when we
-	// want to step down
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	// listen for interrupts or the Linux SIGTERM signal and cancel
-	// our context, which the leader election code will observe and
-	// step down
-	ch := make(chan os.Signal, 1)
-	signal.Notify(ch, os.Interrupt, syscall.SIGTERM)
-	go func() {
-		<-ch
-		log.Println("Received termination, signaling shutdown")
-		cancel()
-	}()
-
-	// we use the Lease lock type since edits to Leases are less common
-	// and fewer objects in the cluster watch "all Leases".
-	id := fmt.Sprintf("%s.opendilab.org", uuid.New().String())
-	lock := &resourcelock.LeaseLock{
-		LeaseMeta: metav1.ObjectMeta{
-			Name:      leaseLockName,
-			Namespace: leaseLockNamespace,
-		},
-		Client: kubeClient.CoordinationV1(),
-		LockConfig: resourcelock.ResourceLockConfig{
-			Identity: id,
-		},
-	}
-
-	// start the leader election code loop
-	leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
-		Lock: lock,
-		// IMPORTANT: you MUST ensure that any code you have that
-		// is protected by the lease must terminate **before**
-		// you call cancel. Otherwise, you could have a background
-		// loop still running and another process could
-		// get elected before your background loop finished, violating
-		// the stated goal of the lease.
-		ReleaseOnCancel: true,
-		LeaseDuration:   60 * time.Second,
-		RenewDeadline:   15 * time.Second,
-		RetryPeriod:     5 * time.Second,
-		Callbacks: leaderelection.LeaderCallbacks{
-			OnStartedLeading: func(ctx context.Context) {
-				// we're notified when we start - this is where you would
-				// usually put your code
-				log.Printf("leader elected: %s\n", id)
-				run(ctx)
-			},
-			OnStoppedLeading: func() {
-				// we can do cleanup here
-				log.Printf("leader lost: %s\n", id)
-				os.Exit(0)
-			},
-			OnNewLeader: func(identity string) {
-				// we're notified when new leader elected
-				if identity == id {
-					// I just got the lock
-					return
-				}
-				log.Printf("new leader elected: %s\n", identity)
-			},
-		},
-	})
-}

cmd/server/server.go

+/*
+Copyright 2021 The OpenDILab authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package server
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/dynamic/dynamicinformer"
+	"k8s.io/client-go/kubernetes"
+	ctrl "sigs.k8s.io/controller-runtime"
+
+	cmdcommon "opendilab.org/di-orchestrator/cmd/common"
+	gpualloc "opendilab.org/di-orchestrator/pkg/common/gpuallocator"
+	serverdynamic "opendilab.org/di-orchestrator/pkg/server/dynamic"
+	serverhttp "opendilab.org/di-orchestrator/pkg/server/http"
+)
+
+var (
+	DefaultAGConfigNamespace = "di-system"
+	DefaultAGConfigName      = "aggregator-config"
+)
+
+type CreateOptions struct {
+	*cmdcommon.GenericFlags
+
+	ServerBindAddress string
+	GPUAllocPolicy    string
+
+	AGConfigNamespace string
+	AGCconfigName     string
+}
+
+func NewCreateOptions() *CreateOptions {
+	return &CreateOptions{
+		GenericFlags:      cmdcommon.NewGenericFlags(),
+		ServerBindAddress: ":8080",
+		GPUAllocPolicy:    gpualloc.SimpleGPUAllocPolicy,
+		AGConfigNamespace: DefaultAGConfigNamespace,
+		AGCconfigName:     DefaultAGConfigName,
+	}
+}
+
+func (o *CreateOptions) AddFlags(cmd *cobra.Command) {
+	cmd.Flags().StringVarP(&o.ServerBindAddress, "server-bind-address", "b", o.ServerBindAddress,
+		"The address for server to bind to.")
+	cmd.Flags().StringVarP(&o.GPUAllocPolicy, "gpu-alloc-policy", "p", o.GPUAllocPolicy,
+		"The policy for server to allocate gpus to pods.")
+
+	cmd.Flags().StringVar(&o.AGConfigNamespace, "agconfig-namespace", o.AGConfigNamespace,
+		"The AggregatorConfig namespace to manage actors and learners.")
+	cmd.Flags().StringVar(&o.AGCconfigName, "agconfig-name", o.AGCconfigName,
+		"The AggregatorConfig name to manage actors and learners.")
+}
+
+// serverCmd represents the server command
+func NewCmdServer() *cobra.Command {
+	o := NewCreateOptions()
+	var serverCmd = &cobra.Command{
+		Use:   "server",
+		Short: "Command to run di-server ",
+		Long: `Run di-server with specified configuration.
+
+Examples:
+	# Start di-server with gpu allocation policy and bind address specified.
+	di-orchestrator server -p simple -b :8080
+`,
+		Run: func(cmd *cobra.Command, args []string) {
+			cobra.CheckErr(runCommand(cmd, o))
+		},
+	}
+
+	o.AddFlags(serverCmd)
+	return serverCmd
+}
+
+func runCommand(cmd *cobra.Command, options *CreateOptions) error {
+	cfg, err := ctrl.GetConfig()
+	if err != nil {
+		return err
+	}
+
+	kubeClient := kubernetes.NewForConfigOrDie(cfg)
+	dynamicClient := dynamic.NewForConfigOrDie(cfg)
+
+	dif := dynamicinformer.NewFilteredDynamicSharedInformerFactory(dynamicClient, serverdynamic.ResyncPeriod, corev1.NamespaceAll, nil)
+
+	dyi := serverdynamic.NewDynamicInformer(dif)
+
+	// start dynamic informer
+	stopCh := make(chan struct{})
+	go dif.Start(stopCh)
+
+	agconfig := fmt.Sprintf("%s/%s", options.AGConfigNamespace, options.AGCconfigName)
+	diServer := serverhttp.NewDIServer(kubeClient, dynamicClient, cmdcommon.Logger, agconfig, dyi, options.GPUAllocPolicy)
+
+	if err := diServer.Start(options.ServerBindAddress); err != nil {
+		return fmt.Errorf("failed to start di-server: %v", err)
+	}
+	return nil
+}

cmd/webhook/main.go → cmd/webhook/webhook.go

@@ -14,27 +14,65 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package main
+package webhook
 
 import (
-	"flag"
-	"os"
-
-	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
-	// to ensure that exec-entrypoint and run can make use of them.
-	_ "k8s.io/client-go/plugin/pkg/client/auth"
-
+	"github.com/spf13/cobra"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
-	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
-	div1alpha1 "opendilab.org/di-orchestrator/api/v1alpha1"
-	//+kubebuilder:scaffold:imports
+	cmdcommon "opendilab.org/di-orchestrator/cmd/common"
+	div1alpha1 "opendilab.org/di-orchestrator/pkg/api/v1alpha1"
 )
 
+type CreateOptions struct {
+	MetricAddress        string
+	ProbeAddress         string
+	EnableLeaderElection bool
+	Port                 int
+}
+
+func NewCreateOptions() *CreateOptions {
+	return &CreateOptions{
+		MetricAddress:        ":8443",
+		ProbeAddress:         ":8080",
+		EnableLeaderElection: false,
+		Port:                 9443,
+	}
+}
+
+func (o *CreateOptions) AddFlags(cmd *cobra.Command) {
+	cmd.Flags().StringVar(&o.MetricAddress, "metric-addr", o.MetricAddress, "The address the metric endpoint binds to.")
+	cmd.Flags().StringVar(&o.ProbeAddress, "probe-addr", o.ProbeAddress, "The address the probe endpoint binds to.")
+	cmd.Flags().BoolVar(&o.EnableLeaderElection, "leader-elect", o.EnableLeaderElection,
+		"Enable leader election for controller manager. "+
+			"Enabling this will ensure there is only one active controller manager.")
+	cmd.Flags().IntVarP(&o.Port, "port", "p", o.Port, "The port the webhook endpoint binds to.")
+}
+
+func NewCmdWebhook() *cobra.Command {
+	o := NewCreateOptions()
+	var webhookCmd = &cobra.Command{
+		Use:   "webhook",
+		Short: "Command to run di-webhook ",
+		Long: `Run di-webhook with specified configuration.
+
+Examples:
+	# Start di-webhook with port specified.
+	di-orchestrator webhook --port 9443
+`,
+		Run: func(cmd *cobra.Command, args []string) {
+			cobra.CheckErr(runCommand(cmd, o))
+		},
+	}
+
+	o.AddFlags(webhookCmd)
+	return webhookCmd
+}
+
 var (
 	scheme   = runtime.NewScheme()
 	setupLog = ctrl.Log.WithName("setup")
@@ -47,54 +85,41 @@ func init() {
 	//+kubebuilder:scaffold:scheme
 }
 
-func main() {
-	var metricsAddr, probeAddr string
-	var enableLeaderElection bool
-	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
-	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
-	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
-		"Enable leader election for controller manager. "+
-			"Enabling this will ensure there is only one active controller manager.")
-
-	opts := zap.Options{
-		Development: true,
-	}
-	opts.BindFlags(flag.CommandLine)
-	flag.Parse()
-
-	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
+func runCommand(cmd *cobra.Command, options *CreateOptions) error {
+	ctrl.SetLogger(cmdcommon.Logger)
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:                 scheme,
-		MetricsBindAddress:     metricsAddr,
-		Port:                   9443,
-		HealthProbeBindAddress: probeAddr,
-		LeaderElection:         enableLeaderElection,
+		MetricsBindAddress:     options.MetricAddress,
+		Port:                   options.Port,
+		HealthProbeBindAddress: options.ProbeAddress,
+		LeaderElection:         options.EnableLeaderElection,
 		LeaderElectionID:       "67841a5d.opendilab.org",
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")
-		os.Exit(1)
+		return err
 	}
 
 	if err = (&div1alpha1.DIJob{}).SetupWebhookWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create webhook", "webhook", "DIJob")
-		os.Exit(1)
+		return err
 	}
 	//+kubebuilder:scaffold:builder
 
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
 		setupLog.Error(err, "unable to set up health check")
-		os.Exit(1)
+		return err
 	}
 	if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
 		setupLog.Error(err, "unable to set up ready check")
-		os.Exit(1)
+		return err
 	}
 
 	setupLog.Info("starting manager")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		setupLog.Error(err, "problem running manager")
-		os.Exit(1)
+		return err
 	}
+	return nil
 }

config/default/kustomization.yaml

@@ -28,7 +28,7 @@ patchesStrategicMerge:
 # Protect the /metrics endpoint by putting it behind auth.
 # If you want your controller-manager to expose the /metrics
 # endpoint w/o any authn/z, please comment the following line.
-- manager_auth_proxy_patch.yaml
+# - manager_auth_proxy_patch.yaml
 
 # Mount the controller config file for loading manager configurations
 # through a ComponentConfig type

config/default/manager_auth_proxy_patch.yaml

@@ -11,6 +11,6 @@ spec:
       containers:
       - name: manager
         args:
-        - "--health-probe-bind-address=:8081"
-        - "--metrics-bind-address=:8080"
+        - "--probe-addr=:8080"
+        - "--metric-addr=:8443"
         - "--leader-elect"

config/di-manager.yaml

@@ -16251,7 +16251,7 @@ spec:
   ports:
   - name: https
     port: 8443
-    targetPort: 8080
+    targetPort: 8443
   selector:
     control-plane: di-operator
 ---
@@ -16270,21 +16270,6 @@ spec:
 ---
 apiVersion: v1
 kind: Service
-metadata:
-  name: di-server-nodeport
-  namespace: di-system
-spec:
-  ports:
-  - nodePort: 32270
-    port: 8080
-    protocol: TCP
-    targetPort: 8080
-  selector:
-    control-plane: di-server
-  type: NodePort
----
-apiVersion: v1
-kind: Service
 metadata:
   name: di-webhook-service
   namespace: di-system
@@ -16314,24 +16299,25 @@ spec:
     spec:
       containers:
       - args:
-        - --health-probe-bind-address=:8081
-        - --metrics-bind-address=:8080
+        - --probe-addr=:8080
+        - --metric-addr=:8443
         - --leader-elect
         command:
-        - /di-operator
-        image: registry.sensetime.com/cloudnative4ai/di-operator:v0.2.2
+        - /di-orchestrator
+        - operator
+        image: registry.sensetime.com/cloudnative4ai/di-orchestrator:v0.3.0
         imagePullPolicy: Always
         livenessProbe:
           httpGet:
             path: /healthz
-            port: 8081
+            port: 8080
           initialDelaySeconds: 15
           periodSeconds: 20
         name: manager
         readinessProbe:
           httpGet:
             path: /readyz
-            port: 8081
+            port: 8080
           initialDelaySeconds: 5
           periodSeconds: 10
         resources:
@@ -16365,12 +16351,11 @@ spec:
       containers:
       - args:
         - --server-bind-address=:8080
-        - --leader-elect
-        - --lease-lock-namespace=di-system
-        - --lease-lock-name=di-server
+        - --gpu-alloc-policy=simple
         command:
-        - /di-server
-        image: registry.sensetime.com/cloudnative4ai/di-server:v0.2.2
+        - /di-orchestrator
+        - server
+        image: registry.sensetime.com/cloudnative4ai/di-orchestrator:v0.3.0
         imagePullPolicy: Always
         livenessProbe:
           httpGet:
@@ -16408,14 +16393,19 @@ spec:
         control-plane: di-webhook
     spec:
       containers:
-      - command:
-        - /di-webhook
-        image: registry.sensetime.com/cloudnative4ai/di-webhook:v0.2.2
+      - args:
+        - --probe-addr=:8080
+        - --metric-addr=:8443
+        - --port=9443
+        command:
+        - /di-orchestrator
+        - webhook
+        image: registry.sensetime.com/cloudnative4ai/di-orchestrator:v0.3.0
         imagePullPolicy: Always
         livenessProbe:
           httpGet:
             path: /healthz
-            port: 8081
+            port: 8080
           initialDelaySeconds: 15
           periodSeconds: 20
         name: webhook
@@ -16426,7 +16416,7 @@ spec:
         readinessProbe:
           httpGet:
             path: /readyz
-            port: 8081
+            port: 8080
           initialDelaySeconds: 5
           periodSeconds: 10
         resources:

config/manager/di_operator.yaml

@@ -24,10 +24,13 @@ spec:
     spec:
       containers:
       - command:
-        - /di-operator
+        - /di-orchestrator
+        - operator
         args:
-        - "--server-address=http://di-server.di-system:8080"
-        image: registry.sensetime.com/cloudnative4ai/di-operator:v0.2.2
+        - "--probe-addr=:8080"
+        - "--metric-addr=:8443"
+        - "--leader-elect"
+        image: registry.sensetime.com/cloudnative4ai/di-orchestrator:v0.3.0
         imagePullPolicy: Always
         name: manager
         securityContext:
@@ -35,13 +38,13 @@ spec:
         livenessProbe:
           httpGet:
             path: /healthz
-            port: 8081
+            port: 8080
           initialDelaySeconds: 15
           periodSeconds: 20
         readinessProbe:
           httpGet:
             path: /readyz
-            port: 8081
+            port: 8080
           initialDelaySeconds: 5
           periodSeconds: 10
         resources:

config/manager/di_server.yaml

@@ -17,13 +17,12 @@ spec:
     spec:
       containers:
       - command:
-        - /di-server
+        - /di-orchestrator
+        - server
         args:
-        - "--server-bind-address=:8080"
-        - "--leader-elect"
-        - "--lease-lock-namespace=di-system"
-        - "--lease-lock-name=di-server"
-        image: registry.sensetime.com/cloudnative4ai/di-server:v0.2.2
+        - --server-bind-address=:8080
+        - --gpu-alloc-policy=simple
+        image: registry.sensetime.com/cloudnative4ai/di-orchestrator:v0.3.0
         imagePullPolicy: Always
         name: server
         securityContext:

config/manager/di_webhook.yaml

@@ -17,8 +17,13 @@ spec:
     spec:
       containers:
       - command:
-        - /di-webhook
-        image: registry.sensetime.com/cloudnative4ai/di-webhook:v0.2.2
+        - /di-orchestrator 
+        - webhook
+        args:
+        - "--probe-addr=:8080"
+        - "--metric-addr=:8443"
+        - --port=9443
+        image: registry.sensetime.com/cloudnative4ai/di-orchestrator:v0.3.0
         imagePullPolicy: Always
         name: webhook
         securityContext:
@@ -26,13 +31,13 @@ spec:
         livenessProbe:
           httpGet:
             path: /healthz
-            port: 8081
+            port: 8080
           initialDelaySeconds: 15
           periodSeconds: 20
         readinessProbe:
           httpGet:
             path: /readyz
-            port: 8081
+            port: 8080
           initialDelaySeconds: 5
           periodSeconds: 10
         resources:

config/manager/kustomization.yaml

@@ -5,12 +5,6 @@ resources:
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
-- name: registry.sensetime.com/cloudnative4ai/di-operator
-  newName: registry.sensetime.com/cloudnative4ai/di-operator
-  newTag: v0.2.2
-- name: registry.sensetime.com/cloudnative4ai/di-server
-  newName: registry.sensetime.com/cloudnative4ai/di-server
-  newTag: v0.2.2
-- name: registry.sensetime.com/cloudnative4ai/di-webhook
-  newName: registry.sensetime.com/cloudnative4ai/di-webhook
-  newTag: v0.2.2
+- name: registry.sensetime.com/cloudnative4ai/di-orchestrator
+  newName: registry.sensetime.com/cloudnative4ai/di-orchestrator
+  newTag: v0.3.0

config/rbac/auth_proxy_service.yaml

@@ -9,6 +9,6 @@ spec:
   ports:
   - name: https
     port: 8443
-    targetPort: 8080
+    targetPort: 8443
   selector:
     control-plane: di-operator

config/rbac/di_server_service.yaml

@@ -11,19 +11,3 @@ spec:
     - protocol: TCP
       port: 8080
       targetPort: 8080
-
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: di-server-nodeport
-  namespace: di-system
-spec:
-  selector:
-    control-plane: di-server
-  type: NodePort
-  ports:
-    - protocol: TCP
-      port: 8080
-      targetPort: 8080
-      nodePort: 32270
\ No newline at end of file

config/samples/agconfig.yaml

@@ -9,7 +9,7 @@ spec:
       spec:
         containers:
         - name: di-container
-          image: opendilab/ding:2021-09-26
+          image: opendilab/ding:nightly-atari
           imagePullPolicy: Always
           env:
           - name: PYTHONUNBUFFERED

config/samples/dijob-cartpole.yaml

@@ -19,7 +19,7 @@ spec:
       spec:
         containers:
         - name: di-container
-          image: opendilab/ding:2021-09-26
+          image: opendilab/ding:nightly-atari
           imagePullPolicy: Always
           env:
           - name: PYTHONUNBUFFERED
@@ -166,7 +166,7 @@ spec:
       spec:
         containers:
         - name: di-container
-          image: opendilab/ding:2021-09-26
+          image: opendilab/ding:nightly-atari
           imagePullPolicy: Always
           env:
           - name: PYTHONUNBUFFERED
@@ -193,7 +193,7 @@ spec:
       spec:
         containers:
         - name: di-container
-          image: opendilab/ding:2021-09-26
+          image: opendilab/ding:nightly-atari
           imagePullPolicy: Always
           env:
           - name: PYTHONUNBUFFERED

config/samples/dijob-qbert.yaml

@@ -19,7 +19,7 @@ spec:
       spec:
         containers:
         - name: di-container
-          image: opendilab/ding:2021-09-26
+          image: opendilab/ding:nightly-atari
           imagePullPolicy: Always
           env:
           - name: PYTHONUNBUFFERED
@@ -169,7 +169,7 @@ spec:
       spec:
         containers:
         - name: di-container
-          image: opendilab/ding:2021-09-26
+          image: opendilab/ding:nightly-atari
           imagePullPolicy: Always
           env:
           - name: PYTHONUNBUFFERED
@@ -199,7 +199,7 @@ spec:
       spec:
         containers:
         - name: di-container
-          image: opendilab/ding:2021-09-26
+          image: opendilab/ding:nightly-atari
           imagePullPolicy: Always
           env:
           - name: PYTHONUNBUFFERED

config/v1.15.5/di-manager.yaml

@@ -320,12 +320,13 @@ spec:
     spec:
       containers:
       - args:
-        - --health-probe-bind-address=:8081
-        - --metrics-bind-address=:8080
+        - --probe-addr=:8080
+        - --metric-addr=:8443
         - --leader-elect
         command:
-        - /di-operator
-        image: registry.sensetime.com/cloudnative4ai/di-operator:v0.2.2
+        - /di-orchestrator
+        - operator
+        image: registry.sensetime.com/cloudnative4ai/di-orchestrator:v0.3.0
         imagePullPolicy: Always
         livenessProbe:
           httpGet:
@@ -371,12 +372,11 @@ spec:
       containers:
       - args:
         - --server-bind-address=:8080
-        - --leader-elect
-        - --lease-lock-namespace=di-system
-        - --lease-lock-name=di-server
+        - --gpu-alloc-policy=simple
         command:
-        - /di-server
-        image: registry.sensetime.com/cloudnative4ai/di-server:v0.2.2
+        - /di-orchestrator
+        - server
+        image: registry.sensetime.com/cloudnative4ai/di-orchestrator:v0.3.0
         imagePullPolicy: Always
         livenessProbe:
           httpGet:
@@ -414,9 +414,14 @@ spec:
         control-plane: di-webhook
     spec:
       containers:
-      - command:
-        - /di-webhook
-        image: registry.sensetime.com/cloudnative4ai/di-webhook:v0.2.2
+      - args:
+        - --probe-addr=:8080
+        - --metric-addr=:8443
+        - --port=9443
+        command:
+        - /di-orchestrator
+        - webhook
+        image: registry.sensetime.com/cloudnative4ai/di-orchestrator:v0.3.0
         imagePullPolicy: Always
         livenessProbe:
           httpGet:

e2e/e2e_suite_test.go

@@ -16,7 +16,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
 
-	div1alpha1 "opendilab.org/di-orchestrator/api/v1alpha1"
+	div1alpha1 "opendilab.org/di-orchestrator/pkg/api/v1alpha1"
 )
 
 func TestE2E(t *testing.T) {

e2e/e2e_test.go

@@ -16,10 +16,10 @@ import (
 	"k8s.io/apimachinery/pkg/util/yaml"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
-	div1alpha1 "opendilab.org/di-orchestrator/api/v1alpha1"
-	dicommon "opendilab.org/di-orchestrator/common"
-	diutil "opendilab.org/di-orchestrator/utils"
-	testutil "opendilab.org/di-orchestrator/utils/testutils"
+	div1alpha1 "opendilab.org/di-orchestrator/pkg/api/v1alpha1"
+	dicommon "opendilab.org/di-orchestrator/pkg/common"
+	diutil "opendilab.org/di-orchestrator/pkg/utils"
+	testutil "opendilab.org/di-orchestrator/pkg/utils/testutils"
 )
 
 const (

go.mod

@@ -6,10 +6,11 @@ require (
 	github.com/deckarep/golang-set v1.7.1
 	github.com/gin-gonic/gin v1.7.7
 	github.com/go-logr/logr v0.3.0
-	github.com/google/uuid v1.1.2
 	github.com/onsi/ginkgo v1.14.1
 	github.com/onsi/gomega v1.10.2
 	github.com/sirupsen/logrus v1.6.0
+	github.com/spf13/cobra v1.2.1
+	github.com/spf13/pflag v1.0.5
 	k8s.io/api v0.20.2
 	k8s.io/apimachinery v0.20.2
 	k8s.io/client-go v0.20.2

main.go

+/*
+Copyright 2021 The OpenDILab authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package main
+
+import "opendilab.org/di-orchestrator/cmd"
+
+func main() {
+	cmd.Execute()
+}

api/v1alpha1/webhook_suite_test.go → pkg/api/v1alpha1/webhook_suite_test.go

@@ -65,10 +65,10 @@ var _ = BeforeSuite(func() {
 
 	By("bootstrapping test environment")
 	testEnv = &envtest.Environment{
-		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "config", "crd", "bases")},
+		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "..", "config", "crd", "bases")},
 		ErrorIfCRDPathMissing: false,
 		WebhookInstallOptions: envtest.WebhookInstallOptions{
-			Paths:            []string{filepath.Join("..", "..", "config", "webhook")},
+			Paths:            []string{filepath.Join("..", "..", "..", "config", "webhook")},
 			LocalServingPort: 8100 + config.GinkgoConfig.ParallelNode,
 		},
 	}

controllers/dijob.go → pkg/controllers/dijob.go

@@ -14,9 +14,9 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
-	div1alpha1 "opendilab.org/di-orchestrator/api/v1alpha1"
-	dicommon "opendilab.org/di-orchestrator/common"
-	diutil "opendilab.org/di-orchestrator/utils"
+	div1alpha1 "opendilab.org/di-orchestrator/pkg/api/v1alpha1"
+	dicommon "opendilab.org/di-orchestrator/pkg/common"
+	diutil "opendilab.org/di-orchestrator/pkg/utils"
 )
 
 func isSucceeded(job *div1alpha1.DIJob) bool {

controllers/dijob_controller.go → pkg/controllers/dijob_controller.go

@@ -34,8 +34,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
-	div1alpha1 "opendilab.org/di-orchestrator/api/v1alpha1"
-	diutil "opendilab.org/di-orchestrator/utils"
+	div1alpha1 "opendilab.org/di-orchestrator/pkg/api/v1alpha1"
+	diutil "opendilab.org/di-orchestrator/pkg/utils"
 )
 
 // DIJobReconciler reconciles a DIJob object

controllers/dijob_controller_test.go → pkg/controllers/dijob_controller_test.go

@@ -12,11 +12,11 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
-	div1alpha1 "opendilab.org/di-orchestrator/api/v1alpha1"
-	dicommon "opendilab.org/di-orchestrator/common"
-	commontypes "opendilab.org/di-orchestrator/common/types"
-	diutil "opendilab.org/di-orchestrator/utils"
-	testutil "opendilab.org/di-orchestrator/utils/testutils"
+	div1alpha1 "opendilab.org/di-orchestrator/pkg/api/v1alpha1"
+	dicommon "opendilab.org/di-orchestrator/pkg/common"
+	commontypes "opendilab.org/di-orchestrator/pkg/common/types"
+	diutil "opendilab.org/di-orchestrator/pkg/utils"
+	testutil "opendilab.org/di-orchestrator/pkg/utils/testutils"
 )
 
 var _ = Describe("DIJob Controller", func() {

controllers/dijob_test.go → pkg/controllers/dijob_test.go

@@ -9,10 +9,10 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 
-	div1alpha1 "opendilab.org/di-orchestrator/api/v1alpha1"
-	dicommon "opendilab.org/di-orchestrator/common"
-	diutil "opendilab.org/di-orchestrator/utils"
-	testutil "opendilab.org/di-orchestrator/utils/testutils"
+	div1alpha1 "opendilab.org/di-orchestrator/pkg/api/v1alpha1"
+	dicommon "opendilab.org/di-orchestrator/pkg/common"
+	diutil "opendilab.org/di-orchestrator/pkg/utils"
+	testutil "opendilab.org/di-orchestrator/pkg/utils/testutils"
 )
 
 var _ = Describe("DIJob Specification", func() {

controllers/pod.go → pkg/controllers/pod.go

@@ -9,9 +9,9 @@ import (
 	"k8s.io/apimachinery/pkg/api/errors"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
-	div1alpha1 "opendilab.org/di-orchestrator/api/v1alpha1"
-	dicommon "opendilab.org/di-orchestrator/common"
-	diutil "opendilab.org/di-orchestrator/utils"
+	div1alpha1 "opendilab.org/di-orchestrator/pkg/api/v1alpha1"
+	dicommon "opendilab.org/di-orchestrator/pkg/common"
+	diutil "opendilab.org/di-orchestrator/pkg/utils"
 )
 
 func (r *DIJobReconciler) reconcilePodsAndServices(ctx context.Context, job *div1alpha1.DIJob, pods []*corev1.Pod, services []*corev1.Service) error {

controllers/status.go → pkg/controllers/status.go

@@ -10,9 +10,9 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
-	div1alpha1 "opendilab.org/di-orchestrator/api/v1alpha1"
-	dicommon "opendilab.org/di-orchestrator/common"
-	diutil "opendilab.org/di-orchestrator/utils"
+	div1alpha1 "opendilab.org/di-orchestrator/pkg/api/v1alpha1"
+	dicommon "opendilab.org/di-orchestrator/pkg/common"
+	diutil "opendilab.org/di-orchestrator/pkg/utils"
 )
 
 const (

controllers/suite_test.go → pkg/controllers/suite_test.go

@@ -35,8 +35,8 @@ import (
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
-	div1alpha1 "opendilab.org/di-orchestrator/api/v1alpha1"
-	testutil "opendilab.org/di-orchestrator/utils/testutils"
+	div1alpha1 "opendilab.org/di-orchestrator/pkg/api/v1alpha1"
+	testutil "opendilab.org/di-orchestrator/pkg/utils/testutils"
 	//+kubebuilder:scaffold:imports
 )
 
@@ -66,7 +66,7 @@ var _ = BeforeSuite(func() {
 
 	By("bootstrapping test environment")
 	testEnv = &envtest.Environment{
-		CRDDirectoryPaths:     []string{filepath.Join("..", "config", "crd", "bases")},
+		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "config", "crd", "bases")},
 		ErrorIfCRDPathMissing: true,
 	}
 

server/dynamic/handler.go → pkg/server/dynamic/handler.go

@@ -8,7 +8,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
-	div1alpha1 "opendilab.org/di-orchestrator/api/v1alpha1"
+	div1alpha1 "opendilab.org/di-orchestrator/pkg/api/v1alpha1"
 )
 
 func GetPodFromObject(obj interface{}) (*corev1.Pod, error) {

server/dynamic/informer.go → pkg/server/dynamic/informer.go

@@ -11,7 +11,7 @@ import (
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/tools/cache"
 
-	div1alpha1 "opendilab.org/di-orchestrator/api/v1alpha1"
+	div1alpha1 "opendilab.org/di-orchestrator/pkg/api/v1alpha1"
 )
 
 var (

server/http/dijob.go → pkg/server/http/dijob.go

@@ -11,10 +11,10 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 
-	div1alpha1 "opendilab.org/di-orchestrator/api/v1alpha1"
-	dicommon "opendilab.org/di-orchestrator/common"
-	commontypes "opendilab.org/di-orchestrator/common/types"
-	diutil "opendilab.org/di-orchestrator/utils"
+	div1alpha1 "opendilab.org/di-orchestrator/pkg/api/v1alpha1"
+	dicommon "opendilab.org/di-orchestrator/pkg/common"
+	commontypes "opendilab.org/di-orchestrator/pkg/common/types"
+	diutil "opendilab.org/di-orchestrator/pkg/utils"
 )
 
 func (s *DIServer) getDIJob(namespace, modulePodName string) (*div1alpha1.DIJob, error) {

server/http/pod.go → pkg/server/http/pod.go

@@ -12,8 +12,8 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 
-	commontypes "opendilab.org/di-orchestrator/common/types"
-	diutil "opendilab.org/di-orchestrator/utils"
+	commontypes "opendilab.org/di-orchestrator/pkg/common/types"
+	diutil "opendilab.org/di-orchestrator/pkg/utils"
 )
 
 func (s *DIServer) getPodsByNames(namespace string, names []string) ([]*corev1.Pod, error) {

server/http/server.go → pkg/server/http/server.go

@@ -17,12 +17,12 @@ import (
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 
-	div1alpha1 "opendilab.org/di-orchestrator/api/v1alpha1"
-	dicommon "opendilab.org/di-orchestrator/common"
-	gpualloc "opendilab.org/di-orchestrator/common/gpuallocator"
-	commontypes "opendilab.org/di-orchestrator/common/types"
-	serverdynamic "opendilab.org/di-orchestrator/server/dynamic"
-	diutil "opendilab.org/di-orchestrator/utils"
+	div1alpha1 "opendilab.org/di-orchestrator/pkg/api/v1alpha1"
+	dicommon "opendilab.org/di-orchestrator/pkg/common"
+	gpualloc "opendilab.org/di-orchestrator/pkg/common/gpuallocator"
+	commontypes "opendilab.org/di-orchestrator/pkg/common/types"
+	serverdynamic "opendilab.org/di-orchestrator/pkg/server/dynamic"
+	diutil "opendilab.org/di-orchestrator/pkg/utils"
 )
 
 var (

server/http/server_test.go → pkg/server/http/server_test.go

@@ -16,11 +16,11 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
-	div1alpha1 "opendilab.org/di-orchestrator/api/v1alpha1"
-	dicommon "opendilab.org/di-orchestrator/common"
-	commontypes "opendilab.org/di-orchestrator/common/types"
-	diutil "opendilab.org/di-orchestrator/utils"
-	testutil "opendilab.org/di-orchestrator/utils/testutils"
+	div1alpha1 "opendilab.org/di-orchestrator/pkg/api/v1alpha1"
+	dicommon "opendilab.org/di-orchestrator/pkg/common"
+	commontypes "opendilab.org/di-orchestrator/pkg/common/types"
+	diutil "opendilab.org/di-orchestrator/pkg/utils"
+	testutil "opendilab.org/di-orchestrator/pkg/utils/testutils"
 )
 
 var _ = Describe("Server Test", func() {

server/http/suite_test.go → pkg/server/http/suite_test.go

@@ -43,9 +43,9 @@ import (
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
-	div1alpha1 "opendilab.org/di-orchestrator/api/v1alpha1"
-	serverdynamic "opendilab.org/di-orchestrator/server/dynamic"
-	testutil "opendilab.org/di-orchestrator/utils/testutils"
+	div1alpha1 "opendilab.org/di-orchestrator/pkg/api/v1alpha1"
+	serverdynamic "opendilab.org/di-orchestrator/pkg/server/dynamic"
+	testutil "opendilab.org/di-orchestrator/pkg/utils/testutils"
 	//+kubebuilder:scaffold:imports
 )
 
@@ -83,7 +83,7 @@ var _ = BeforeSuite(func() {
 
 	By("bootstrapping test environment")
 	testEnv = &envtest.Environment{
-		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "config", "crd", "bases")},
+		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "..", "config", "crd", "bases")},
 		ErrorIfCRDPathMissing: true,
 	}
 

utils/replicas_util.go → pkg/utils/replicas_util.go

@@ -9,7 +9,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 
-	dicommon "opendilab.org/di-orchestrator/common"
+	dicommon "opendilab.org/di-orchestrator/pkg/common"
 )
 
 func BuildPodAndService(

utils/testutils/dijob.go → pkg/utils/testutils/dijob.go

@@ -9,9 +9,9 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
-	div1alpha1 "opendilab.org/di-orchestrator/api/v1alpha1"
-	dicommon "opendilab.org/di-orchestrator/common"
-	diutil "opendilab.org/di-orchestrator/utils"
+	div1alpha1 "opendilab.org/di-orchestrator/pkg/api/v1alpha1"
+	dicommon "opendilab.org/di-orchestrator/pkg/common"
+	diutil "opendilab.org/di-orchestrator/pkg/utils"
 )
 
 func NewDIJob() *div1alpha1.DIJob {

utils/testutils/pod.go → pkg/utils/testutils/pod.go

@@ -14,7 +14,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
-	dicommon "opendilab.org/di-orchestrator/common"
+	dicommon "opendilab.org/di-orchestrator/pkg/common"
 )
 
 func NewPod(name, jobName string, ownRefer metav1.OwnerReference) *corev1.Pod {

utils/util.go → pkg/utils/util.go

@@ -16,9 +16,9 @@ import (
 	utilrand "k8s.io/apimachinery/pkg/util/rand"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
-	div1alpha1 "opendilab.org/di-orchestrator/api/v1alpha1"
-	dicommon "opendilab.org/di-orchestrator/common"
-	commontypes "opendilab.org/di-orchestrator/common/types"
+	div1alpha1 "opendilab.org/di-orchestrator/pkg/api/v1alpha1"
+	dicommon "opendilab.org/di-orchestrator/pkg/common"
+	commontypes "opendilab.org/di-orchestrator/pkg/common/types"
 )
 
 const (