[werft] Create GitHub relase on 'publish-release=true'

.werft/build.js

@@ -5,6 +5,7 @@ const { sleep } = require('./util/util.js');
 const { wipeAndRecreateNamespace, setKubectlContextNamespace, deleteNonNamespaceObjects } = require('./util/kubectl.js');
 const { issueAndInstallCertficate } = require('./util/certs.js');
 const { reportBuildFailureInSlack } = require('./util/slack.js');
+const semver = require('semver');
 
 const GCLOUD_SERVICE_ACCOUNT_PATH = "/mnt/secrets/gcp-sa/service-account.json";
 
@@ -102,9 +103,38 @@ async function build(context, version) {
     }
     exec(`leeway build --werft=true -Dversion=${version} -DremoveSources=false -DimageRepoBase=${imageRepo}`, buildEnv);
     if (publishRelease) {
-        publishHelmChart("gcr.io/gitpod-io/self-hosted");
-        exec(`leeway run --werft=true install/installer:publish-as-latest -Dversion=${version} -DimageRepoBase=${imageRepo}`)
-        exec(`gcloud auth activate-service-account --key-file "${GCLOUD_SERVICE_ACCOUNT_PATH}"`);
+        try {
+            werft.phase("publish", "checking version semver compliance...");
+            if (!semver.valid(version)) {
+                // make this an explicit error as early as possible. Is required by helm Charts.yaml/version
+                throw new Error(`'${version}' is not semver compliant and thus cannot used for Self-Hosted releases!`)
+            }
+
+            werft.phase("publish", "publishing docker images...");
+            exec(`leeway run --werft=true install/installer:publish-as-latest -Dversion=${version} -DimageRepoBase=${imageRepo}`);
+
+            werft.phase("publish", "publishing Helm chart...");
+            publishHelmChart("gcr.io/gitpod-io/self-hosted", version);
+
+            werft.phase("publish", `preparing GitHub release files...`);
+            const releaseFilesTmpDir = exec("mktemp -d", { silent: true }).stdout.trim();
+            const releaseTarName = "release.tar.gz";
+            exec(`leeway build --werft=true chart:release-tars -Dversion=${version} -DimageRepoBase=${imageRepo} --save ${releaseFilesTmpDir}/${releaseTarName}`);
+            exec(`cd ${releaseFilesTmpDir} && tar xzf ${releaseTarName} && rm -f ${releaseTarName}`);
+
+            werft.phase("publish", `publishing GitHub release ${version}...`);
+            const prereleaseFlag = semver.prerelease(version) !== null ? "-prerelease" : "";
+            const tag = `v${version}`;
+            const releaseBranch = context.Repository.ref;
+            const description = `Gitpod Self-Hosted ${version}<br/><br/>Docs: https://www.gitpod.io/docs/self-hosted/latest/self-hosted/`;
+            exec(`github-release ${prereleaseFlag} gitpod-io/gitpod ${tag} ${releaseBranch} '${description}' "${releaseFilesTmpDir}/*"`);
+
+            werft.done('publish');
+        } catch (err) {
+            werft.fail('publish', err);
+        } finally {
+            exec(`gcloud auth activate-service-account --key-file "${GCLOUD_SERVICE_ACCOUNT_PATH}"`);
+        }
     }
     // gitTag(`build/${version}`);
 
@@ -298,7 +328,7 @@ async function deployToDev(deploymentConfig, workspaceFeatureFlags, dynamicCPULi
 /**
  * Publish Charts
  */
-async function publishHelmChart(imageRepoBase) {
+async function publishHelmChart(imageRepoBase, version) {
     werft.phase("publish-charts", "Publish charts");
     [
         "gcloud config set project gitpod-io",

.werft/build.yaml

@@ -73,6 +73,12 @@ pod:
         secretKeyRef:
           name: slack-path
           key: slackPath
+    # used for GitHub releases (NOTE: for some reasons the token contains a trailing \n, is trimmed below)
+    - name: GITHUB_TOKEN
+      valueFrom:
+        secretKeyRef:
+          name: github-sh-release-token
+          key: token
     # - name: GITPOD_TEST_TOKEN_GITHUB
     #   valueFrom:
     #     secretKeyRef:
@@ -95,10 +101,12 @@ pod:
         sleep 1
         set -Eeuo pipefail
 
+        export GITHUB_TOKEN=$(echo $GITHUB_TOKEN | xargs)
+
         export DOCKER_HOST=tcp://$NODENAME:2375
         sudo chown -R gitpod:gitpod /workspace
 
-        npm install shelljs
+        npm install shelljs semver
         printf '{{ toJson . }}' > context.json
 
         {{- if .Annotations.debug }}

.werft/wipe-devstaging.yaml

@@ -14,7 +14,7 @@ pod:
       secretName: gcp-sa-gitpod-dev-deployer
   containers:
   - name: wipe-devstaging
-    image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:gpl-bump-helm.12
+    image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:gpl-sh-gh-release.3
     workingDir: /workspace
     imagePullPolicy: Always
     volumeMounts:

chart/BUILD.yaml

@@ -9,14 +9,22 @@ packages:
       - "**/*.tpl"
       - "secrets/**/*"
       - "config/**/*"
+      # Required for semver check below
+      - test-semver.js
     argdeps:
       - version
       - imageRepoBase
     config:
       commands:
+        # Chart.version has to be semver compliant for "helm package to work".
+        # We use this in two modes:
+        #  - publishing "Gitpod Self-Hosted" releases: version _is_ semver compliant
+        #  - publishing "branch builds": version is <branch-name>.<nr> (_not_ semver compliant)
+        # To fix this locally (without poisoning the leeway upstream deps) we detect semver compliance here, and in case of non-compliance, prefix with "0.1.0-".
+        # Note: This assumes all clients of the resulting "branch builds" package do not care about the actual version used.
+        - ["sh", "-c", "(npm install semver && node test-semver.js ${version}) && yq w -i Chart.yaml version ${version} || yq w -i Chart.yaml version 0.1.0-${version}"]
+        - ["rm", "-f", "test-semver.js"]
         # set values
-        # Chart.version has to be semver compliant
-        - ["sh", "-c", "yq w -i Chart.yaml version 0.1.0-${version}"]
         - ["yq", "w", "-i", "values.yaml", "version", "${version}"]
         - ["yq", "w", "-i", "values.yaml", "imagePrefix", "${imageRepoBase}/"]
         # prepare chart
@@ -27,3 +35,17 @@ packages:
         - ["helm", "dependency", "build", "gitpod"]
         - ["helm", "package", "gitpod", "--destination", "."]
         - ["rm", "-rf", "yq", "BUILD.yaml", "original_files.txt"]
+  - name: release-tars
+    type: generic
+    argdeps:
+    - version
+    deps:
+    - chart/config/db/init:release-tar
+    - install:release-tar
+    - :helm
+    config:
+      commands:
+      # re-wrap/mv into named tars
+      - ["sh", "-c", "tar czf gitpod-database-init-${version}.tar.gz -C chart-config-db-init--release-tar . && rm -rf chart-config-db-init--release-tar"]
+      - ["sh", "-c", "tar czf gitpod-terraform-${version}.tar.gz -C install--release-tar . && rm -rf install--release-tar"]
+      - ["sh", "-c", "mv chart--helm/*.tgz gitpod-helm-${version}.tar.gz && rm -rf chart--helm"]
\ No newline at end of file

chart/config/db/init/BUILD.yaml

@@ -8,3 +8,11 @@ packages:
     type: generic
     config:
       commands: [["ls"]]
+  - name: release-tar
+    srcs:
+     - "*.sql"
+    type: generic
+    config:
+      commands:
+      # exclude as it's for test installations only
+      - ["rm", "-f", "00-testdb-user.sql"]

chart/test-semver.js

+const semver = require('semver');
+if (!semver.valid(process.argv[2])) {
+    process.exit(1)
+}
\ No newline at end of file

dev/image/Dockerfile

@@ -66,6 +66,9 @@ RUN cd /usr/bin && curl -fsSL https://github.com/csweichel/werft/releases/downlo
 # yq - jq for YAML files
 RUN cd /usr/bin && curl -fsSL https://github.com/mikefarah/yq/releases/download/2.4.0/yq_linux_amd64 > yq && chmod +x yq
 
+# release helper
+RUN cd /usr/bin && curl -fsSL https://github.com/c4milo/github-release/releases/download/v1.1.0/github-release_v1.1.0_linux_amd64.tar.gz | tar xz
+
 ### Protobuf
 RUN set -ex \
     && tmpdir=$(mktemp -d) \

install/BUILD.yaml

@@ -7,3 +7,10 @@ packages:
       - install/docker/examples/gitpod-gitlab/gitlab:docker
       - install/docker/gitpod-image:docker
       - install/installer:docker
+  - name: release-tar
+    srcs:
+    - "aws-terraform"
+    - "gcp-terraform"
+    type: generic
+    config:
+      commands: [["ls"]]