[8.16] [ci] Click to deploy cloud (#205623) (#208045)

# Backport

This will backport the following commits from `main` to `8.16`:
- [[ci] Click to deploy cloud
(#205623)](#205623)

<!--- Backport version: 9.6.4 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sorenlouv/backport)

<!--BACKPORT [{"author":{"name":"Alex
Szabo","email":"[email protected]"},"sourceCommit":{"committedDate":"2025-01-23T09:26:03Z","message":"[ci]
Click to deploy cloud (#205623)\n\n## Summary\nSimilar to
https://github.com/elastic/kibana/pull/195581\n\nAdds a pipeline that
builds Kibana and starts cloud deployment without\ngoing through the CI
test suites (as in normal pull-request pipeline\nruns). It can be useful
if a developer would like to save time/compute\non
re-building/re-testing the whole project before deploying to
the\ncloud.\n\nAdded labels (`ci:cloud-deploy / ci:cloud-redeploy`) are
required\nsimilarly to the usual CI flow.\n\nRelated to:
https://github.com/elastic/kibana-operations/issues/121","sha":"e36833b3a60b62f794f47951f5ceae842d6c44b3","branchLabelMapping":{"^v9.0.0$":"main","^v8.18.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["Team:Operations","release_note:skip","v9.0.0","backport:all-open"],"title":"[ci]
Click to deploy
cloud","number":205623,"url":"https://github.com/elastic/kibana/pull/205623","mergeCommit":{"message":"[ci]
Click to deploy cloud (#205623)\n\n## Summary\nSimilar to
https://github.com/elastic/kibana/pull/195581\n\nAdds a pipeline that
builds Kibana and starts cloud deployment without\ngoing through the CI
test suites (as in normal pull-request pipeline\nruns). It can be useful
if a developer would like to save time/compute\non
re-building/re-testing the whole project before deploying to
the\ncloud.\n\nAdded labels (`ci:cloud-deploy / ci:cloud-redeploy`) are
required\nsimilarly to the usual CI flow.\n\nRelated to:
https://github.com/elastic/kibana-operations/issues/121","sha":"e36833b3a60b62f794f47951f5ceae842d6c44b3"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/205623","number":205623,"mergeCommit":{"message":"[ci]
Click to deploy cloud (#205623)\n\n## Summary\nSimilar to
https://github.com/elastic/kibana/pull/195581\n\nAdds a pipeline that
builds Kibana and starts cloud deployment without\ngoing through the CI
test suites (as in normal pull-request pipeline\nruns). It can be useful
if a developer would like to save time/compute\non
re-building/re-testing the whole project before deploying to
the\ncloud.\n\nAdded labels (`ci:cloud-deploy / ci:cloud-redeploy`) are
required\nsimilarly to the usual CI flow.\n\nRelated to:
https://github.com/elastic/kibana-operations/issues/121","sha":"e36833b3a60b62f794f47951f5ceae842d6c44b3"}}]}]
BACKPORT-->

Author: delanni

.buildkite/package-lock.json

@@ -15,12 +15,14 @@
     "globby": "^11.1.0",
     "js-yaml": "^4.1.0",
     "minimatch": "^5.0.1",
+    "minimist": "^1.2.8",
     "tslib": "*"
   },
   "devDependencies": {
     "@types/chai": "^4.3.3",
     "@types/js-yaml": "^4.0.9",
     "@types/minimatch": "^3.0.5",
+    "@types/minimist": "^1.2.5",
     "@types/mocha": "^10.0.1",
     "@types/node": "^15.12.2",
     "chai": "^4.3.10",

.buildkite/package.json

@@ -0,0 +1,45 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-deploy-cloud-from-pr
+  description: 'Builds Kibana and initiates a Kibana cloud deployment from a PR'
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-deploy-cloud-from-pr'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  system: buildkite
+  owner: 'group:kibana-operations'
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / deploy cloud from PR
+      description: 'Builds Kibana and initiates a Kibana cloud deployment from a PR'
+    spec:
+      env:
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'false'
+
+      allow_rebuilds: false
+      branch_configuration: main
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/build_pr_and_deploy_cloud.yml
+      provider_settings:
+        build_pull_requests: true
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: true
+        trigger_mode: none
+      cancel_intermediate_builds: true
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: BUILD_AND_READ
+      tags:
+        - kibana

.buildkite/pipeline-resource-definitions/kibana-deploy-cloud.yml

@@ -9,6 +9,8 @@
 
 import { Octokit, RestEndpointMethodTypes } from '@octokit/rest';
 
+export const KIBANA_COMMENT_SIGIL = 'kbn-message-context';
+
 const github = new Octokit({
   auth: process.env.GITHUB_TOKEN,
 });
@@ -93,6 +95,76 @@ export const doAnyChangesMatch = async (
   return anyFilesMatchRequired;
 };
 
+export function addComment(
+  comment: string,
+  owner = process.env.GITHUB_PR_BASE_OWNER,
+  repo = process.env.GITHUB_PR_BASE_REPO,
+  prNumber: undefined | string | number = process.env.GITHUB_PR_NUMBER
+) {
+  if (!owner || !repo || !prNumber) {
+    throw Error(
+      "Couldn't retrieve Github PR info from environment variables in order to add a comment"
+    );
+  }
+
+  return github.issues.createComment({
+    owner,
+    repo,
+    issue_number: typeof prNumber === 'number' ? prNumber : parseInt(prNumber, 10),
+    body: comment,
+  });
+}
+
+export async function upsertComment(
+  messageOpts: {
+    commentBody: string;
+    commentContext: string;
+    clearPrevious: boolean;
+  },
+  owner = process.env.GITHUB_PR_BASE_OWNER,
+  repo = process.env.GITHUB_PR_BASE_REPO,
+  prNumber: undefined | string | number = process.env.GITHUB_PR_NUMBER
+) {
+  const { commentBody, commentContext, clearPrevious } = messageOpts;
+  if (!owner || !repo || !prNumber) {
+    throw Error(
+      "Couldn't retrieve Github PR info from environment variables in order to add a comment"
+    );
+  }
+  if (!commentContext) {
+    throw Error('Comment context is required when updating a comment');
+  }
+
+  const commentMarker = `<!-- ${KIBANA_COMMENT_SIGIL}:${commentContext} -->`;
+  const body = `${commentMarker}\n${commentBody}`;
+
+  const existingComment = (
+    await github.paginate(github.issues.listComments, {
+      owner,
+      repo,
+      issue_number: typeof prNumber === 'number' ? prNumber : parseInt(prNumber, 10),
+    })
+  ).find((comment) => comment.body?.includes(commentMarker));
+
+  if (!existingComment) {
+    return addComment(body, owner, repo, prNumber);
+  } else if (clearPrevious) {
+    await github.issues.deleteComment({
+      owner,
+      repo,
+      comment_id: existingComment.id,
+    });
+    return addComment(body, owner, repo, prNumber);
+  } else {
+    return github.issues.updateComment({
+      owner,
+      repo,
+      comment_id: existingComment.id,
+      body,
+    });
+  }
+}
+
 export function getGithubClient() {
   return github;
 }

.buildkite/pipeline-utils/github/github.ts

@@ -0,0 +1,82 @@
+env:
+  ELASTIC_PR_COMMENTS_ENABLED: 'true'
+  ELASTIC_GITHUB_BUILD_COMMIT_STATUS_ENABLED: 'true'
+  GITHUB_BUILD_COMMIT_STATUS_CONTEXT: kibana-deploy-cloud-from-pr
+
+steps:
+  - group: 'Cloud Deployment'
+    if: "build.env('GITHUB_PR_LABELS') =~ /(ci:cloud-deploy|ci:cloud-redeploy)/"
+
+    steps:
+      - command: .buildkite/scripts/lifecycle/pre_build.sh
+        label: Pre-Build
+        timeout_in_minutes: 10
+        agents:
+          provider: gcp
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          machineType: n2-standard-2
+        retry:
+          automatic:
+            - exit_status: '*'
+              limit: 1
+
+      - command: |
+          ts-node .buildkite/scripts/lifecycle/comment_on_pr.ts \
+            --message "PR Cloud deployment started at: $BUILDKITE_BUILD_URL" \
+            --context "cloud-deploy-job" \
+            --clear-previous
+        label: Comment with job URL
+        agents:
+          provider: gcp
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          machineType: n2-standard-2
+        timeout_in_minutes: 5
+
+      - command: .buildkite/scripts/steps/build_kibana.sh
+        label: Build Kibana Distribution
+        agents:
+          provider: gcp
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          machineType: n2-standard-8
+          preemptible: true
+          diskSizeGb: 125
+        if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+        timeout_in_minutes: 90
+        retry:
+          automatic:
+            - exit_status: '-1'
+              limit: 3
+
+      - wait: ~
+
+      - command: .buildkite/scripts/steps/cloud/build_and_deploy.sh
+        label: 'Build and Deploy to Cloud'
+        agents:
+          provider: gcp
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          machineType: n2-standard-2
+          preemptible: true
+        timeout_in_minutes: 30
+        retry:
+          automatic:
+            - exit_status: '-1'
+              limit: 3
+
+      - wait: ~
+
+      - command: |
+          ts-node .buildkite/scripts/lifecycle/comment_on_pr.ts \
+            --message "Cloud deployment initiated, see credentials at: $BUILDKITE_BUILD_URL" \
+            --context "cloud-deploy-job" \
+            --clear-previous
+        label: Comment with job URL
+        agents:
+          provider: gcp
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          machineType: n2-standard-2
+        timeout_in_minutes: 5

.buildkite/pipelines/build_pr_and_deploy_cloud.yml

@@ -0,0 +1,78 @@
+env:
+  ELASTIC_PR_COMMENTS_ENABLED: 'true'
+  ELASTIC_GITHUB_BUILD_COMMIT_STATUS_ENABLED: 'true'
+  GITHUB_BUILD_COMMIT_STATUS_CONTEXT: kibana-deploy-project-from-pr
+
+steps:
+  - group: 'Project Deployment'
+    if: "build.env('GITHUB_PR_LABELS') =~ /ci:project-deploy-(elasticsearch|observability|security)/"
+
+    steps:
+      - command: .buildkite/scripts/lifecycle/pre_build.sh
+        label: Pre-Build
+        timeout_in_minutes: 10
+        agents:
+          provider: gcp
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          machineType: n2-standard-2
+        retry:
+          automatic:
+            - exit_status: '*'
+              limit: 1
+
+      - command: |
+          ts-node .buildkite/scripts/lifecycle/comment_on_pr.ts \
+            --message "PR Project deployment started at: $BUILDKITE_BUILD_URL" \
+            --context "project-deploy-job" \
+            --clear-previous
+        label: Comment with job URL
+        agents:
+          provider: gcp
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          machineType: n2-standard-2
+        timeout_in_minutes: 5
+
+      - wait: ~
+
+      - command: .buildkite/scripts/steps/artifacts/docker_image.sh
+        label: 'Build Project Image'
+        key: build_project_image
+        agents:
+          provider: gcp
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          machineType: n2-standard-16
+          preemptible: true
+        timeout_in_minutes: 60
+        retry:
+          automatic:
+            - exit_status: '-1'
+              limit: 3
+
+      - wait: ~
+      - command: .buildkite/scripts/steps/serverless/deploy.sh
+        label: 'Deploy Project'
+        agents:
+          provider: gcp
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          machineType: n2-standard-4
+          preemptible: true
+        timeout_in_minutes: 10
+
+      - wait: ~
+
+      - command: |
+          ts-node .buildkite/scripts/lifecycle/comment_on_pr.ts \
+            --message "Project deployed, see credentials at: $BUILDKITE_BUILD_URL" \
+            --context "project-deploy-job" \
+            --clear-previous
+        label: Comment with job URL
+        agents:
+          provider: gcp
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          machineType: n2-standard-2
+        timeout_in_minutes: 5