TUN-9800: Migrate cloudflared-ci pipelines to Gitlab CI

## Summary This commit migrates the cloduflared ci pipelines, that built, tested and component tested the linux binaries to gitlab ci. The only thing that is remaining to move from teamcity to gitlab are now the release pipelines that run on master. Relates to TUN-9800
2026-06-21 19:47:39 +08:00 · 2025-09-11 11:33:24 +01:00
parent d9e13ab2ab
commit 173396be90
14 changed files with 209 additions and 83 deletions
--- a/.ci/ci-image.gitlab-ci.yml
+++ b/.ci/ci-image.gitlab-ci.yml
@@ -8,10 +8,9 @@ include:
    inputs:
      stage: pre-build
      jobPrefix: ci-image
-      # runOnChangesTo: [".ci/image/**"]
-      # runOnMR: true
-      # runOnBranches: '^master$'
-      runOnBranches: "^.+$"
+      runOnChangesTo: [".ci/image/**"]
+      runOnMR: true
+      runOnBranches: '^master$'
      commentImageRefs: false
      runner: vm-linux-x86-4cpu-8gb
      EXTRA_DIB_ARGS: "--manifest=.ci/image/.docker-images"
@@ -23,9 +22,8 @@ include:
    inputs:
      stage: pre-build
      jobPrefix: ci-image
-      # runOnMR: true
-      # runOnBranches: '^master$'
-      runOnBranches: "^.+$"
+      runOnMR: true
+      runOnBranches: '^master$'
      IMAGE_PATH: "$REGISTRY_HOST/stash/tun/cloudflared/ci-image/master"
      VARIABLE_NAME: BUILD_IMAGE
      needs:
--- a/.ci/commons.gitlab-ci.yml
+++ b/.ci/commons.gitlab-ci.yml
@@ -5,13 +5,19 @@
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
      when: always
    - when: never
-  # Rules to run the job only on branches that are not master. This is needed because for now
-  # we need to keep a similar behavior due to the integration with teamcity, which requires us
-  # to not trigger pipelines on tags and/or merge requests.
-  run-on-branch:
+  # Rules to run the job only on merge requests
+  run-on-mr:
    - if: $CI_COMMIT_TAG
      when: never
-    - if: $CI_PIPELINE_SOURCE != "merge_request_event" && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: always
+    - when: never
+  # Rules to run the job on merge_requests and master branch
+  run-always:
+    - if: $CI_COMMIT_TAG
+      when: never
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_COMMIT_BRANCH != null && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
      when: always
    - when: never

@@ -28,4 +34,20 @@
      else
        echo "No tag present — skipping."
        exit 0
-      fi
+      fi
+
+.component-tests:
+  image: $BUILD_IMAGE
+  rules:
+    - !reference [.default-rules, run-always]
+  variables:
+    COMPONENT_TESTS_CONFIG: component-test-config.yaml
+    COMPONENT_TESTS_CONFIG_CONTENT: Y2xvdWRmbGFyZWRfYmluYXJ5OiBjbG91ZGZsYXJlZC5leGUKY3JlZGVudGlhbHNfZmlsZTogY3JlZC5qc29uCm9yaWdpbmNlcnQ6IGNlcnQucGVtCnpvbmVfZG9tYWluOiBhcmdvdHVubmVsdGVzdC5jb20Kem9uZV90YWc6IDQ4Nzk2ZjFlNzBiYjc2NjljMjliYjUxYmEyODJiZjY1
+  secrets:
+    DNS_API_TOKEN:
+      vault: gitlab/cloudflare/tun/cloudflared/_dev/_terraform_atlantis/component_tests_token/data@kv
+      file: false
+    COMPONENT_TESTS_ORIGINCERT:
+      vault: gitlab/cloudflare/tun/cloudflared/_dev/component_tests_cert_pem/data@kv
+      file: false
+  cache: {}
--- a/.ci/image/Dockerfile
+++ b/.ci/image/Dockerfile
@@ -7,8 +7,9 @@ RUN apt-get update && \
    apt-get install --no-install-recommends --allow-downgrades -y \
        build-essential \
        git \
-        go-boring=1.24.4-1 \
+        go-boring=1.24.6-1 \
        libffi-dev \
+        procps \
        python3-dev \
        python3-pip \
        python3-setuptools \
--- a/.ci/linux.gitlab-ci.yml
+++ b/.ci/linux.gitlab-ci.yml
@@ -0,0 +1,90 @@
+.golang-inputs: &golang_inputs
+  runOnMR: true
+  runOnBranches: '^master$'
+  outputDir: artifacts
+  runner: linux-x86-8cpu-16gb
+  stage: build
+  golangVersion: "boring-1.24"
+  CGO_ENABLED: 1
+
+include:
+  ###################
+  ### Linux Build ###
+  ###################
+  - component: $CI_SERVER_FQDN/cloudflare/ci/golang/boring-make@~latest
+    inputs:
+      <<: *golang_inputs
+      jobPrefix: linux-build
+      GOLANG_MAKE_TARGET: ci-build
+
+  ########################
+  ### Linux FIPS Build ###
+  ########################
+  - component: $CI_SERVER_FQDN/cloudflare/ci/golang/boring-make@~latest
+    inputs:
+      <<: *golang_inputs
+      jobPrefix: linux-fips-build
+      GOLANG_MAKE_TARGET: ci-fips-build
+
+  #################
+  ### Unit Tests ##
+  #################
+  - component: $CI_SERVER_FQDN/cloudflare/ci/golang/boring-make@~latest
+    inputs:
+      <<: *golang_inputs
+      stage: test
+      jobPrefix: test
+      GOLANG_MAKE_TARGET: ci-test
+
+  ######################
+  ### Unit Tests FIPS ##
+  ######################
+  - component: $CI_SERVER_FQDN/cloudflare/ci/golang/boring-make@~latest
+    inputs:
+      <<: *golang_inputs
+      stage: test
+      jobPrefix: test-fips
+      GOLANG_MAKE_TARGET: ci-fips-test
+
+  #################
+  ### Vuln Check ##
+  #################
+  - component: $CI_SERVER_FQDN/cloudflare/ci/golang/boring-make@~latest
+    inputs:
+      <<: *golang_inputs
+      runOnBranches: '^$'
+      stage: validate
+      jobPrefix: vulncheck
+      GOLANG_MAKE_TARGET: vulncheck
+
+#################################
+### Run Linux Component Tests ###
+#################################
+component-tests-linux: &component-tests-linux
+  stage: test
+  extends: .component-tests
+  needs:
+    - ci-image-get-image-ref
+    - linux-build-boring-make
+  script:
+    - ./.ci/scripts/component-tests.sh
+  variables: &component-tests-variables
+    CI: 1
+    COMPONENT_TESTS_CONFIG_CONTENT: Y2xvdWRmbGFyZWRfYmluYXJ5OiBjbG91ZGZsYXJlZApjcmVkZW50aWFsc19maWxlOiBjcmVkLmpzb24Kb3JpZ2luY2VydDogY2VydC5wZW0Kem9uZV9kb21haW46IGFyZ290dW5uZWx0ZXN0LmNvbQp6b25lX3RhZzogNDg3OTZmMWU3MGJiNzY2OWMyOWJiNTFiYTI4MmJmNjU=
+  tags:
+    - linux-x86-8cpu-16gb
+  artifacts:
+    reports:
+      junit: report.xml
+
+######################################
+### Run Linux FIPS Component Tests ###
+######################################
+component-tests-linux-fips:
+  <<: *component-tests-linux
+  needs:
+    - ci-image-get-image-ref
+    - linux-fips-build-boring-make
+  variables:
+    <<: *component-tests-variables
+    COMPONENT_TESTS_FIPS: 1
--- a/.ci/mac.gitlab-ci.yml
+++ b/.ci/mac.gitlab-ci.yml
@@ -6,7 +6,7 @@ include:
 ###############################
 .mac-build-defaults: &mac-build-defaults
  rules:
-    - !reference [.default-rules, run-on-branch]
+    - !reference [.default-rules, run-on-mr]
  tags:
    - "macstadium-${RUNNER_ARCH}"
  parallel:
--- a/.ci/scripts/component-tests.sh
+++ b/.ci/scripts/component-tests.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+set -e -o pipefail
+
+# Fetch cloudflared from the artifacts folder
+mv ./artifacts/cloudflared ./cloudflared
+
+python3 -m venv env
+. env/bin/activate
+
+pip install --upgrade -r component-tests/requirements.txt
+
+# Creates and routes a Named Tunnel for this build. Also constructs
+# config file from env vars.
+python3 component-tests/setup.py --type create
+
+# Define the cleanup function
+cleanup() {
+    # The Named Tunnel is deleted and its route unprovisioned here.
+    python3 component-tests/setup.py --type cleanup
+}
+
+# The trap will call the cleanup function on script exit
+trap cleanup EXIT
+
+pytest component-tests -o log_cli=true --log-cli-level=INFO --junit-xml=report.xml
--- a/.ci/scripts/fmt-check.sh
+++ b/.ci/scripts/fmt-check.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e -o pipefail
+
+OUTPUT=$(go run -mod=readonly golang.org/x/tools/cmd/goimports@v0.30.0 -l -d -local github.com/cloudflare/cloudflared $(go list -mod=vendor -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc))
+
+if [ -n "$OUTPUT" ] ; then
+  PAGER=$(which colordiff || echo cat)
+  echo
+  echo "Code formatting issues found, use 'make fmt' to correct them"
+  echo
+  echo "$OUTPUT" | $PAGER
+  exit 1
+fi
--- a/.ci/scripts/windows/component-test.ps1
+++ b/.ci/scripts/windows/component-test.ps1
@@ -31,7 +31,7 @@ Write-Host "Running component tests"
 try {
    python -m pip --disable-pip-version-check install --upgrade -r component-tests/requirements.txt --use-pep517
    python component-tests/setup.py --type create
-    python -m pytest component-tests -o log_cli=true --log-cli-level=INFO
+    python -m pytest component-tests -o log_cli=true --log-cli-level=INFO --junit-xml=report.xml
    if ($LASTEXITCODE -ne 0) {
        throw "Failed component tests"
    }
--- a/.ci/scripts/windows/go-wrapper.ps1
+++ b/.ci/scripts/windows/go-wrapper.ps1
@@ -3,7 +3,7 @@ Param(
    [string]$ScriptToExecute
 )

-# This script its a wrapper that downloads a specific version
+# The script is a wrapper that downloads a specific version
 # of go, adds it to the PATH and executes a script with that go
 # version in the path.

--- a/.ci/windows.gitlab-ci.yml
+++ b/.ci/windows.gitlab-ci.yml
@@ -6,7 +6,7 @@ include:
 ###################################
 .windows-build-defaults: &windows-build-defaults
  rules:
-    - !reference [.default-rules, run-on-branch]
+    - !reference [.default-rules, run-always]
  tags:
    - windows-x86
  cache: {}
@@ -27,27 +27,20 @@ build-cloudflared-windows:
 ### Load Environment Variables for Component Tests ###
 ######################################################
 load-windows-env-variables:
-  rules:
-    - !reference [.default-rules, run-on-branch]
  stage: pre-build
+  extends: .component-tests
  script:
-    - echo "COMPONENT_TESTS_CONFIG=component-test-config.yaml" >> windows.env
-    - echo "COMPONENT_TESTS_CONFIG_CONTENT=Y2xvdWRmbGFyZWRfYmluYXJ5OiBjbG91ZGZsYXJlZC5leGUKY3JlZGVudGlhbHNfZmlsZTogY3JlZC5qc29uCm9yaWdpbmNlcnQ6IGNlcnQucGVtCnpvbmVfZG9tYWluOiBhcmdvdHVubmVsdGVzdC5jb20Kem9uZV90YWc6IDQ4Nzk2ZjFlNzBiYjc2NjljMjliYjUxYmEyODJiZjY1" >> windows.env
+    - echo "COMPONENT_TESTS_CONFIG=$COMPONENT_TESTS_CONFIG" >> windows.env
+    - echo "COMPONENT_TESTS_CONFIG_CONTENT=$COMPONENT_TESTS_CONFIG_CONTENT" >> windows.env
    - echo "DNS_API_TOKEN=$DNS_API_TOKEN" >> windows.env
    # We have to encode the `COMPONENT_TESTS_ORIGINCERT` secret, because it content is a file, otherwise we can't export it using gitlab
    - echo "COMPONENT_TESTS_ORIGINCERT=$(echo "$COMPONENT_TESTS_ORIGINCERT" | base64 -w0)" >> windows.env
-  secrets:
-    DNS_API_TOKEN:
-      vault: gitlab/cloudflare/tun/cloudflared/_dev/_terraform_atlantis/component_tests_token/data@kv
-      file: false
-    COMPONENT_TESTS_ORIGINCERT:
-      vault: gitlab/cloudflare/tun/cloudflared/_dev/component_tests_cert_pem/data@kv
-      file: false
+  variables:
+    COMPONENT_TESTS_CONFIG_CONTENT: Y2xvdWRmbGFyZWRfYmluYXJ5OiBjbG91ZGZsYXJlZC5leGUKY3JlZGVudGlhbHNfZmlsZTogY3JlZC5qc29uCm9yaWdpbmNlcnQ6IGNlcnQucGVtCnpvbmVfZG9tYWluOiBhcmdvdHVubmVsdGVzdC5jb20Kem9uZV90YWc6IDQ4Nzk2ZjFlNzBiYjc2NjljMjliYjUxYmEyODJiZjY1
  artifacts:
    access: 'none'
    reports:
      dotenv: windows.env
-  cache: {}

 ###################################
 ### Run Windows Component Tests ###
@@ -60,6 +53,9 @@ component-tests-cloudflared-windows:
    # We have to decode the secret we encoded on the `load-windows-env-variables` job
    - $env:COMPONENT_TESTS_ORIGINCERT = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($env:COMPONENT_TESTS_ORIGINCERT))
    - powershell -ExecutionPolicy Bypass -File ".\.ci\scripts\windows\go-wrapper.ps1" "${GO_VERSION}" ".\.ci\scripts\windows\component-test.ps1"
+  artifacts:
+    reports:
+      junit: report.xml

 ################################
 ### Package Windows Binaries ###