Add CICD-Pipeline to build and push Docker Image

.ci_cd/ci_cd.json

@@ -1,23 +1,9 @@
 {
  "push": {
- "master": {
- "distribute": {
- "docker": "develop,latest"
- },
+ "developement": {
  "deploy": {
- "managerTag": "develop",
- "environment": "staging"
+
  }
  }
- },
- "release": {
- "distribute": {
- "docker": "latest,$version",
- "maven": "$version"
- },
- "deploy": {
- "managerTag": "latest",
- "environment": "production"
- }
  }
 }

.github/workflows/ci_cd.yml

@@ -58,7 +58,7 @@ on:
  # Push on master excluding tags
  push:
  branches:
- - 'master'
+ - 'feature/cicd_pipeline'
  tags-ignore:
  - '*.*'
 
@@ -93,6 +93,12 @@ on:
  secrets:
  SECRETS:
  required: false
+
+# needed for aws auth
+permissions:
+ id-token: write # This is required for requesting the JWT
+ contents: read # This is required for actions/checkout
+
 
 jobs:
 
@@ -176,7 +182,7 @@ jobs:
  # Check which files have changed to only run appropriate tests and checks
  - name: Backend files changed
  id: backend-files-changed
- if: github.event_name == 'pull_request'
+ # if: github.event_name == 'pull_request'
  uses: tj-actions/changed-files@v35
  with:
  files_ignore: |
@@ -192,7 +198,7 @@ jobs:
  # Check which files have changed to only run appropriate tests and checks
  - name: UI files changed
  id: ui-files-changed
- if: github.event_name == 'pull_request'
+ # if: github.event_name == 'pull_request'
  uses: tj-actions/changed-files@v35
  with:
  files: |
@@ -379,9 +385,9 @@ jobs:
  deploys = eventConfig['deploy'] if 'deploy' in eventConfig else {}
  if 'distribute' in eventConfig:
  if 'docker' in eventConfig['distribute']:
- dockerPublishTags = eventConfig['distribute']['docker']
+  dockerPublishTags = eventConfig['distribute']['docker']
  if 'maven' in eventConfig['distribute']:
- mavenPublishTag = eventConfig['distribute']['maven']
+  mavenPublishTag = eventConfig['distribute']['maven']
 
  if dockerPublishTags is not None and isMainRepo == 'true':
  dockerPublishTags = dockerPublishTags.replace("$version", refName)
@@ -567,7 +573,7 @@ jobs:
  uses: docker/setup-buildx-action@v2
  with:
  version: latest
- install: true
+ install: true # set docker buildx build as synonym for docker build
 
  - name: Set up JDK 17 and gradle cache
  id: java
@@ -758,131 +764,171 @@ jobs:
  run: |
  ${{ steps.deployment-docker-command.outputs.value }}
 
- - name: Do deployments
- if: steps.deployments.outputs.value != ''
- shell: python
- run: |
- import json
- import os
- import sys
- import subprocess
-
- deployments = os.getenv("DEPLOYMENTS")
- deployments = deployments.split(";")
- managerRef = os.getenv("MANAGER_REF")
- deploymentRef = os.getenv("DEPLOYMENT_REF")
- isCustomProject = os.getenv("IS_CUSTOM_PROJECT")
- inputsAndSecrets = json.loads(os.getenv("INPUTS_AND_SECRETS"))
- ipv4 = os.getenv("IPV4")
- ipv6 = os.getenv("IPV6")
-
- failure = False
-
- # Determine deploy script to use
- deployScript = ".ci_cd/deploy.sh"
-
- if not os.path.exists(deployScript) and isCustomProject == 'true':
- deployScript = "openremote/.ci_cd/deploy.sh"
-
- if not os.path.exists(deployScript):
- os.system(f"Deploy script not found '{deployScript}'")
- sys.exit(1)
-
- for deployment in deployments:
- dep = deployment.split(":")
- env = dep[0]
- managerTag = dep[1]
- managerTagFound = True
-
- os.putenv("MANAGER_TAG", managerTag)
- os.putenv("ENVIRONMENT", env)
-
- # Clean stale ssh credentials and temp files
- os.system("rm temp.env 2>/dev/null")
- os.system("rm ssh.key 2>/dev/null")
- os.system("rm -r temp 2>/dev/null")
- os.system("mkdir temp")
-
- # ------------------------------------------------------
- # Output env variables to temp env file for POSIX shell
- # ------------------------------------------------------
-
- # Output inputs and secrets (spacial handling for SSH_KEY and some other variables)
- # _$ENV_ prefixed keys are output last (to override any non env specific keys)
- environment = (env if env else "").upper()
- prefix = "_" + environment + "_"
-
- for key, value in inputsAndSecrets.items():
- if "." in key:
- continue
+ - name: Configure AWS Credentials
+ uses: aws-actions/configure-aws-credentials@v4
+ with:
+ role-to-assume: arn:aws:iam::205672091018:role/GitHubAction-AssumeRoleWithAction
+ role-session-name: GitHub_to_AWS_via_FederatedOIDC_meterverse
+ aws-region: ${{ env.AWS_REGION }}
+ env:
+ AWS_REGION: eu-central-1
+
+ - name: Login to Amazon ECR
+ id: login-ecr
+ uses: aws-actions/amazon-ecr-login@v1
 
- envFile = "temp/env"
+ - name: Set current date as env variable
+ run: echo "NOW=$(TZ='Europe/Berlin' date +'%Y-%m-%dT%H-%M-%S')" >> $GITHUB_ENV
 
- # Look for temp and env prefixed keys
- if key.startswith("_"):
- if key.startswith("_TEMP_"):
- key = key.replace("_TEMP_", "")
- envFile = "temp.env"
- elif key.startswith(prefix):
- key = key.replace(prefix, "")
- else:
- continue
+ - name: Build, Tag, and Push image to Amazon ECR
+ id: build-image
+ if: steps.deployments.outputs.value != ''
+ env:
+ ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+ ECR_REPOSITORY: meterverse_manager
+ ECR_IMAGE_TAG:
+ IMAGE_TAG: ${{ github.sha }}
+ run: | 
+ docker build --provenance=false --platform linux/amd64 --build-arg GIT_COMMIT=$IMAGE_TAG \
+ -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG \
+ -t $ECR_REGISTRY/$ECR_REPOSITORY:$NOW \
+ -t $ECR_REGISTRY/$ECR_REPOSITORY:latest \
+ manager/build/install/manager -o type=registry
+ echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT
 
- if key == "github_token":
- continue
- else: 
- os.system(f"echo 'Secret found {key}...'")
- if key == "SSH_KEY":
- os.system(f"echo \"{value}\" > ssh.key")
- else:
- lines = len(value.split("\n"))
- if lines > 1:
- os.system(f"echo '{key}='\"'\"'' >> {envFile}")
- os.system(f"echo '{value}'\"'\"'' >> {envFile}")
- else:
- os.system(f"echo '{key}='\"'\"'{value}'\"'\"'' >> {envFile}")
+
+ # delete image from github actions after push:
+ # docker rmi $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+ docker image prune -f
 
- # Output env file if exists
- if os.path.exists(".ci_cd/env/.env"):
- os.system(f"echo 'Outputting .ci_cd/env/.env to temp/env'")
- os.system("cat .ci_cd/env/.env >> temp/env")
+ # echo "Docker images:"
+ # docker images
+
+ # - name: Do deployments
+ # if: steps.deployments.outputs.value != ''
+ # shell: python
+ # run: |
+ # import json
+ # import os
+ # import sys
+ # import subprocess
+
+ # deployments = os.getenv("DEPLOYMENTS")
+ # deployments = deployments.split(";")
+ # managerRef = os.getenv("MANAGER_REF")
+ # deploymentRef = os.getenv("DEPLOYMENT_REF")
+ # isCustomProject = os.getenv("IS_CUSTOM_PROJECT")
+ # inputsAndSecrets = json.loads(os.getenv("INPUTS_AND_SECRETS"))
+ # ipv4 = os.getenv("IPV4")
+ # ipv6 = os.getenv("IPV6")
+
+ # failure = False
+
+ # # Determine deploy script to use
+ # deployScript = ".ci_cd/deploy.sh"
+
+ # if not os.path.exists(deployScript) and isCustomProject == 'true':
+ # deployScript = "openremote/.ci_cd/deploy.sh"
+
+ # if not os.path.exists(deployScript):
+ # os.system(f"Deploy script not found '{deployScript}'")
+ # sys.exit(1)
+
+ # for deployment in deployments:
+ # dep = deployment.split(":")
+ # env = dep[0]
+ # managerTag = dep[1]
+ # managerTagFound = True
+
+ # os.putenv("MANAGER_TAG", managerTag)
+ # os.putenv("ENVIRONMENT", env)
+
+ # # Clean stale ssh credentials and temp files
+ # os.system("rm temp.env 2>/dev/null")
+ # os.system("rm ssh.key 2>/dev/null")
+ # os.system("rm -r temp 2>/dev/null")
+ # os.system("mkdir temp")
+
+ # # ------------------------------------------------------
+ # # Output env variables to temp env file for POSIX shell
+ # # ------------------------------------------------------
+
+ # # Output inputs and secrets (spacial handling for SSH_KEY and some other variables)
+ # # _$ENV_ prefixed keys are output last (to override any non env specific keys)
+ # environment = (env if env else "").upper()
+ # prefix = "_" + environment + "_"
+
+ # for key, value in inputsAndSecrets.items():
+ # if "." in key:
+ # continue
+
+ # envFile = "temp/env"
+
+ # # Look for temp and env prefixed keys
+ # if key.startswith("_"):
+ # if key.startswith("_TEMP_"):
+ # key = key.replace("_TEMP_", "")
+ # envFile = "temp.env"
+ # elif key.startswith(prefix):
+ # key = key.replace(prefix, "")
+ # else:
+ # continue
+
+ # if key == "github_token":
+ # continue
+ # else: 
+ # os.system(f"echo 'Secret found {key}...'")
+ # if key == "SSH_KEY":
+ # os.system(f"echo \"{value}\" > ssh.key")
+ # else:
+ # lines = len(value.split("\n"))
+ # if lines > 1:
+ # os.system(f"echo '{key}='\"'\"'' >> {envFile}")
+ # os.system(f"echo '{value}'\"'\"'' >> {envFile}")
+ # else:
+ # os.system(f"echo '{key}='\"'\"'{value}'\"'\"'' >> {envFile}")
+
+ # # Output env file if exists
+ # if os.path.exists(".ci_cd/env/.env"):
+ # os.system(f"echo 'Outputting .ci_cd/env/.env to temp/env'")
+ # os.system("cat .ci_cd/env/.env >> temp/env")
 
- # Output environment specific env file if exists
- if env is not None and env != '' and os.path.exists(f".ci_cd/env/{env}.env"):
- os.system(f"echo 'Outputting .ci_cd/env/{env}.env to temp/env'")
- os.system(f"cat .ci_cd/env/{env}.env >> temp/env")
+ #  # Output environment specific env file if exists
+ #  if env is not None and env != '' and os.path.exists(f".ci_cd/env/{env}.env"):
+ #  os.system(f"echo 'Outputting .ci_cd/env/{env}.env to temp/env'")
+ #  os.system(f"cat .ci_cd/env/{env}.env >> temp/env")
 
- # Set CIDR environment variable
- if ipv4 is not None and ipv4 != '':
- os.putenv("CIDR", ipv4 + '/32')
- elif ipv6 is not None and ipv6 != '':
- os.putenv("CIDR", ipv6 + '/64')
+ #  # Set CIDR environment variable
+ #  if ipv4 is not None and ipv4 != '':
+ #  os.putenv("CIDR", ipv4 + '/32')
+ #  elif ipv6 is not None and ipv6 != '':
+ #  os.putenv("CIDR", ipv6 + '/64')
 
- # Execute deploy script
- os.system(f"echo 'Executing deploy script for deployment: managerTag={managerTag} deploymentTag={deploymentRef} environment={env}'")
- # Uncomment this in combination with the SSH debug step afterwards to debug deployment script
- #sys.exit(0)
- result = subprocess.run(f"bash {deployScript}", shell=True)
-
- if result.returncode != 0:
- os.system(f"echo 'Deployment failed: managerTag={managerTag} deploymentTag={deploymentRef} environment={env}'")
- failure = True
- continue
-
- if failure == True:
- os.system("echo 'One or more deployments failed'")
- sys.exit(1)
- env:
- IS_CUSTOM_PROJECT: ${{ steps.check_custom_project.outputs.files_exists }}
- REPO_NAME: ${{ github.repository }}
- DEPLOYMENTS: ${{ steps.deployments.outputs.value }}
- MANAGER_DOCKER_BUILD_PATH: ${{ steps.manager-docker-command.outputs.buildPath }}
- DEPLOYMENT_DOCKER_BUILD_PATH: ${{ steps.deployment-docker-command.outputs.buildPath }}
- MANAGER_REF: ${{ steps.manager-docker-command.outputs.refTag }}
- DEPLOYMENT_REF: ${{ steps.deployment-docker-command.outputs.refTag }}
- INPUTS_AND_SECRETS: ${{ toJSON(steps.inputs-and-secrets.outputs) }}
- IPV4: ${{ steps.ip-address.outputs.ipv4 }}
- IPV6: ${{ steps.ip-address.outputs.ipv6 }}
+ #  # Execute deploy script
+ #  os.system(f"echo 'Executing deploy script for deployment: managerTag={managerTag} deploymentTag={deploymentRef} environment={env}'")
+ #  # Uncomment this in combination with the SSH debug step afterwards to debug deployment script
+ #  #sys.exit(0)
+ #  result = subprocess.run(f"bash {deployScript}", shell=True)
+
+ #  if result.returncode != 0:
+ #  os.system(f"echo 'Deployment failed: managerTag={managerTag} deploymentTag={deploymentRef} environment={env}'")
+ #  failure = True
+ #  continue
+
+ #  if failure == True:
+ #  os.system("echo 'One or more deployments failed'")
+ #  sys.exit(1)
+ #  env:
+ #  IS_CUSTOM_PROJECT: ${{ steps.check_custom_project.outputs.files_exists }}
+ #  REPO_NAME: ${{ github.repository }}
+ #  DEPLOYMENTS: ${{ steps.deployments.outputs.value }}
+ #  MANAGER_DOCKER_BUILD_PATH: ${{ steps.manager-docker-command.outputs.buildPath }}
+ #  DEPLOYMENT_DOCKER_BUILD_PATH: ${{ steps.deployment-docker-command.outputs.buildPath }}
+ #  MANAGER_REF: ${{ steps.manager-docker-command.outputs.refTag }}
+ #  DEPLOYMENT_REF: ${{ steps.deployment-docker-command.outputs.refTag }}
+ #  INPUTS_AND_SECRETS: ${{ toJSON(steps.inputs-and-secrets.outputs) }}
+ #  IPV4: ${{ steps.ip-address.outputs.ipv4 }}
+ #  IPV6: ${{ steps.ip-address.outputs.ipv6 }}
 
 # - name: Setup upterm session
 # uses: lhotari/action-upterm@v1