fix(ci): Instant rollback process (#5460)

* fix(ci): Get back after workflow creation in the list * fix(ci): Condition fix * fix(ci): working dir for one step has been deleted * fix(ci): Token update * fix(ci): Token update * fix(ci): Token update * fix(ci): Token update * fix(ci): + environment * fix(ci): fix environment * fix(ci): fix environment * fix(ci): Environment name * fix(ci): Environment name fix * fix(ci): Environment Development * fix(ci): Inbound_mail service name has been changed * fix(ci): Inbound_mail service name has been changed * fix(ci): Variables have been updated * fix(ci): Variable previous_task_definition_arn to GITHUB_ENV * fix(ci): Variable update * fix(ci): Variables restructure * fix(ci): Debug mode * fix(ci): List fix * fix(ci): Refactor * fix(ci): Cut extra word * fix(ci): Cut extra word * fix(ci): Variables fix * fix(ci): Indexing * fix(ci): Indexing from 0 * fix(ci): Indexing from 0 * fix(ci): Fix logic * fix(ci): Fix logic * fix(ci): Error description * fix(ci): Syntax fix * fix(ci): Exclude | * fix(ci): Workflow name * fix(ci): Workflow name * feat(ci): Matrix for both regions support * feat(ci): Matrix for both regions support * feat(ci): New setting to use hash commit * feat(ci): Fix ID of step * feat(ci): Syntax fix * feat(ci): Netlify rollback job * feat(ci): Netlify rollback job by using a commit hash * feat(ci): Extra step has been deleted * feat(ci): Condition fix * feat(ci): New id step * feat(ci): Error message * feat(ci): Exit 1 has been deleted * feat(ci): Comments have been deleted * feat(ci): Logic fix * feat(ci): The changes after comments
novuhq · May 8, 2024 · 04ef4ad · 04ef4ad
1 parent 2a7e964
commit 04ef4ad
Showing 1 changed file with 277 additions and 1 deletion.
diff --git a/.github/workflows/rollback.yml b/.github/workflows/rollback.yml
@@ -1,11 +1,287 @@
 name: Rollback
+run-name: Rollback the ${{ inputs.service }} service in the ${{ inputs.environment }} environment
 
 on:
  workflow_dispatch:
+ inputs:
+ service:
+ type: choice
+ description: Select service to rollback.
+ options:
+ - inbound_mail
+ - api
+ - web
+ - webhook
+ - widget
+ - worker
+ - ws
+ environment:
+ type: choice
+ description: Select the environment
+ options:
+ - Production
+ - Development
+ region:
+ type: choice
+ description: Select the environment region. Required only in production.
+ options:
+ - [EU,US]
+ - [EU]
+ - [US]
+ mode:
+ type: choice
+ description: The Rollback mode. You can roll back to the previously deployed version or to the version that has the current commit hash of this branch in an image tag name or a deployment info.
+ options:
+ - Previous Version
+ - Commit Hash
 
 jobs:
  ecs:
+ if: contains(fromJson('["api", "inbound_mail", "webhook", "worker", "ws"]'), github.event.inputs.service)
  runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ region: ${{ fromJSON(github.event.inputs.region) }}
  timeout-minutes: 60
+ environment: ${{ github.event.inputs.environment }}
+ permissions:
+ contents: read
+ packages: write
+ deployments: write
  steps:
- - run: echo "Rolling back"
+ - run: echo "Rolling back ${{ github.event.inputs.service }} in ${{ github.event.inputs.environment }}"
+
+ - id: commit
+ if: contains(fromJson('["Commit Hash"]'), github.event.inputs.mode)
+ uses: prompt/actions-commit-hash@v3
+
+ - name: Prepare variables
+ id: variables
+ run: |
+ if [[ "${{ matrix.region }}" == "EU" && "${{ github.event.inputs.environment }}" == "Production" ]]; then
+ echo "Using Terraform Workspace: novu-prod-eu"
+ echo "TF_WORKSPACE=novu-prod-eu" >> $GITHUB_ENV
+ elif [[ "${{ matrix.region }}" == "US" && "${{ github.event.inputs.environment }}" == "Production" ]]; then
+ echo "Using Terraform Workspace: novu-prod"
+ echo "TF_WORKSPACE=novu-prod" >> $GITHUB_ENV
+ elif [[ "${{ matrix.region }}" == "EU" && "${{ github.event.inputs.environment }}" == "Development" ]]; then
+ echo "Using Terraform Workspace: novu-dev"
+ echo "TF_WORKSPACE=novu-dev" >> $GITHUB_ENV
+ elif [[ "${{ matrix.region }}" == "US" && "${{ github.event.inputs.environment }}" == "Development" ]]; then
+ echo "Using Terraform Workspace: novu-dev"
+ echo "TF_WORKSPACE=novu-dev" >> $GITHUB_ENV
+ echo "Error: Development environment doesn't exist in the US region." >&2
+ exit 1
+ else
+ echo "Using Terraform Workspace: novu-dev"
+ echo "TF_WORKSPACE=novu-dev" >> $GITHUB_ENV
+ fi
+
+ - name: Checkout cloud infra
+ uses: actions/checkout@master
+ with:
+ repository: novuhq/cloud-infra
+ token: ${{ secrets.GH_PACKAGES }}
+ path: cloud-infra
+
+ - name: Terraform setup
+ uses: hashicorp/setup-terraform@v1
+ with:
+ cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }}
+ terraform_version: 1.5.5
+ terraform_wrapper: false
+
+ - name: Terraform Init
+ working-directory: cloud-infra/terraform/novu/aws
+ run: terraform init
+
+ - name: Terraform get output
+ working-directory: cloud-infra/terraform/novu/aws
+ id: terraform
+ env:
+ SERVICE_NAME: ${{ github.event.inputs.service }}
+ run: |
+ echo "ecs_container_name=$(terraform output -json ${{ env.SERVICE_NAME }}_ecs_container_name | jq -r .)" >> $GITHUB_ENV
+ echo "ecs_service=$(terraform output -json ${{ env.SERVICE_NAME }}_ecs_service | jq -r .)" >> $GITHUB_ENV
+ echo "ecs_cluster=$(terraform output -json ${{ env.SERVICE_NAME }}_ecs_cluster | jq -r .)" >> $GITHUB_ENV
+ echo "task_name=$(terraform output -json ${{ env.SERVICE_NAME }}_task_name | jq -r .)" >> $GITHUB_ENV
+ echo "aws_region=$(terraform output -json aws_region | jq -r .)" >> $GITHUB_ENV
+
+ - name: Configure AWS credentials
+ uses: aws-actions/configure-aws-credentials@v4
+ with:
+ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ aws-region: ${{ env.aws_region }}
+
+ - name: ECS get output
+ if: contains(fromJson('["Previous Version"]'), github.event.inputs.mode)
+ id: ecs-output
+ run: |
+ echo "Retrieving current_task_definition_arn..."
+ current_task_definition_arn=$(aws ecs describe-services --cluster ${{ env.ecs_cluster }} --services ${{ env.ecs_service }} --query 'services[0].taskDefinition' --output text)
+ echo "current_task_definition_arn=$current_task_definition_arn" >> $GITHUB_ENV
+ 
+ echo "Retrieving task_definition_family..."
+ task_definition_family=$(aws ecs describe-task-definition --task-definition ${{ env.task_name }} --query 'taskDefinition.family' --output text)
+ echo "task_definition_family=$task_definition_family" >> $GITHUB_ENV
+ 
+ echo "Retrieving task_definition_list..."
+ task_definition_list=$(aws ecs list-task-definitions --family-prefix "${task_definition_family}" --output text --sort DESC | grep 'TASKDEFINITIONARNS' | cut -f 2)
+ task_definition_list_formatted=$(echo "$task_definition_list" | tr '\n' '|') # Replace newline with '|'
+ echo "task_definition_list=$task_definition_list_formatted" >> $GITHUB_ENV
+
+ if [ -n "$task_definition_list" ]; then
+ echo "Retrieving previous_task_definition_arn..."
+ index=$(echo "$task_definition_list" | grep -n "$current_task_definition_arn" | cut -d ':' -f 1)
+ if [ -n "$index" ]; then
+ if [ "$index" -ge 1 ]; then # Greater than or equal to 1
+ previous_index=$((index + 1))
+ previous_task_definition_arn=$(echo "$task_definition_list" | sed -n "${previous_index}p")
+ echo "previous_task_definition_arn=$previous_task_definition_arn" >> $GITHUB_ENV
+ else
+ echo "Invalid index value: $index"
+ fi
+ else
+ echo "Previous task definition not found. It seems to me someone deleted the current task from the list and that is why I can't find the previous task."
+ exit 1
+ fi
+ else
+ echo "No task definitions found."
+ exit 1
+ fi
+
+ - name: ECS get output by using commit hash
+ if: contains(fromJson('["Commit Hash"]'), github.event.inputs.mode)
+ id: ecs-output-commit-hash
+ env:
+ IMAGE_TAG: ${{ steps.commit.outputs.hash }}
+ run: |
+ task_definition_family=$(aws ecs describe-task-definition --task-definition ${{ env.task_name }} --query 'taskDefinition.family' --output text)
+ task_definition_arns=$(aws ecs list-task-definitions --family-prefix "${task_definition_family}" --query 'taskDefinitionArns' --output text --sort DESC)
+ found=false
+ for arn in $(echo "$task_definition_arns" | tr '\t' '\n' | head -n 20); do
+ task_definition=$(aws ecs describe-task-definition --task-definition $arn)
+ if echo "$task_definition" | grep -q "$IMAGE_TAG"; then
+ echo "Found task definition with image tag $IMAGE_TAG: $arn"
+ found=true
+ needed_arn=$arn
+ break
+ fi
+ done
+ if [ "$found" = false ]; then
+ echo "Error: Task definition with image tag $IMAGE_TAG not found within the last 20 tasks."
+ exit 1
+ fi
+ current_task_definition_arn=$(aws ecs describe-services --cluster ${{ env.ecs_cluster }} --services ${{ env.ecs_service }} --query 'services[0].taskDefinition' --output text)
+ echo "current_task_definition_arn=$current_task_definition_arn" >> $GITHUB_ENV
+ echo "previous_task_definition_arn=$needed_arn" >> $GITHUB_ENV
+ echo "Your task definition ARN is $needed_arn"
+
+
+ - name: Rollback a service to the previous task definition
+ id: rollback
+ env:
+ PREVIOUS_TASK: ${{ env.previous_task_definition_arn }}
+ CURRENT_TASK: ${{ env.current_task_definition_arn }}
+ run: |
+ aws ecs update-service --cluster ${{ env.ecs_cluster }} --service ${{ env.ecs_service }} --task-definition ${{ env.PREVIOUS_TASK }}
+ aws ecs wait services-stable --cluster ${{ env.ecs_cluster }} --service ${{ env.ecs_service }}
+ echo "After Rollback:"
+ echo "The previous task definition: $(echo $CURRENT_TASK | awk -F'task-definition/' '{print $2}')"
+ echo "The current task definition: $(echo $PREVIOUS_TASK | awk -F'task-definition/' '{print $2}')"
+
+
+ netlify:
+ if: contains(fromJson('["web", "widget"]'), github.event.inputs.service)
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ region: ${{ fromJSON(github.event.inputs.region) }}
+ timeout-minutes: 60
+ environment: ${{ github.event.inputs.environment }}
+ permissions:
+ contents: read
+ packages: write
+ deployments: write
+ env:
+ NETLIFY_ACCESS_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
+ steps:
+ - run: echo "Rolling back ${{ github.event.inputs.service }} in ${{ github.event.inputs.environment }}"
+
+ - id: commit-netlify
+ if: contains(fromJson('["Commit Hash"]'), github.event.inputs.mode)
+ uses: prompt/actions-commit-hash@v3
+
+ - name: Prepare variables
+ id: variables
+ run: |
+ if [[ "${{ github.event.inputs.service }}" == "widget" && "${{ github.event.inputs.environment }}" == "Development" && "${{ matrix.region }}" == "EU" ]]; then
+ echo "Using netlify_site_id: b9147448-b835-4eb1-a2f0-11102f611f5f"
+ echo "netlify_site_id=b9147448-b835-4eb1-a2f0-11102f611f5f" >> $GITHUB_ENV
+ elif [[ "${{ github.event.inputs.service }}" == "web" && "${{ github.event.inputs.environment }}" == "Development" && "${{ matrix.region }}" == "EU" ]]; then
+ echo "Using netlify_site_id: 45396446-dc86-4ad6-81e4-86d3eb78d06f"
+ echo "netlify_site_id=45396446-dc86-4ad6-81e4-86d3eb78d06f" >> $GITHUB_ENV
+ elif [[ "${{ github.event.inputs.environment }}" == "Development" && "${{ matrix.region }}" == "US" ]]; then
+ echo "Error: Development environment doesn't exist in the US region." >&2
+ exit 1
+ elif [[ "${{ github.event.inputs.service }}" == "web" && "${{ github.event.inputs.environment }}" == "Production" && "${{ matrix.region }}" == "EU" ]]; then
+ echo "Using netlify_site_id: d2e8b860-7016-4202-9256-ebca0f13259a"
+ echo "netlify_site_id=d2e8b860-7016-4202-9256-ebca0f13259a" >> $GITHUB_ENV
+ elif [[ "${{ github.event.inputs.service }}" == "web" && "${{ github.event.inputs.environment }}" == "Production" && "${{ matrix.region }}" == "US" ]]; then
+ echo "Using netlify_site_id: 8639d8b9-81f9-44c3-b885-585a7fd2b5ff"
+ echo "netlify_site_id=8639d8b9-81f9-44c3-b885-585a7fd2b5ff" >> $GITHUB_ENV
+ elif [[ "${{ github.event.inputs.service }}" == "widget" && "${{ github.event.inputs.environment }}" == "Production" && "${{ matrix.region }}" == "EU" ]]; then
+ echo "Using netlify_site_id: 20a64bdd-1934-4284-875f-862410c69a3b"
+ echo "netlify_site_id=20a64bdd-1934-4284-875f-862410c69a3b" >> $GITHUB_ENV
+ elif [[ "${{ github.event.inputs.service }}" == "widget" && "${{ github.event.inputs.environment }}" == "Production" && "${{ matrix.region }}" == "US" ]]; then
+ echo "Using netlify_site_id: 6f927fd4-dcb0-4cf3-8c0b-8c5539d0d034"
+ echo "netlify_site_id=6f927fd4-dcb0-4cf3-8c0b-8c5539d0d034" >> $GITHUB_ENV
+ fi
+
+ - name: Get Current Deploy ID
+ if: contains(fromJson('["Previous Version"]'), github.event.inputs.mode)
+ id: get_current_deploy
+ env:
+ NETLIFY_SITE_ID: ${{ env.netlify_site_id }}
+ run: |
+ response=$(curl -s -H "Authorization: Bearer $NETLIFY_ACCESS_TOKEN" "https://api.netlify.com/api/v1/sites/${NETLIFY_SITE_ID}")
+ current_deploy_id=$(echo "$response" | jq -r '.published_deploy.id')
+ echo "current_deploy_id=$current_deploy_id" >> $GITHUB_ENV
+
+ - name: Find Previous Production Deployments and Determine Previous Deploy ID
+ if: contains(fromJson('["Previous Version"]'), github.event.inputs.mode)
+ id: previous_deploy_id
+ env:
+ NETLIFY_SITE_ID: ${{ env.netlify_site_id }}
+ run: |
+ response=$(curl -s -H "Authorization: Bearer $NETLIFY_ACCESS_TOKEN" "https://api.netlify.com/api/v1/sites/${NETLIFY_SITE_ID}/deploys?per_page=100")
+ deploy_ids=$(echo "$response" | jq -r '.[] | select(.context == "production" and .state == "ready" and .published_at != null) | .id' | sort)
+ current_index=$(echo "$deploy_ids" | grep -n "$current_deploy_id" | cut -d ":" -f 1)
+ previous_index=$((current_index - 1))
+ previous_deploy_id=$(echo "$deploy_ids" | sed "${previous_index}q;d")
+ echo "previous_deploy_id=$previous_deploy_id" >> $GITHUB_ENV
+
+ - name: Determine Previous Deploy ID
+ if: contains(fromJson('["Commit Hash"]'), github.event.inputs.mode)
+ env:
+ NETLIFY_SITE_ID: ${{ env.netlify_site_id }}
+ COMMIT_REF: ${{ steps.commit-netlify.outputs.hash }}
+ run: |
+ response=$(curl -s -H "Authorization: Bearer $NETLIFY_ACCESS_TOKEN" "https://api.netlify.com/api/v1/sites/$NETLIFY_SITE_ID/deploys")
+ deploy_id=$(echo "$response" | jq -r ".[] | select(.commit_ref == \"$COMMIT_REF\") | .id")
+ if [ -n "$deploy_id" ]; then
+ echo "Deploy ID for commit $COMMIT_REF: $deploy_id"
+ echo "previous_deploy_id=$deploy_id" >> $GITHUB_ENV
+ else
+ echo "Deploy not found for commit $COMMIT_REF"
+ exit 1
+ fi
+
+ - name: Rollback to Previous Deploy
+ if: env.previous_deploy_id != null
+ env:
+ NETLIFY_SITE_ID: ${{ env.netlify_site_id }}
+ run: |
+ echo "Restoring previous deploy..."
+ curl -X POST -H "Authorization: Bearer $NETLIFY_ACCESS_TOKEN" "https://api.netlify.com/api/v1/sites/${{ env.netlify_site_id }}/deploys/${{ env.previous_deploy_id }}/restore"