Merge branch 'main' into szymon/versioning

.github/workflows/dispatcher.yml

@@ -0,0 +1,26 @@
+name: ChatOps Dispatcher
+
+on:
+ issue_comment:
+ types:
+ - created
+
+jobs:
+ slash-command-dispatch:
+ name: Slash Command Dispatcher
+ runs-on: ubuntu-latest
+ permissions:
+ pull-requests: write
+ steps:
+ - name: Slash Command Dispatch
+ uses: peter-evans/slash-command-dispatch@v4
+ with:
+ token: ${{ secrets.WANDB_RELEASE_TOKEN }}
+ reaction-token: ${{ secrets.WANDB_RELEASE_TOKEN }}
+ commands: |
+ test
+ destroy
+ help
+ permission: maintain
+ issue-type: pull-request
+ event-type-suffix: -command

.github/workflows/handler-help.yaml

@@ -0,0 +1,27 @@
+name: Pull Request Help Handler
+
+on:
+ repository_dispatch:
+ types:
+ - help-command
+
+jobs:
+ help:
+ name: Run help
+ runs-on: ubuntu-latest
+ permissions:
+ pull-requests: write
+ steps:
+ - name: Update comment
+ uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0
+ with:
+ token: ${{ secrets.WANDB_RELEASE_TOKEN }}
+ repository: ${{ github.event.client_payload.github.payload.repository.full_name }}
+ comment-id: ${{ github.event.client_payload.github.payload.comment.id }}
+ body: |
+ > | Command | Description |
+ > | ------- | ----------- |
+ > | /test [destroy=false] | Run the Terraform test workflow on the modules in the tests/ The named argument "destroy=false" will disable the destruction of test infrastructure for debugging purposes. |
+ > | /destroy | Destroy any resources that may still be in Terraform state from previous tests |
+ > | /help | Shows this help message |
+ reactions: confused

.github/workflows/handler-test.yml

@@ -0,0 +1,102 @@
+name: Pull Request Test Handler
+
+on:
+ repository_dispatch:
+ types:
+ - test-command
+
+env:
+ GOOGLE_SERVICE_ACCOUNT: "[email protected]"
+ TF_TOKEN_app_terraform_io: ${{ secrets.TFE_TOKEN }}
+
+jobs:
+ public-dns-with-cloud-dns:
+ name: Public DNS with Cloud DNS
+ runs-on: ubuntu-latest
+
+ defaults:
+ run:
+ working-directory: ./tests/public-dns
+
+ steps:
+ - name: Create URL to the run output
+ id: vars
+ run: echo "run-url=https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" >> $GITHUB_OUTPUT
+
+ - name: Display Action URL
+ if: ${{ always() }}
+ uses: peter-evans/create-or-update-comment@v4
+ with:
+ token: ${{ secrets.WANDB_RELEASE_TOKEN }}
+ repository: ${{ github.event.client_payload.github.payload.repository.full_name }}
+ comment-id: ${{ github.event.client_payload.github.payload.comment.id }}
+ body: |
+ ${{ format('### Public DNS Terraform Google - Test Report', job.status == 'success' && ':white_check_mark:' || ':x:') }}
+
+ ${{ format(':link: [Action Summary Page]({0})', steps.vars.outputs.run-url) }}
+
+ - name: Checkout code
+ uses: actions/checkout@v2
+ with:
+ repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }}
+ ref: ${{ github.event.client_payload.pull_request.head.sha }}
+ persist-credentials: false
+
+ - name: Setup Terraform
+ uses: hashicorp/setup-terraform@v1
+ with:
+ terraform_version: "~1"
+
+ - name: Install k6 for load testing
+ env:
+ K6_URL: https://github.com/loadimpact/k6/releases/download/v0.31.1/k6-v0.31.1-linux64.tar.gz
+ run: |
+ sudo apt-get install jq
+ curl -L $K6_URL | tar -xz --strip-components=1
+
+ - name: Authenticate with Google Cloud
+ id: auth
+ uses: google-github-actions/auth@v2
+ with:
+ credentials_json: "${{ secrets.GOOGLE_TESTING_SA_CREDENTIALS }}"
+
+ - name: Set up Cloud SDK
+ uses: google-github-actions/setup-gcloud@v2
+
+ - name: Terraform Init
+ id: init
+ run: terraform init -input=false -no-color
+
+ - name: Terraform Validate
+ id: validate
+ run: terraform validate -no-color
+
+ - name: Terraform Apply
+ id: apply
+ run: terraform apply -auto-approve -input=false -no-color
+
+ - name: Retrieve Health Check URL
+ id: retrieve-health-check-url
+ run: |
+ terraform output -no-color -raw health_check_url
+
+ - name: Terraform Destroy
+ id: destroy
+ if: ${{ always() && github.event.client_payload.slash_command.args.named.destroy != 'false' }}
+ run: terraform destroy -auto-approve -input=false -no-color
+
+ - name: Update comment
+ if: ${{ always() }}
+ uses: peter-evans/create-or-update-comment@v4
+ with:
+ token: ${{ secrets.WANDB_RELEASE_TOKEN }}
+ repository: ${{ github.event.client_payload.github.payload.repository.full_name }}
+ comment-id: ${{ github.event.client_payload.github.payload.comment.id }}
+ body: |
+ ${{ format('- {0} Terraform Init', steps.init.outcome == 'success' && ':white_check_mark:' || ':x:') }}
+
+ ${{ format('- {0} Terraform Validate', steps.validate.outcome == 'success' && ':white_check_mark:' || ':x:') }}
+
+ ${{ format('- {0} Terraform Apply', steps.apply.outcome == 'success' && ':white_check_mark:' || ':x:') }}
+
+ ${{ github.event.client_payload.slash_command.args.named.destroy != 'false' && format('- {0} Terraform Destroy', steps.destroy.outcome == 'success' && ':white_check_mark:' || ':x:') || '' }}

.github/workflows/tf-lint.yaml

@@ -0,0 +1,24 @@
+name: Terraform Lint Check
+
+on: [push]
+
+jobs:
+ tflint:
+ name: Run tflint
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v2
+
+ - name: Setup Terraform
+ uses: hashicorp/setup-terraform@v1
+ with:
+ terraform_version: "~1"
+
+ - name: Install tflint
+ run: |
+ curl -s https://raw.githubusercontent.com/terraform-linters/tflint/master/install_linux.sh | bash
+
+ - name: Run tflint
+ run: tflint

main.tf

@@ -21,12 +21,11 @@ module "project_factory_project_services" {
 }
 
 locals {
- fqdn = var.subdomain == null ? var.domain_name : "${var.subdomain}.${var.domain_name}"
- url_prefix = var.ssl ? "https" : "http"
- url = "${local.url_prefix}://${local.fqdn}"
- internal_app_port = 32543
- create_bucket = var.bucket_name == ""
- create_network = var.network == null
+ fqdn = var.subdomain == null ? var.domain_name : "${var.subdomain}.${var.domain_name}"
+ url_prefix = var.ssl ? "https" : "http"
+ url = "${local.url_prefix}://${local.fqdn}"
+ create_bucket = var.bucket_name == ""
+ create_network = var.network == null
 }
 
 module "service_accounts" {
@@ -45,7 +44,7 @@ module "kms" {
 }
 
 locals {
- crypto_key = var.use_internal_queue ? null : module.kms.0.crypto_key
+ crypto_key = var.use_internal_queue ? null : module.kms[0].crypto_key
 }
 
 module "storage" {
@@ -72,9 +71,9 @@ module "networking" {
 }
 
 locals {
- network_connection = try(module.networking.0.connection, { network = var.network })
- network = try(module.networking.0.network, { self_link = var.network })
- subnetwork = try(module.networking.0.subnetwork, { self_link = var.subnetwork })
+ network_connection = try(module.networking[0].connection, { network = var.network })
+ network = try(module.networking[0].network, { self_link = var.network })
+ subnetwork = try(module.networking[0].subnetwork, { self_link = var.subnetwork })
 }
 
 module "app_gke" {
@@ -128,10 +127,10 @@ module "redis" {
 }
 
 locals {
- redis_certificate = var.create_redis ? module.redis.0.ca_cert : null
- redis_connection_string = var.create_redis ? "redis://:${module.redis.0.auth_string}@${module.redis.0.connection_string}?tls=true&ttlInSeconds=604800&caCertPath=/etc/ssl/certs/server_ca.pem" : null
- bucket = local.create_bucket ? module.storage.0.bucket_name : var.bucket_name
- bucket_queue = var.use_internal_queue ? "internal://" : "pubsub:/${module.storage.0.bucket_queue_name}"
+ redis_certificate = var.create_redis ? module.redis[0].ca_cert : null
+ redis_connection_string = var.create_redis ? "redis://:${module.redis[0].auth_string}@${module.redis[0].connection_string}?tls=true&ttlInSeconds=604800&caCertPath=/etc/ssl/certs/server_ca.pem" : null
+ bucket = local.create_bucket ? module.storage[0].bucket_name : var.bucket_name
+ bucket_queue = var.use_internal_queue ? "internal://" : "pubsub:/${module.storage[0].bucket_queue_name}"
  project_id = module.project_factory_project_services.project_id
  secret_store_source = "gcp-secretmanager://${local.project_id}?namespace=${var.namespace}"
 }
@@ -218,10 +217,10 @@ module "wandb" {
  }
 
  redis = var.create_redis ? {
- password = module.redis.0.auth_string
- host = module.redis.0.host
- port = module.redis.0.port
- caCert = module.redis.0.ca_cert
+ password = module.redis[0].auth_string
+ host = module.redis[0].host
+ port = module.redis[0].port
+ caCert = module.redis[0].ca_cert
  params = {
  tls = true
  ttlInSeconds = 604800

tests/public-dns/README.md

@@ -0,0 +1,11 @@
+# TEST: Public DNS deployment of W&B
+
+## About This Test
+
+## Prerequisites
+
+## How This Test Is Used
+
+This test is leveraged by this repository's continuous integration setup which
+leverages workspaces in a Terraform Cloud workspaces as a remote backend so that
+Terraform state is preserved.

tests/public-dns/main.tf

@@ -0,0 +1,66 @@
+provider "google" {}
+provider "google-beta" {}
+
+resource "random_pet" "main" {
+ length = 1
+ prefix = "tgw-pd"
+ separator = "-"
+}
+
+variable "license" {
+ type = string
+}
+
+locals {
+ labels = {
+ oktodelete = "true"
+ department = "engineering"
+ product = "server"
+ repository = "terraform-google-wandb"
+ description = "public-dns"
+ environment = "test"
+ }
+
+ subdomain = "tgw-pd"
+
+ project = "playground-111"
+}
+
+data "google_dns_managed_zone" "default" {
+ project = local.project
+ name = "wandb-ml"
+}
+
+# Create A record which points to lb ip address
+resource "google_dns_record_set" "a" {
+ project = local.project
+ name = "${local.subdomain}.${data.google_dns_managed_zone.default.dns_name}"
+ managed_zone = data.google_dns_managed_zone.default.name
+ type = "A"
+ ttl = 60
+ rrdatas = [module.wandb.address]
+}
+
+module "wandb" {
+ source = "../.."
+
+ namespace = random_pet.main.id
+ license = var.license
+ domain_name = "wandb.ml"
+ subdomain = local.subdomain
+
+ create_redis = true
+ use_internal_queue = true
+
+ deletion_protection = false
+
+ labels = local.labels
+}
+
+output "url" {
+ value = module.wandb.url
+}
+
+output "health_check_url" {
+ value = "${module.wandb.url}/health"
+}

tests/public-dns/versions.tf

@@ -0,0 +1,11 @@
+terraform {
+ required_version = ">= 0.14"
+
+ cloud {
+ organization = "weights-and-biases"
+
+ workspaces {
+ name = "terraform-google-wandb-public-dns"
+ }
+ }
+}