feat(security): multiple encryption options, API tokens, easier setup (…

…#125)

* (wip) encryption

* feat: api tokens

* chore: add api token generation command

* fix: e2e tests

* chore: set timeout for e2e job

* fix: e2e tests, remove client-side certs

* chore: address PR review comments

* fix: token tests

* chore: address review comments and fix tests

.github/workflows/test.yml

@@ -102,6 +102,7 @@ jobs:
 
  e2e:
  runs-on: ubuntu-latest
+ timeout-minutes: 5
  env:
  DATABASE_URL: postgresql://hatchet:[email protected]:5431/hatchet
 
@@ -153,14 +154,17 @@ jobs:
  . .env
  set +a
 
- go run ./cmd/hatchet-admin seed
+ go run ./cmd/hatchet-admin quickstart
 
- go run ./cmd/hatchet-engine &
- go run ./cmd/hatchet-api &
+ go run ./cmd/hatchet-engine --config ./generated/ &
+ go run ./cmd/hatchet-api --config ./generated/ &
  sleep 30
 
  - name: Test
- run: go test -tags e2e ./... -p 1 -v -failfast
+ run: |
+ export HATCHET_CLIENT_TOKEN="$(go run ./cmd/hatchet-admin token create --config ./generated/ --tenant-id 707d0855-80ab-4e1f-a156-f1c4546cbf52)"
+
+ go test -tags e2e ./... -p 1 -v -failfast
 
  - name: Teardown
  run: docker compose down

.gitignore

@@ -85,3 +85,5 @@ tmp
 
 postgres-data
 rabbitmq.conf
+
+*encryption-keys

.pre-commit-config.yaml

@@ -0,0 +1,7 @@
+repos:
+ - repo: https://github.com/gitguardian/ggshield
+ rev: v1.23.0
+ hooks:
+ - id: ggshield
+ language_version: python3
+ stages: [commit]

CONTRIBUTING.md

@@ -8,7 +8,7 @@
 - `docker-compose`
 - [`Taskfile`](https://taskfile.dev/installation/)
 - The following additional devtools:
- - `protoc`: `brew install protobuf@25` 
+ - `protoc`: `brew install protobuf@25`
  - `caddy` and `nss`: `brew install caddy nss`
 
 ### Setup
@@ -19,7 +19,9 @@
 
 3. Generate certificates needed for communicating between the Hatchet client and engine: `task generate-certs`
 
-4. Create environment variables:
+4. Generate keysets for encryption: `task generate-local-encryption-keys`
+
+5. Create environment variables:
 
 ```sh
 alias randstring='f() { openssl rand -base64 69 | tr -d "\n" | tr -d "=+/" | cut -c1-$1 };f'
@@ -30,29 +32,55 @@ SERVER_TLS_CERT_FILE=./hack/dev/certs/cluster.pem
 SERVER_TLS_KEY_FILE=./hack/dev/certs/cluster.key
 SERVER_TLS_ROOT_CA_FILE=./hack/dev/certs/ca.cert
 
+SERVER_ENCRYPTION_MASTER_KEYSET_FILE=./hack/dev/encryption-keys/master.key
+SERVER_ENCRYPTION_JWT_PRIVATE_KEYSET_FILE=./hack/dev/encryption-keys/private_ec256.key
+SERVER_ENCRYPTION_JWT_PUBLIC_KEYSET_FILE=./hack/dev/encryption-keys/public_ec256.key
+
 SERVER_PORT=8080
 SERVER_URL=https://app.dev.hatchet-tools.com
 
 SERVER_AUTH_COOKIE_SECRETS="$(randstring 16) $(randstring 16)"
 SERVER_AUTH_COOKIE_DOMAIN=app.dev.hatchet-tools.com
 SERVER_AUTH_COOKIE_INSECURE=false
 SERVER_AUTH_SET_EMAIL_VERIFIED=true
+
+SERVER_LOGGER_LEVEL=debug
+SERVER_LOGGER_FORMAT=console
+DATABASE_LOGGER_LEVEL=debug
+DATABASE_LOGGER_FORMAT=console
 EOF
 ```
 
-5. Migrate the database: `task prisma-migrate`
+6. Migrate the database: `task prisma-migrate`
 
-6. Generate all files: `task generate`
+7. Generate all files: `task generate`
 
-7. Seed the database: `task seed-dev`
+8. Seed the database: `task seed-dev`
 
-8. Start the Hatchet engine, API server, dashboard, and Prisma studio:
+9. Start the Hatchet engine, API server, dashboard, and Prisma studio:
 
 ```sh
 task start-dev
 ```
 
-10. To create and test workflows, run the examples in the `./examples` directory. You will need to add the tenant (output from the `task seed-dev` command) to the `.env` file in each example directory.
+### Creating and testing workflows
+
+To create and test workflows, run the examples in the `./examples` directory.
+
+You will need to add the tenant (output from the `task seed-dev` command) to the `.env` file in each example directory. An example `.env` file for the `./examples/simple` directory can be generated via:
+
+```sh
+alias get_token='go run ./cmd/hatchet-admin token create --name local --tenant-id 707d0855-80ab-4e1f-a156-f1c4546cbf52'
+
+cat > ./examples/simple/.env <<EOF
+HATCHET_CLIENT_TENANT_ID=707d0855-80ab-4e1f-a156-f1c4546cbf52
+HATCHET_CLIENT_TLS_ROOT_CA_FILE=../../hack/dev/certs/ca.cert
+HATCHET_CLIENT_TLS_SERVER_NAME=cluster
+HATCHET_CLIENT_TOKEN="$(get_token)"
+EOF
+```
+
+This example can then be run via `go run main.go` from the `./examples/simple` directory.
 
 ### Logging
 
@@ -83,3 +111,37 @@ OTEL_EXPORTER_OTLP_HEADERS=<optional-headers>
 # optional
 OTEL_EXPORTER_OTLP_ENDPOINT=<collector-url>
 ```
+
+### CloudKMS
+
+CloudKMS can be used to generate master encryption keys:
+
+```
+gcloud kms keyrings create "development" --location "global"
+gcloud kms keys create "development" --location "global" --keyring "development" --purpose "encryption"
+gcloud kms keys list --location "global" --keyring "development"
+```
+
+From the last step, copy the Key URI and set the following environment variable:
+
+```
+SERVER_ENCRYPTION_CLOUDKMS_KEY_URI=gcp-kms://projects/<PROJECT>/locations/global/keyRings/development/cryptoKeys/development
+```
+
+Generate a service account in GCP which can encrypt/decrypt on CloudKMS, then download a service account JSON file and set it via:
+
+```
+SERVER_ENCRYPTION_CLOUDKMS_CREDENTIALS_JSON='{...}'
+```
+
+## Issues
+
+### Query engine leakage
+
+Sometimes the spawned query engines from Prisma don't get killed when hot reloading. You can run `task kill-query-engines` on OSX to kill the query engines.
+
+Make sure you call `.Disconnect` on the database config object when writing CLI commands which interact with the database. If you don't, and you try to wrap these CLI commands in a new command, it will never exit, for example:
+
+```
+export HATCHET_CLIENT_TOKEN="$(go run ./cmd/hatchet-admin token create --tenant-id <tenant>)"
+```

Taskfile.yaml

@@ -1,9 +1,6 @@
 version: "3"
 
 tasks:
- write-default-env:
- cmds:
- - sh ./hack/dev/write-default-env.sh
  set-etc-hosts:
  cmds:
  - sudo sh ./hack/dev/manage-hosts.sh add 127.0.0.1 app.dev.hatchet-tools.com
@@ -46,7 +43,13 @@ tasks:
  - task: generate-api-client
  generate-certs:
  cmds:
- - sh ./hack/dev/generate-temporal-certs.sh ./hack/dev/certs
+ - sh ./hack/dev/generate-x509-certs.sh ./hack/dev/certs
+ generate-local-encryption-keys:
+ cmds:
+ - sh ./hack/dev/generate-local-encryption-keys.sh ./hack/dev/encryption-keys
+ generate-dev-api-token:
+ cmds:
+ - sh ./hack/dev/generate-dev-api-token.sh
  generate-api-server:
  cmds:
  - sh ./hack/oas/generate-server.sh

api-contracts/dispatcher/dispatcher.proto

@@ -15,17 +15,14 @@ service Dispatcher {
 }
 
 message WorkerRegisterRequest {
- // the tenant id
- string tenantId = 1;
-
  // the name of the worker
- string workerName = 2;
+ string workerName = 1;
 
  // a list of actions that this worker can run
- repeated string actions = 3;
+ repeated string actions = 2;
 
  // (optional) the services for this worker
- repeated string services = 4;
+ repeated string services = 3;
 }
 
 message WorkerRegisterResponse {
@@ -74,19 +71,13 @@ message AssignedAction {
 }
 
 message WorkerListenRequest {
- // the tenant id
- string tenantId = 1;
-
  // the id of the worker
- string workerId = 2;
+ string workerId = 1;
 }
 
 message WorkerUnsubscribeRequest {
- // the tenant id to unsubscribe from
- string tenantId = 1;
-
  // the id of the worker
- string workerId = 2;
+ string workerId = 1;
 }
 
 message WorkerUnsubscribeResponse {
@@ -105,34 +96,31 @@ enum ActionEventType {
 }
 
 message ActionEvent {
- // the tenant id
- string tenantId = 1;
-
  // the id of the worker
- string workerId = 2;
+ string workerId = 1;
 
  // the id of the job
- string jobId = 3;
+ string jobId = 2;
 
  // the job run id
- string jobRunId = 4;
+ string jobRunId = 3;
 
  // the id of the step
- string stepId = 5;
+ string stepId = 4;
 
  // the step run id
- string stepRunId = 6;
+ string stepRunId = 5;
 
  // the action id
- string actionId = 7;
+ string actionId = 6;
 
- google.protobuf.Timestamp eventTimestamp = 8;
+ google.protobuf.Timestamp eventTimestamp = 7;
 
  // the step event type
- ActionEventType eventType = 9;
+ ActionEventType eventType = 8;
 
  // the event payload
- string eventPayload = 10;
+ string eventPayload = 9;
 }
 
 message ActionEventResponse {

api-contracts/events/events.proto

@@ -30,28 +30,22 @@ message Event {
 }
 
 message PushEventRequest {
- // the tenant id
- string tenantId = 1;
-
  // the key for the event
- string key = 2;
+ string key = 1;
 
  // the payload for the event
- string payload = 3;
+ string payload = 2;
 
  // when the event was generated
- google.protobuf.Timestamp eventTimestamp = 4;
+ google.protobuf.Timestamp eventTimestamp = 3;
 }
 
 message ListEventRequest {
- // (required) the tenant id
- string tenantId = 1;
-
  // (optional) the number of events to skip
- int32 offset = 2; 
+ int32 offset = 1; 
 
  // (optional) the key for the event
- string key = 3;
+ string key = 2;
 }
 
 message ListEventResponse {
@@ -60,9 +54,6 @@ message ListEventResponse {
 }
 
 message ReplayEventRequest {
- // the tenant id
- string tenantId = 1;
-
  // the event id to replay
- string eventId = 2;
+ string eventId = 1;
 }

api-contracts/openapi/components/schemas/_index.yaml

@@ -108,3 +108,11 @@ WorkerList:
  $ref: "./worker.yaml#/WorkerList"
 Worker:
  $ref: "./worker.yaml#/Worker"
+APIToken:
+ $ref: "./api_tokens.yaml#/APIToken"
+CreateAPITokenRequest:
+ $ref: "./api_tokens.yaml#/CreateAPITokenRequest"
+CreateAPITokenResponse:
+ $ref: "./api_tokens.yaml#/CreateAPITokenResponse"
+ListAPITokensResponse:
+ $ref: "./api_tokens.yaml#/ListAPITokensResponse"

api-contracts/openapi/components/schemas/api_tokens.yaml

@@ -0,0 +1,45 @@
+APIToken:
+ type: object
+ properties:
+ metadata:
+ $ref: "./metadata.yaml#/APIResourceMeta"
+ name:
+ type: string
+ description: The name of the API token.
+ maxLength: 255
+ expiresAt:
+ type: string
+ format: date-time
+ description: When the API token expires.
+ required:
+ - metadata
+ - name
+ - expiresAt
+
+CreateAPITokenRequest:
+ type: object
+ properties:
+ name:
+ type: string
+ description: A name for the API token.
+ maxLength: 255
+ required:
+ - name
+
+CreateAPITokenResponse:
+ type: object
+ properties:
+ token:
+ type: string
+ description: The API token.
+ required:
+ - token
+
+ListAPITokensResponse:
+ properties:
+ pagination:
+ $ref: "./metadata.yaml#/PaginationResponse"
+ rows:
+ items:
+ $ref: "#/APIToken"
+ type: array

api-contracts/openapi/openapi.yaml

@@ -48,6 +48,10 @@ paths:
  $ref: "./paths/tenant/tenant.yaml#/invites"
  /api/v1/tenants/{tenant}/invites/{tenant-invite}:
  $ref: "./paths/tenant/tenant.yaml#/inviteScoped"
+ /api/v1/tenants/{tenant}/api-tokens:
+ $ref: "./paths/api-tokens/api_tokens.yaml#/withTenant"
+ /api/v1/api-tokens/{api-token}:
+ $ref: "./paths/api-tokens/api_tokens.yaml#/revoke"
  /api/v1/tenants/{tenant}/events:
  $ref: "./paths/event/event.yaml#/withTenant"
  /api/v1/tenants/{tenant}/events/replay: