Merge pull request #236 from jembi/PLAT-731-jempi-clustering

Plat-731 Jempi Clustering

.env.cluster

@@ -81,6 +81,14 @@ GF_SERVER_DOMAIN=grafana.domain
 JEMPI_WEB_INSTANCES=3
 REACT_APP_JEMPI_BASE_URL=https://jempi-api.domain/JeMPI
 JEMPI_SESSION_SECURE=true
+JEMPI_REPMGR_PARTNER_NODES=jempi-postgresql-01,jempi-postgresql-02,jempi-postgresql-03
+JEMPI_ASYNC_RECEIVER_INSTANCES=1
+JEMPI_SYNC_RECEIVER_INSTANCES=1
+JEMPI_PRE_PROCESSOR_INSTANCES=1
+JEMPI_CONTROLLER_INSTANCES=1
+JEMPI_EM_CALCULATOR_INSTANCES=1
+JEMPI_LINKER_INSTANCES=1
+JEMPI_API_INSTANCES=3
 
 # Resource limits
 OPENHIM_MEMORY_LIMIT=4G

client-registry-jempi/docker-compose.api.yml

@@ -14,9 +14,9 @@ services:
  JEMPI_FILE_IMPORT_MAX_SIZE_BYTE: ${JEMPI_FILE_IMPORT_MAX_SIZE_BYTE}
  JEMPI_SESSION_SECURE: ${JEMPI_SESSION_SECURE}
  JEMPI_SESSION_DOMAIN_NAME: ${DOMAIN_NAME}
- postgres.server: jempi-postgresql
+ postgres.server: ${JEMPI_REPMGR_PARTNER_NODES}
  deploy:
- replicas: 1
+ replicas: ${JEMPI_API_INSTANCES}
  resources:
  limits:
  memory: ${JEMPI_API_MEMORY_LIMIT}

client-registry-jempi/docker-compose.combined-cluster.yml

@@ -0,0 +1,72 @@
+version: '3.9'
+
+services:
+ jempi-postgresql-01:
+ deploy:
+ placement:
+ constraints:
+ - "node.labels.name==node-1"
+
+ jempi-postgresql-02:
+ image: bitnami/postgresql-repmgr:15.2.0
+ environment:
+ POSTGRESQL_USERNAME: ${POSTGRESQL_USERNAME}
+ POSTGRESQL_DATABASE: ${POSTGRESQL_DATABASE}
+ REPMGR_PASSWORD: ${JEMPI_REPMGR_PASSWORD}
+ REPMGR_PRIMARY_HOST: ${JEMPI_REPMGR_PRIMARY_HOST}
+ REPMGR_PARTNER_NODES: ${JEMPI_REPMGR_PARTNER_NODES}
+ REPMGR_NODE_NAME: jempi-postgresql-02
+ REPMGR_NODE_NETWORK_NAME: jempi-postgresql-02
+ ALLOW_EMPTY_PASSWORD: "yes"
+ deploy:
+ placement:
+ constraints:
+ - "node.labels.name==node-2"
+ mode: replicated
+ replicas: 1
+ resources:
+ limits:
+ cpus: ${JEMPI_POSTGRES_CPU_LIMIT}
+ memory: ${JEMPI_POSTGRES_MEMORY_LIMIT}
+ reservations:
+ cpus: ${JEMPI_POSTGRES_CPU_RESERVE}
+ memory: ${JEMPI_POSTGRES_MEMORY_RESERVE}
+ volumes:
+ - "jempi-psql-02-data:/bitnami/postgresql"
+ configs:
+ - target: /docker-entrypoint-initdb.d/jempi_psql_init_db.sql
+ source: jempi_psql_init_db.sql
+
+ jempi-postgresql-03:
+ image: bitnami/postgresql-repmgr:15.2.0
+ environment:
+ POSTGRESQL_USERNAME: ${POSTGRESQL_USERNAME}
+ POSTGRESQL_DATABASE: ${POSTGRESQL_DATABASE}
+ REPMGR_PASSWORD: ${JEMPI_REPMGR_PASSWORD}
+ REPMGR_PRIMARY_HOST: ${JEMPI_REPMGR_PRIMARY_HOST}
+ REPMGR_PARTNER_NODES: ${JEMPI_REPMGR_PARTNER_NODES}
+ REPMGR_NODE_NAME: jempi-postgresql-03
+ REPMGR_NODE_NETWORK_NAME: jempi-postgresql-03
+ ALLOW_EMPTY_PASSWORD: "yes"
+ deploy:
+ placement:
+ constraints:
+ - "node.labels.name==node-3"
+ mode: replicated
+ replicas: 1
+ resources:
+ limits:
+ cpus: ${JEMPI_POSTGRES_CPU_LIMIT}
+ memory: ${JEMPI_POSTGRES_MEMORY_LIMIT}
+ reservations:
+ cpus: ${JEMPI_POSTGRES_CPU_RESERVE}
+ memory: ${JEMPI_POSTGRES_MEMORY_RESERVE}
+ volumes:
+ - "jempi-psql-03-data:/bitnami/postgresql"
+ configs:
+ - target: /docker-entrypoint-initdb.d/jempi_psql_init_db.sql
+ source: jempi_psql_init_db.sql
+
+volumes:
+ jempi-psql-02-data:
+ jempi-psql-03-data:

client-registry-jempi/docker-compose.combined-dev.yml

@@ -22,7 +22,7 @@ services:
  protocol: tcp
  mode: host
 
- jempi-postgresql:
+ jempi-postgresql-01:
  ports:
  - published: 5435
  target: 5432

client-registry-jempi/docker-compose.combined.yml

@@ -6,7 +6,7 @@ services:
  environment:
  kafka.bootstrap.servers: ${KAFKA_HOSTS}
  deploy:
- replicas: 1
+ replicas: ${JEMPI_ASYNC_RECEIVER_INSTANCES}
  resources:
  limits:
  memory: ${JEMPI_ASYNC_RECEIVER_MEMORY_LIMIT}
@@ -21,7 +21,7 @@ services:
  environment:
  kafka.bootstrap.servers: ${KAFKA_HOSTS}
  deploy:
- replicas: 1
+ replicas: ${JEMPI_SYNC_RECEIVER_INSTANCES}
  resources:
  limits:
  memory: ${JEMPI_SYNC_RECEIVER_MEMORY_LIMIT}
@@ -36,7 +36,7 @@ services:
  environment:
  kafka.bootstrap.servers: ${KAFKA_HOSTS}
  deploy:
- replicas: 1
+ replicas: ${JEMPI_PRE_PROCESSOR_INSTANCES}
  resources:
  limits:
  memory: ${JEMPI_PRE_PROCESSOR_MEMORY_LIMIT}
@@ -51,7 +51,7 @@ services:
  environment:
  kafka.bootstrap.servers: ${KAFKA_HOSTS}
  deploy:
- replicas: 1
+ replicas: ${JEMPI_CONTROLLER_INSTANCES}
  resources:
  limits:
  memory: ${JEMPI_CONTROLLER_MEMORY_LIMIT}
@@ -66,7 +66,7 @@ services:
  environment:
  kafka.bootstrap.servers: ${KAFKA_HOSTS}
  deploy:
- replicas: 1
+ replicas: ${JEMPI_EM_CALCULATOR_INSTANCES}
  resources:
  limits:
  memory: ${JEMPI_EM_CALCULATOR_MEMORY_LIMIT}
@@ -81,7 +81,7 @@ services:
  environment:
  kafka.bootstrap.servers: ${KAFKA_HOSTS}
  deploy:
- replicas: 1
+ replicas: ${JEMPI_LINKER_INSTANCES}
  resources:
  limits:
  memory: ${JEMPI_LINKER_MEMORY_LIMIT}
@@ -91,11 +91,16 @@ services:
  kafka:
  default:
 
- jempi-postgresql:
- image: bitnami/postgresql:15.2.0
+ jempi-postgresql-01:
+ image: bitnami/postgresql-repmgr:15.2.0
  environment:
  POSTGRESQL_USERNAME: ${POSTGRESQL_USERNAME}
  POSTGRESQL_DATABASE: ${POSTGRESQL_DATABASE}
+ REPMGR_PASSWORD: ${JEMPI_REPMGR_PASSWORD}
+ REPMGR_PRIMARY_HOST: ${JEMPI_REPMGR_PRIMARY_HOST}
+ REPMGR_PARTNER_NODES: ${JEMPI_REPMGR_PARTNER_NODES}
+ REPMGR_NODE_NAME: jempi-postgresql-01
+ REPMGR_NODE_NETWORK_NAME: jempi-postgresql-01
  ALLOW_EMPTY_PASSWORD: "yes"
  deploy:
  mode: replicated
@@ -112,10 +117,7 @@ services:
  configs:
  - target: /docker-entrypoint-initdb.d/jempi_psql_init_db.sql
  source: jempi_psql_init_db.sql
- networks:
- kafka:
- default:
-
+
 volumes:
  jempi-psql-01-data:
 

client-registry-jempi/docker-compose.dgraph-zero-cluster.yml

@@ -6,3 +6,44 @@ services:
  placement:
  constraints:
  - node.labels.name == node-1
+ command: dgraph zero --my=jempi-zero-01:5080 --replicas 3 --bindall --raft "idx=1"
+
+ jempi-zero-02:
+ image: dgraph/dgraph:v22.0.0
+ volumes:
+ - jempi-zero-02-data:/dgraph
+ deploy:
+ replicas: 1
+ placement:
+ constraints:
+ - node.labels.name == node-2
+ resources:
+ limits:
+ memory: ${JEMPI_ZERO_MEMORY_LIMIT}
+ reservations:
+ memory: ${JEMPI_ZERO_MEMORY_RESERVE}
+ restart_policy:
+ condition: on-failure
+ command: dgraph zero --my=jempi-zero-02:5080 --replicas 3 --peer=jempi-zero-01:5080 --raft "idx=2"
+
+ jempi-zero-03:
+ image: dgraph/dgraph:v22.0.0
+ volumes:
+ - jempi-zero-03-data:/dgraph
+ deploy:
+ replicas: 1
+ placement:
+ constraints:
+ - node.labels.name == node-3
+ resources:
+ limits:
+ memory: ${JEMPI_ZERO_MEMORY_LIMIT}
+ reservations:
+ memory: ${JEMPI_ZERO_MEMORY_RESERVE}
+ restart_policy:
+ condition: on-failure
+ command: dgraph zero --my=jempi-zero-03:5080 --replicas 3 --peer=jempi-zero-01:5080 --raft "idx=3"
+
+volumes:
+ jempi-zero-02-data:
+ jempi-zero-03-data:

client-registry-jempi/docker-compose.dgraph-zero.yml

@@ -9,14 +9,12 @@ services:
  replicas: 1
  resources:
  limits:
- memory: ${JEMPI_ZERO_01_MEMORY_LIMIT}
+ memory: ${JEMPI_ZERO_MEMORY_LIMIT}
  reservations:
- memory: ${JEMPI_ZERO_01_MEMORY_RESERVE}
+ memory: ${JEMPI_ZERO_MEMORY_RESERVE}
  restart_policy:
  condition: on-failure
  command: dgraph zero --my=jempi-zero-01:5080 --replicas 1
 
 volumes:
  jempi-zero-01-data:
-
-

client-registry-jempi/docker-compose.web.yml

@@ -24,7 +24,6 @@ services:
  keycloak:
  default:
 
-
 networks:
  reverse-proxy:
  name: reverse-proxy_public

client-registry-jempi/package-metadata.json

@@ -11,8 +11,8 @@
  ],
  "environmentVariables": {
  "KAFKA_HOSTS": "kafka-01:9092",
- "JEMPI_ZERO_01_MEMORY_LIMIT": "3G",
- "JEMPI_ZERO_01_MEMORY_RESERVE": "500M",
+ "JEMPI_ZERO_MEMORY_LIMIT": "3G",
+ "JEMPI_ZERO_MEMORY_RESERVE": "500M",
  "JEMPI_ALPHA_01_MEMORY_LIMIT": "3G",
  "JEMPI_ALPHA_01_MEMORY_RESERVE": "500M",
  "JEMPI_ALPHA_02_MEMORY_LIMIT": "3G",
@@ -35,6 +35,7 @@
  "JEMPI_SYNC_RECEIVER_MEMORY_RESERVE": "500M",
  "JEMPI_API_MEMORY_LIMIT": "3G",
  "JEMPI_API_MEMORY_RESERVE": "500M",
+ "JEMPI_API_INSTANCES": 1,
  "JEMPI_KAFKA_TOPICS": "JeMPI-async-preprocessor,JeMPI-patient-controller,JeMPI-patient-em,JeMPI-patient-linker,JeMPI-mu-linker,JeMPI-notifications",
  "JEMPI_ASYNC_RECEIVER_IMAGE_TAG": "0.2.0",
  "JEMPI_SYNC_RECEIVER_IMAGE_TAG": "0.2.0",
@@ -43,6 +44,12 @@
  "JEMPI_EM_CALCULATOR_IMAGE_TAG": "0.2.0",
  "JEMPI_LINKER_IMAGE_TAG": "0.2.0",
  "JEMPI_API_IMAGE_TAG": "0.2.0",
+ "JEMPI_ASYNC_RECEIVER_INSTANCES": 1,
+ "JEMPI_SYNC_RECEIVER_INSTANCES": 1,
+ "JEMPI_PRE_PROCESSOR_INSTANCES": 1,
+ "JEMPI_CONTROLLER_INSTANCES": 1,
+ "JEMPI_EM_CALCULATOR_INSTANCES": 1,
+ "JEMPI_LINKER_INSTANCES": 1,
  "JEMPI_OPENHIM_PASSWORD": "instant101",
  "JEMPI_SESSION_SECRET": "c05ll3lesrinf39t7mc5h6un6r0c69lgfno69dsak3vabeqamouq4328cuaekros401ajdpkh60rrt",
  "JEMPI_FILE_IMPORT_MAX_SIZE_BYTE": 128000000,
@@ -60,6 +67,9 @@
  "KC_JEMPI_ROOT_URL": "http://localhost:3033",
  "POSTGRESQL_DATABASE": "notifications",
  "POSTGRESQL_USERNAME": "postgres",
+ "JEMPI_REPMGR_PASSWORD": "dev_password_only",
+ "JEMPI_REPMGR_PRIMARY_HOST": "jempi-postgresql-01",
+ "JEMPI_REPMGR_PARTNER_NODES": "jempi-postgresql-01",
  "JEMPI_POSTGRES_CPU_LIMIT": "0",
  "JEMPI_POSTGRES_CPU_RESERVE": "0.05",
  "JEMPI_POSTGRES_MEMORY_LIMIT": "3G",

client-registry-jempi/swarm.sh

@@ -35,6 +35,7 @@ function initialize_package() {
  local dgraph_dev_compose_param=""
  local dgraph_zero_dev_compose_param=""
  local combined_dev_compose_param=""
+ local combined_cluster_compose_param=""
  local api_dev_compose_param=""
  local web_dev_compose_param=""
  local dgraph_cluster_compose_param=""
@@ -54,6 +55,7 @@ function initialize_package() {
  if [[ "$CLUSTERED_MODE" == "true" ]]; then
  dgraph_cluster_compose_param="docker-compose.dgraph-cluster.yml"
  dgraph_zero_cluster_compose_param="docker-compose.dgraph-zero-cluster.yml"
+ combined_cluster_compose_param="docker-compose.combined-cluster.yml"
  fi
 
  (
@@ -66,7 +68,7 @@ function initialize_package() {
  docker::deploy_service $STACK "${COMPOSE_FILE_PATH}" "docker-compose.dgraph.yml" "$dgraph_dev_compose_param" "$dgraph_cluster_compose_param"
 
  log info "Deploy other combined services"
- docker::deploy_service $STACK "${COMPOSE_FILE_PATH}" "docker-compose.combined.yml" "$combined_dev_compose_param"
+ docker::deploy_service $STACK "${COMPOSE_FILE_PATH}" "docker-compose.combined.yml" "$combined_dev_compose_param" "$combined_cluster_compose_param"
 
  log info "Deploy JeMPI API"
  docker::deploy_service $STACK "${COMPOSE_FILE_PATH}" "docker-compose.api.yml" "$api_dev_compose_param"