From f1b9a9115f6d46399bc7c010f2205e2d9e2a890d Mon Sep 17 00:00:00 2001
From: Arran Standish <arranstandish@hotmail.com>
Date: Mon, 15 May 2023 07:56:12 +0200
Subject: [PATCH 1/6] Add inital JemPI clustering config

---
 .env.cluster                                  |  3 +
 client-registry-jempi/docker-compose.api.yml  |  4 +-
 .../docker-compose.combined-cluster.yml       | 72 +++++++++++++++++++
 .../docker-compose.combined-dev.yml           |  2 +-
 .../docker-compose.combined.yml               | 26 +++----
 .../docker-compose.dgraph-zero-cluster.yml    | 41 +++++++++++
 .../docker-compose.dgraph-zero.yml            |  6 +-
 client-registry-jempi/docker-compose.web.yml  |  1 -
 client-registry-jempi/package-metadata.json   |  9 ++-
 client-registry-jempi/swarm.sh                |  4 +-
 10 files changed, 145 insertions(+), 23 deletions(-)
 create mode 100644 client-registry-jempi/docker-compose.combined-cluster.yml

diff --git a/.env.cluster b/.env.cluster
index 81765e82..f83ee1d6 100644
--- a/.env.cluster
+++ b/.env.cluster
@@ -81,6 +81,9 @@ GF_SERVER_DOMAIN=grafana.domain
 JEMPI_WEB_INSTANCES=3
 REACT_APP_JEMPI_BASE_URL=https://jempi-api.domain/JeMPI
 JEMPI_SESSION_SECURE=true
+JEMPI_REPMGR_PARTNER_NODES=jempi-postgresql-01,jempi-postgresql-02,jempi-postgresql-03
+JEMPI_COMBINED_INSTANCES=3
+JEMPI_API_INSTANCES=3
 
 # Resource limits
 OPENHIM_MEMORY_LIMIT=4G
diff --git a/client-registry-jempi/docker-compose.api.yml b/client-registry-jempi/docker-compose.api.yml
index af57c17a..7f32f9db 100644
--- a/client-registry-jempi/docker-compose.api.yml
+++ b/client-registry-jempi/docker-compose.api.yml
@@ -14,9 +14,9 @@ services:
       JEMPI_FILE_IMPORT_MAX_SIZE_BYTE: ${JEMPI_FILE_IMPORT_MAX_SIZE_BYTE}
       JEMPI_SESSION_SECURE: ${JEMPI_SESSION_SECURE}
       JEMPI_SESSION_DOMAIN_NAME: ${DOMAIN_NAME}
-      postgres.server: jempi-postgresql
+      postgres.server: ${JEMPI_REPMGR_PARTNER_NODES}
     deploy:
-      replicas: 1
+      replicas: ${JEMPI_API_INSTANCES}
       resources:
         limits:
           memory: ${JEMPI_API_MEMORY_LIMIT}
diff --git a/client-registry-jempi/docker-compose.combined-cluster.yml b/client-registry-jempi/docker-compose.combined-cluster.yml
new file mode 100644
index 00000000..ea3bd37d
--- /dev/null
+++ b/client-registry-jempi/docker-compose.combined-cluster.yml
@@ -0,0 +1,72 @@
+version: '3.9'
+
+services:
+  jempi-postgresql-01:
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==node-1"
+  
+  jempi-postgresql-02:
+    image: bitnami/postgresql-repmgr:15.2.0
+    environment:
+      POSTGRESQL_USERNAME: ${POSTGRESQL_USERNAME}
+      POSTGRESQL_DATABASE: ${POSTGRESQL_DATABASE}
+      REPMGR_PASSWORD: ${JEMPI_REPMGR_PASSWORD}
+      REPMGR_PRIMARY_HOST: ${JEMPI_REPMGR_PRIMARY_HOST}
+      REPMGR_PARTNER_NODES: ${JEMPI_REPMGR_PARTNER_NODES}
+      REPMGR_NODE_NAME: jempi-postgresql-02
+      REPMGR_NODE_NETWORK_NAME: jempi-postgresql-02
+      ALLOW_EMPTY_PASSWORD: "yes"
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==node-2"
+      mode: replicated
+      replicas: 1
+      resources:
+        limits:
+          cpus: ${JEMPI_POSTGRES_CPU_LIMIT}
+          memory: ${JEMPI_POSTGRES_MEMORY_LIMIT}
+        reservations:
+          cpus: ${JEMPI_POSTGRES_CPU_RESERVE}
+          memory: ${JEMPI_POSTGRES_MEMORY_RESERVE}
+    volumes:
+      - "jempi-psql-02-data:/bitnami/postgresql"
+    configs:
+      - target: /docker-entrypoint-initdb.d/jempi_psql_init_db.sql
+        source: jempi_psql_init_db.sql
+
+  jempi-postgresql-03:
+    image: bitnami/postgresql-repmgr:15.2.0
+    environment:
+      POSTGRESQL_USERNAME: ${POSTGRESQL_USERNAME}
+      POSTGRESQL_DATABASE: ${POSTGRESQL_DATABASE}
+      REPMGR_PASSWORD: ${JEMPI_REPMGR_PASSWORD}
+      REPMGR_PRIMARY_HOST: ${JEMPI_REPMGR_PRIMARY_HOST}
+      REPMGR_PARTNER_NODES: ${JEMPI_REPMGR_PARTNER_NODES}
+      REPMGR_NODE_NAME: jempi-postgresql-03
+      REPMGR_NODE_NETWORK_NAME: jempi-postgresql-03
+      ALLOW_EMPTY_PASSWORD: "yes"
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==node-3"
+      mode: replicated
+      replicas: 1
+      resources:
+        limits:
+          cpus: ${JEMPI_POSTGRES_CPU_LIMIT}
+          memory: ${JEMPI_POSTGRES_MEMORY_LIMIT}
+        reservations:
+          cpus: ${JEMPI_POSTGRES_CPU_RESERVE}
+          memory: ${JEMPI_POSTGRES_MEMORY_RESERVE}
+    volumes:
+      - "jempi-psql-03-data:/bitnami/postgresql"
+    configs:
+      - target: /docker-entrypoint-initdb.d/jempi_psql_init_db.sql
+        source: jempi_psql_init_db.sql
+
+volumes:
+  jempi-psql-02-data:
+  jempi-psql-03-data:
diff --git a/client-registry-jempi/docker-compose.combined-dev.yml b/client-registry-jempi/docker-compose.combined-dev.yml
index 619144ce..1f33a0f1 100644
--- a/client-registry-jempi/docker-compose.combined-dev.yml
+++ b/client-registry-jempi/docker-compose.combined-dev.yml
@@ -22,7 +22,7 @@ services:
         protocol: tcp
         mode: host
 
-  jempi-postgresql:
+  jempi-postgresql-01:
     ports:
       - published: 5435
         target: 5432
diff --git a/client-registry-jempi/docker-compose.combined.yml b/client-registry-jempi/docker-compose.combined.yml
index d6e8b5ce..3e6daa2c 100644
--- a/client-registry-jempi/docker-compose.combined.yml
+++ b/client-registry-jempi/docker-compose.combined.yml
@@ -6,7 +6,7 @@ services:
     environment:
       kafka.bootstrap.servers: ${KAFKA_HOSTS}
     deploy:
-      replicas: 1
+      replicas: ${JEMPI_COMBINED_INSTANCES}
       resources:
         limits:
           memory: ${JEMPI_ASYNC_RECEIVER_MEMORY_LIMIT}
@@ -21,7 +21,7 @@ services:
     environment:
       kafka.bootstrap.servers: ${KAFKA_HOSTS}
     deploy:
-      replicas: 1
+      replicas: ${JEMPI_COMBINED_INSTANCES}
       resources:
         limits:
           memory: ${JEMPI_SYNC_RECEIVER_MEMORY_LIMIT}
@@ -36,7 +36,7 @@ services:
     environment:
       kafka.bootstrap.servers: ${KAFKA_HOSTS}
     deploy:
-      replicas: 1
+      replicas: ${JEMPI_COMBINED_INSTANCES}
       resources:
         limits:
           memory: ${JEMPI_PRE_PROCESSOR_MEMORY_LIMIT}
@@ -51,7 +51,7 @@ services:
     environment:
       kafka.bootstrap.servers: ${KAFKA_HOSTS}
     deploy:
-      replicas: 1
+      replicas: ${JEMPI_COMBINED_INSTANCES}
       resources:
         limits:
           memory: ${JEMPI_CONTROLLER_MEMORY_LIMIT}
@@ -66,7 +66,7 @@ services:
     environment:
       kafka.bootstrap.servers: ${KAFKA_HOSTS}
     deploy:
-      replicas: 1
+      replicas: ${JEMPI_COMBINED_INSTANCES}
       resources:
         limits:
           memory: ${JEMPI_EM_CALCULATOR_MEMORY_LIMIT}
@@ -81,7 +81,7 @@ services:
     environment:
       kafka.bootstrap.servers: ${KAFKA_HOSTS}
     deploy:
-      replicas: 1
+      replicas: ${JEMPI_COMBINED_INSTANCES}
       resources:
         limits:
           memory: ${JEMPI_LINKER_MEMORY_LIMIT}
@@ -91,11 +91,16 @@ services:
       kafka:
       default:
 
-  jempi-postgresql:
-    image: bitnami/postgresql:15.2.0
+  jempi-postgresql-01:
+    image: bitnami/postgresql-repmgr:15.2.0
     environment:
       POSTGRESQL_USERNAME: ${POSTGRESQL_USERNAME}
       POSTGRESQL_DATABASE: ${POSTGRESQL_DATABASE}
+      REPMGR_PASSWORD: ${JEMPI_REPMGR_PASSWORD}
+      REPMGR_PRIMARY_HOST: ${JEMPI_REPMGR_PRIMARY_HOST}
+      REPMGR_PARTNER_NODES: ${JEMPI_REPMGR_PARTNER_NODES}
+      REPMGR_NODE_NAME: jempi-postgresql-01
+      REPMGR_NODE_NETWORK_NAME: jempi-postgresql-01
       ALLOW_EMPTY_PASSWORD: "yes"
     deploy:
       mode: replicated
@@ -112,10 +117,7 @@ services:
     configs:
       - target: /docker-entrypoint-initdb.d/jempi_psql_init_db.sql
         source: jempi_psql_init_db.sql
-    networks:
-      kafka:
-      default:
-
+  
 volumes:
   jempi-psql-01-data:
 
diff --git a/client-registry-jempi/docker-compose.dgraph-zero-cluster.yml b/client-registry-jempi/docker-compose.dgraph-zero-cluster.yml
index e6714d6c..8441119c 100644
--- a/client-registry-jempi/docker-compose.dgraph-zero-cluster.yml
+++ b/client-registry-jempi/docker-compose.dgraph-zero-cluster.yml
@@ -6,3 +6,44 @@ services:
       placement:
         constraints:
           - node.labels.name == node-1
+    command: dgraph zero --my=jempi-zero-01:5080 --replicas 3 --bindall --raft "idx=1"
+  
+  jempi-zero-02:
+    image: dgraph/dgraph:v22.0.0
+    volumes:
+      - jempi-zero-02-data:/dgraph
+    deploy:
+      replicas: 1
+      placement:
+        constraints:
+          - node.labels.name == node-2
+      resources:
+        limits:
+          memory: ${JEMPI_ZERO_MEMORY_LIMIT}
+        reservations:
+          memory: ${JEMPI_ZERO_MEMORY_RESERVE}
+      restart_policy:
+        condition: on-failure
+    command: dgraph zero --my=jempi-zero-02:5080 --replicas 3 --peer=jempi-zero-01:5080 --raft "idx=2"
+
+  jempi-zero-03:
+    image: dgraph/dgraph:v22.0.0
+    volumes:
+      - jempi-zero-03-data:/dgraph
+    deploy:
+      replicas: 1
+      placement:
+        constraints:
+          - node.labels.name == node-3
+      resources:
+        limits:
+          memory: ${JEMPI_ZERO_MEMORY_LIMIT}
+        reservations:
+          memory: ${JEMPI_ZERO_MEMORY_RESERVE}
+      restart_policy:
+        condition: on-failure
+    command: dgraph zero --my=jempi-zero-03:5080 --replicas=3 --peer=jempi-zero-01:5080 --raft "idx=3"
+
+volumes:
+  jempi-zero-02-data:
+  jempi-zero-03-data:
diff --git a/client-registry-jempi/docker-compose.dgraph-zero.yml b/client-registry-jempi/docker-compose.dgraph-zero.yml
index 4739a3ba..ac1974e3 100644
--- a/client-registry-jempi/docker-compose.dgraph-zero.yml
+++ b/client-registry-jempi/docker-compose.dgraph-zero.yml
@@ -9,14 +9,12 @@ services:
       replicas: 1
       resources:
         limits:
-          memory: ${JEMPI_ZERO_01_MEMORY_LIMIT}
+          memory: ${JEMPI_ZERO_MEMORY_LIMIT}
         reservations:
-          memory: ${JEMPI_ZERO_01_MEMORY_RESERVE}
+          memory: ${JEMPI_ZERO_MEMORY_RESERVE}
       restart_policy:
         condition: on-failure
     command: dgraph zero --my=jempi-zero-01:5080 --replicas 1
 
 volumes:
   jempi-zero-01-data:
-
-
diff --git a/client-registry-jempi/docker-compose.web.yml b/client-registry-jempi/docker-compose.web.yml
index 172f1937..53a7b5ba 100644
--- a/client-registry-jempi/docker-compose.web.yml
+++ b/client-registry-jempi/docker-compose.web.yml
@@ -24,7 +24,6 @@ services:
       keycloak:
       default:
 
-
 networks:
   reverse-proxy:
     name: reverse-proxy_public
diff --git a/client-registry-jempi/package-metadata.json b/client-registry-jempi/package-metadata.json
index de9f1f64..15e7bbac 100644
--- a/client-registry-jempi/package-metadata.json
+++ b/client-registry-jempi/package-metadata.json
@@ -11,8 +11,8 @@
   ],
   "environmentVariables": {
     "KAFKA_HOSTS": "kafka-01:9092",
-    "JEMPI_ZERO_01_MEMORY_LIMIT": "3G",
-    "JEMPI_ZERO_01_MEMORY_RESERVE": "500M",
+    "JEMPI_ZERO_MEMORY_LIMIT": "3G",
+    "JEMPI_ZERO_MEMORY_RESERVE": "500M",
     "JEMPI_ALPHA_01_MEMORY_LIMIT": "3G",
     "JEMPI_ALPHA_01_MEMORY_RESERVE": "500M",
     "JEMPI_ALPHA_02_MEMORY_LIMIT": "3G",
@@ -35,6 +35,7 @@
     "JEMPI_SYNC_RECEIVER_MEMORY_RESERVE": "500M",
     "JEMPI_API_MEMORY_LIMIT": "3G",
     "JEMPI_API_MEMORY_RESERVE": "500M",
+    "JEMPI_API_INSTANCES": 1,
     "JEMPI_KAFKA_TOPICS": "JeMPI-async-preprocessor,JeMPI-patient-controller,JeMPI-patient-em,JeMPI-patient-linker,JeMPI-mu-linker,JeMPI-notifications",
     "JEMPI_ASYNC_RECEIVER_IMAGE_TAG": "0.2.0",
     "JEMPI_SYNC_RECEIVER_IMAGE_TAG": "0.2.0",
@@ -43,6 +44,7 @@
     "JEMPI_EM_CALCULATOR_IMAGE_TAG": "0.2.0",
     "JEMPI_LINKER_IMAGE_TAG": "0.2.0",
     "JEMPI_API_IMAGE_TAG": "0.2.0",
+    "JEMPI_COMBINED_INSTANCES": 1,
     "JEMPI_OPENHIM_PASSWORD": "instant101",
     "JEMPI_SESSION_SECRET": "c05ll3lesrinf39t7mc5h6un6r0c69lgfno69dsak3vabeqamouq4328cuaekros401ajdpkh60rrt",
     "JEMPI_FILE_IMPORT_MAX_SIZE_BYTE": 128000000,
@@ -60,6 +62,9 @@
     "KC_JEMPI_ROOT_URL": "http://localhost:3033",
     "POSTGRESQL_DATABASE": "notifications",
     "POSTGRESQL_USERNAME": "postgres",
+    "JEMPI_REPMGR_PASSWORD": "dev_password_only",
+    "JEMPI_REPMGR_PRIMARY_HOST": "jempi-postgresql-01",
+    "JEMPI_REPMGR_PARTNER_NODES": "jempi-postgresql-01",
     "JEMPI_POSTGRES_CPU_LIMIT": "0",
     "JEMPI_POSTGRES_CPU_RESERVE": "0.05",
     "JEMPI_POSTGRES_MEMORY_LIMIT": "3G",
diff --git a/client-registry-jempi/swarm.sh b/client-registry-jempi/swarm.sh
index 93cd7794..5ef5ce66 100644
--- a/client-registry-jempi/swarm.sh
+++ b/client-registry-jempi/swarm.sh
@@ -35,6 +35,7 @@ function initialize_package() {
   local dgraph_dev_compose_param=""
   local dgraph_zero_dev_compose_param=""
   local combined_dev_compose_param=""
+  local combined_cluster_compose_param=""
   local api_dev_compose_param=""
   local web_dev_compose_param=""
   local dgraph_cluster_compose_param=""
@@ -54,6 +55,7 @@ function initialize_package() {
   if [[ "$CLUSTERED_MODE" == "true" ]]; then
     dgraph_cluster_compose_param="docker-compose.dgraph-cluster.yml"
     dgraph_zero_cluster_compose_param="docker-compose.dgraph-zero-cluster.yml"
+    combined_cluster_compose_param="docker-compose.combined-cluster.yml"
   fi
 
   (
@@ -66,7 +68,7 @@ function initialize_package() {
     docker::deploy_service $STACK "${COMPOSE_FILE_PATH}" "docker-compose.dgraph.yml" "$dgraph_dev_compose_param" "$dgraph_cluster_compose_param"
 
     log info "Deploy other combined services"
-    docker::deploy_service $STACK "${COMPOSE_FILE_PATH}" "docker-compose.combined.yml" "$combined_dev_compose_param"
+    docker::deploy_service $STACK "${COMPOSE_FILE_PATH}" "docker-compose.combined.yml" "$combined_dev_compose_param" "$combined_cluster_compose_param"
 
     log info "Deploy JeMPI API"
     docker::deploy_service $STACK "${COMPOSE_FILE_PATH}" "docker-compose.api.yml" "$api_dev_compose_param"

From 34fb0c35af3aff44f35807dd3415087fa4033e2f Mon Sep 17 00:00:00 2001
From: Arran Standish <arranstandish@hotmail.com>
Date: Mon, 15 May 2023 12:54:32 +0200
Subject: [PATCH 2/6] Update dgraph zero-3 to follow the same replica setting
 convention of 1 and 2

---
 client-registry-jempi/docker-compose.dgraph-zero-cluster.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client-registry-jempi/docker-compose.dgraph-zero-cluster.yml b/client-registry-jempi/docker-compose.dgraph-zero-cluster.yml
index 8441119c..853f7c7f 100644
--- a/client-registry-jempi/docker-compose.dgraph-zero-cluster.yml
+++ b/client-registry-jempi/docker-compose.dgraph-zero-cluster.yml
@@ -42,7 +42,7 @@ services:
           memory: ${JEMPI_ZERO_MEMORY_RESERVE}
       restart_policy:
         condition: on-failure
-    command: dgraph zero --my=jempi-zero-03:5080 --replicas=3 --peer=jempi-zero-01:5080 --raft "idx=3"
+    command: dgraph zero --my=jempi-zero-03:5080 --replicas 3 --peer=jempi-zero-01:5080 --raft "idx=3"
 
 volumes:
   jempi-zero-02-data:

From af8d337bbf087c75ee183ead91379e6bcdf25c4e Mon Sep 17 00:00:00 2001
From: Arran Standish <arranstandish@hotmail.com>
Date: Wed, 17 May 2023 10:26:17 +0200
Subject: [PATCH 3/6] Add jempi cluster tests and update jempi single tests to
 reflect keycloak prereq

---
 .../cluster-mode/jempi.cluster.feature        | 97 +++++++++++++++++++
 .../features/single-mode/jempi.feature        | 12 ++-
 test/cucumber/package.json                    |  1 +
 3 files changed, 106 insertions(+), 4 deletions(-)
 create mode 100644 test/cucumber/features/cluster-mode/jempi.cluster.feature

diff --git a/test/cucumber/features/cluster-mode/jempi.cluster.feature b/test/cucumber/features/cluster-mode/jempi.cluster.feature
new file mode 100644
index 00000000..c951f357
--- /dev/null
+++ b/test/cucumber/features/cluster-mode/jempi.cluster.feature
@@ -0,0 +1,97 @@
+Feature: Client Registry JeMPI?
+  Does the Client Registry JeMPI package work as expected
+
+  Scenario: Init Client Registry JeMPI
+    Given I use parameters "package init -n=client-registry-jempi --dev --env-file=.env.cluster"
+        When I launch the platform with params
+    Then The service "mongo-1" should be started with 1 replica
+    And The service "mongo-1" should be started with 1 replica
+    And The service "mongo-1" should be started with 1 replica
+    And The service "openhim-core" should be started with 3 replica
+    And The service "openhim-console" should be started with 3 replica
+    And The service "kafka-01" should be started with 1 replica
+    And The service "kafka-02" should be started with 1 replica
+    And The service "kafka-03" should be started with 1 replica
+    And The service "kafdrop" should be started with 1 replica
+    And The service "kafka-minion" should be started with 1 replica
+    And The service "keycloak-postgres-1" should be started with 1 replica
+    And The service "keycloak-postgres-2" should be started with 1 replica
+    And The service "keycloak-postgres-3" should be started with 1 replica
+    And The service "identity-access-manager-keycloak" should be started with 1 replica
+    And The service "jempi-ratel" should be started with 1 replica
+    And The service "jempi-alpha-01" should be started with 1 replica
+    And The service "jempi-alpha-02" should be started with 1 replica
+    And The service "jempi-alpha-03" should be started with 1 replica
+    And The service "jempi-zero-01" should be started with 1 replica
+    And The service "jempi-zero-02" should be started with 1 replica
+    And The service "jempi-zero-03" should be started with 1 replica
+    And The service "jempi-async-receiver" should be started with 3 replica
+    And The service "jempi-async-receiver" should be connected to the networks
+      | kafka_public | jempi_default |
+    And The service "jempi-sync-receiver" should be started with 3 replica
+    And The service "jempi-sync-receiver" should be connected to the networks
+      | kafka_public | jempi_default |
+    And The service "jempi-pre-processor" should be started with 3 replica
+    And The service "jempi-pre-processor" should be connected to the networks
+      | kafka_public | jempi_default |
+    And The service "jempi-controller" should be started with 3 replica
+    And The service "jempi-controller" should be connected to the networks
+      | kafka_public | jempi_default |
+    # jempi-em-calculator is not ready for testing yet
+    # And The service "jempi-em-calculator" should be started with 3 replica
+    # And The service "jempi-em-calculator" should be connected to the networks
+    #   | kafka_public | jempi_default |
+    And The service "jempi-linker" should be started with 3 replica
+    And The service "jempi-linker" should be connected to the networks
+      | kafka_public | jempi_default |
+    And The service "jempi-api" should be started with 3 replica
+    And The service "jempi-api" should be connected to the networks
+      | kafka_public | jempi_default |
+    And The service "jempi-postgresql-01" should be started with 1 replica
+    And The service "jempi-postgresql-02" should be started with 1 replica
+    And The service "jempi-postgresql-03" should be started with 1 replica
+    And The service "jempi-web" should be started with 3 replica
+    And The service "jempi-web" should be connected to the networks
+      | reverse-proxy_public | keycloak_public | jempi_default |
+
+  Scenario: Destroy Client Registry JeMPI
+    Given I use parameters "package destroy -n=client-registry-jempi --dev --env-file=.env.cluster"
+    When I launch the platform with params
+    Then The service "client-registry-jempi" should be removed
+    And The service "kafka-01" should be removed
+    And The service "kafka-02" should be removed
+    And The service "kafka-03" should be removed
+    And The service "kafdrop" should be removed
+    And The service "kafka-minion" should be removed
+    And The service "keycloak-postgres-1" should be removed
+    And The service "keycloak-postgres-2" should be removed
+    And The service "keycloak-postgres-3" should be removed
+    And The service "identity-access-manager-keycloak" should be removed
+    And The service "jempi-ratel" should be removed
+    And The service "jempi-alpha-01" should be removed
+    And The service "jempi-alpha-02" should be removed
+    And The service "jempi-alpha-03" should be removed
+    And The service "jempi-async-receiver" should be removed
+    And The service "jempi-sync-receiver" should be removed
+    And The service "jempi-pre-processor" should be removed
+    And The service "jempi-controller" should be removed
+    And The service "jempi-em-calculator" should be removed
+    And The service "jempi-linker" should be removed
+    And The service "jempi-zero-01" should be removed
+    And The service "jempi-zero-02" should be removed
+    And The service "jempi-zero-03" should be removed
+    And The service "jempi-api" should be removed
+    And The service "jempi-web" should be removed
+    And The service "jempi-postgresql-01" should be removed
+    And The service "jempi-postgresql-02" should be removed
+    And The service "jempi-postgresql-03" should be removed
+    And The service "mongo-1" should be removed
+    And The service "mongo-2" should be removed
+    And The service "mongo-3" should be removed
+    And The service "openhim-core" should be removed
+    And The service "openhim-console" should be removed
+    And There should be 0 service
+    And There should be 0 volume
+    And There should be 0 config
+    And There should not be network
+      | keycloak_public | reverse-proxy_public | kafka_public |
diff --git a/test/cucumber/features/single-mode/jempi.feature b/test/cucumber/features/single-mode/jempi.feature
index acf5e026..d4fdfa8a 100644
--- a/test/cucumber/features/single-mode/jempi.feature
+++ b/test/cucumber/features/single-mode/jempi.feature
@@ -9,6 +9,9 @@ Feature: Client Registry JeMPI?
     And The service "openhim-console" should be started with 1 replica
     And The service "kafka-01" should be started with 1 replica
     And The service "kafdrop" should be started with 1 replica
+    And The service "kafka-minion" should be started with 1 replica
+    And The service "keycloak-postgres-1" should be started with 1 replica
+    And The service "identity-access-manager-keycloak" should be started with 1 replica
     And The service "jempi-ratel" should be started with 1 replica
     And The service "jempi-alpha-01" should be started with 1 replica
     And The service "jempi-alpha-02" should be started with 1 replica
@@ -36,9 +39,7 @@ Feature: Client Registry JeMPI?
     And The service "jempi-api" should be started with 1 replica
     And The service "jempi-api" should be connected to the networks
       | kafka_public | jempi_default |
-    And The service "jempi-postgresql" should be started with 1 replica
-    And The service "jempi-postgresql" should be connected to the networks
-      | kafka_public | jempi_default |
+    And The service "jempi-postgresql-01" should be started with 1 replica
     And The service "jempi-web" should be started with 1 replica
     And The service "jempi-web" should be connected to the networks
       | reverse-proxy_public | keycloak_public | jempi_default |
@@ -49,6 +50,9 @@ Feature: Client Registry JeMPI?
     Then The service "client-registry-jempi" should be removed
     And The service "kafka-01" should be removed
     And The service "kafdrop" should be removed
+    And The service "kafka-minion" should be removed
+    And The service "keycloak-postgres-1" should be removed
+    And The service "identity-access-manager-keycloak" should be removed
     And The service "jempi-ratel" should be removed
     And The service "jempi-alpha-01" should be removed
     And The service "jempi-alpha-02" should be removed
@@ -62,7 +66,7 @@ Feature: Client Registry JeMPI?
     And The service "jempi-zero-01" should be removed
     And The service "jempi-api" should be removed
     And The service "jempi-web" should be removed
-    And The service "jempi-postgresql" should be removed
+    And The service "jempi-postgresql-01" should be removed
     And The service "mongo-1" should be removed
     And The service "openhim-core" should be removed
     And The service "openhim-console" should be removed
diff --git a/test/cucumber/package.json b/test/cucumber/package.json
index 947a850a..b9684654 100644
--- a/test/cucumber/package.json
+++ b/test/cucumber/package.json
@@ -24,6 +24,7 @@
     "test:single:jsreport": "cucumber-js 'features/single-mode/jsreport.feature'",
     "test:single:superset": "cucumber-js 'features/single-mode/superset.feature'",
     "test:single:jempi": "cucumber-js 'features/single-mode/jempi.feature'",
+    "test:cluster:jempi": "cucumber-js 'features/cluster-mode/jempi.cluster.feature'",
     "test:single:mpi-mediator": "cucumber-js 'features/single-mode/mpi-mediator.feature'",
     "test:single:monitoring": "cucumber-js 'features/single-mode/monitoring.feature'",
     "test:cluster:monitoring": "cucumber-js 'features/cluster-mode/monitoring.feature'"

From a0c73151eb8cd52d110d55e915f54a62278e4a35 Mon Sep 17 00:00:00 2001
From: Arran Standish <arranstandish@hotmail.com>
Date: Thu, 18 May 2023 10:07:58 +0200
Subject: [PATCH 4/6] Fix JemPI cluster unit tests

---
 test/cucumber/features/cluster-mode/jempi.cluster.feature     | 4 ++--
 .../features/cluster-mode/openhim-packages.cluster.feature    | 3 +--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/test/cucumber/features/cluster-mode/jempi.cluster.feature b/test/cucumber/features/cluster-mode/jempi.cluster.feature
index c951f357..07dc2412 100644
--- a/test/cucumber/features/cluster-mode/jempi.cluster.feature
+++ b/test/cucumber/features/cluster-mode/jempi.cluster.feature
@@ -2,8 +2,8 @@ Feature: Client Registry JeMPI?
   Does the Client Registry JeMPI package work as expected
 
   Scenario: Init Client Registry JeMPI
-    Given I use parameters "package init -n=client-registry-jempi --dev --env-file=.env.cluster"
-        When I launch the platform with params
+    Given I use parameters "package up -n=client-registry-jempi --dev --env-file=.env.cluster"
+    When I launch the platform with params
     Then The service "mongo-1" should be started with 1 replica
     And The service "mongo-1" should be started with 1 replica
     And The service "mongo-1" should be started with 1 replica
diff --git a/test/cucumber/features/cluster-mode/openhim-packages.cluster.feature b/test/cucumber/features/cluster-mode/openhim-packages.cluster.feature
index aa95fa00..ca7c0ce8 100644
--- a/test/cucumber/features/cluster-mode/openhim-packages.cluster.feature
+++ b/test/cucumber/features/cluster-mode/openhim-packages.cluster.feature
@@ -23,14 +23,13 @@ Feature: Openhim and its dependent packages?
     And There should be 5 services
 
   Scenario: Destroy Openhim and its dependent packages
-    Given I use parameters "package destroy -n=interoperability-layer-openhim,mpi-mediator --only --dev --env-file=.env.cluster"
+    Given I use parameters "package destroy -n=interoperability-layer-openhim --dev --env-file=.env.cluster"
     When I launch the platform with params
     Then The service "mongo-1" should be removed
     And The service "mongo-2" should be removed
     And The service "mongo-3" should be removed
     And The service "openhim-core" should be removed
     And The service "openhim-console" should be removed
-    And The service "mpi-mediator" should be removed
     And There should be 0 service
     And There should be 0 volume
     And There should be 0 config

From 0e14e208d2c15ef3774082d0f8194bfde63dffa9 Mon Sep 17 00:00:00 2001
From: Arran Standish <arranstandish@hotmail.com>
Date: Wed, 24 May 2023 08:30:43 +0200
Subject: [PATCH 5/6] Split out JemPI combined replicas to own variables

---
 .env.cluster                                         |  7 ++++++-
 client-registry-jempi/docker-compose.combined.yml    | 12 ++++++------
 client-registry-jempi/package-metadata.json          |  7 ++++++-
 .../features/cluster-mode/jempi.cluster.feature      | 10 +++++-----
 4 files changed, 23 insertions(+), 13 deletions(-)

diff --git a/.env.cluster b/.env.cluster
index f83ee1d6..c13bed3d 100644
--- a/.env.cluster
+++ b/.env.cluster
@@ -82,7 +82,12 @@ JEMPI_WEB_INSTANCES=3
 REACT_APP_JEMPI_BASE_URL=https://jempi-api.domain/JeMPI
 JEMPI_SESSION_SECURE=true
 JEMPI_REPMGR_PARTNER_NODES=jempi-postgresql-01,jempi-postgresql-02,jempi-postgresql-03
-JEMPI_COMBINED_INSTANCES=3
+JEMPI_ASYNC_RECEIVER_INSTANCES=1
+JEMPI_SYNC_RECEIVER_INSTANCES=1
+JEMPI_PRE_PROCESSOR_INSTANCES=1
+JEMPI_CONTROLLER_INSTANCES=1
+JEMPI_EM_CALCULATOR_INSTANCES=1
+JEMPI_LINKER_INSTANCES=1
 JEMPI_API_INSTANCES=3
 
 # Resource limits
diff --git a/client-registry-jempi/docker-compose.combined.yml b/client-registry-jempi/docker-compose.combined.yml
index 3e6daa2c..7741a17d 100644
--- a/client-registry-jempi/docker-compose.combined.yml
+++ b/client-registry-jempi/docker-compose.combined.yml
@@ -6,7 +6,7 @@ services:
     environment:
       kafka.bootstrap.servers: ${KAFKA_HOSTS}
     deploy:
-      replicas: ${JEMPI_COMBINED_INSTANCES}
+      replicas: ${JEMPI_ASYNC_RECEIVER_INSTANCE}
       resources:
         limits:
           memory: ${JEMPI_ASYNC_RECEIVER_MEMORY_LIMIT}
@@ -21,7 +21,7 @@ services:
     environment:
       kafka.bootstrap.servers: ${KAFKA_HOSTS}
     deploy:
-      replicas: ${JEMPI_COMBINED_INSTANCES}
+      replicas: ${JEMPI_SYNC_RECEIVER_INSTANCES}
       resources:
         limits:
           memory: ${JEMPI_SYNC_RECEIVER_MEMORY_LIMIT}
@@ -36,7 +36,7 @@ services:
     environment:
       kafka.bootstrap.servers: ${KAFKA_HOSTS}
     deploy:
-      replicas: ${JEMPI_COMBINED_INSTANCES}
+      replicas: ${JEMPI_PRE_PROCESSOR_INSTANCES}
       resources:
         limits:
           memory: ${JEMPI_PRE_PROCESSOR_MEMORY_LIMIT}
@@ -51,7 +51,7 @@ services:
     environment:
       kafka.bootstrap.servers: ${KAFKA_HOSTS}
     deploy:
-      replicas: ${JEMPI_COMBINED_INSTANCES}
+      replicas: ${JEMPI_CONTROLLER_INSTANCES}
       resources:
         limits:
           memory: ${JEMPI_CONTROLLER_MEMORY_LIMIT}
@@ -66,7 +66,7 @@ services:
     environment:
       kafka.bootstrap.servers: ${KAFKA_HOSTS}
     deploy:
-      replicas: ${JEMPI_COMBINED_INSTANCES}
+      replicas: ${JEMPI_EM_CALCULATOR_INSTANCES}
       resources:
         limits:
           memory: ${JEMPI_EM_CALCULATOR_MEMORY_LIMIT}
@@ -81,7 +81,7 @@ services:
     environment:
       kafka.bootstrap.servers: ${KAFKA_HOSTS}
     deploy:
-      replicas: ${JEMPI_COMBINED_INSTANCES}
+      replicas: ${JEMPI_LINKER_INSTANCES}
       resources:
         limits:
           memory: ${JEMPI_LINKER_MEMORY_LIMIT}
diff --git a/client-registry-jempi/package-metadata.json b/client-registry-jempi/package-metadata.json
index 15e7bbac..2a21dcec 100644
--- a/client-registry-jempi/package-metadata.json
+++ b/client-registry-jempi/package-metadata.json
@@ -44,7 +44,12 @@
     "JEMPI_EM_CALCULATOR_IMAGE_TAG": "0.2.0",
     "JEMPI_LINKER_IMAGE_TAG": "0.2.0",
     "JEMPI_API_IMAGE_TAG": "0.2.0",
-    "JEMPI_COMBINED_INSTANCES": 1,
+    "JEMPI_ASYNC_RECEIVER_INSTANCES": 1,
+    "JEMPI_SYNC_RECEIVER_INSTANCES": 1,
+    "JEMPI_PRE_PROCESSOR_INSTANCES": 1,
+    "JEMPI_CONTROLLER_INSTANCES": 1,
+    "JEMPI_EM_CALCULATOR_INSTANCES": 1,
+    "JEMPI_LINKER_INSTANCES": 1,
     "JEMPI_OPENHIM_PASSWORD": "instant101",
     "JEMPI_SESSION_SECRET": "c05ll3lesrinf39t7mc5h6un6r0c69lgfno69dsak3vabeqamouq4328cuaekros401ajdpkh60rrt",
     "JEMPI_FILE_IMPORT_MAX_SIZE_BYTE": 128000000,
diff --git a/test/cucumber/features/cluster-mode/jempi.cluster.feature b/test/cucumber/features/cluster-mode/jempi.cluster.feature
index 07dc2412..e1a3bf95 100644
--- a/test/cucumber/features/cluster-mode/jempi.cluster.feature
+++ b/test/cucumber/features/cluster-mode/jempi.cluster.feature
@@ -25,23 +25,23 @@ Feature: Client Registry JeMPI?
     And The service "jempi-zero-01" should be started with 1 replica
     And The service "jempi-zero-02" should be started with 1 replica
     And The service "jempi-zero-03" should be started with 1 replica
-    And The service "jempi-async-receiver" should be started with 3 replica
+    And The service "jempi-async-receiver" should be started with 1 replica
     And The service "jempi-async-receiver" should be connected to the networks
       | kafka_public | jempi_default |
-    And The service "jempi-sync-receiver" should be started with 3 replica
+    And The service "jempi-sync-receiver" should be started with 1 replica
     And The service "jempi-sync-receiver" should be connected to the networks
       | kafka_public | jempi_default |
-    And The service "jempi-pre-processor" should be started with 3 replica
+    And The service "jempi-pre-processor" should be started with 1 replica
     And The service "jempi-pre-processor" should be connected to the networks
       | kafka_public | jempi_default |
-    And The service "jempi-controller" should be started with 3 replica
+    And The service "jempi-controller" should be started with 1 replica
     And The service "jempi-controller" should be connected to the networks
       | kafka_public | jempi_default |
     # jempi-em-calculator is not ready for testing yet
     # And The service "jempi-em-calculator" should be started with 3 replica
     # And The service "jempi-em-calculator" should be connected to the networks
     #   | kafka_public | jempi_default |
-    And The service "jempi-linker" should be started with 3 replica
+    And The service "jempi-linker" should be started with 1 replica
     And The service "jempi-linker" should be connected to the networks
       | kafka_public | jempi_default |
     And The service "jempi-api" should be started with 3 replica

From 4b908d0f96b264996f7953be389c6d31b5a1f211 Mon Sep 17 00:00:00 2001
From: arran-standish <125864621+arran-standish@users.noreply.github.com>
Date: Wed, 24 May 2023 09:46:14 +0200
Subject: [PATCH 6/6] Fix async receiver replica spelling

---
 client-registry-jempi/docker-compose.combined.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client-registry-jempi/docker-compose.combined.yml b/client-registry-jempi/docker-compose.combined.yml
index 7741a17d..dd345eb8 100644
--- a/client-registry-jempi/docker-compose.combined.yml
+++ b/client-registry-jempi/docker-compose.combined.yml
@@ -6,7 +6,7 @@ services:
     environment:
       kafka.bootstrap.servers: ${KAFKA_HOSTS}
     deploy:
-      replicas: ${JEMPI_ASYNC_RECEIVER_INSTANCE}
+      replicas: ${JEMPI_ASYNC_RECEIVER_INSTANCES}
       resources:
         limits:
           memory: ${JEMPI_ASYNC_RECEIVER_MEMORY_LIMIT}