Cu 86bygcqxd images placement for data store dashboards monitoring

.github/workflows/run-tests.sh

@@ -25,43 +25,47 @@ for package in "${CHANGED_FILES[@]}"; do
  fi
 done
 
+function run_test() {
+ DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud HOST=$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":$1
+}
+
+# Run the basic funtional end to end tests for the CDR recipe
+run_test "recipe"
+
 if [[ ${#changed_packages[@]} -eq 0 ]] || [[ "${!changed_packages[*]}" == *"utils"* ]] || [[ "${!changed_packages[*]}" == *"features/steps"* ]] || [[ "${!changed_packages[*]}" == *"infrastructure"* ]] ; then
- openhim_ran="true"
- DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":openhim
+ openhimRan="true"
+ run_test "openhim"
 else
  for folder_name in "${!changed_packages[@]}"; do
  echo "$folder_name was changed"
 
  if [[ $folder_name == *"clickhouse"* ]]; then
- DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":clickhouse
+ run_test "clickhouse"
  elif [[ $folder_name == *"elastic"* ]] || [[ $folder_name == *"kibana"* ]] || [[ $folder_name == *"logstash"* ]]; then
- DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":elk
+ run_test "elk"
  elif [[ $folder_name == *"kafka"* ]] || [[ $folder_name == *"monitoring"* ]]; then
- DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":kafka
+ run_test "kafka"
  elif [[ $folder_name == *"openhim"* ]] && [[ $openhimRan == "false" ]]; then
  openhimRan="true"
- DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":openhim
+ run_test "openhim"
  elif [[ $folder_name == *"reverse-proxy"* ]]; then
- DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":nginx
+ run_test "nginx"
  elif [[ $folder_name == *"hapi"* ]]; then
- DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":hapi
+ run_test "hapi"
  elif [[ $folder_name == *"santempi"* ]]; then
- DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":sante
+ run_test "sante"
  elif [[ $folder_name == *"monitoring"* ]]; then
- DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":monitoring
+ run_test "monitoring"
  elif [[ $folder_name == *"keycloak"* ]]; then
- DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":keycloak
+ run_test "keycloak"
  elif [[ $folder_name == *"superset"* ]] && [[ $NODE_MODE == "single" ]]; then
- DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":superset
+ run_test "superset"
  elif [[ $folder_name == *"jsreport"* ]] && [[ $NODE_MODE == "single" ]]; then
- DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":jsreport
+ run_test "jsreport"
  elif [[ $folder_name == *"mpi-mediator"* ]] && [[ $NODE_MODE == "single" ]]; then
- DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":mpi-mediator
+ run_test "mpi-mediator"
  elif [[ $folder_name == *"jempi"* ]] && [[ $NODE_MODE == "single" ]]; then
- DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":jempi
+ run_test "jempi"
  fi
  done
 fi
-
-# Run the basic funtional end to end tests for the CDR recipe
-DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud HOST=$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":recipe

analytics-datastore-clickhouse/docker-compose.cluster.yml

@@ -1,17 +1,17 @@
-version: '3.9'
+version: "3.9"
 
 services:
  analytics-datastore-clickhouse-01:
- image: clickhouse/clickhouse-server
+ image: ${CLICKHOUSE_IMAGE}
  ulimits:
  noFile: 262144
  volumes:
  - clickhouse-data-01:/var/lib/clickhouse/
  hostname: analytics-datastore-clickhouse-01
- deploy: 
- placement: 
- constraints: 
- - "node.labels.name==node-1"
+ deploy:
+ placement:
+ constraints:
+ - "node.labels.name==${ANALYTICS_DATASTORE_CLICKHOUSE_01_PLACEMENT}"
  configs:
  - target: /etc/clickhouse-server/config.d/docker_related_config.xml
  source: docker_related_config.xml
@@ -36,12 +36,12 @@ services:
  default:
 
  analytics-datastore-clickhouse-02:
- image: clickhouse/clickhouse-server
+ image: ${CLICKHOUSE_IMAGE}
  hostname: analytics-datastore-clickhouse-02
- deploy: 
- placement: 
- constraints: 
- - "node.labels.name==node-2"
+ deploy:
+ placement:
+ constraints:
+ - "node.labels.name==${ANALYTICS_DATASTORE_CLICKHOUSE_02_PLACEMENT}"
  ulimits:
  noFile: 262144
  volumes:
@@ -70,16 +70,16 @@ services:
  default:
 
  analytics-datastore-clickhouse-03:
- image: clickhouse/clickhouse-server
+ image: ${CLICKHOUSE_IMAGE}
  hostname: analytics-datastore-clickhouse-03
- deploy: 
- placement: 
- constraints: 
- - "node.labels.name==node-3"
+ deploy:
+ placement:
+ constraints:
+ - "node.labels.name==${ANALYTICS_DATASTORE_CLICKHOUSE_03_PLACEMENT}"
  ulimits:
  noFile: 262144
  volumes:
- - clickhouse-data-03:/var/lib/clickhouse/ 
+ - clickhouse-data-03:/var/lib/clickhouse/
  configs:
  - target: /etc/clickhouse-server/config.d/docker_related_config.xml
  source: docker_related_config.xml
@@ -109,7 +109,7 @@ services:
  ulimits:
  noFile: 262144
  volumes:
- - clickhouse-data-04:/var/lib/clickhouse/ 
+ - clickhouse-data-04:/var/lib/clickhouse/
  configs:
  - target: /etc/clickhouse-server/config.d/docker_related_config.xml
  source: docker_related_config.xml
@@ -136,77 +136,77 @@ volumes:
  clickhouse-data-02:
  clickhouse-data-03:
  clickhouse-data-04:
- 
+
 configs:
  docker_related_config.xml:
  file: ./cluster_configs/docker_related_config.xml
  name: docker_related_config.xml-${docker_related_config_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_enable_keeper_01.xml:
  file: ./cluster_configs/enable_keeper_01.xml
  name: enable_keeper_01.xml-${enable_keeper_01_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_enable_keeper_02.xml:
  file: ./cluster_configs/enable_keeper_02.xml
  name: enable_keeper_02.xml-${enable_keeper_02_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_enable_keeper_03.xml:
  file: ./cluster_configs/enable_keeper_03.xml
  name: enable_keeper_03.xml-${enable_keeper_03_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_macros_01.xml:
  file: ./cluster_configs/macros_01.xml
  name: macros_01.xml-${macros_01_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_macros_02.xml:
  file: ./cluster_configs/macros_02.xml
  name: macros_02.xml-${macros_02_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_macros_03.xml:
  file: ./cluster_configs/macros_03.xml
  name: macros_03.xml-${macros_03_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_macros_04.xml:
  file: ./cluster_configs/macros_04.xml
  name: macros_04.xml-${macros_04_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_remote_servers.xml:
  file: ./cluster_configs/remote_servers.xml
  name: remote_servers.xml-${remote_servers_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_use_keeper.xml:
  file: ./cluster_configs/use_keeper.xml
  name: use_keeper.xml-${use_keeper_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_metric_log.xml:
  file: ./general_configs/metric_log.xml
  name: metric_log.xml.xml-${metric_log_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_part_log.xml:
  file: ./general_configs/part_log.xml
  name: part_log.xml.xml-${part_log_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_query_log.xml:
  file: ./general_configs/query_log.xml
  name: query_log.xml.xml-${query_log_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_trace_log.xml:
  file: ./general_configs/trace_log.xml
  name: trace_log.xml.xml-${trace_log_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
 
 networks:

analytics-datastore-clickhouse/docker-compose.yml

@@ -1,8 +1,8 @@
-version: '3.9'
+version: "3.9"
 
 services:
  analytics-datastore-clickhouse:
- image: clickhouse/clickhouse-server
+ image: ${CLICKHOUSE_IMAGE}
  ulimits:
  noFile: 262144
  volumes:
@@ -15,7 +15,7 @@ services:
  - target: /etc/clickhouse-server/config.d/query_log.xml
  source: clickhouse_query_log.xml
  - target: /etc/clickhouse-server/config.d/trace_log.xml
- source: clickhouse_trace_log.xml 
+ source: clickhouse_trace_log.xml
  networks:
  public:
  default:
@@ -27,22 +27,22 @@ configs:
  clickhouse_metric_log.xml:
  file: ./general_configs/metric_log.xml
  name: metric_log.xml.xml-${metric_log_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_part_log.xml:
  file: ./general_configs/part_log.xml
  name: part_log.xml.xml-${part_log_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_query_log.xml:
  file: ./general_configs/query_log.xml
  name: query_log.xml.xml-${query_log_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_trace_log.xml:
  file: ./general_configs/trace_log.xml
  name: trace_log.xml.xml-${trace_log_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
 
 networks:

analytics-datastore-clickhouse/package-metadata.json

@@ -7,6 +7,10 @@
  "dependencies": [],
  "environmentVariables": {
  "CLICKHOUSE_HOST": "analytics-datastore-clickhouse",
- "CLICKHOUSE_PORT": "8123"
+ "CLICKHOUSE_PORT": "8123",
+ "CLICKHOUSE_IMAGE": "clickhouse/clickhouse-server:23.8.14.6",
+ "ANALYTICS_DATASTORE_CLICKHOUSE_01_PLACEMENT": "node-1",
+ "ANALYTICS_DATASTORE_CLICKHOUSE_02_PLACEMENT": "node-2",
+ "ANALYTICS_DATASTORE_CLICKHOUSE_03_PLACEMENT": "node-3"
  }
 }

analytics-datastore-elastic-search/docker-compose.certs.yml

@@ -2,7 +2,7 @@ version: "3.9"
 
 services:
  create_certs:
- image: docker.elastic.co/elasticsearch/elasticsearch:7.13.0
+ image: ${ES_IMAGE}
  deploy:
  placement:
  constraints: