Cu 86bygcqxd images placement for data store dashboards monitoring

.github/workflows/run-tests.sh

@@ -25,6 +25,9 @@ for package in "${CHANGED_FILES[@]}"; do
  fi
 done
 
+# Run the basic funtional end to end tests for the CDR recipe
+DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud HOST=$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":recipe
+
 if [[ ${#changed_packages[@]} -eq 0 ]] || [[ "${!changed_packages[*]}" == *"utils"* ]] || [[ "${!changed_packages[*]}" == *"features/steps"* ]] || [[ "${!changed_packages[*]}" == *"infrastructure"* ]] ; then
  openhim_ran="true"
  DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":openhim
@@ -34,7 +37,8 @@ else
 
  if [[ $folder_name == *"clickhouse"* ]]; then
  DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":clickhouse
- elif [[ $folder_name == *"elastic"* ]] || [[ $folder_name == *"kibana"* ]] || [[ $folder_name == *"logstash"* ]]; then
+ el
+ if [[ $folder_name == *"elastic"* ]] || [[ $folder_name == *"kibana"* ]] || [[ $folder_name == *"logstash"* ]]; then
  DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":elk
  elif [[ $folder_name == *"kafka"* ]] || [[ $folder_name == *"monitoring"* ]]; then
  DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":kafka
@@ -62,6 +66,3 @@ else
  fi
  done
 fi
-
-# Run the basic funtional end to end tests for the CDR recipe
-DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud HOST=$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":recipe

analytics-datastore-clickhouse/docker-compose.cluster.yml

@@ -1,17 +1,17 @@
-version: '3.9'
+version: "3.9"
 
 services:
  analytics-datastore-clickhouse-01:
- image: clickhouse/clickhouse-server
+ image: ${CLICKHOUSE_IMAGE}
  ulimits:
  noFile: 262144
  volumes:
  - clickhouse-data-01:/var/lib/clickhouse/
  hostname: analytics-datastore-clickhouse-01
- deploy: 
- placement: 
- constraints: 
- - "node.labels.name==node-1"
+ deploy:
+ placement:
+ constraints:
+ - "node.labels.name==${ANALYTICS_DATASTORE_CLICKHOUSE_01_PLACEMENT}"
  configs:
  - target: /etc/clickhouse-server/config.d/docker_related_config.xml
  source: docker_related_config.xml
@@ -36,12 +36,12 @@ services:
  default:
 
  analytics-datastore-clickhouse-02:
- image: clickhouse/clickhouse-server
+ image: ${CLICKHOUSE_IMAGE}
  hostname: analytics-datastore-clickhouse-02
- deploy: 
- placement: 
- constraints: 
- - "node.labels.name==node-2"
+ deploy:
+ placement:
+ constraints:
+ - "node.labels.name==${ANALYTICS_DATASTORE_CLICKHOUSE_02_PLACEMENT}"
  ulimits:
  noFile: 262144
  volumes:
@@ -70,16 +70,16 @@ services:
  default:
 
  analytics-datastore-clickhouse-03:
- image: clickhouse/clickhouse-server
+ image: ${CLICKHOUSE_IMAGE}
  hostname: analytics-datastore-clickhouse-03
- deploy: 
- placement: 
- constraints: 
- - "node.labels.name==node-3"
+ deploy:
+ placement:
+ constraints:
+ - "node.labels.name==${ANALYTICS_DATASTORE_CLICKHOUSE_03_PLACEMENT}"
  ulimits:
  noFile: 262144
  volumes:
- - clickhouse-data-03:/var/lib/clickhouse/ 
+ - clickhouse-data-03:/var/lib/clickhouse/
  configs:
  - target: /etc/clickhouse-server/config.d/docker_related_config.xml
  source: docker_related_config.xml
@@ -109,7 +109,7 @@ services:
  ulimits:
  noFile: 262144
  volumes:
- - clickhouse-data-04:/var/lib/clickhouse/ 
+ - clickhouse-data-04:/var/lib/clickhouse/
  configs:
  - target: /etc/clickhouse-server/config.d/docker_related_config.xml
  source: docker_related_config.xml
@@ -136,77 +136,77 @@ volumes:
  clickhouse-data-02:
  clickhouse-data-03:
  clickhouse-data-04:
- 
+
 configs:
  docker_related_config.xml:
  file: ./cluster_configs/docker_related_config.xml
  name: docker_related_config.xml-${docker_related_config_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_enable_keeper_01.xml:
  file: ./cluster_configs/enable_keeper_01.xml
  name: enable_keeper_01.xml-${enable_keeper_01_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_enable_keeper_02.xml:
  file: ./cluster_configs/enable_keeper_02.xml
  name: enable_keeper_02.xml-${enable_keeper_02_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_enable_keeper_03.xml:
  file: ./cluster_configs/enable_keeper_03.xml
  name: enable_keeper_03.xml-${enable_keeper_03_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_macros_01.xml:
  file: ./cluster_configs/macros_01.xml
  name: macros_01.xml-${macros_01_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_macros_02.xml:
  file: ./cluster_configs/macros_02.xml
  name: macros_02.xml-${macros_02_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_macros_03.xml:
  file: ./cluster_configs/macros_03.xml
  name: macros_03.xml-${macros_03_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_macros_04.xml:
  file: ./cluster_configs/macros_04.xml
  name: macros_04.xml-${macros_04_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_remote_servers.xml:
  file: ./cluster_configs/remote_servers.xml
  name: remote_servers.xml-${remote_servers_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_use_keeper.xml:
  file: ./cluster_configs/use_keeper.xml
  name: use_keeper.xml-${use_keeper_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_metric_log.xml:
  file: ./general_configs/metric_log.xml
  name: metric_log.xml.xml-${metric_log_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_part_log.xml:
  file: ./general_configs/part_log.xml
  name: part_log.xml.xml-${part_log_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_query_log.xml:
  file: ./general_configs/query_log.xml
  name: query_log.xml.xml-${query_log_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_trace_log.xml:
  file: ./general_configs/trace_log.xml
  name: trace_log.xml.xml-${trace_log_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
 
 networks:

analytics-datastore-clickhouse/docker-compose.yml

@@ -1,8 +1,8 @@
-version: '3.9'
+version: "3.9"
 
 services:
  analytics-datastore-clickhouse:
- image: clickhouse/clickhouse-server
+ image: ${CLICKHOUSE_IMAGE}
  ulimits:
  noFile: 262144
  volumes:
@@ -15,7 +15,7 @@ services:
  - target: /etc/clickhouse-server/config.d/query_log.xml
  source: clickhouse_query_log.xml
  - target: /etc/clickhouse-server/config.d/trace_log.xml
- source: clickhouse_trace_log.xml 
+ source: clickhouse_trace_log.xml
  networks:
  public:
  default:
@@ -27,22 +27,22 @@ configs:
  clickhouse_metric_log.xml:
  file: ./general_configs/metric_log.xml
  name: metric_log.xml.xml-${metric_log_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_part_log.xml:
  file: ./general_configs/part_log.xml
  name: part_log.xml.xml-${part_log_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_query_log.xml:
  file: ./general_configs/query_log.xml
  name: query_log.xml.xml-${query_log_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
  clickhouse_trace_log.xml:
  file: ./general_configs/trace_log.xml
  name: trace_log.xml.xml-${trace_log_xml_DIGEST:?err}
- labels: 
+ labels:
  name: clickhouse
 
 networks:

analytics-datastore-clickhouse/package-metadata.json

@@ -7,6 +7,10 @@
  "dependencies": [],
  "environmentVariables": {
  "CLICKHOUSE_HOST": "analytics-datastore-clickhouse",
- "CLICKHOUSE_PORT": "8123"
+ "CLICKHOUSE_PORT": "8123",
+ "CLICKHOUSE_IMAGE": "clickhouse/clickhouse-server:23.8.14.6",
+ "ANALYTICS_DATASTORE_CLICKHOUSE_01_PLACEMENT": "node-1",
+ "ANALYTICS_DATASTORE_CLICKHOUSE_02_PLACEMENT": "node-2",
+ "ANALYTICS_DATASTORE_CLICKHOUSE_03_PLACEMENT": "node-3"
  }
 }

analytics-datastore-elastic-search/docker-compose.certs.yml

@@ -2,7 +2,7 @@ version: "3.9"
 
 services:
  create_certs:
- image: docker.elastic.co/elasticsearch/elasticsearch:7.13.0
+ image: ${ES_IMAGE}
  deploy:
  placement:
  constraints:

analytics-datastore-elastic-search/docker-compose.cluster.yml

@@ -1,8 +1,8 @@
-version: '3.9'
+version: "3.9"
 
 services:
  analytics-datastore-elastic-search-01:
- image: docker.elastic.co/elasticsearch/elasticsearch:7.13.0
+ image: ${ES_IMAGE}
  ulimits:
  memlock:
  soft: -1
@@ -16,20 +16,20 @@ services:
  memory: ${ES_MEMORY_RESERVE}
  placement:
  constraints:
- - "node.labels.name==node-1"
+ - "node.labels.name==${ES_01_PLACEMENT}"
  environment:
  node.name: es01
  cluster.name: es-cluster
  discovery.seed_hosts: analytics-datastore-elastic-search-02,analytics-datastore-elastic-search-03
  cluster.initial_master_nodes: es01
  xpack.license.self_generated.type: basic
- bootstrap.memory_lock: 'true'
- xpack.security.enabled: 'true'
- xpack.security.http.ssl.enabled: 'false'
+ bootstrap.memory_lock: "true"
+ xpack.security.enabled: "true"
+ xpack.security.http.ssl.enabled: "false"
  xpack.security.http.ssl.key: certs/es01/es01.key
  xpack.security.http.ssl.certificate_authorities: certs/ca/ca.crt
  xpack.security.http.ssl.certificate: certs/es01/es01.crt
- xpack.security.transport.ssl.enabled: 'true'
+ xpack.security.transport.ssl.enabled: "true"
  xpack.security.transport.ssl.verification_mode: certificate
  xpack.security.transport.ssl.certificate_authorities: certs/ca/ca.crt
  xpack.security.transport.ssl.certificate: certs/es01/es01.crt
@@ -52,7 +52,7 @@ services:
  public:
 
  analytics-datastore-elastic-search-02:
- image: docker.elastic.co/elasticsearch/elasticsearch:7.13.0
+ image: ${ES_IMAGE}
  ulimits:
  memlock:
  soft: -1
@@ -66,20 +66,20 @@ services:
  memory: ${ES_MEMORY_RESERVE}
  placement:
  constraints:
- - "node.labels.name==node-2"
+ - "node.labels.name==${ES_02_PLACEMENT}"
  environment:
  node.name: es02
  cluster.name: es-cluster
  discovery.seed_hosts: analytics-datastore-elastic-search-01,analytics-datastore-elastic-search-03
  cluster.initial_master_nodes: es01
- bootstrap.memory_lock: 'true'
- xpack.security.enabled: 'true'
+ bootstrap.memory_lock: "true"
+ xpack.security.enabled: "true"
  xpack.license.self_generated.type: basic
- xpack.security.http.ssl.enabled: 'false'
+ xpack.security.http.ssl.enabled: "false"
  xpack.security.http.ssl.key: certs/es02/es02.key
  xpack.security.http.ssl.certificate_authorities: certs/ca/ca.crt
  xpack.security.http.ssl.certificate: certs/es02/es02.crt
- xpack.security.transport.ssl.enabled: 'true'
+ xpack.security.transport.ssl.enabled: "true"
  xpack.security.transport.ssl.verification_mode: certificate
  xpack.security.transport.ssl.certificate_authorities: certs/ca/ca.crt
  xpack.security.transport.ssl.certificate: certs/es02/es02.crt
@@ -102,7 +102,7 @@ services:
  public:
 
  analytics-datastore-elastic-search-03:
- image: docker.elastic.co/elasticsearch/elasticsearch:7.13.0
+ image: ${ES_IMAGE}
  ulimits:
  memlock:
  soft: -1
@@ -116,20 +116,20 @@ services:
  memory: ${ES_MEMORY_RESERVE}
  placement:
  constraints:
- - "node.labels.name==node-3"
+ - "node.labels.name==${ES_03_PLACEMENT}"
  environment:
  node.name: es03
  cluster.name: es-cluster
  discovery.seed_hosts: analytics-datastore-elastic-search-01,analytics-datastore-elastic-search-02
  cluster.initial_master_nodes: es01
- bootstrap.memory_lock: 'true'
- xpack.security.enabled: 'true'
+ bootstrap.memory_lock: "true"
+ xpack.security.enabled: "true"
  xpack.license.self_generated.type: basic
- xpack.security.http.ssl.enabled: 'false'
+ xpack.security.http.ssl.enabled: "false"
  xpack.security.http.ssl.key: certs/es03/es03.key
  xpack.security.http.ssl.certificate_authorities: certs/ca/ca.crt
  xpack.security.http.ssl.certificate: certs/es03/es03.crt
- xpack.security.transport.ssl.enabled: 'true'
+ xpack.security.transport.ssl.enabled: "true"
  xpack.security.transport.ssl.verification_mode: certificate
  xpack.security.transport.ssl.certificate_authorities: certs/ca/ca.crt
  xpack.security.transport.ssl.certificate: certs/es03/es03.crt

analytics-datastore-elastic-search/docker-compose.yml

@@ -1,8 +1,8 @@
-version: '3.9'
+version: "3.9"
 
 services:
  analytics-datastore-elastic-search:
- image: docker.elastic.co/elasticsearch/elasticsearch:7.13.0
+ image: ${ES_IMAGE}
  ulimits:
  memlock:
  soft: -1
@@ -16,9 +16,9 @@ services:
  environment:
  node.name: es01
  discovery.type: single-node
- bootstrap.memory_lock: 'true'
- xpack.security.enabled: 'true'
- xpack.monitoring.collection.enabled: 'true'
+ bootstrap.memory_lock: "true"
+ xpack.security.enabled: "true"
+ xpack.monitoring.collection.enabled: "true"
  ES_JAVA_OPTS: ${ES_HEAP_SIZE}
  search.max_buckets: 1000000
  search.default_search_timeout: -1