diff --git a/.env.cluster b/.env.cluster index 5e8e70b7..0c501267 100644 --- a/.env.cluster +++ b/.env.cluster @@ -19,6 +19,8 @@ OPENHIM_MONGO_ATNAURL=mongodb://mongo-1:27017,mongo-2:27017,mongo-3:27017/openhi # FHIR Datastore - HAPI FHIR HAPI_FHIR_INSTANCES=3 +# (pool size * instances) should be less than 100 +HF_MAX_POOL_SIZE=30 REPMGR_PARTNER_NODES=postgres-1,postgres-2,postgres-3 POSTGRES_REPLICA_SET=postgres-1:5432,postgres-2:5432,postgres-3:5432 HAPI_DB_SET=pgpool-1:5432,pgpool-2:5432,pgpool-3:5432 diff --git a/.env.remote b/.env.remote index 882cf140..bee8a87c 100644 --- a/.env.remote +++ b/.env.remote @@ -15,6 +15,8 @@ ES_HOSTS="\"analytics-datastore-elastic-search-01:9200","analytics-datastore-ela CLICKHOUSE_HOST=analytics-datastore-clickhouse-01 # FHIR Datastore - HAPI FHIR +HAPI_FHIR_INSTANCES=3 +HF_MAX_POOL_SIZE=30 REPMGR_PARTNER_NODES=postgres-1,postgres-2,postgres-3 POSTGRES_REPLICA_SET=postgres-1:5432,postgres-2:5432,postgres-3:5432 HAPI_DB_SET=postgres-1:5432,postgres-2:5432,postgres-3:5432 diff --git a/config.yaml b/config.yaml index 4471237d..a4047aba 100644 --- a/config.yaml +++ b/config.yaml @@ -1,5 +1,5 @@ projectName: platform -image: jembi/platform:2.4.1 +image: jembi/platform:2.4.2 logPath: /tmp/logs packages: diff --git a/fhir-datastore-hapi-fhir/config/custom_pgpool.conf b/fhir-datastore-hapi-fhir/config/custom_pgpool.conf index 88a7ce68..a0885000 100644 --- a/fhir-datastore-hapi-fhir/config/custom_pgpool.conf +++ b/fhir-datastore-hapi-fhir/config/custom_pgpool.conf @@ -1,14 +1,9 @@ -failover_on_backend_error='on' -detach_false_primary='on' auto_failback='on' health_check_period=25 health_check_max_retries=3 health_check_user='postgres' -reserved_connections=1 -max_pool=6 +max_pool=5 client_idle_limit=300 -connection_life_time=300 -child_max_connections=25 use_watchdog='on' hostname0='pgpool-1' diff --git a/fhir-datastore-hapi-fhir/config/custom_postgresql.conf b/fhir-datastore-hapi-fhir/config/custom_postgresql.conf new file mode 100644 index 00000000..f4543c51 --- /dev/null +++ b/fhir-datastore-hapi-fhir/config/custom_postgresql.conf @@ -0,0 +1,2 @@ +max_connections = 250 +shared_buffers = 768MB diff --git a/fhir-datastore-hapi-fhir/docker-compose-pgpool.cluster.yml b/fhir-datastore-hapi-fhir/docker-compose-pgpool.cluster.yml new file mode 100644 index 00000000..3c054910 --- /dev/null +++ b/fhir-datastore-hapi-fhir/docker-compose-pgpool.cluster.yml @@ -0,0 +1,126 @@ +version: '3.9' + +services: + pgpool-1: + image: bitnami/pgpool:4.4.3 + deploy: + placement: + constraints: + - "node.labels.name==node-1" + replicas: 1 + resources: + limits: + cpus: ${HF_PGPOOL_CPU_LIMIT} + memory: ${HF_PGPOOL_MEMORY_LIMIT} + reservations: + cpus: ${HF_PGPOOL_CPU_RESERVE} + memory: ${HF_PGPOOL_MEMORY_RESERVE} + configs: + - target: /config/custom_pgpool.conf + source: pgpool.conf + - target: /opt/bitnami/pgpool/conf/pgpool_node_id + source: pgpool_node_id0.conf + environment: + PGPOOL_BACKEND_NODES: ${HF_PGPOOL_NODES} + PGPOOL_BACKEND_APPLICATION_NAMES: ${REPMGR_PARTNER_NODES} + PGPOOL_SR_CHECK_USER: postgres + PGPOOL_SR_CHECK_PASSWORD: ${HAPI_POSTGRES_PASSWORD} + PGPOOL_POSTGRES_USERNAME: postgres + PGPOOL_POSTGRES_PASSWORD: ${HAPI_POSTGRES_PASSWORD} + PGPOOL_ADMIN_USERNAME: ${HF_PGPOOL_USERNAME} + PGPOOL_ADMIN_PASSWORD: ${HF_PGPOOL_PASSWORD} + PGPOOL_POSTGRES_CUSTOM_USERS: ${HF_POSTGRESQL_USERNAME} + PGPOOL_POSTGRES_CUSTOM_PASSWORDS: ${HF_POSTGRESQL_PASSWORD} + PGPOOL_ENABLE_LOAD_BALANCING: "no" + PGPOOL_AUTO_FAILBACK: "yes" + PGPOOL_USER_CONF_FILE: "/config/custom_pgpool.conf" + + pgpool-2: + image: bitnami/pgpool:4.4.3 + deploy: + placement: + constraints: + - "node.labels.name==node-2" + replicas: 1 + resources: + limits: + cpus: ${HF_PGPOOL_CPU_LIMIT} + memory: ${HF_PGPOOL_MEMORY_LIMIT} + reservations: + cpus: ${HF_PGPOOL_CPU_RESERVE} + memory: ${HF_PGPOOL_MEMORY_RESERVE} + configs: + - target: /config/custom_pgpool.conf + source: pgpool.conf + - target: /opt/bitnami/pgpool/conf/pgpool_node_id + source: pgpool_node_id1.conf + environment: + PGPOOL_BACKEND_NODES: ${HF_PGPOOL_NODES} + PGPOOL_BACKEND_APPLICATION_NAMES: ${REPMGR_PARTNER_NODES} + PGPOOL_SR_CHECK_USER: postgres + PGPOOL_SR_CHECK_PASSWORD: ${HAPI_POSTGRES_PASSWORD} + PGPOOL_POSTGRES_USERNAME: postgres + PGPOOL_POSTGRES_PASSWORD: ${HAPI_POSTGRES_PASSWORD} + PGPOOL_ADMIN_USERNAME: ${HF_PGPOOL_USERNAME} + PGPOOL_ADMIN_PASSWORD: ${HF_PGPOOL_PASSWORD} + PGPOOL_POSTGRES_CUSTOM_USERS: ${HF_POSTGRESQL_USERNAME} + PGPOOL_POSTGRES_CUSTOM_PASSWORDS: ${HF_POSTGRESQL_PASSWORD} + PGPOOL_ENABLE_LOAD_BALANCING: "no" + PGPOOL_AUTO_FAILBACK: "yes" + PGPOOL_USER_CONF_FILE: "/config/custom_pgpool.conf" + + pgpool-3: + image: bitnami/pgpool:4.4.3 + deploy: + placement: + constraints: + - "node.labels.name==node-3" + replicas: 1 + resources: + limits: + cpus: ${HF_PGPOOL_CPU_LIMIT} + memory: ${HF_PGPOOL_MEMORY_LIMIT} + reservations: + cpus: ${HF_PGPOOL_CPU_RESERVE} + memory: ${HF_PGPOOL_MEMORY_RESERVE} + configs: + - target: /config/custom_pgpool.conf + source: pgpool.conf + - target: /opt/bitnami/pgpool/conf/pgpool_node_id + source: pgpool_node_id2.conf + environment: + PGPOOL_BACKEND_NODES: ${HF_PGPOOL_NODES} + PGPOOL_BACKEND_APPLICATION_NAMES: ${REPMGR_PARTNER_NODES} + PGPOOL_SR_CHECK_USER: postgres + PGPOOL_SR_CHECK_PASSWORD: ${HAPI_POSTGRES_PASSWORD} + PGPOOL_POSTGRES_USERNAME: postgres + PGPOOL_POSTGRES_PASSWORD: ${HAPI_POSTGRES_PASSWORD} + PGPOOL_ADMIN_USERNAME: ${HF_PGPOOL_USERNAME} + PGPOOL_ADMIN_PASSWORD: ${HF_PGPOOL_PASSWORD} + PGPOOL_POSTGRES_CUSTOM_USERS: ${HF_POSTGRESQL_USERNAME} + PGPOOL_POSTGRES_CUSTOM_PASSWORDS: ${HF_POSTGRESQL_PASSWORD} + PGPOOL_ENABLE_LOAD_BALANCING: "no" + PGPOOL_AUTO_FAILBACK: "yes" + PGPOOL_USER_CONF_FILE: "/config/custom_pgpool.conf" + +configs: + pgpool.conf: + name: pgpool_conf-${pgpool_conf_DIGEST:?err} + file: ./config/custom_pgpool.conf + labels: + name: hapi-fhir + pgpool_node_id0.conf: + name: pgpool_node_id0-${pgpool_node_id0_DIGEST:?err} + file: ./config/pgpool_node_id0 + labels: + name: hapi-fhir + pgpool_node_id1.conf: + name: pgpool_node_id1-${pgpool_node_id1_DIGEST:?err} + file: ./config/pgpool_node_id1 + labels: + name: hapi-fhir + pgpool_node_id2.conf: + name: pgpool_node_id2-${pgpool_node_id2_DIGEST:?err} + file: ./config/pgpool_node_id2 + labels: + name: hapi-fhir diff --git a/fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml b/fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml index 87ea782d..fa1913ba 100644 --- a/fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml +++ b/fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml @@ -22,8 +22,13 @@ services: REPMGR_PRIMARY_HOST: ${REPMGR_PRIMARY_HOST} REPMGR_PARTNER_NODES: ${REPMGR_PARTNER_NODES} POSTGRESQL_POSTGRES_PASSWORD: ${HAPI_POSTGRES_PASSWORD} + REPMGR_FAILOVER: ${HF_POSTGRES_FAILOVER} + REPMGR_DEGRADED_MONITORING_TIMEOUT: ${HF_POSTGRES_DEGRADED_MONITORING_TIMEOUT} volumes: - 'hapi-postgres-2-data:/bitnami/postgresql' + configs: + - target: /bitnami/postgresql/conf/conf.d/custom_postgresql.conf + source: postgresql.conf deploy: placement: constraints: @@ -53,8 +58,13 @@ services: REPMGR_PRIMARY_HOST: ${REPMGR_PRIMARY_HOST} REPMGR_PARTNER_NODES: ${REPMGR_PARTNER_NODES} POSTGRESQL_POSTGRES_PASSWORD: ${HAPI_POSTGRES_PASSWORD} + REPMGR_FAILOVER: ${HF_POSTGRES_FAILOVER} + REPMGR_DEGRADED_MONITORING_TIMEOUT: ${HF_POSTGRES_DEGRADED_MONITORING_TIMEOUT} volumes: - 'hapi-postgres-3-data:/bitnami/postgresql' + configs: + - target: /bitnami/postgresql/conf/conf.d/custom_postgresql.conf + source: postgresql.conf deploy: placement: constraints: @@ -71,130 +81,6 @@ services: default: pg_backup_net: - pgpool-1: - image: bitnami/pgpool:4.4.3 - deploy: - placement: - max_replicas_per_node: 1 - resources: - limits: - cpus: ${HF_PGPOOL_CPU_LIMIT} - memory: ${HF_PGPOOL_MEMORY_LIMIT} - reservations: - cpus: ${HF_PGPOOL_CPU_RESERVE} - memory: ${HF_PGPOOL_MEMORY_RESERVE} - configs: - - target: /config/custom_pgpool.conf - source: pgpool.conf - - target: /opt/bitnami/pgpool/conf/pgpool_node_id - source: pgpool_node_id0.conf - environment: - PGPOOL_BACKEND_NODES: ${HF_PGPOOL_NODES} - PGPOOL_BACKEND_APPLICATION_NAMES: ${REPMGR_PARTNER_NODES} - PGPOOL_SR_CHECK_USER: postgres - PGPOOL_SR_CHECK_PASSWORD: ${HAPI_POSTGRES_PASSWORD} - PGPOOL_POSTGRES_USERNAME: postgres - PGPOOL_POSTGRES_PASSWORD: ${HAPI_POSTGRES_PASSWORD} - PGPOOL_ADMIN_USERNAME: ${HF_PGPOOL_USERNAME} - PGPOOL_ADMIN_PASSWORD: ${HF_PGPOOL_PASSWORD} - PGPOOL_POSTGRES_CUSTOM_USERS: ${HF_POSTGRESQL_USERNAME} - PGPOOL_POSTGRES_CUSTOM_PASSWORDS: ${HF_POSTGRESQL_PASSWORD} - PGPOOL_ENABLE_LOAD_BALANCING: "no" - PGPOOL_AUTO_FAILBACK: "yes" - PGPOOL_USER_CONF_FILE: "/config/custom_pgpool.conf" - networks: - default: - - pgpool-2: - image: bitnami/pgpool:4.4.3 - deploy: - placement: - max_replicas_per_node: 1 - resources: - limits: - cpus: ${HF_PGPOOL_CPU_LIMIT} - memory: ${HF_PGPOOL_MEMORY_LIMIT} - reservations: - cpus: ${HF_PGPOOL_CPU_RESERVE} - memory: ${HF_PGPOOL_MEMORY_RESERVE} - configs: - - target: /config/custom_pgpool.conf - source: pgpool.conf - - target: /opt/bitnami/pgpool/conf/pgpool_node_id - source: pgpool_node_id1.conf - environment: - PGPOOL_BACKEND_NODES: ${HF_PGPOOL_NODES} - PGPOOL_BACKEND_APPLICATION_NAMES: ${REPMGR_PARTNER_NODES} - PGPOOL_SR_CHECK_USER: postgres - PGPOOL_SR_CHECK_PASSWORD: ${HAPI_POSTGRES_PASSWORD} - PGPOOL_POSTGRES_USERNAME: postgres - PGPOOL_POSTGRES_PASSWORD: ${HAPI_POSTGRES_PASSWORD} - PGPOOL_ADMIN_USERNAME: ${HF_PGPOOL_USERNAME} - PGPOOL_ADMIN_PASSWORD: ${HF_PGPOOL_PASSWORD} - PGPOOL_POSTGRES_CUSTOM_USERS: ${HF_POSTGRESQL_USERNAME} - PGPOOL_POSTGRES_CUSTOM_PASSWORDS: ${HF_POSTGRESQL_PASSWORD} - PGPOOL_ENABLE_LOAD_BALANCING: "no" - PGPOOL_AUTO_FAILBACK: "yes" - PGPOOL_USER_CONF_FILE: "/config/custom_pgpool.conf" - networks: - default: - - pgpool-3: - image: bitnami/pgpool:4.4.3 - deploy: - placement: - max_replicas_per_node: 1 - resources: - limits: - cpus: ${HF_PGPOOL_CPU_LIMIT} - memory: ${HF_PGPOOL_MEMORY_LIMIT} - reservations: - cpus: ${HF_PGPOOL_CPU_RESERVE} - memory: ${HF_PGPOOL_MEMORY_RESERVE} - configs: - - target: /config/custom_pgpool.conf - source: pgpool.conf - - target: /opt/bitnami/pgpool/conf/pgpool_node_id - source: pgpool_node_id2.conf - environment: - PGPOOL_BACKEND_NODES: ${HF_PGPOOL_NODES} - PGPOOL_BACKEND_APPLICATION_NAMES: ${REPMGR_PARTNER_NODES} - PGPOOL_SR_CHECK_USER: postgres - PGPOOL_SR_CHECK_PASSWORD: ${HAPI_POSTGRES_PASSWORD} - PGPOOL_POSTGRES_USERNAME: postgres - PGPOOL_POSTGRES_PASSWORD: ${HAPI_POSTGRES_PASSWORD} - PGPOOL_ADMIN_USERNAME: ${HF_PGPOOL_USERNAME} - PGPOOL_ADMIN_PASSWORD: ${HF_PGPOOL_PASSWORD} - PGPOOL_POSTGRES_CUSTOM_USERS: ${HF_POSTGRESQL_USERNAME} - PGPOOL_POSTGRES_CUSTOM_PASSWORDS: ${HF_POSTGRESQL_PASSWORD} - PGPOOL_ENABLE_LOAD_BALANCING: "no" - PGPOOL_AUTO_FAILBACK: "yes" - PGPOOL_USER_CONF_FILE: "/config/custom_pgpool.conf" - networks: - default: - -configs: - pgpool.conf: - name: pgpool_conf-${pgpool_conf_DIGEST:?err} - file: ./config/custom_pgpool.conf - labels: - name: hapi-fhir - pgpool_node_id0.conf: - name: pgpool_node_id0-${pgpool_node_id0_DIGEST:?err} - file: ./config/pgpool_node_id0 - labels: - name: hapi-fhir - pgpool_node_id1.conf: - name: pgpool_node_id1-${pgpool_node_id1_DIGEST:?err} - file: ./config/pgpool_node_id1 - labels: - name: hapi-fhir - pgpool_node_id2.conf: - name: pgpool_node_id2-${pgpool_node_id2_DIGEST:?err} - file: ./config/pgpool_node_id2 - labels: - name: hapi-fhir - volumes: hapi-postgres-2-data: hapi-postgres-3-data: diff --git a/fhir-datastore-hapi-fhir/docker-compose-postgres.yml b/fhir-datastore-hapi-fhir/docker-compose-postgres.yml index d338f52f..4045cf57 100644 --- a/fhir-datastore-hapi-fhir/docker-compose-postgres.yml +++ b/fhir-datastore-hapi-fhir/docker-compose-postgres.yml @@ -14,8 +14,13 @@ services: REPMGR_PRIMARY_HOST: ${REPMGR_PRIMARY_HOST} REPMGR_PARTNER_NODES: ${REPMGR_PARTNER_NODES} POSTGRESQL_POSTGRES_PASSWORD: ${HAPI_POSTGRES_PASSWORD} + REPMGR_FAILOVER: ${HF_POSTGRES_FAILOVER} + REPMGR_DEGRADED_MONITORING_TIMEOUT: ${HF_POSTGRES_DEGRADED_MONITORING_TIMEOUT} volumes: - "hapi-postgres-1-data:/bitnami/postgresql" + configs: + - target: /bitnami/postgresql/conf/conf.d/custom_postgresql.conf + source: postgresql.conf deploy: replicas: 1 resources: @@ -33,6 +38,13 @@ services: volumes: hapi-postgres-1-data: +configs: + postgresql.conf: + name: postgresql_conf-${postgresql_conf_DIGEST:?err} + file: ./config/custom_postgresql.conf + labels: + name: hapi-fhir + networks: default: hapi-fhir-postgres: diff --git a/fhir-datastore-hapi-fhir/docker-compose.yml b/fhir-datastore-hapi-fhir/docker-compose.yml index 8546fd48..010b5aee 100644 --- a/fhir-datastore-hapi-fhir/docker-compose.yml +++ b/fhir-datastore-hapi-fhir/docker-compose.yml @@ -2,7 +2,7 @@ version: '3.9' services: hapi-fhir: - image: jembi/hapi:v6.0.1-wget + image: jembi/hapi:${HF_IMAGE_TAG} networks: mpi: public: @@ -13,7 +13,7 @@ services: - spring.datasource.password=${HF_POSTGRESQL_PASSWORD} - spring.datasource.driverClassName=org.postgresql.Driver - spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.PostgreSQL95Dialect - - spring.datasource.hikari.maximumPoolSize=250 + - spring.datasource.hikari.maximumPoolSize=${HF_MAX_POOL_SIZE} - spring.datasource.hikari.minimumIdle=20 - spring.datasource.hikari.connectionTimeout=60000 - spring.datasource.hikari.idleTimeout=600000 @@ -36,8 +36,8 @@ services: healthcheck: test: /bin/wget --no-verbose --tries=1 --spider http://localhost:8080/fhir/Organization?identifier=urn:healthcheck interval: 30s - timeout: 30s - retries: 2 + timeout: 5s + retries: 5 start_period: 2m networks: diff --git a/fhir-datastore-hapi-fhir/package-metadata.json b/fhir-datastore-hapi-fhir/package-metadata.json index e267fb88..686c415f 100644 --- a/fhir-datastore-hapi-fhir/package-metadata.json +++ b/fhir-datastore-hapi-fhir/package-metadata.json @@ -32,6 +32,11 @@ "HF_PGPOOL_MEMORY_LIMIT": "500M", "HF_PGPOOL_CPU_RESERVE": "0.05", "HF_PGPOOL_MEMORY_RESERVE": "50M", - "HF_JAVA_OPTS": "-Xmx2g" + "HF_JAVA_OPTS": "-Xmx2g", + "HF_IMAGE_TAG": "v6.0.1-wget", + "HF_MAX_POOL_SIZE": "80", + "HF_PGPOOL_ENABLED": "true", + "HF_POSTGRES_FAILOVER": "automatic", + "HF_POSTGRES_DEGRADED_MONITORING_TIMEOUT": "5" } } diff --git a/fhir-datastore-hapi-fhir/swarm.sh b/fhir-datastore-hapi-fhir/swarm.sh index 23376542..eeecad52 100644 --- a/fhir-datastore-hapi-fhir/swarm.sh +++ b/fhir-datastore-hapi-fhir/swarm.sh @@ -51,6 +51,10 @@ function initialize_package() { ( docker::deploy_service "$STACK" "${COMPOSE_FILE_PATH}" "docker-compose-postgres.yml" "$postgres_cluster_compose_filename" "$postgres_dev_compose_filename" + if [[ "${CLUSTERED_MODE}" == "true" && "${HF_PGPOOL_ENABLED}" == "true" ]]; then + docker::deploy_service "$STACK" "${COMPOSE_FILE_PATH}" "docker-compose-pgpool.cluster.yml" + fi + docker::deploy_service "$STACK" "${COMPOSE_FILE_PATH}" "docker-compose.yml" "$hapi_fhir_dev_compose_filename" ) || {