Merge pull request #213 from jembi/logging

Logging storage service for platform monitoring
jembi · Feb 24, 2023 · cc42a24 · cc42a24
2 parents d0e0e7f + 42226fb
commit cc42a24
Show file tree

Hide file tree

Showing 21 changed files with 1,363 additions and 60 deletions.
diff --git a/.env.cluster b/.env.cluster
@@ -28,7 +28,7 @@ REPMGR_PARTNER_NODES=santempi-psql-1,santempi-psql-2,santempi-psql-3
 # Reverse Proxy - Nginx
 REVERSE_PROXY_INSTANCES=3
 DOMAIN_NAME=domain
-SUBDOMAINS=openhimcomms.domain,openhimcore.domain,openhimconsole.domain,kibana.domain,reports.domain,santewww.domain,santempi.domain,superset.domain,keycloak.domain,grafana.domain
+SUBDOMAINS=openhimcomms.domain,openhimcore.domain,openhimconsole.domain,kibana.domain,reports.domain,santewww.domain,santempi.domain,superset.domain,keycloak.domain,grafana.domain,minio.domain
 STAGING=true
 INSECURE=false
 

diff --git a/.github/workflows/run-tests.sh b/.github/workflows/run-tests.sh
@@ -49,6 +49,8 @@ else
  DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":hapi
  elif [[ $folder_name == *"santempi"* ]]; then
  DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":sante
+ elif [[ $folder_name == *"monitoring"* ]]; then
+ DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":monitoring
  fi
  done
 fi
diff --git a/identity-access-manager-keycloak/docker-compose.yml b/identity-access-manager-keycloak/docker-compose.yml
@@ -2,8 +2,14 @@ version: '3.9'
 
 services:
  identity-access-manager-keycloak:
- image: keycloak/keycloak
- command: ["start", "--proxy=edge", "--hostname-url=${KC_FRONTEND_URL}", "--import-realm"]
+ image: keycloak/keycloak:20.0
+ command:
+ [
+ "start",
+ "--proxy=edge",
+ "--hostname-url=${KC_FRONTEND_URL}",
+ "--import-realm"
+ ]
  hostname: identity-access-manager-keycloak
  healthcheck:
  test: curl --fail http://localhost:8080/health/ready || exit 1

diff --git a/infrastructure/ansible/roles/docker/files/docker-daemon.json b/infrastructure/ansible/roles/docker/files/docker-daemon.json
@@ -2,6 +2,7 @@
  "log-driver": "json-file",
  "log-opts": {
  "max-size": "10m",
- "max-file": "3"
+ "max-file": "3",
+ "labels-regex": "^.+"
  }
 }
diff --git a/monitoring/docker-compose.cluster.yml b/monitoring/docker-compose.cluster.yml
@@ -29,6 +29,96 @@ services:
  - '--web.console.templates=/etc/prometheus/consoles'
  - '--web.enable-lifecycle'
 
+ minio-01:
+ deploy:
+ placement:
+ constraints:
+ - "node.labels.name==node-1"
+
+ minio-02:
+ image: quay.io/minio/minio:RELEASE.2022-10-24T18-35-07Z
+ entrypoint: sh
+ command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}'
+ environment:
+ MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
+ MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
+ healthcheck:
+ test:
+ [
+ "CMD",
+ "curl",
+ "-f",
+ "http://localhost:9000/minio/health/live"
+ ]
+ interval: 30s
+ timeout: 20s
+ retries: 3
+ hostname: minio-02
+ volumes:
+ - minio-02-data1:/data1
+ - minio-02-data2:/data2
+ deploy:
+ placement:
+ constraints:
+ - "node.labels.name==node-1"
+ replicas: 1
+
+ minio-03:
+ image: quay.io/minio/minio:RELEASE.2022-10-24T18-35-07Z
+ entrypoint: sh
+ command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}'
+ environment:
+ MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
+ MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
+ healthcheck:
+ test:
+ [
+ "CMD",
+ "curl",
+ "-f",
+ "http://localhost:9000/minio/health/live"
+ ]
+ interval: 30s
+ timeout: 20s
+ retries: 3
+ hostname: minio-03
+ volumes:
+ - minio-03-data1:/data1
+ - minio-03-data2:/data2
+ deploy:
+ placement:
+ constraints:
+ - "node.labels.name==node-2"
+ replicas: 1
+
+ minio-04:
+ image: quay.io/minio/minio:RELEASE.2022-10-24T18-35-07Z
+ entrypoint: sh
+ command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}'
+ environment:
+ MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
+ MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
+ healthcheck:
+ test:
+ [
+ "CMD",
+ "curl",
+ "-f",
+ "http://localhost:9000/minio/health/live"
+ ]
+ interval: 30s
+ timeout: 20s
+ retries: 3
+ hostname: minio-04
+ volumes:
+ - minio-04-data1:/data1
+ - minio-04-data2:/data2
+ deploy:
+ placement:
+ constraints:
+ - "node.labels.name==node-3"
+ replicas: 1
+
 configs:
  prometheus.yml:
  file: ./prometheus/prometheus.yml
@@ -38,3 +128,9 @@ configs:
 
 volumes:
  prometheus_data_backup:
+ minio-02-data1:
+ minio-02-data2:
+ minio-03-data1:
+ minio-03-data2:
+ minio-04-data1:
+ minio-04-data2:
diff --git a/monitoring/docker-compose.dev.yml b/monitoring/docker-compose.dev.yml
@@ -12,3 +12,15 @@ services:
  - target: 9090
  published: 9090
  mode: host
+
+ loki:
+ ports:
+ - target: 3100
+ published: 3100
+ mode: host
+
+ minio-01:
+ ports:
+ - target: 9001
+ published: 9005
+ mode: host
diff --git a/monitoring/docker-compose.yml b/monitoring/docker-compose.yml
@@ -4,7 +4,7 @@ services:
  grafana:
  image: grafana/grafana-oss:9.2.3
  volumes:
- - grafana_data:/var/lib/grafana
+ - grafana-data:/var/lib/grafana
  environment:
  GF_SECURITY_ADMIN_USER: ${GF_SECURITY_ADMIN_USER}
  GF_SECURITY_ADMIN_PASSWORD: ${GF_SECURITY_ADMIN_PASSWORD}
@@ -49,12 +49,14 @@ services:
  source: kminion-groups_rev1.json
  - target: /etc/grafana/provisioning/dashboards/applications/kminion-topic_rev1.json
  source: kminion-topic_rev1.json
+ - target: /etc/grafana/provisioning/dashboards/containers/logging-universal-dashboard_rev1.json
+ source: logging-universal-dashboard_rev1.json
 
  prometheus:
  image: prom/prometheus:v2.38.0
  user: root
  volumes:
- - prometheus_data:/prometheus
+ - prometheus-data:/prometheus
  - /var/run/docker.sock:/var/run/docker.sock:ro
  configs:
  - target: /etc/prometheus/prometheus.yml
@@ -88,11 +90,58 @@ services:
  deploy:
  mode: global
 
- prometheus-kafka-adapter:
- image: telefonica/prometheus-kafka-adapter:1.8.0
+ loki:
+ image: grafana/loki:2.6.1
+ volumes:
+ - loki-data:/tmp/loki
+ environment:
+ MO_SECURITY_ADMIN_USER: ${MO_SECURITY_ADMIN_USER}
+ MO_SECURITY_ADMIN_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
+ configs:
+ - target: /etc/loki/loki-config.yml
+ source: loki-config.yml
+ command: -config.file=/etc/loki/loki-config.yml -config.expand-env=true
+ deploy:
+ labels:
+ - prometheus-job-service=loki
+ - prometheus-address=loki:3100
+
+ promtail:
+ image: grafana/promtail:2.6.1
+ volumes:
+ - /var/lib/docker/containers:/host/containers
+ - /var/log:/var/log:ro
+ configs:
+ - target: /etc/promtail/promtail-config.yml
+ source: promtail-config.yml
+ command: -config.file=/etc/promtail/promtail-config.yml
+ deploy:
+ mode: global
+
+ minio-01:
+ image: quay.io/minio/minio:RELEASE.2022-10-24T18-35-07Z
+ entrypoint: sh
+ command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...${NUM_MINIO_SERVERS}}/data{1...2}'
  environment:
- - KAFKA_BROKER_LIST=kafka:9092
- - KAFKA_COMPRESSION=gzip
+ MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
+ MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
+ healthcheck:
+ test:
+ [
+ "CMD",
+ "curl",
+ "-f",
+ "http://localhost:9000/minio/health/live"
+ ]
+ interval: 30s
+ timeout: 20s
+ retries: 3
+ hostname: minio-01
+ volumes:
+ - minio-01-data1:/data1
+ - minio-01-data2:/data2
+ deploy:
+ replicas: 1
 
 configs:
  grafana.ini:
@@ -135,12 +184,30 @@ configs:
  name: kminion-topic_rev1.json-${kminion_topic_rev1_json_DIGEST:?err}
  labels:
  name: grafana
+ logging-universal-dashboard_rev1.json:
+ file: ./grafana/dashboards/containers/logging-universal-dashboard_rev1.json
+ name: logging-universal-dashboard_rev1.json-${logging_universal_dashboard_rev1_json_DIGEST:?err}
+ labels:
+ name: grafana
  prometheus.yml:
  file: ./prometheus/prometheus.yml
  name: prometheus.yml-${prometheus_yml_DIGEST:?err}
  labels:
  name: prometheus
+ loki-config.yml:
+ file: ./loki/loki-config.yml
+ name: loki-config.yml-${loki_config_yml_DIGEST:?err}
+ labels:
+ name: loki
+ promtail-config.yml:
+ file: ./promtail/promtail-config.yml
+ name: promtail-config.yml-${promtail_config_yml_DIGEST:?err}
+ labels:
+ name: promtail
 
 volumes:
- prometheus_data:
- grafana_data:
+ prometheus-data:
+ grafana-data:
+ loki-data:
+ minio-01-data1:
+ minio-01-data2: