Merge pull request #233 from hubmapconsortium/1.12

PR to align with ingest 1.12 PROD release
hubmapconsortium · Jul 26, 2020 · 6f70a9a · 6f70a9a
2 parents be1df5d + faf1895
commit 6f70a9a
Show file tree

Hide file tree

Showing 33 changed files with 1,205 additions and 4,879 deletions.
diff --git a/README.md b/README.md
@@ -4,7 +4,7 @@ The HuBMAP Data Ingest Portal UI is a web application built with [React](https:/
 
 And the backend ingest API is located in this repository at `src/ingest-api`, this is a restful web service exposing calls needed for the ingest UI React application. The API is documented [here](http://smart-api.info/ui/2628cdd76b9994d89ad98ac92a82c18b).
 
-## Development and deployment environments
+## Localhost development and remote deployment environments
 
 We have the following 4 development and deployment environments:
 
@@ -18,9 +18,24 @@ We have the following 4 development and deployment environments:
 
 This option allows you to setup all the pieces in a containerized environment with docker and docker-compose. This requires to have the [HuBMAP Gateway](https://github.com/hubmapconsortium/gateway) running locally before starting building this docker compose project. Please follow the [instructions](https://github.com/hubmapconsortium/gateway#workflow-of-setting-up-multiple-hubmap-docker-compose-projects). It also requires the Gateway project to be configured accordingly.
 
-### Deployment on dev, test, stage, and prod
+### Remote deployment
 
-In localhost mode, all the docker containers are running on the same host machine. However, the ingest-api will be deployed on a separare host machine for dev, test, and prod mode due to different deployment requirements. 
+In localhost mode, all the docker containers are running on the same host machine. However, the ingest-api will be deployed on a separare host machine for dev, test, stage, and prod mode due to different deployment requirements. 
+
+There are a few configurable environment variables to keep in mind:
+
+- `COMMONS_BRANCH`: build argument only to be used during image creation. We can specify which [commons](https://github.com/hubmapconsortium/commons) branch to use during the image creation. Default to master branch if not set or null.
+- `HOST_UID`: the user id on the host machine to be mapped to the container. Default to 1000 if not set or null.
+- `HOST_GID`: the user's group id on the host machine to be mapped to the container. Default to 1000 if not set or null.
+
+We can set and verify the environment variable like below:
+
+````
+export COMMONS_BRANCH=devel
+echo $COMMONS_BRANCH
+````
+
+Note: Environment variables set like this are only stored temporally. When you exit the running instance of bash by exiting the terminal, they get discarded. So for rebuilding the docker image, we'll need to make sure to set the environment variables again if necessary.
 
 Before we go ahead to start building the docker image, we can do a check to see if the required configuration file is in place:
 
@@ -29,6 +44,12 @@ cd docker
 ./ingest-api-docker.sh dev check
 ````
 
+We can also validate and view the details of corresponding compose file:
+
+````
+./ingest-api-docker.sh dev config
+````
+
 Building the docker images and starting/stopping the contianers require to use docker daemon, you'll probably need to use `sudo` in the following steps. 
 
 To build the docker image of ingest-api:

diff --git a/docker/README.md b/docker/README.md
diff --git a/docker/docker-compose-ingest-api.dev.yml b/docker/docker-compose-ingest-api.dev.yml
@@ -4,9 +4,13 @@ version: "3.7"
 services:
 
  ingest-api:
- build: ./ingest-api-dev
+ build: 
+ context: ./ingest-api-dev
+ args:
+ # The commons github branch to be used during image build (default to master if not set or null)
+ - COMMONS_BRANCH=${COMMONS_BRANCH:-master}
  # Build the image with name and tag
- image: ingest-api:1.11
+ image: ingest-api:1.12
  hostname: ingest-api
  container_name: ingest-api
  init: true
@@ -20,9 +24,25 @@ services:
  # ingest-api on http 7777 to be used by airflow
  - "7777:7777"
  environment:
+ # This base user `hive` is used to run the application process within the container
+ # It's also used for ACL settings of ingest-api app.cfg
+ # Both UID and GID of user `hive` being used by the container
  # uid=68728(hive) gid=23629(hive) groups=23629(hive),1011(docker)
- - HOST_GID=${HOST_GID:-23629}
- - HOST_UID=${HOST_UID:-68728}
+ - BASE_USER_NAME=${BASE_USER_NAME:-hive}
+ - BASE_USER_UID=${BASE_USER_UID:-68728}
+ - BASE_USER_GID=${BASE_USER_GID:-23629}
+ # User `shirey`, group `hubmap`, and group `hubseq` are used for ACL settings of ingest-api app.cfg
+ # Only UID of user `shirey` being used by the container
+ # uid=50069(shirey) gid=23653(pitthive) groups=992(docker),23653(pitthive),23684(hivetest)
+ - ADMIN_FILE_USER_NAME=${ADMIN_FILE_USER_NAME:-shirey}
+ - ADMIN_FILE_USER_UID=${ADMIN_FILE_USER_UID:-50069}
+ # GID of group `hubmap` and GID of group `hubseq` being used by the container
+ # hubmap:*:24357:blood
+ - CONSORTIUM_FILE_GROUP_NAME=${CONSORTIUM_FILE_GROUP_NAME:-hubmap}
+ - CONSORTIUM_FILE_GROUP_GID=${CONSORTIUM_FILE_GROUP_GID:-24357}
+ # hubseq:*:24358:blood
+ - GENOMIC_DATA_FILE_GROUP_NAME=${GENOMIC_DATA_FILE_GROUP_NAME:-hubseq}
+ - GENOMIC_DATA_FILE_GROUP_GID=${GENOMIC_DATA_FILE_GROUP_GID:-24358}
  volumes:
  # Mount the source code to container
  - "../src/ingest-api:/usr/src/app/src"
@@ -38,6 +58,8 @@ services:
  - "/hive/hubmap-dev/scratch:/hubmap-scratch"
  # Mount Globus data
  - "/hive/hubmap-dev/lz:/hive/hubmap-dev/lz"
+ - "/hive/hubmap-dev/public:/hive/hubmap-dev/public"
+ - "/hive/hubmap-dev/consortium:/hive/hubmap-dev/consortium"
  # assets
  - "/hive/hubmap-dev/assets:/usr/src/assets"
  networks:

diff --git a/docker/docker-compose-ingest-api.localhost.yml b/docker/docker-compose-ingest-api.localhost.yml
@@ -4,9 +4,13 @@ version: "3.7"
 services:
 
  ingest-api:
- build: ./ingest-api-localhost
+ build: 
+ context: ./ingest-api-localhost
+ args:
+ # The commons github branch to be used during image build (default to master if not set or null)
+ - COMMONS_BRANCH=${COMMONS_BRANCH:-master}
  # Build the image with name and tag
- image: ingest-api:1.11
+ image: ingest-api:1.12
  hostname: ingest-api
  container_name: ingest-api
  environment:

diff --git a/docker/docker-compose-ingest-api.prod.yml b/docker/docker-compose-ingest-api.prod.yml
@@ -4,9 +4,13 @@ version: "3.7"
 services:
 
  ingest-api:
- build: ./ingest-api-prod
+ build: 
+ context: ./ingest-api-prod
+ args:
+ # The commons github branch to be used during image build (default to master if not set or null)
+ - COMMONS_BRANCH=${COMMONS_BRANCH:-master}
  # Build the image with name and tag
- image: ingest-api:1.11
+ image: ingest-api:1.12
  hostname: ingest-api
  container_name: ingest-api
  init: true
@@ -22,9 +26,25 @@ services:
  # ingest-api on http 7777 to be used by airflow
  - "7777:7777"
  environment:
+ # This base user `hive` is used to run the application process within the container
+ # It's also used for ACL settings of ingest-api app.cfg
+ # Both UID and GID of user `hive` being used by the container
  # uid=68728(hive) gid=23629(hive) groups=23629(hive),1011(docker)
- - HOST_GID=${HOST_GID:-23629}
- - HOST_UID=${HOST_UID:-68728}
+ - BASE_USER_NAME=${BASE_USER_NAME:-hive}
+ - BASE_USER_UID=${BASE_USER_UID:-68728}
+ - BASE_USER_GID=${BASE_USER_GID:-23629}
+ # User `shirey`, group `hubmap`, and group `hubseq` are used for ACL settings of ingest-api app.cfg
+ # Only UID of user `shirey` being used by the container
+ # uid=50069(shirey) gid=23653(pitthive) groups=992(docker),23653(pitthive),23684(hivetest)
+ - ADMIN_FILE_USER_NAME=${ADMIN_FILE_USER_NAME:-shirey}
+ - ADMIN_FILE_USER_UID=${ADMIN_FILE_USER_UID:-50069}
+ # GID of group `hubmap` and GID of group `hubseq` being used by the container
+ # hubmap:*:24357:blood
+ - CONSORTIUM_FILE_GROUP_NAME=${CONSORTIUM_FILE_GROUP_NAME:-hubmap}
+ - CONSORTIUM_FILE_GROUP_GID=${CONSORTIUM_FILE_GROUP_GID:-24357}
+ # hubseq:*:24358:blood
+ - GENOMIC_DATA_FILE_GROUP_NAME=${GENOMIC_DATA_FILE_GROUP_NAME:-hubseq}
+ - GENOMIC_DATA_FILE_GROUP_GID=${GENOMIC_DATA_FILE_GROUP_GID:-24358}
  volumes:
  # Mount the app config to container in order to keep it outside of the image
  - "../src/ingest-api/instance:/usr/src/app/src/instance"
@@ -42,6 +62,8 @@ services:
  - "/hive/users/hive/hubmap/hivevm193-prod/venv/lib/python3.6/site-packages/airflow/www:/airflow-static"
  # Mount Globus data
  - "/hive/hubmap/lz:/hive/hubmap/lz"
+ - "/hive/hubmap/public:/hive/hubmap/public"
+ - "/hive/hubmap/consortium:/hive/hubmap/consortium"
  # assets
  - "/hive/hubmap/assets:/usr/src/assets"
  networks:

diff --git a/docker/docker-compose-ingest-api.stage.yml b/docker/docker-compose-ingest-api.stage.yml
@@ -4,9 +4,13 @@ version: "3.7"
 services:
 
  ingest-api:
- build: ./ingest-api-stage
+ build: 
+ context: ./ingest-api-stage
+ args:
+ # The commons github branch to be used during image build (default to master if not set or null)
+ - COMMONS_BRANCH=${COMMONS_BRANCH:-master}
  # Build the image with name and tag
- image: ingest-api:1.11
+ image: ingest-api:1.12
  hostname: ingest-api
  container_name: ingest-api
  init: true
@@ -22,9 +26,25 @@ services:
  # ingest-api on http 7777 to be used by airflow
  - "7777:7777"
  environment:
+ # This base user `hive` is used to run the application process within the container
+ # It's also used for ACL settings of ingest-api app.cfg
+ # Both UID and GID of user `hive` being used by the container
  # uid=68728(hive) gid=23629(hive) groups=23629(hive),1011(docker)
- - HOST_GID=${HOST_GID:-23629}
- - HOST_UID=${HOST_UID:-68728}
+ - BASE_USER_NAME=${BASE_USER_NAME:-hive}
+ - BASE_USER_UID=${BASE_USER_UID:-68728}
+ - BASE_USER_GID=${BASE_USER_GID:-23629}
+ # User `shirey`, group `hubmap`, and group `hubseq` are used for ACL settings of ingest-api app.cfg
+ # Only UID of user `shirey` being used by the container
+ # uid=50069(shirey) gid=23653(pitthive) groups=992(docker),23653(pitthive),23684(hivetest)
+ - ADMIN_FILE_USER_NAME=${ADMIN_FILE_USER_NAME:-shirey}
+ - ADMIN_FILE_USER_UID=${ADMIN_FILE_USER_UID:-50069}
+ # GID of group `hubmap` and GID of group `hubseq` being used by the container
+ # hubmap:*:24357:blood
+ - CONSORTIUM_FILE_GROUP_NAME=${CONSORTIUM_FILE_GROUP_NAME:-hubmap}
+ - CONSORTIUM_FILE_GROUP_GID=${CONSORTIUM_FILE_GROUP_GID:-24357}
+ # hubseq:*:24358:blood
+ - GENOMIC_DATA_FILE_GROUP_NAME=${GENOMIC_DATA_FILE_GROUP_NAME:-hubseq}
+ - GENOMIC_DATA_FILE_GROUP_GID=${GENOMIC_DATA_FILE_GROUP_GID:-24358}
  volumes:
  # Mount the app config to container in order to keep it outside of the image
  - "../src/ingest-api/instance:/usr/src/app/src/instance"
@@ -42,6 +62,8 @@ services:
  - "/hive/users/hive/hubmap/hivevm195-stage/venv/lib/python3.6/site-packages/airflow/www:/airflow-static"
  # Mount Globus data
  - "/hive/hubmap-stage/protected:/hive/hubmap-stage/protected"
+ - "/hive/hubmap-stage/public:/hive/hubmap-stage/public"
+ - "/hive/hubmap-stage/consortium:/hive/hubmap-stage/consortium"
  # assets
  - "/hive/hubmap-stage/assets:/usr/src/assets"
  networks:

diff --git a/docker/docker-compose-ingest-api.test.yml b/docker/docker-compose-ingest-api.test.yml
@@ -4,9 +4,13 @@ version: "3.7"
 services:
 
  ingest-api:
- build: ./ingest-api-test
+ build: 
+ context: ./ingest-api-test
+ args:
+ # The commons github branch to be used during image build (default to master if not set or null)
+ - COMMONS_BRANCH=${COMMONS_BRANCH:-master}
  # Build the image with name and tag
- image: ingest-api:1.11
+ image: ingest-api:1.12
  hostname: ingest-api
  container_name: ingest-api
  init: true
@@ -23,9 +27,25 @@ services:
  # ingest-api on http 7777 to be used by airflow
  - "7777:7777"
  environment:
+ # This base user `hive` is used to run the application process within the container
+ # It's also used for ACL settings of ingest-api app.cfg
+ # Both UID and GID of user `hive` being used by the container
  # uid=68728(hive) gid=23629(hive) groups=23629(hive),1011(docker)
- - HOST_GID=${HOST_GID:-23629}
- - HOST_UID=${HOST_UID:-68728}
+ - BASE_USER_NAME=${BASE_USER_NAME:-hive}
+ - BASE_USER_UID=${BASE_USER_UID:-68728}
+ - BASE_USER_GID=${BASE_USER_GID:-23629}
+ # User `shirey`, group `hubmap`, and group `hubseq` are used for ACL settings of ingest-api app.cfg
+ # Only UID of user `shirey` being used by the container
+ # uid=50069(shirey) gid=23653(pitthive) groups=992(docker),23653(pitthive),23684(hivetest)
+ - ADMIN_FILE_USER_NAME=${ADMIN_FILE_USER_NAME:-shirey}
+ - ADMIN_FILE_USER_UID=${ADMIN_FILE_USER_UID:-50069}
+ # GID of group `hubmap` and GID of group `hubseq` being used by the container
+ # hubmap:*:24357:blood
+ - CONSORTIUM_FILE_GROUP_NAME=${CONSORTIUM_FILE_GROUP_NAME:-hubmap}
+ - CONSORTIUM_FILE_GROUP_GID=${CONSORTIUM_FILE_GROUP_GID:-24357}
+ # hubseq:*:24358:blood
+ - GENOMIC_DATA_FILE_GROUP_NAME=${GENOMIC_DATA_FILE_GROUP_NAME:-hubseq}
+ - GENOMIC_DATA_FILE_GROUP_GID=${GENOMIC_DATA_FILE_GROUP_GID:-24358}
  volumes:
  # Mount the app config to container in order to keep it outside of the image
  - "../src/ingest-api/instance:/usr/src/app/src/instance"
@@ -45,6 +65,8 @@ services:
  - "/hive/users/hive/hubmap/hivevm192-test/venv/lib/python3.6/site-packages/flower:/flower-static"
  # Mount Globus data
  - "/hive/hubmap-test/lz:/hive/hubmap-test/lz"
+ - "/hive/hubmap-test/public:/hive/hubmap-test/public"
+ - "/hive/hubmap-test/consortium:/hive/hubmap-test/consortium"
  # assets
  - "/hive/hubmap-test/assets:/usr/src/assets"
  networks:

diff --git a/docker/docker-compose-ingest-ui.dev.yml b/docker/docker-compose-ingest-ui.dev.yml
@@ -6,7 +6,7 @@ services:
  ingest-ui:
  build: ./ingest-ui
  # Build the image with name and tag
- image: ingest-ui:1.11
+ image: ingest-ui:1.12
  hostname: ingest-ui
  container_name: ingest-ui
  environment:

diff --git a/docker/docker-compose-ingest-ui.localhost.yml b/docker/docker-compose-ingest-ui.localhost.yml
@@ -6,7 +6,7 @@ services:
  ingest-ui:
  build: ./ingest-ui
  # Build the image with name and tag
- image: ingest-ui:1.11
+ image: ingest-ui:1.12
  hostname: ingest-ui
  container_name: ingest-ui
  environment:

diff --git a/docker/docker-compose-ingest-ui.prod.yml b/docker/docker-compose-ingest-ui.prod.yml
@@ -6,7 +6,7 @@ services:
  ingest-ui:
  build: ./ingest-ui
  # Build the image with name and tag
- image: ingest-ui:1.11
+ image: ingest-ui:1.12
  hostname: ingest-ui
  container_name: ingest-ui
  environment:

diff --git a/docker/docker-compose-ingest-ui.stage.yml b/docker/docker-compose-ingest-ui.stage.yml
@@ -6,7 +6,7 @@ services:
  ingest-ui:
  build: ./ingest-ui
  # Build the image with name and tag
- image: ingest-ui:1.11
+ image: ingest-ui:1.12
  hostname: ingest-ui
  container_name: ingest-ui
  environment:

diff --git a/docker/docker-compose-ingest-ui.test.yml b/docker/docker-compose-ingest-ui.test.yml
@@ -6,7 +6,7 @@ services:
  ingest-ui:
  build: ./ingest-ui
  # Build the image with name and tag
- image: ingest-ui:1.11
+ image: ingest-ui:1.12
  hostname: ingest-ui
  container_name: ingest-ui
  environment:

diff --git a/docker/ingest-api-dev/Dockerfile b/docker/ingest-api-dev/Dockerfile
@@ -3,6 +3,10 @@ FROM hubmap/api-base-image:latest
 
 LABEL description="HuBMAP Ingest API Service"
 
+# The commons branch to be used in requirements.txt during image build
+# Default is master branch specified in docker-compose.yml if not set before the build
+ARG COMMONS_BRANCH
+
 # Change to directory that contains the Dockerfile
 WORKDIR /usr/src/app