From 1a4776636bf6b585d4a19a6721d9d57b02c88ca4 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 14 Jul 2020 19:51:44 +0200 Subject: Migrate from links to docker-compose networks This change migrates the docker-compose configuration from using links to using custom networks since links have been deprecated for some time. --- docker-compose.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'docker-compose.yml') diff --git a/docker-compose.yml b/docker-compose.yml index 6338e3d0..b2072be9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,8 +6,8 @@ services: restart: on-failure ports: - "8081:8081" - links: - - mongo + networks: + - backend depends_on: - mongo environment: @@ -56,6 +56,8 @@ services: - OPENDC_DB - OPENDC_DB_USERNAME - OPENDC_DB_PASSWORD + networks: + - backend # Comment out for public deployment ports: - 27017:27017 @@ -66,10 +68,10 @@ services: mongo-express: image: mongo-express restart: on-failure - links: - - mongo + networks: + - backend depends_on: - - mongo + - mongo ports: - 8082:8081 environment: @@ -79,3 +81,6 @@ services: volumes: mongo-volume: external: false + +networks: + backend: {} -- cgit v1.2.3 From 02997b2522b9c66072b16f1425c02e81e0085e3c Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 14 Jul 2020 21:10:56 +0200 Subject: Rename web-server to API This change renames the web-server component to API in order to be more descriptive of its role. The OpenDC API bridges between the frontend on one side and the database and simulator on the other side. --- docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'docker-compose.yml') diff --git a/docker-compose.yml b/docker-compose.yml index b2072be9..6dc01f67 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,8 @@ version: "3" services: - frontend: - build: ./ - image: frontend + api: + build: ./api + image: api restart: on-failure ports: - "8081:8081" -- cgit v1.2.3 From a196ba2c08bd16479134ab542f2560b75f19424f Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 15 Jul 2020 15:45:02 +0200 Subject: Make frontend independent of API This change makes the frontend independent of the API by removing the static file serving logic from the API server. Instead, we can serve the frontend as static HTML over CDNs. --- docker-compose.yml | 61 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 26 deletions(-) (limited to 'docker-compose.yml') diff --git a/docker-compose.yml b/docker-compose.yml index 6dc01f67..837f9019 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,21 @@ version: "3" services: + frontend: + build: + context: ./frontend + args: + - REACT_APP_OAUTH_CLIENT_ID=${OPENDC_OAUTH_CLIENT_ID} + image: frontend + restart: on-failure + ports: + - "8081:80" + networks: + - backend + api: build: ./api image: api restart: on-failure - ports: - - "8081:8081" networks: - backend depends_on: @@ -20,34 +30,33 @@ services: - OPENDC_DB_HOST=mongo - OPENDC_FLASK_SECRET - OPENDC_OAUTH_CLIENT_ID - - REACT_APP_OAUTH_CLIENT_ID=${OPENDC_OAUTH_CLIENT_ID} - OPENDC_ROOT_DIR - OPENDC_SERVER_BASE_URL -# TODO: Implement new database interaction on the simulator side -# simulator: -# build: -# context: ./opendc-simulator -# dockerfile: opendc-model-odc/setup/Dockerfile -# image: simulator -# restart: on-failure -# links: -# - mongo -# depends_on: -# - mongo -# environment: -# - PERSISTENCE_URL=jdbc:mysql://mariadb:3306/opendc -# - PERSISTENCE_USER=opendc -# - PERSISTENCE_PASSWORD=opendcpassword -# - COLLECT_MACHINE_STATES=ON -# - COLLECT_TASK_STATES=ON -# - COLLECT_STAGE_MEASUREMENTS=OFF -# - COLLECT_TASK_METRICS=OFF -# - COLLECT_JOB_METRICS=OFF + # TODO: Implement new database interaction on the simulator side + # simulator: + # build: + # context: ./opendc-simulator + # dockerfile: opendc-model-odc/setup/Dockerfile + # image: simulator + # restart: on-failure + # links: + # - mongo + # depends_on: + # - mongo + # environment: + # - PERSISTENCE_URL=jdbc:mysql://mariadb:3306/opendc + # - PERSISTENCE_USER=opendc + # - PERSISTENCE_PASSWORD=opendcpassword + # - COLLECT_MACHINE_STATES=ON + # - COLLECT_TASK_STATES=ON + # - COLLECT_STAGE_MEASUREMENTS=OFF + # - COLLECT_TASK_METRICS=OFF + # - COLLECT_JOB_METRICS=OFF mongo: build: - context: database + context: database restart: on-failure environment: - MONGO_INITDB_ROOT_USERNAME @@ -79,8 +88,8 @@ services: ME_CONFIG_MONGODB_ADMINPASSWORD: "${MONGO_INITDB_ROOT_PASSWORD}" volumes: - mongo-volume: - external: false + mongo-volume: + external: false networks: backend: {} -- cgit v1.2.3 From fc5405bab041545f4b7f04faa22fb21cc84f5c43 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 16 Jul 2020 22:30:57 +0200 Subject: Add docker-compose service for simulator This change re-adds the simulator service for the docker-compose configuration, such that it will listen for incoming jobs from the API. --- docker-compose.yml | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) (limited to 'docker-compose.yml') diff --git a/docker-compose.yml b/docker-compose.yml index 837f9019..5e45ea59 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: "3" +version: "3.8" services: frontend: build: @@ -33,26 +33,26 @@ services: - OPENDC_ROOT_DIR - OPENDC_SERVER_BASE_URL - # TODO: Implement new database interaction on the simulator side - # simulator: - # build: - # context: ./opendc-simulator - # dockerfile: opendc-model-odc/setup/Dockerfile - # image: simulator - # restart: on-failure - # links: - # - mongo - # depends_on: - # - mongo - # environment: - # - PERSISTENCE_URL=jdbc:mysql://mariadb:3306/opendc - # - PERSISTENCE_USER=opendc - # - PERSISTENCE_PASSWORD=opendcpassword - # - COLLECT_MACHINE_STATES=ON - # - COLLECT_TASK_STATES=ON - # - COLLECT_STAGE_MEASUREMENTS=OFF - # - COLLECT_TASK_METRICS=OFF - # - COLLECT_JOB_METRICS=OFF + simulator: + build: ./simulator + image: simulator + restart: on-failure + networks: + - backend + depends_on: + - mongo + volumes: + - type: bind + source: ./traces + target: /home/gradle/simulator/traces + - type: volume + source: results-volume + target: /home/gradle/simulator/results + environment: + - OPENDC_DB + - OPENDC_DB_USERNAME + - OPENDC_DB_PASSWORD + - OPENDC_DB_HOST=mongo mongo: build: @@ -90,6 +90,7 @@ services: volumes: mongo-volume: external: false + results-volume: networks: backend: {} -- cgit v1.2.3 From 0a895abfe307fbb6a28ceac6a07c5ac4863627fd Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 17 Jul 2020 17:25:45 +0200 Subject: Add data processing pipeline via Spark This change adds support for processing the experimental results by means of a Spark data processing pipeline. --- docker-compose.yml | 55 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 5 deletions(-) (limited to 'docker-compose.yml') diff --git a/docker-compose.yml b/docker-compose.yml index 5e45ea59..ecd2fceb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,20 +39,64 @@ services: restart: on-failure networks: - backend + - spark depends_on: - mongo + - spark-master + - spark-worker volumes: - type: bind source: ./traces target: /home/gradle/simulator/traces - type: volume - source: results-volume - target: /home/gradle/simulator/results + source: results-volume + target: /results + # Override to root as the simulator won't be able to access the volume otherwise + user: root environment: - OPENDC_DB - OPENDC_DB_USERNAME - OPENDC_DB_PASSWORD - OPENDC_DB_HOST=mongo + - OPENDC_OUTPUT=/results + - OPENDC_SPARK=spark://spark-master:7077 + + spark-master: + image: docker.io/bitnami/spark:3-debian-10 + networks: + - spark + environment: + - SPARK_MODE=master + - SPARK_RPC_AUTHENTICATION_ENABLED=no + - SPARK_RPC_ENCRYPTION_ENABLED=no + - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no + - SPARK_SSL_ENABLED=no + # Comment out for public deployment + ports: + - "8080:8080" + - "7077:7077" + + spark-worker: + image: docker.io/bitnami/spark:3-debian-10 + networks: + - spark + depends_on: + - spark-master + volumes: + - type: volume + source: results-volume + target: /results + environment: + - SPARK_MODE=worker + - SPARK_MASTER_URL=spark://spark-master:7077 + - SPARK_WORKER_MEMORY=2G + - SPARK_WORKER_CORES=2 + - SPARK_RPC_AUTHENTICATION_ENABLED=no + - SPARK_RPC_ENCRYPTION_ENABLED=no + - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no + - SPARK_SSL_ENABLED=no + ports: + - "7081:8081" mongo: build: @@ -69,10 +113,10 @@ services: - backend # Comment out for public deployment ports: - - 27017:27017 + - "27017:27017" # Uncomment for persistent deployment - #volumes: - # - mongo-volume:/data/db + volumes: + - mongo-volume:/data/db mongo-express: image: mongo-express @@ -94,3 +138,4 @@ volumes: networks: backend: {} + spark: {} -- cgit v1.2.3 From bde8b51fc40a02e6e8514ff428a748a133502c34 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Sat, 18 Jul 2020 16:47:32 +0200 Subject: Default to local Spark instance --- docker-compose.yml | 42 ------------------------------------------ 1 file changed, 42 deletions(-) (limited to 'docker-compose.yml') diff --git a/docker-compose.yml b/docker-compose.yml index ecd2fceb..d4954f58 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,11 +39,8 @@ services: restart: on-failure networks: - backend - - spark depends_on: - mongo - - spark-master - - spark-worker volumes: - type: bind source: ./traces @@ -59,44 +56,6 @@ services: - OPENDC_DB_PASSWORD - OPENDC_DB_HOST=mongo - OPENDC_OUTPUT=/results - - OPENDC_SPARK=spark://spark-master:7077 - - spark-master: - image: docker.io/bitnami/spark:3-debian-10 - networks: - - spark - environment: - - SPARK_MODE=master - - SPARK_RPC_AUTHENTICATION_ENABLED=no - - SPARK_RPC_ENCRYPTION_ENABLED=no - - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no - - SPARK_SSL_ENABLED=no - # Comment out for public deployment - ports: - - "8080:8080" - - "7077:7077" - - spark-worker: - image: docker.io/bitnami/spark:3-debian-10 - networks: - - spark - depends_on: - - spark-master - volumes: - - type: volume - source: results-volume - target: /results - environment: - - SPARK_MODE=worker - - SPARK_MASTER_URL=spark://spark-master:7077 - - SPARK_WORKER_MEMORY=2G - - SPARK_WORKER_CORES=2 - - SPARK_RPC_AUTHENTICATION_ENABLED=no - - SPARK_RPC_ENCRYPTION_ENABLED=no - - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no - - SPARK_SSL_ENABLED=no - ports: - - "7081:8081" mongo: build: @@ -138,4 +97,3 @@ volumes: networks: backend: {} - spark: {} -- cgit v1.2.3 From 4f5c582927574928c7f9df96ae52b448ba77028b Mon Sep 17 00:00:00 2001 From: Georgios Andreadis Date: Mon, 20 Jul 2020 11:28:45 +0200 Subject: Add port forwarding for frontend dev setups --- docker-compose.yml | 3 +++ 1 file changed, 3 insertions(+) (limited to 'docker-compose.yml') diff --git a/docker-compose.yml b/docker-compose.yml index d4954f58..4b9856e4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,9 @@ services: build: ./api image: api restart: on-failure + # Comment out these 2 lines for deployment + ports: + - "8081:8081" networks: - backend depends_on: -- cgit v1.2.3 From b27510cbe9de29edb0d1d8be58a49383b836ded5 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Mon, 20 Jul 2020 11:55:31 +0200 Subject: Add deployment information regarding traces --- docker-compose.yml | 4 ---- 1 file changed, 4 deletions(-) (limited to 'docker-compose.yml') diff --git a/docker-compose.yml b/docker-compose.yml index 4b9856e4..6b5c979c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -51,8 +51,6 @@ services: - type: volume source: results-volume target: /results - # Override to root as the simulator won't be able to access the volume otherwise - user: root environment: - OPENDC_DB - OPENDC_DB_USERNAME @@ -76,7 +74,6 @@ services: # Comment out for public deployment ports: - "27017:27017" - # Uncomment for persistent deployment volumes: - mongo-volume:/data/db @@ -95,7 +92,6 @@ services: volumes: mongo-volume: - external: false results-volume: networks: -- cgit v1.2.3 From 2d62b339a1ee532d85152e9e4ecfb11048af1923 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Mon, 20 Jul 2020 12:20:57 +0200 Subject: Fix docker-compose configuration --- docker-compose.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'docker-compose.yml') diff --git a/docker-compose.yml b/docker-compose.yml index 6b5c979c..c3e62317 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,10 +7,12 @@ services: - REACT_APP_OAUTH_CLIENT_ID=${OPENDC_OAUTH_CLIENT_ID} image: frontend restart: on-failure - ports: - - "8081:80" networks: - backend + depends_on: + - api + ports: + - "8081:80" api: build: ./api @@ -18,7 +20,7 @@ services: restart: on-failure # Comment out these 2 lines for deployment ports: - - "8081:8081" + - "8082:8081" networks: - backend depends_on: @@ -47,7 +49,7 @@ services: volumes: - type: bind source: ./traces - target: /home/gradle/simulator/traces + target: /app/traces - type: volume source: results-volume target: /results @@ -85,7 +87,7 @@ services: depends_on: - mongo ports: - - 8082:8081 + - "8083:8081" environment: ME_CONFIG_MONGODB_ADMINUSERNAME: "${MONGO_INITDB_ROOT_USERNAME}" ME_CONFIG_MONGODB_ADMINPASSWORD: "${MONGO_INITDB_ROOT_PASSWORD}" -- cgit v1.2.3