diff options
50 files changed, 1863 insertions, 442 deletions
@@ -1,4 +1,4 @@ -FROM openjdk:15-slim +FROM openjdk:16-slim MAINTAINER OpenDC Maintainers <opendc@atlarge-research.com> # Obtain (cache) Gradle wrapper @@ -11,7 +11,7 @@ RUN ./gradlew --version COPY ./ /app/ RUN ./gradlew --no-daemon :installDist -FROM openjdk:15-slim +FROM openjdk:16-slim COPY --from=0 /app/build/install /opt/ COPY --from=0 /app/traces /opt/opendc/traces WORKDIR /opt/opendc diff --git a/docker-compose.yml b/docker-compose.yml index b6246613..b5d8cb98 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,10 +47,11 @@ services: source: ./traces target: /app/traces environment: - - OPENDC_DB - - OPENDC_DB_USERNAME - - OPENDC_DB_PASSWORD - - OPENDC_DB_HOST=mongo + - OPENDC_API_URL=${OPENDC_API_BASE_URL} + - AUTH0_DOMAIN=${OPENDC_AUTH0_DOMAIN} + - AUTH0_AUDIENCE=${OPENDC_AUTH0_AUDIENCE} + - AUTH0_CLIENT_ID=${OPENDC_AUTH0_CLIENT_ID_RUNNER} + - AUTH0_CLIENT_SECRET=${OPENDC_AUTH0_CLIENT_SECRET_RUNNER} - SENTRY_DSN=${OPENDC_SIMULATOR_SENTRY_DSN} - SENTRY_ENVIRONMENT diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 5c0f9ce3..8553787d 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -6,6 +6,8 @@ log4j = "2.14.1" opentelemetry-main = "1.2.0" opentelemetry-metrics = "1.2.0-alpha" hadoop = "3.3.0" +ktor = "1.6.1" +jackson = "2.12.3" [libraries] kotlinx-coroutines = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version = "1.5.0" } @@ -35,7 +37,8 @@ clikt = { module = "com.github.ajalt.clikt:clikt", version = "3.2.0" } progressbar = { module = "me.tongfei:progressbar", version = "0.9.0" } # Format -jackson-module-kotlin = { module = "com.fasterxml.jackson.module:jackson-module-kotlin", version = "2.12.3" } +jackson-module-kotlin = { module = "com.fasterxml.jackson.module:jackson-module-kotlin", version.ref = "jackson" } +jackson-datatype-jsr310 = { module = "com.fasterxml.jackson.datatype:jackson-datatype-jsr310", version.ref = "jackson" } parquet = { module = "org.apache.parquet:parquet-avro", version = "1.12.0" } yaml = { module = "org.yaml:snakeyaml", version = "1.28" } config = { module = "com.typesafe:config", version = "1.4.1" } @@ -44,7 +47,10 @@ config = { module = "com.typesafe:config", version = "1.4.1" } kotlinx-benchmark-runtime-jvm = { module = "org.jetbrains.kotlinx:kotlinx-benchmark-runtime-jvm", version = "0.3.1" } # Other -mongodb = { module = "org.mongodb:mongodb-driver-sync", version = "4.2.3" } classgraph = { module = "io.github.classgraph:classgraph", version = "4.8.108" } hadoop-common = { module = "org.apache.hadoop:hadoop-common", version.ref = "hadoop" } hadoop-mapreduce-client-core = { module = "org.apache.hadoop:hadoop-mapreduce-client-core", version.ref = "hadoop" } +ktor-client-cio = { module = "io.ktor:ktor-client-cio", version.ref = "ktor" } +ktor-client-auth = { module = "io.ktor:ktor-client-auth", version.ref = "ktor" } +ktor-client-jackson = { module = "io.ktor:ktor-client-jackson", version.ref = "ktor" } +ktor-client-mock = { module = "io.ktor:ktor-client-mock", version.ref = "ktor" } diff --git a/opendc-web/opendc-web-api/app.py b/opendc-web/opendc-web-api/app.py index 5916046b..36c80b7a 100755 --- a/opendc-web/opendc-web-api/app.py +++ b/opendc-web/opendc-web-api/app.py @@ -10,6 +10,7 @@ from flask_restful import Api from flask_swagger_ui import get_swaggerui_blueprint from marshmallow import ValidationError +from opendc.api.jobs import JobList, Job from opendc.api.portfolios import Portfolio, PortfolioScenarios from opendc.api.prefabs import Prefab, PrefabList from opendc.api.projects import ProjectList, Project, ProjectTopologies, ProjectPortfolios @@ -60,6 +61,8 @@ def setup_api(app): api.add_resource(TraceList, '/traces/') api.add_resource(Trace, '/traces/<string:trace_id>') api.add_resource(SchedulerList, '/schedulers/') + api.add_resource(JobList, '/jobs/') + api.add_resource(Job, '/jobs/<string:job_id>') @app.errorhandler(AuthError) def handle_auth_error(ex): @@ -89,6 +92,7 @@ def setup_swagger(app): }, oauth_config={ 'clientId': os.environ.get("AUTH0_DOCS_CLIENT_ID", ""), + 'additionalQueryStringParams': {'audience': os.environ.get("AUTH0_AUDIENCE", "https://api.opendc.org/v2/")}, } ) app.register_blueprint(swaggerui_blueprint) diff --git a/opendc-web/opendc-web-api/conftest.py b/opendc-web/opendc-web-api/conftest.py index 430262f1..958a5894 100644 --- a/opendc-web/opendc-web-api/conftest.py +++ b/opendc-web/opendc-web-api/conftest.py @@ -8,7 +8,7 @@ from flask import _request_ctx_stack, g from opendc.database import Database -def decorator(f): +def requires_auth_mock(f): @wraps(f) def decorated_function(*args, **kwargs): _request_ctx_stack.top.current_user = {'sub': 'test'} @@ -16,13 +16,24 @@ def decorator(f): return decorated_function +def requires_scope_mock(required_scope): + def decorator(f): + @wraps(f) + def decorated_function(*args, **kwargs): + return f(*args, **kwargs) + return decorated_function + return decorator + + @pytest.fixture def client(): """Returns a Flask API client to interact with.""" # Disable authorization for test API endpoints from opendc import exts - exts.requires_auth = decorator + exts.requires_auth = requires_auth_mock + exts.requires_scope = requires_scope_mock + exts.has_scope = lambda x: False from app import create_app diff --git a/opendc-web/opendc-web-api/opendc/api/jobs.py b/opendc-web/opendc-web-api/opendc/api/jobs.py new file mode 100644 index 00000000..6fb0522b --- /dev/null +++ b/opendc-web/opendc-web-api/opendc/api/jobs.py @@ -0,0 +1,105 @@ +# Copyright (c) 2021 AtLarge Research +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +from flask import request +from flask_restful import Resource +from marshmallow import fields, Schema, validate +from werkzeug.exceptions import BadRequest, Conflict + +from opendc.exts import requires_auth, requires_scope +from opendc.models.scenario import Scenario + + +def convert_to_job(scenario): + """Convert a scenario to a job. + """ + return JobSchema().dump({ + '_id': scenario['_id'], + 'scenarioId': scenario['_id'], + 'state': scenario['simulation']['state'], + 'heartbeat': scenario['simulation'].get('heartbeat', None), + 'results': scenario.get('results', {}) + }) + + +class JobSchema(Schema): + """ + Schema representing a simulation job. + """ + _id = fields.String(dump_only=True) + scenarioId = fields.String(dump_only=True) + state = fields.String(required=True, + validate=validate.OneOf(["QUEUED", "CLAIMED", "RUNNING", "FINISHED", "FAILED"])) + heartbeat = fields.DateTime() + results = fields.Dict() + + +class JobList(Resource): + """ + Resource representing the list of available jobs. + """ + method_decorators = [requires_auth, requires_scope('runner')] + + def get(self): + """Get all available jobs.""" + jobs = Scenario.get_jobs() + data = list(map(convert_to_job, jobs.obj)) + return {'data': data} + + +class Job(Resource): + """ + Resource representing a single job. + """ + method_decorators = [requires_auth, requires_scope('runner')] + + def get(self, job_id): + """Get the details of a single job.""" + job = Scenario.from_id(job_id) + job.check_exists() + data = convert_to_job(job.obj) + return {'data': data} + + def post(self, job_id): + """Update the details of a single job.""" + action = JobSchema(only=('state', 'results')).load(request.json) + + job = Scenario.from_id(job_id) + job.check_exists() + + old_state = job.obj['simulation']['state'] + new_state = action['state'] + + if old_state == new_state: + data = job.update_state(new_state) + elif (old_state, new_state) == ('QUEUED', 'CLAIMED'): + data = job.update_state('CLAIMED') + elif (old_state, new_state) == ('CLAIMED', 'RUNNING'): + data = job.update_state('RUNNING') + elif (old_state, new_state) == ('RUNNING', 'FINISHED'): + data = job.update_state('FINISHED', results=action.get('results', None)) + elif old_state in ('CLAIMED', 'RUNNING') and new_state == 'FAILED': + data = job.update_state('FAILED') + else: + raise BadRequest('Invalid state transition') + + if not data: + raise Conflict('State conflict') + + return {'data': convert_to_job(data)} diff --git a/opendc-web/opendc-web-api/opendc/api/portfolios.py b/opendc-web/opendc-web-api/opendc/api/portfolios.py index b07e9da5..eea82289 100644 --- a/opendc-web/opendc-web-api/opendc/api/portfolios.py +++ b/opendc-web/opendc-web-api/opendc/api/portfolios.py @@ -22,7 +22,7 @@ from flask import request from flask_restful import Resource from marshmallow import Schema, fields -from opendc.exts import requires_auth, current_user +from opendc.exts import requires_auth, current_user, has_scope from opendc.models.portfolio import Portfolio as PortfolioModel, PortfolioSchema from opendc.models.project import Project from opendc.models.scenario import ScenarioSchema, Scenario @@ -42,9 +42,12 @@ class Portfolio(Resource): portfolio = PortfolioModel.from_id(portfolio_id) portfolio.check_exists() - portfolio.check_user_access(current_user['sub'], False) - data = portfolio.obj + # Users with scope runner can access all portfolios + if not has_scope('runner'): + portfolio.check_user_access(current_user['sub'], False) + + data = PortfolioSchema().dump(portfolio.obj) return {'data': data} def put(self, portfolio_id): @@ -63,7 +66,7 @@ class Portfolio(Resource): portfolio.set_property('targets.repeatsPerScenario', result['portfolio']['targets']['repeatsPerScenario']) portfolio.update() - data = portfolio.obj + data = PortfolioSchema().dump(portfolio.obj) return {'data': data} def delete(self, portfolio_id): @@ -84,7 +87,8 @@ class Portfolio(Resource): project.update() old_object = portfolio.delete() - return {'data': old_object} + data = PortfolioSchema().dump(old_object) + return {'data': data} class PutSchema(Schema): """ @@ -125,7 +129,7 @@ class PortfolioScenarios(Resource): portfolio.obj['scenarioIds'].append(scenario.get_id()) portfolio.update() - data = scenario.obj + data = ScenarioSchema().dump(scenario.obj) return {'data': data} class PostSchema(Schema): diff --git a/opendc-web/opendc-web-api/opendc/api/prefabs.py b/opendc-web/opendc-web-api/opendc/api/prefabs.py index 7bb17e7d..730546ba 100644 --- a/opendc-web/opendc-web-api/opendc/api/prefabs.py +++ b/opendc-web/opendc-web-api/opendc/api/prefabs.py @@ -24,7 +24,6 @@ from flask_restful import Resource from marshmallow import Schema, fields from opendc.models.prefab import Prefab as PrefabModel, PrefabSchema -from opendc.database import Database from opendc.exts import current_user, requires_auth, db @@ -56,14 +55,15 @@ class PrefabList(Resource): result = schema.load(request.json) prefab = PrefabModel(result['prefab']) - prefab.set_property('datetimeCreated', Database.datetime_to_string(datetime.now())) - prefab.set_property('datetimeLastEdited', Database.datetime_to_string(datetime.now())) + prefab.set_property('datetimeCreated', datetime.now()) + prefab.set_property('datetimeLastEdited', datetime.now()) user_id = current_user['sub'] prefab.set_property('authorId', user_id) prefab.insert() - return {'data': prefab.obj} + data = PrefabSchema().dump(prefab.obj) + return {'data': data} class PostSchema(Schema): """ @@ -83,7 +83,8 @@ class Prefab(Resource): prefab = PrefabModel.from_id(prefab_id) prefab.check_exists() prefab.check_user_access(current_user['sub']) - return {'data': prefab.obj} + data = PrefabSchema().dump(prefab.obj) + return {'data': data} def put(self, prefab_id): """Update a prefab's name and/or contents.""" @@ -97,10 +98,11 @@ class Prefab(Resource): prefab.set_property('name', result['prefab']['name']) prefab.set_property('rack', result['prefab']['rack']) - prefab.set_property('datetime_last_edited', Database.datetime_to_string(datetime.now())) + prefab.set_property('datetimeLastEdited', datetime.now()) prefab.update() - return {'data': prefab.obj} + data = PrefabSchema().dump(prefab.obj) + return {'data': data} def delete(self, prefab_id): """Delete this Prefab.""" @@ -111,7 +113,8 @@ class Prefab(Resource): old_object = prefab.delete() - return {'data': old_object} + data = PrefabSchema().dump(old_object) + return {'data': data} class PutSchema(Schema): """ diff --git a/opendc-web/opendc-web-api/opendc/api/projects.py b/opendc-web/opendc-web-api/opendc/api/projects.py index 8c44b680..05f02a84 100644 --- a/opendc-web/opendc-web-api/opendc/api/projects.py +++ b/opendc-web/opendc-web-api/opendc/api/projects.py @@ -27,7 +27,6 @@ from opendc.models.portfolio import Portfolio, PortfolioSchema from opendc.models.topology import Topology, TopologySchema from opendc.models.project import Project as ProjectModel, ProjectSchema from opendc.exts import current_user, requires_auth -from opendc.database import Database class ProjectList(Resource): @@ -40,7 +39,8 @@ class ProjectList(Resource): """Get the authorized projects of the user""" user_id = current_user['sub'] projects = ProjectModel.get_for_user(user_id) - return {'data': projects} + data = ProjectSchema().dump(projects, many=True) + return {'data': data} def post(self): """Create a new project, and return that new project.""" @@ -53,8 +53,8 @@ class ProjectList(Resource): topology.insert() project = ProjectModel(result['project']) - project.set_property('datetimeCreated', Database.datetime_to_string(datetime.now())) - project.set_property('datetimeLastEdited', Database.datetime_to_string(datetime.now())) + project.set_property('datetimeCreated', datetime.now()) + project.set_property('datetimeLastEdited', datetime.now()) project.set_property('topologyIds', [topology.get_id()]) project.set_property('portfolioIds', []) project.set_property('authorizations', [{'userId': user_id, 'level': 'OWN'}]) @@ -63,7 +63,8 @@ class ProjectList(Resource): topology.set_property('projectId', project.get_id()) topology.update() - return {'data': project.obj} + data = ProjectSchema().dump(project.obj) + return {'data': data} class Project(Resource): @@ -79,7 +80,8 @@ class Project(Resource): project.check_exists() project.check_user_access(current_user['sub'], False) - return {'data': project.obj} + data = ProjectSchema().dump(project.obj) + return {'data': data} def put(self, project_id): """Update a project's name.""" @@ -92,10 +94,11 @@ class Project(Resource): project.check_user_access(current_user['sub'], True) project.set_property('name', result['project']['name']) - project.set_property('datetimeLastEdited', Database.datetime_to_string(datetime.now())) + project.set_property('datetimeLastEdited', datetime.now()) project.update() - return {'data': project.obj} + data = ProjectSchema().dump(project.obj) + return {'data': data} def delete(self, project_id): """Delete this Project.""" @@ -113,8 +116,8 @@ class Project(Resource): portfolio.delete() old_object = project.delete() - - return {'data': old_object} + data = ProjectSchema().dump(old_object) + return {'data': data} class PutSchema(Schema): """ @@ -148,10 +151,11 @@ class ProjectTopologies(Resource): topology.insert() project.obj['topologyIds'].append(topology.get_id()) - project.set_property('datetimeLastEdited', Database.datetime_to_string(datetime.now())) + project.set_property('datetimeLastEdited', datetime.now()) project.update() - return {'data': topology.obj} + data = TopologySchema().dump(topology.obj) + return {'data': data} class PutSchema(Schema): """ diff --git a/opendc-web/opendc-web-api/opendc/api/scenarios.py b/opendc-web/opendc-web-api/opendc/api/scenarios.py index b566950a..eacb0b49 100644 --- a/opendc-web/opendc-web-api/opendc/api/scenarios.py +++ b/opendc-web/opendc-web-api/opendc/api/scenarios.py @@ -24,7 +24,7 @@ from marshmallow import Schema, fields from opendc.models.scenario import Scenario as ScenarioModel, ScenarioSchema from opendc.models.portfolio import Portfolio -from opendc.exts import current_user, requires_auth +from opendc.exts import current_user, requires_auth, has_scope class Scenario(Resource): @@ -37,8 +37,12 @@ class Scenario(Resource): """Get scenario by identifier.""" scenario = ScenarioModel.from_id(scenario_id) scenario.check_exists() - scenario.check_user_access(current_user['sub'], False) - data = scenario.obj + + # Users with scope runner can access all scenarios + if not has_scope('runner'): + scenario.check_user_access(current_user['sub'], False) + + data = ScenarioSchema().dump(scenario.obj) return {'data': data} def put(self, scenario_id): @@ -54,7 +58,7 @@ class Scenario(Resource): scenario.set_property('name', result['scenario']['name']) scenario.update() - data = scenario.obj + data = ScenarioSchema().dump(scenario.obj) return {'data': data} def delete(self, scenario_id): @@ -72,7 +76,8 @@ class Scenario(Resource): portfolio.update() old_object = scenario.delete() - return {'data': old_object} + data = ScenarioSchema().dump(old_object) + return {'data': data} class PutSchema(Schema): """ diff --git a/opendc-web/opendc-web-api/opendc/api/topologies.py b/opendc-web/opendc-web-api/opendc/api/topologies.py index eedf049d..c0b2e7ee 100644 --- a/opendc-web/opendc-web-api/opendc/api/topologies.py +++ b/opendc-web/opendc-web-api/opendc/api/topologies.py @@ -24,10 +24,9 @@ from flask import request from flask_restful import Resource from marshmallow import Schema, fields -from opendc.database import Database from opendc.models.project import Project from opendc.models.topology import Topology as TopologyModel, TopologySchema -from opendc.exts import current_user, requires_auth +from opendc.exts import current_user, requires_auth, has_scope class Topology(Resource): @@ -42,8 +41,12 @@ class Topology(Resource): """ topology = TopologyModel.from_id(topology_id) topology.check_exists() - topology.check_user_access(current_user['sub'], False) - data = topology.obj + + # Users with scope runner can access all topologies + if not has_scope('runner'): + topology.check_user_access(current_user['sub'], False) + + data = TopologySchema().dump(topology.obj) return {'data': data} def put(self, topology_id): @@ -60,10 +63,10 @@ class Topology(Resource): topology.set_property('name', result['topology']['name']) topology.set_property('rooms', result['topology']['rooms']) - topology.set_property('datetimeLastEdited', Database.datetime_to_string(datetime.now())) + topology.set_property('datetimeLastEdited', datetime.now()) topology.update() - data = topology.obj + data = TopologySchema().dump(topology.obj) return {'data': data} def delete(self, topology_id): @@ -84,7 +87,8 @@ class Topology(Resource): project.update() old_object = topology.delete() - return {'data': old_object} + data = TopologySchema().dump(old_object) + return {'data': data} class PutSchema(Schema): """ diff --git a/opendc-web/opendc-web-api/opendc/api/traces.py b/opendc-web/opendc-web-api/opendc/api/traces.py index f685f00c..6be8c5e5 100644 --- a/opendc-web/opendc-web-api/opendc/api/traces.py +++ b/opendc-web/opendc-web-api/opendc/api/traces.py @@ -21,7 +21,7 @@ from flask_restful import Resource from opendc.exts import requires_auth -from opendc.models.trace import Trace as TraceModel +from opendc.models.trace import Trace as TraceModel, TraceSchema class TraceList(Resource): @@ -33,7 +33,7 @@ class TraceList(Resource): def get(self): """Get all available Traces.""" traces = TraceModel.get_all() - data = traces.obj + data = TraceSchema().dump(traces.obj, many=True) return {'data': data} @@ -47,5 +47,5 @@ class Trace(Resource): """Get trace information by identifier.""" trace = TraceModel.from_id(trace_id) trace.check_exists() - data = trace.obj + data = TraceSchema().dump(trace.obj) return {'data': data} diff --git a/opendc-web/opendc-web-api/opendc/auth.py b/opendc-web/opendc-web-api/opendc/auth.py index 1870f01c..6db06fb1 100644 --- a/opendc-web/opendc-web-api/opendc/auth.py +++ b/opendc-web/opendc-web-api/opendc/auth.py @@ -42,8 +42,7 @@ def get_token(): if parts[0].lower() != "bearer": raise AuthError({ "code": "invalid_header", - "description": "Authorization header must start with" - " Bearer" + "description": "Authorization header must start with Bearer" }, 401) if len(parts) == 1: raise AuthError({"code": "invalid_header", "description": "Token not found"}, 401) diff --git a/opendc-web/opendc-web-api/opendc/database.py b/opendc-web/opendc-web-api/opendc/database.py index f9a33b66..dd6367f2 100644 --- a/opendc-web/opendc-web-api/opendc/database.py +++ b/opendc-web/opendc-web-api/opendc/database.py @@ -19,9 +19,8 @@ # SOFTWARE. import urllib.parse -from datetime import datetime -from pymongo import MongoClient +from pymongo import MongoClient, ReturnDocument DATETIME_STRING_FORMAT = '%Y-%m-%dT%H:%M:%S' CONNECTION_POOL = None @@ -77,6 +76,12 @@ class Database: """Updates an existing object.""" return getattr(self.opendc_db, collection).update({'_id': _id}, obj) + def fetch_and_update(self, query, update, collection): + """Updates an existing object.""" + return getattr(self.opendc_db, collection).find_one_and_update(query, + update, + return_document=ReturnDocument.AFTER) + def delete_one(self, query, collection): """Deletes one object matching the given query. @@ -90,13 +95,3 @@ class Database: The query needs to be in json format, i.e.: `{'name': prefab_name}`. """ getattr(self.opendc_db, collection).delete_many(query) - - @staticmethod - def datetime_to_string(datetime_to_convert): - """Return a database-compatible string representation of the given datetime object.""" - return datetime_to_convert.strftime(DATETIME_STRING_FORMAT) - - @staticmethod - def string_to_datetime(string_to_convert): - """Return a datetime corresponding to the given string representation.""" - return datetime.strptime(string_to_convert, DATETIME_STRING_FORMAT) diff --git a/opendc-web/opendc-web-api/opendc/exts.py b/opendc-web/opendc-web-api/opendc/exts.py index f088a29c..17dacd5e 100644 --- a/opendc-web/opendc-web-api/opendc/exts.py +++ b/opendc-web/opendc-web-api/opendc/exts.py @@ -2,10 +2,11 @@ import os from functools import wraps from flask import g, _request_ctx_stack +from jose import jwt from werkzeug.local import LocalProxy from opendc.database import Database -from opendc.auth import AuthContext, AsymmetricJwtAlgorithm, get_token +from opendc.auth import AuthContext, AsymmetricJwtAlgorithm, get_token, AuthError def get_db(): @@ -34,8 +35,7 @@ def get_auth_context(): _auth_context = AuthContext( alg=AsymmetricJwtAlgorithm(jwks_url=f"https://{os.environ['AUTH0_DOMAIN']}/.well-known/jwks.json"), issuer=f"https://{os.environ['AUTH0_DOMAIN']}/", - audience=os.environ['AUTH0_AUDIENCE'] - ) + audience=os.environ['AUTH0_AUDIENCE']) g.auth_context = _auth_context return _auth_context @@ -46,7 +46,6 @@ auth_context = LocalProxy(get_auth_context) def requires_auth(f): """Decorator to determine if the Access Token is valid. """ - @wraps(f) def decorated(*args, **kwargs): token = get_token() @@ -58,3 +57,36 @@ def requires_auth(f): current_user = LocalProxy(lambda: getattr(_request_ctx_stack.top, 'current_user', None)) + + +def has_scope(required_scope): + """Determines if the required scope is present in the Access Token + Args: + required_scope (str): The scope required to access the resource + """ + token = get_token() + unverified_claims = jwt.get_unverified_claims(token) + if unverified_claims.get("scope"): + token_scopes = unverified_claims["scope"].split() + for token_scope in token_scopes: + if token_scope == required_scope: + return True + return False + + +def requires_scope(required_scope): + """Determines if the required scope is present in the Access Token + Args: + required_scope (str): The scope required to access the resource + """ + def decorator(f): + @wraps(f) + def decorated(*args, **kwargs): + if not has_scope(required_scope): + raise AuthError({ + "code": "Unauthorized", + "description": "You don't have access to this resource" + }, 403) + return f(*args, **kwargs) + return decorated + return decorator diff --git a/opendc-web/opendc-web-api/opendc/models/portfolio.py b/opendc-web/opendc-web-api/opendc/models/portfolio.py index aff1d3f0..1643e23e 100644 --- a/opendc-web/opendc-web-api/opendc/models/portfolio.py +++ b/opendc-web/opendc-web-api/opendc/models/portfolio.py @@ -16,7 +16,7 @@ class PortfolioSchema(Schema): """ Schema representing a portfolio. """ - _id = fields.String() + _id = fields.String(dump_only=True) projectId = fields.String() name = fields.String(required=True) scenarioIds = fields.List(fields.String()) diff --git a/opendc-web/opendc-web-api/opendc/models/prefab.py b/opendc-web/opendc-web-api/opendc/models/prefab.py index d83ef4cb..5e4b81dc 100644 --- a/opendc-web/opendc-web-api/opendc/models/prefab.py +++ b/opendc-web/opendc-web-api/opendc/models/prefab.py @@ -9,7 +9,8 @@ class PrefabSchema(Schema): """ Schema for a Prefab. """ - _id = fields.String() + _id = fields.String(dump_only=True) + authorId = fields.String(dump_only=True) name = fields.String(required=True) datetimeCreated = fields.DateTime() datetimeLastEdited = fields.DateTime() diff --git a/opendc-web/opendc-web-api/opendc/models/project.py b/opendc-web/opendc-web-api/opendc/models/project.py index ee84c73e..f2b3b564 100644 --- a/opendc-web/opendc-web-api/opendc/models/project.py +++ b/opendc-web/opendc-web-api/opendc/models/project.py @@ -1,20 +1,29 @@ -from marshmallow import Schema, fields +from marshmallow import Schema, fields, validate from werkzeug.exceptions import Forbidden from opendc.models.model import Model from opendc.exts import db +class ProjectAuthorizations(Schema): + """ + Schema representing a project authorization. + """ + userId = fields.String(required=True) + level = fields.String(required=True, validate=validate.OneOf(["VIEW", "EDIT", "OWN"])) + + class ProjectSchema(Schema): """ Schema representing a Project. """ - _id = fields.String() + _id = fields.String(dump_only=True) name = fields.String(required=True) datetimeCreated = fields.DateTime() datetimeLastEdited = fields.DateTime() topologyIds = fields.List(fields.String()) portfolioIds = fields.List(fields.String()) + authorizations = fields.List(fields.Nested(ProjectAuthorizations)) class Project(Model): diff --git a/opendc-web/opendc-web-api/opendc/models/scenario.py b/opendc-web/opendc-web-api/opendc/models/scenario.py index 2911b1ae..0fb6c453 100644 --- a/opendc-web/opendc-web-api/opendc/models/scenario.py +++ b/opendc-web/opendc-web-api/opendc/models/scenario.py @@ -1,15 +1,12 @@ +from datetime import datetime + from marshmallow import Schema, fields + +from opendc.exts import db from opendc.models.model import Model from opendc.models.portfolio import Portfolio -class SimulationSchema(Schema): - """ - Simulation details. - """ - state = fields.String() - - class TraceSchema(Schema): """ Schema for specifying the trace of a scenario. @@ -38,10 +35,9 @@ class ScenarioSchema(Schema): """ Schema representing a scenario. """ - _id = fields.String() + _id = fields.String(dump_only=True) portfolioId = fields.String() name = fields.String(required=True) - simulation = fields.Nested(SimulationSchema) trace = fields.Nested(TraceSchema) topology = fields.Nested(TopologySchema) operational = fields.Nested(OperationalSchema) @@ -62,3 +58,21 @@ class Scenario(Model): """ portfolio = Portfolio.from_id(self.obj['portfolioId']) portfolio.check_user_access(user_id, edit_access) + + @classmethod + def get_jobs(cls): + """Obtain the scenarios that have been queued. + """ + return cls(db.fetch_all({'simulation.state': 'QUEUED'}, cls.collection_name)) + + def update_state(self, new_state, results=None): + """Atomically update the state of the Scenario. + """ + update = {'$set': {'simulation.state': new_state, 'simulation.heartbeat': datetime.now()}} + if results: + update['$set']['results'] = results + return db.fetch_and_update( + query={'_id': self.obj['_id'], 'simulation.state': self.obj['simulation']['state']}, + update=update, + collection=self.collection_name + ) diff --git a/opendc-web/opendc-web-api/opendc/models/topology.py b/opendc-web/opendc-web-api/opendc/models/topology.py index c6354ae6..71d2cade 100644 --- a/opendc-web/opendc-web-api/opendc/models/topology.py +++ b/opendc-web/opendc-web-api/opendc/models/topology.py @@ -72,8 +72,8 @@ class TopologySchema(Schema): """ Schema representing a datacenter topology. """ - _id = fields.String() - projectId = fields.String() + _id = fields.String(dump_only=True) + projectId = fields.String(dump_only=True) name = fields.String(required=True) rooms = fields.List(fields.Nested(RoomSchema), required=True) diff --git a/opendc-web/opendc-web-api/opendc/models/trace.py b/opendc-web/opendc-web-api/opendc/models/trace.py index 2f6e4926..69287f29 100644 --- a/opendc-web/opendc-web-api/opendc/models/trace.py +++ b/opendc-web/opendc-web-api/opendc/models/trace.py @@ -1,6 +1,15 @@ +from marshmallow import Schema, fields + from opendc.models.model import Model +class TraceSchema(Schema): + """Schema for a Trace.""" + _id = fields.String(dump_only=True) + name = fields.String() + type = fields.String() + + class Trace(Model): """Model representing a Trace.""" diff --git a/opendc-web/opendc-web-api/static/schema.yml b/opendc-web/opendc-web-api/static/schema.yml index 99e88095..6a07ae52 100644 --- a/opendc-web/opendc-web-api/static/schema.yml +++ b/opendc-web/opendc-web-api/static/schema.yml @@ -965,7 +965,7 @@ paths: application/json: schema: properties: - project: + prefab: $ref: "#/components/schemas/Prefab" description: Prefab's new properties. required: true @@ -1040,6 +1040,135 @@ paths: "application/json": schema: $ref: "#/components/schemas/NotFound" + /jobs: + get: + tags: + - jobs + description: Get all available jobs to run. + responses: + "200": + description: Successfully retrieved available jobs. + content: + "application/json": + schema: + type: object + required: + - data + properties: + data: + type: array + items: + $ref: "#/components/schemas/Job" + "401": + description: Unauthorized. + content: + "application/json": + schema: + $ref: "#/components/schemas/Unauthorized" + "/jobs/{jobId}": + get: + tags: + - jobs + description: Get this Job. + parameters: + - name: jobId + in: path + description: Job's ID. + required: true + schema: + type: string + responses: + "200": + description: Successfully retrieved Job. + content: + "application/json": + schema: + type: object + required: + - data + properties: + data: + $ref: "#/components/schemas/Job" + "401": + description: Unauthorized. + content: + "application/json": + schema: + $ref: "#/components/schemas/Unauthorized" + "403": + description: Forbidden from retrieving Job. + content: + "application/json": + schema: + $ref: "#/components/schemas/Forbidden" + "404": + description: Job not found. + content: + "application/json": + schema: + $ref: "#/components/schemas/NotFound" + post: + tags: + - jobs + description: Update this Job. + parameters: + - name: jobId + in: path + description: Job's ID. + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + properties: + job: + $ref: "#/components/schemas/Job" + description: Job's new properties. + required: true + responses: + "200": + description: Successfully updated Job. + content: + "application/json": + schema: + type: object + required: + - data + properties: + data: + $ref: "#/components/schemas/Job" + "400": + description: Missing or incorrectly typed parameter. + content: + "application/json": + schema: + $ref: "#/components/schemas/Invalid" + "401": + description: Unauthorized. + content: + "application/json": + schema: + $ref: "#/components/schemas/Unauthorized" + "403": + description: Forbidden from retrieving Job. + content: + "application/json": + schema: + $ref: "#/components/schemas/Forbidden" + "404": + description: Job not found. + content: + "application/json": + schema: + $ref: "#/components/schemas/NotFound" + "409": + description: State conflict. + content: + "application/json": + schema: + $ref: "#/components/schemas/Invalid" components: securitySchemes: auth0: @@ -1051,6 +1180,7 @@ components: tokenUrl: https://opendc.eu.auth0.com/oauth/token scopes: openid: Grants access to user_id + runner: Grants access to runner jobs schemas: Unauthorized: type: object @@ -1153,6 +1283,10 @@ components: object: type: object properties: + _id: + type: string + name: + type: string capacity: type: integer powerCapacityW: @@ -1162,52 +1296,70 @@ components: items: type: object properties: + _id: + type: string position: type: integer - cpuItems: + cpus: type: array items: type: object properties: + _id: + type: string name: type: string clockRateMhz: type: integer numberOfCores: type: integer - gpuItems: + energyConsumptionW: + type: integer + gpus: type: array items: type: object properties: + _id: + type: string name: type: string clockRateMhz: type: integer numberOfCores: type: integer - memoryItems: + energyConsumptionW: + type: integer + memories: type: array items: type: object properties: + _id: + type: string name: type: string speedMbPerS: type: integer sizeMb: type: integer - storageItems: + energyConsumptionW: + type: integer + storages: type: array items: - type: integer + type: object properties: + _id: + type: string name: type: string speedMbPerS: type: integer sizeMb: type: integer + energyConsumptionW: + type: integer Portfolio: type: object properties: @@ -1239,13 +1391,6 @@ components: type: string name: type: string - simulation: - type: object - properties: - state: - type: string - results: - type: object trace: type: object properties: @@ -1267,6 +1412,19 @@ components: type: boolean schedulerName: type: string + Job: + type: object + properties: + _id: + type: string + scenarioId: + type: string + state: + type: string + heartbeat: + type: string + results: + type: object Trace: type: object properties: diff --git a/opendc-web/opendc-web-api/tests/api/test_jobs.py b/opendc-web/opendc-web-api/tests/api/test_jobs.py new file mode 100644 index 00000000..2efe6933 --- /dev/null +++ b/opendc-web/opendc-web-api/tests/api/test_jobs.py @@ -0,0 +1,139 @@ +# Copyright (c) 2021 AtLarge Research +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# +from datetime import datetime + +from opendc.exts import db + +test_id = 24 * '1' +test_id_2 = 24 * '2' + + +def test_get_jobs(client, mocker): + mocker.patch.object(db, 'fetch_all', return_value=[ + {'_id': 'a', 'scenarioId': 'x', 'portfolioId': 'y', 'simulation': {'state': 'QUEUED'}} + ]) + res = client.get('/jobs/') + assert '200' in res.status + + +def test_get_job_non_existing(client, mocker): + mocker.patch.object(db, 'fetch_one', return_value=None) + assert '404' in client.get(f'/jobs/{test_id}').status + + +def test_get_job(client, mocker): + mocker.patch.object(db, 'fetch_one', return_value={ + '_id': 'a', 'scenarioId': 'x', 'portfolioId': 'y', 'simulation': {'state': 'QUEUED'} + }) + res = client.get(f'/jobs/{test_id}') + assert '200' in res.status + + +def test_update_job_nop(client, mocker): + mocker.patch.object(db, 'fetch_one', return_value={ + '_id': test_id, 'scenarioId': 'x', 'portfolioId': 'y', 'simulation': {'state': 'QUEUED'} + }) + update_mock = mocker.patch.object(db, 'fetch_and_update', return_value={ + '_id': test_id, 'scenarioId': 'x', 'portfolioId': 'y', + 'simulation': {'state': 'QUEUED', 'heartbeat': datetime.now()} + }) + res = client.post(f'/jobs/{test_id}', json={'state': 'QUEUED'}) + assert '200' in res.status + update_mock.assert_called_once() + + +def test_update_job_invalid_state(client, mocker): + mocker.patch.object(db, 'fetch_one', return_value={ + '_id': test_id, 'scenarioId': 'x', 'portfolioId': 'y', 'simulation': {'state': 'QUEUED'} + }) + res = client.post(f'/jobs/{test_id}', json={'state': 'FINISHED'}) + assert '400' in res.status + + +def test_update_job_claim(client, mocker): + mocker.patch.object(db, 'fetch_one', return_value={ + '_id': test_id, 'scenarioId': 'x', 'portfolioId': 'y', 'simulation': {'state': 'QUEUED'} + }) + update_mock = mocker.patch.object(db, 'fetch_and_update', return_value={ + '_id': test_id, 'scenarioId': 'x', 'portfolioId': 'y', + 'simulation': {'state': 'CLAIMED', 'heartbeat': datetime.now()} + }) + res = client.post(f'/jobs/{test_id}', json={'state': 'CLAIMED'}) + assert '200' in res.status + update_mock.assert_called_once() + + +def test_update_job_conflict(client, mocker): + mocker.patch.object(db, 'fetch_one', return_value={ + '_id': test_id, 'scenarioId': 'x', 'portfolioId': 'y', 'simulation': {'state': 'QUEUED'} + }) + update_mock = mocker.patch.object(db, 'fetch_and_update', return_value=None) + res = client.post(f'/jobs/{test_id}', json={'state': 'CLAIMED'}) + assert '409' in res.status + update_mock.assert_called_once() + + +def test_update_job_run(client, mocker): + mocker.patch.object(db, 'fetch_one', return_value={ + '_id': test_id, 'scenarioId': 'x', 'portfolioId': 'y', 'simulation': {'state': 'CLAIMED'} + }) + update_mock = mocker.patch.object(db, 'fetch_and_update', return_value={ + '_id': test_id, 'scenarioId': 'x', 'portfolioId': 'y', + 'simulation': {'state': 'RUNNING', 'heartbeat': datetime.now()} + }) + res = client.post(f'/jobs/{test_id}', json={'state': 'RUNNING'}) + assert '200' in res.status + update_mock.assert_called_once() + + +def test_update_job_finished(client, mocker): + mocker.patch.object(db, 'fetch_one', return_value={ + '_id': test_id, 'scenarioId': 'x', 'portfolioId': 'y', 'simulation': {'state': 'RUNNING'} + }) + update_mock = mocker.patch.object(db, 'fetch_and_update', return_value={ + '_id': test_id, 'scenarioId': 'x', 'portfolioId': 'y', + 'simulation': {'state': 'FINISHED', 'heartbeat': datetime.now()} + }) + res = client.post(f'/jobs/{test_id}', json={'state': 'FINISHED'}) + assert '200' in res.status + update_mock.assert_called_once() + + +def test_update_job_failed(client, mocker): + mocker.patch.object(db, 'fetch_one', return_value={ + '_id': test_id, 'scenarioId': 'x', 'portfolioId': 'y', 'simulation': {'state': 'RUNNING'} + }) + update_mock = mocker.patch.object(db, 'fetch_and_update', return_value={ + '_id': test_id, 'scenarioId': 'x', 'portfolioId': 'y', + 'simulation': {'state': 'FAILED', 'heartbeat': datetime.now()} + }) + res = client.post(f'/jobs/{test_id}', json={'state': 'FAILED'}) + assert '200' in res.status + update_mock.assert_called_once() diff --git a/opendc-web/opendc-web-runner/build.gradle.kts b/opendc-web/opendc-web-runner/build.gradle.kts index f2b2ba23..1f705b79 100644 --- a/opendc-web/opendc-web-runner/build.gradle.kts +++ b/opendc-web/opendc-web-runner/build.gradle.kts @@ -25,11 +25,12 @@ description = "Experiment runner for OpenDC" /* Build configuration */ plugins { `kotlin-conventions` + `testing-conventions` application } application { - mainClass.set("org.opendc.runner.web.MainKt") + mainClass.set("org.opendc.web.runner.MainKt") } dependencies { @@ -42,12 +43,13 @@ dependencies { implementation(libs.kotlin.logging) implementation(libs.clikt) - implementation(libs.jackson.module.kotlin) { - exclude(group = "org.jetbrains.kotlin", module = "kotlin-reflect") - } - implementation(kotlin("reflect")) implementation(libs.sentry.log4j2) - implementation(libs.mongodb) + implementation(libs.ktor.client.cio) + implementation(libs.ktor.client.auth) + implementation(libs.ktor.client.jackson) + implementation(libs.jackson.datatype.jsr310) runtimeOnly(libs.log4j.slf4j) + + testImplementation(libs.ktor.client.mock) } diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/ScenarioManager.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/ScenarioManager.kt deleted file mode 100644 index a3907051..00000000 --- a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/ScenarioManager.kt +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.runner.web - -import com.mongodb.client.MongoCollection -import com.mongodb.client.model.Filters -import com.mongodb.client.model.Updates -import org.bson.Document -import org.bson.types.ObjectId -import java.time.Instant - -/** - * Manages the queue of scenarios that need to be processed. - */ -public class ScenarioManager(private val collection: MongoCollection<Document>) { - /** - * Find the next scenario that the simulator needs to process. - */ - public fun findNext(): Document? { - return collection - .find(Filters.eq("simulation.state", "QUEUED")) - .first() - } - - /** - * Claim the scenario in the database with the specified id. - */ - public fun claim(id: ObjectId): Boolean { - val res = collection.findOneAndUpdate( - Filters.and( - Filters.eq("_id", id), - Filters.eq("simulation.state", "QUEUED") - ), - Updates.combine( - Updates.set("simulation.state", "RUNNING"), - Updates.set("simulation.heartbeat", Instant.now()) - ) - ) - return res != null - } - - /** - * Update the heartbeat of the specified scenario. - */ - public fun heartbeat(id: ObjectId) { - collection.findOneAndUpdate( - Filters.and( - Filters.eq("_id", id), - Filters.eq("simulation.state", "RUNNING") - ), - Updates.set("simulation.heartbeat", Instant.now()) - ) - } - - /** - * Mark the scenario as failed. - */ - public fun fail(id: ObjectId) { - collection.findOneAndUpdate( - Filters.eq("_id", id), - Updates.combine( - Updates.set("simulation.state", "FAILED"), - Updates.set("simulation.heartbeat", Instant.now()) - ) - ) - } - - /** - * Persist the specified results. - */ - public fun finish(id: ObjectId, results: List<WebExperimentMonitor.Result>) { - collection.findOneAndUpdate( - Filters.eq("_id", id), - Updates.combine( - Updates.set("simulation.state", "FINISHED"), - Updates.unset("simulation.time"), - Updates.set("results.total_requested_burst", results.map { it.totalRequestedBurst }), - Updates.set("results.total_granted_burst", results.map { it.totalGrantedBurst }), - Updates.set("results.total_overcommitted_burst", results.map { it.totalOvercommittedBurst }), - Updates.set("results.total_interfered_burst", results.map { it.totalInterferedBurst }), - Updates.set("results.mean_cpu_usage", results.map { it.meanCpuUsage }), - Updates.set("results.mean_cpu_demand", results.map { it.meanCpuDemand }), - Updates.set("results.mean_num_deployed_images", results.map { it.meanNumDeployedImages }), - Updates.set("results.max_num_deployed_images", results.map { it.maxNumDeployedImages }), - Updates.set("results.total_power_draw", results.map { it.totalPowerDraw }), - Updates.set("results.total_failure_slices", results.map { it.totalFailureSlices }), - Updates.set("results.total_failure_vm_slices", results.map { it.totalFailureVmSlices }), - Updates.set("results.total_vms_submitted", results.map { it.totalVmsSubmitted }), - Updates.set("results.total_vms_queued", results.map { it.totalVmsQueued }), - Updates.set("results.total_vms_finished", results.map { it.totalVmsFinished }), - Updates.set("results.total_vms_failed", results.map { it.totalVmsFailed }) - ) - ) - } -} diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/TopologyParser.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/TopologyParser.kt deleted file mode 100644 index 2135ee1d..00000000 --- a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/TopologyParser.kt +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.runner.web - -import com.mongodb.client.AggregateIterable -import com.mongodb.client.MongoCollection -import com.mongodb.client.model.Aggregates -import com.mongodb.client.model.Field -import com.mongodb.client.model.Filters -import com.mongodb.client.model.Projections -import org.bson.Document -import org.bson.types.ObjectId -import org.opendc.format.environment.EnvironmentReader -import org.opendc.format.environment.MachineDef -import org.opendc.simulator.compute.model.MachineModel -import org.opendc.simulator.compute.model.MemoryUnit -import org.opendc.simulator.compute.model.ProcessingNode -import org.opendc.simulator.compute.model.ProcessingUnit -import org.opendc.simulator.compute.power.LinearPowerModel -import java.util.* - -/** - * A helper class that converts the MongoDB topology into an OpenDC environment. - */ -public class TopologyParser(private val collection: MongoCollection<Document>) { - - /** - * Parse the topology from the specified [id]. - */ - public fun read(id: ObjectId): EnvironmentReader { - val nodes = mutableListOf<MachineDef>() - val random = Random(0) - - for (machine in fetchMachines(id)) { - val clusterId = machine.get("rack_id").toString() - val position = machine.getInteger("position") - - val processors = machine.getList("cpus", Document::class.java).flatMap { cpu -> - val cores = cpu.getInteger("numberOfCores") - val speed = cpu.get("clockRateMhz", Number::class.java).toDouble() - // TODO Remove hardcoding of vendor - val node = ProcessingNode("Intel", "amd64", cpu.getString("name"), cores) - List(cores) { coreId -> - ProcessingUnit(node, coreId, speed) - } - } - val memoryUnits = machine.getList("memories", Document::class.java).map { memory -> - MemoryUnit( - "Samsung", - memory.getString("name"), - memory.get("speedMbPerS", Number::class.java).toDouble(), - memory.get("sizeMb", Number::class.java).toLong() - ) - } - - val energyConsumptionW = machine.getList("cpus", Document::class.java).sumOf { it.getInteger("energyConsumptionW") }.toDouble() - - nodes.add( - MachineDef( - UUID(random.nextLong(), random.nextLong()), - "node-$clusterId-$position", - mapOf("cluster" to clusterId), - MachineModel(processors, memoryUnits), - LinearPowerModel(2 * energyConsumptionW, energyConsumptionW * 0.5) - ) - ) - } - - return object : EnvironmentReader { - override fun read(): List<MachineDef> = nodes - override fun close() {} - } - } - - /** - * Fetch the metadata of the topology. - */ - private fun fetchName(id: ObjectId): String { - return collection.aggregate( - listOf( - Aggregates.match(Filters.eq("_id", id)), - Aggregates.project(Projections.include("name")) - ) - ) - .first()!! - .getString("name") - } - - /** - * Fetch a topology from the database with the specified [id]. - */ - private fun fetchMachines(id: ObjectId): AggregateIterable<Document> { - return collection.aggregate( - listOf( - Aggregates.match(Filters.eq("_id", id)), - Aggregates.project(Projections.fields(Document("racks", "\$rooms.tiles.rack"))), - Aggregates.unwind("\$racks"), - Aggregates.unwind("\$racks"), - Aggregates.replaceRoot("\$racks"), - Aggregates.addFields(Field("machines.rack_id", "\$_id")), - Aggregates.unwind("\$machines"), - Aggregates.replaceRoot("\$machines") - ) - ) - } -} diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/ApiClient.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/ApiClient.kt new file mode 100644 index 00000000..9f2656c4 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/ApiClient.kt @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client + +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import io.ktor.client.* +import io.ktor.client.features.auth.* +import io.ktor.client.features.auth.providers.* +import io.ktor.client.features.json.* +import io.ktor.client.request.* +import io.ktor.client.statement.* +import io.ktor.http.* +import org.opendc.web.client.model.* +import java.net.URI + +/** + * Client implementation for the OpenDC REST API (version 2). + * + * @param baseUrl The base url of the API. + * @param auth The authentication configuration for the client. + * @param client The HTTP client to use. + */ +public class ApiClient( + private val baseUrl: URI, + private val auth: AuthConfiguration, + private val audience: String = "https://api.opendc.org/v2/", + client: HttpClient = HttpClient {} +) : AutoCloseable { + /** + * The Ktor [HttpClient] that is used to communicate with the REST API. + */ + private val client = client.config { + install(JsonFeature) { + serializer = JacksonSerializer { + registerModule(JavaTimeModule()) + } + } + install(Auth) { + bearer { + loadTokens { requestToken() } + refreshTokens { requestToken() } + } + } + expectSuccess = false + } + + /** + * Retrieve the topology with the specified [id]. + */ + public suspend fun getPortfolio(id: String): Portfolio? { + val url = URLBuilder(Url(baseUrl)) + .path("portfolios", id) + .build() + return when (val result = client.get<ApiResult<Portfolio>>(url)) { + is ApiResult.Success -> result.data + else -> null + } + } + + /** + * Retrieve the scenario with the specified [id]. + */ + public suspend fun getScenario(id: String): Scenario? { + val url = URLBuilder(Url(baseUrl)) + .path("scenarios", id) + .build() + return when (val result = client.get<ApiResult<Scenario>>(url)) { + is ApiResult.Success -> result.data + else -> null + } + } + + /** + * Retrieve the topology with the specified [id]. + */ + public suspend fun getTopology(id: String): Topology? { + val url = URLBuilder(Url(baseUrl)) + .path("topologies", id) + .build() + return when (val result = client.get<ApiResult<Topology>>(url)) { + is ApiResult.Success -> result.data + else -> null + } + } + + /** + * Retrieve the available jobs. + */ + public suspend fun getJobs(): List<Job> { + val url = URLBuilder(Url(baseUrl)) + .path("jobs") + .build() + return when (val result = client.get<ApiResult<List<Job>>>(url)) { + is ApiResult.Success -> result.data + else -> emptyList() + } + } + + /** + * Update the specified job. + * + * @param id The identifier of the job. + * @param state The new state of the job. + * @param results The results of the job. + */ + public suspend fun updateJob(id: String, state: SimulationState, results: Map<String, Any> = emptyMap()): Boolean { + val url = URLBuilder(Url(baseUrl)) + .path("jobs", id) + .build() + + data class Request( + val state: SimulationState, + val results: Map<String, Any> + ) + + val res = client.post<HttpResponse> { + url(url) + contentType(ContentType.Application.Json) + body = Request(state, results) + } + return res.status.isSuccess() + } + + /** + * Request the auth token for the API. + */ + private suspend fun requestToken(): BearerTokens { + data class Request( + val audience: String, + @JsonProperty("grant_type") + val grantType: String, + @JsonProperty("client_id") + val clientId: String, + @JsonProperty("client_secret") + val clientSecret: String + ) + + data class Response( + @JsonProperty("access_token") + val accessToken: String, + @JsonProperty("token_type") + val tokenType: String, + val scope: String = "", + @JsonProperty("expires_in") + val expiresIn: Long + ) + + val result = client.post<Response> { + url(Url("https://${auth.domain}/oauth/token")) + contentType(ContentType.Application.Json) + body = Request(audience, "client_credentials", auth.clientId, auth.clientSecret) + } + + return BearerTokens(result.accessToken, "") + } + + override fun close() = client.close() +} diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/ApiResult.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/ApiResult.kt new file mode 100644 index 00000000..a3df01c5 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/ApiResult.kt @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client + +import com.fasterxml.jackson.annotation.JsonSubTypes +import com.fasterxml.jackson.annotation.JsonTypeInfo + +/** + * Generic response model for the OpenDC API. + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.DEDUCTION) +@JsonSubTypes(JsonSubTypes.Type(ApiResult.Success::class), JsonSubTypes.Type(ApiResult.Failure::class)) +public sealed class ApiResult<out T> { + /** + * A response indicating everything is okay. + */ + public data class Success<out T>(val data: T) : ApiResult<T>() + + /** + * A response indicating a failure. + */ + public data class Failure<out T>(val message: String, val errors: List<String> = emptyList()) : ApiResult<T>() +} diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/AuthConfiguration.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/AuthConfiguration.kt new file mode 100644 index 00000000..5dbf2f59 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/AuthConfiguration.kt @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client + +/** + * The authentication configuration for the API client. + */ +public data class AuthConfiguration( + val domain: String, + val clientId: String, + val clientSecret: String +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Job.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Job.kt new file mode 100644 index 00000000..eeb65e49 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Job.kt @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonProperty +import java.time.LocalDateTime + +/** + * A description of a simulation job. + */ +public data class Job( + @JsonProperty("_id") + val id: String, + val scenarioId: String, + val state: SimulationState, + val heartbeat: LocalDateTime, + val results: Map<String, Any> +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Machine.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Machine.kt new file mode 100644 index 00000000..c6757c5c --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Machine.kt @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * A machine in a rack. + */ +@JsonIgnoreProperties("id_legacy") +public data class Machine( + @JsonProperty("_id") + val id: String, + val position: Int, + val cpus: List<ProcessingUnit> = emptyList(), + val gpus: List<ProcessingUnit> = emptyList(), + @JsonProperty("memories") + val memory: List<MemoryUnit> = emptyList(), + @JsonProperty("storages") + val storage: List<MemoryUnit> = emptyList() +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/MemoryUnit.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/MemoryUnit.kt new file mode 100644 index 00000000..11e794e8 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/MemoryUnit.kt @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * A memory unit in a system. + */ +public data class MemoryUnit( + @JsonProperty("_id") + val id: String, + val name: String, + val speedMbPerS: Double, + val sizeMb: Double, + val energyConsumptionW: Double +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/OperationalPhenomena.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/OperationalPhenomena.kt new file mode 100644 index 00000000..ef5b4902 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/OperationalPhenomena.kt @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +/** + * Object describing the enabled operational phenomena for a scenario. + */ +public data class OperationalPhenomena( + val failuresEnabled: Boolean, + val performanceInterferenceEnabled: Boolean, + val schedulerName: String +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Portfolio.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Portfolio.kt new file mode 100644 index 00000000..6904920b --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Portfolio.kt @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * A portfolio in OpenDC. + */ +public data class Portfolio( + @JsonProperty("_id") + val id: String, + val projectId: String, + val name: String, + @JsonProperty("scenarioIds") + val scenarios: Set<String>, + val targets: PortfolioTargets +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/PortfolioTargets.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/PortfolioTargets.kt new file mode 100644 index 00000000..07c11c19 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/PortfolioTargets.kt @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +/** + * The targets of a portfolio. + */ +public data class PortfolioTargets(val enabledMetrics: Set<String>, val repeatsPerScenario: Int) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ProcessingUnit.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ProcessingUnit.kt new file mode 100644 index 00000000..449b5c43 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ProcessingUnit.kt @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * A CPU model. + */ +public data class ProcessingUnit( + @JsonProperty("_id") + val id: String, + val name: String, + val clockRateMhz: Double, + val numberOfCores: Int, + val energyConsumptionW: Double +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Rack.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Rack.kt new file mode 100644 index 00000000..a0464388 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Rack.kt @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * A rack in a datacenter. + */ +@JsonIgnoreProperties("id_legacy") +public class Rack( + @JsonProperty("_id") + val id: String, + val name: String, + val capacity: Int, + val powerCapacityW: Double, + val machines: List<Machine> +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Room.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Room.kt new file mode 100644 index 00000000..e961d6db --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Room.kt @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * A room in a datacenter. + */ +@JsonIgnoreProperties("id_legacy") +public data class Room( + @JsonProperty("_id") + val id: String, + val name: String, + val tiles: Set<RoomTile> +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/RoomTile.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/RoomTile.kt new file mode 100644 index 00000000..3bee3204 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/RoomTile.kt @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * A room tile. + */ +@JsonIgnoreProperties("id_legacy") +public data class RoomTile( + @JsonProperty("_id") + val id: String, + val positionX: Double, + val positionY: Double, + val rack: Rack? = null +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Scenario.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Scenario.kt new file mode 100644 index 00000000..851ff980 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Scenario.kt @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * A simulation scenario. + */ +public data class Scenario( + @JsonProperty("_id") + val id: String, + val portfolioId: String, + val name: String, + val trace: ScenarioTrace, + val topology: ScenarioTopology, + @JsonProperty("operational") + val operationalPhenomena: OperationalPhenomena +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ScenarioTopology.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ScenarioTopology.kt new file mode 100644 index 00000000..2b90f7ef --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ScenarioTopology.kt @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +/** + * The topology details for a scenario. + */ +public data class ScenarioTopology(val topologyId: String) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ScenarioTrace.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ScenarioTrace.kt new file mode 100644 index 00000000..adff6d97 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ScenarioTrace.kt @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +/** + * The trace details of a scenario. + */ +public data class ScenarioTrace(val traceId: String, val loadSamplingFraction: Double) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/SimulationState.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/SimulationState.kt new file mode 100644 index 00000000..2eadd747 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/SimulationState.kt @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +/** + * The state of a simulation job. + */ +public enum class SimulationState { + QUEUED, CLAIMED, RUNNING, FINISHED, FAILED +} diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Topology.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Topology.kt new file mode 100644 index 00000000..b59aba42 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Topology.kt @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * Model for an OpenDC topology. + */ +@JsonIgnoreProperties("id_legacy", "datacenter_id_legacy", "datetimeLastUpdated", "datetimeLastEdited") +public data class Topology( + @JsonProperty("_id") + val id: String, + val projectId: String, + val name: String, + val rooms: Set<Room>, +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/Main.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/Main.kt index d0b97d90..5b5ef802 100644 --- a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/Main.kt +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/Main.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020 AtLarge Research + * Copyright (c) 2021 AtLarge Research * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -20,27 +20,18 @@ * SOFTWARE. */ -package org.opendc.runner.web +package org.opendc.web.runner import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.parameters.options.* import com.github.ajalt.clikt.parameters.types.file -import com.github.ajalt.clikt.parameters.types.int import com.github.ajalt.clikt.parameters.types.long -import com.mongodb.MongoClientSettings -import com.mongodb.MongoCredential -import com.mongodb.ServerAddress -import com.mongodb.client.MongoClients -import com.mongodb.client.MongoDatabase -import com.mongodb.client.model.Filters import io.opentelemetry.api.metrics.MeterProvider import io.opentelemetry.sdk.metrics.SdkMeterProvider import io.opentelemetry.sdk.metrics.export.MetricProducer import kotlinx.coroutines.* import kotlinx.coroutines.channels.Channel import mu.KotlinLogging -import org.bson.Document -import org.bson.types.ObjectId import org.opendc.compute.service.scheduler.FilterScheduler import org.opendc.compute.service.scheduler.filters.ComputeCapabilitiesFilter import org.opendc.compute.service.scheduler.filters.ComputeFilter @@ -51,70 +42,71 @@ import org.opendc.experiments.capelin.trace.ParquetTraceReader import org.opendc.experiments.capelin.trace.PerformanceInterferenceReader import org.opendc.experiments.capelin.trace.RawParquetTraceReader import org.opendc.format.environment.EnvironmentReader +import org.opendc.format.environment.MachineDef import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel +import org.opendc.simulator.compute.model.MachineModel +import org.opendc.simulator.compute.model.MemoryUnit +import org.opendc.simulator.compute.model.ProcessingNode +import org.opendc.simulator.compute.model.ProcessingUnit +import org.opendc.simulator.compute.power.LinearPowerModel import org.opendc.simulator.core.runBlockingSimulation import org.opendc.telemetry.sdk.toOtelClock +import org.opendc.web.client.ApiClient +import org.opendc.web.client.AuthConfiguration +import org.opendc.web.client.model.Scenario +import org.opendc.web.client.model.Topology import java.io.File +import java.net.URI +import java.util.* import kotlin.random.Random import kotlin.random.asJavaRandom +import org.opendc.web.client.model.Portfolio as ClientPortfolio private val logger = KotlinLogging.logger {} /** * Represents the CLI command for starting the OpenDC web runner. */ -@OptIn(ExperimentalCoroutinesApi::class) class RunnerCli : CliktCommand(name = "runner") { /** - * The name of the database to use. + * The URL to the OpenDC API. */ - private val mongoDb by option( - "--mongo-db", - help = "name of the database to use", - envvar = "OPENDC_DB" + private val apiUrl by option( + "--api-url", + help = "url to the OpenDC API", + envvar = "OPENDC_API_URL" ) - .default("opendc") + .convert { URI(it) } + .default(URI("https://api.opendc.org/v2")) /** - * The database host to connect to. + * The auth domain to use. */ - private val mongoHost by option( - "--mongo-host", - help = "database host to connect to", - envvar = "OPENDC_DB_HOST" + private val authDomain by option( + "--auth-domain", + help = "auth domain of the OpenDC API", + envvar = "AUTH0_DOMAIN" ) - .default("localhost") - - /** - * The database port to connect to. - */ - private val mongoPort by option( - "--mongo-port", - help = "database port to connect to", - envvar = "OPENDC_DB_PORT" - ) - .int() - .default(27017) + .required() /** - * The database user to connect with. + * The auth client ID to use. */ - private val mongoUser by option( - "--mongo-user", - help = "database user to connect with", - envvar = "OPENDC_DB_USER" + private val authClientId by option( + "--auth-id", + help = "auth client id of the OpenDC API", + envvar = "AUTH0_CLIENT_ID" ) - .default("opendc") + .required() /** - * The database password to connect with. + * The auth client secret to use. */ - private val mongoPassword by option( - "--mongo-password", - help = "database password to connect with", - envvar = "OPENDC_DB_PASSWORD" + private val authClientSecret by option( + "--auth-secret", + help = "auth client secret of the OpenDC API", + envvar = "AUTH0_CLIENT_SECRET" ) - .convert { it.toCharArray() } .required() /** @@ -137,43 +129,25 @@ class RunnerCli : CliktCommand(name = "runner") { envvar = "OPENDC_RUN_TIMEOUT" ) .long() - .default(60 * 3) // Experiment may run for a maximum of three minutes - - /** - * Connect to the user-specified database. - */ - private fun createDatabase(): MongoDatabase { - val credential = MongoCredential.createScramSha1Credential( - mongoUser, - mongoDb, - mongoPassword - ) - - val settings = MongoClientSettings.builder() - .credential(credential) - .applyToClusterSettings { it.hosts(listOf(ServerAddress(mongoHost, mongoPort))) } - .build() - val client = MongoClients.create(settings) - return client.getDatabase(mongoDb) - } + .default(60L * 3) // Experiment may run for a maximum of three minutes /** * Run a single scenario. */ - private suspend fun runScenario(portfolio: Document, scenario: Document, topologyParser: TopologyParser): List<WebExperimentMonitor.Result> { - val id = scenario.getObjectId("_id") + private suspend fun runScenario(portfolio: ClientPortfolio, scenario: Scenario, environment: EnvironmentReader): List<WebExperimentMonitor.Result> { + val id = scenario.id logger.info { "Constructing performance interference model" } val traceDir = File( tracePath, - scenario.getEmbedded(listOf("trace", "traceId"), String::class.java) + scenario.trace.traceId ) val traceReader = RawParquetTraceReader(traceDir) val interferenceGroups = let { val path = File(traceDir, "performance-interference-model.json") - val operational = scenario.get("operational", Document::class.java) - val enabled = operational.getBoolean("performanceInterferenceEnabled") + val operational = scenario.operationalPhenomena + val enabled = operational.performanceInterferenceEnabled if (!enabled || !path.exists()) { return@let null @@ -182,11 +156,8 @@ class RunnerCli : CliktCommand(name = "runner") { PerformanceInterferenceReader(path.inputStream()).use { reader -> reader.read() } } - val targets = portfolio.get("targets", Document::class.java) - val topologyId = scenario.getEmbedded(listOf("topology", "topologyId"), ObjectId::class.java) - val environment = topologyParser.read(topologyId) - - val results = (0 until targets.getInteger("repeatsPerScenario")).map { repeat -> + val targets = portfolio.targets + val results = (0 until targets.repeatsPerScenario).map { repeat -> logger.info { "Starting repeat $repeat" } withTimeout(runTimeout * 1000) { val interferenceModel = interferenceGroups?.let { VmInterferenceModel(it, Random(repeat.toLong()).asJavaRandom()) } @@ -203,7 +174,7 @@ class RunnerCli : CliktCommand(name = "runner") { * Run a single repeat. */ private suspend fun runRepeat( - scenario: Document, + scenario: Scenario, repeat: Int, environment: EnvironmentReader, traceReader: RawParquetTraceReader, @@ -214,9 +185,8 @@ class RunnerCli : CliktCommand(name = "runner") { try { runBlockingSimulation { val seed = repeat - val traceDocument = scenario.get("trace", Document::class.java) - val workloadName = traceDocument.getString("traceId") - val workloadFraction = traceDocument.get("loadSamplingFraction", Number::class.java).toDouble() + val workloadName = scenario.trace.traceId + val workloadFraction = scenario.trace.loadSamplingFraction val seeder = Random(seed) @@ -228,9 +198,9 @@ class RunnerCli : CliktCommand(name = "runner") { .build() val metricProducer = meterProvider as MetricProducer - val operational = scenario.get("operational", Document::class.java) + val operational = scenario.operationalPhenomena val allocationPolicy = - when (val policyName = operational.getString("schedulerName")) { + when (val policyName = operational.schedulerName) { "mem" -> FilterScheduler( filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), weighers = listOf(MemoryWeigher() to -1.0) @@ -275,7 +245,7 @@ class RunnerCli : CliktCommand(name = "runner") { Workload(workloadName, workloadFraction), seed ) - val failureFrequency = if (operational.getBoolean("failuresEnabled", false)) 24.0 * 7 else 0.0 + val failureFrequency = if (operational.failuresEnabled) 24.0 * 7 else 0.0 withComputeService(clock, meterProvider, environment, allocationPolicy, interferenceModel) { scheduler -> val failureDomain = if (failureFrequency > 0) { @@ -315,17 +285,14 @@ class RunnerCli : CliktCommand(name = "runner") { return monitor.getResult() } - private val POLL_INTERVAL = 5000L // ms = 5 s + private val POLL_INTERVAL = 30000L // ms = 30 s private val HEARTBEAT_INTERVAL = 60000L // ms = 1 min override fun run(): Unit = runBlocking(Dispatchers.Default) { logger.info { "Starting OpenDC web runner" } - logger.info { "Connecting to MongoDB instance" } - val database = createDatabase() - val manager = ScenarioManager(database.getCollection("scenarios")) - val portfolios = database.getCollection("portfolios") - val topologies = database.getCollection("topologies") - val topologyParser = TopologyParser(topologies) + + val client = ApiClient(baseUrl = apiUrl, AuthConfiguration(authDomain, authClientId, authClientSecret)) + val manager = ScenarioManager(client) logger.info { "Watching for queued scenarios" } @@ -337,7 +304,7 @@ class RunnerCli : CliktCommand(name = "runner") { continue } - val id = scenario.getObjectId("_id") + val id = scenario.id logger.info { "Found queued scenario $id: attempting to claim" } @@ -350,14 +317,16 @@ class RunnerCli : CliktCommand(name = "runner") { // Launch heartbeat process val heartbeat = launch { while (true) { - delay(HEARTBEAT_INTERVAL) manager.heartbeat(id) + delay(HEARTBEAT_INTERVAL) } } try { - val portfolio = portfolios.find(Filters.eq("_id", scenario.getObjectId("portfolioId"))).first()!! - val results = runScenario(portfolio, scenario, topologyParser) + val scenarioModel = client.getScenario(id)!! + val portfolio = client.getPortfolio(scenarioModel.portfolioId)!! + val environment = convert(client.getTopology(scenarioModel.topology.topologyId)!!) + val results = runScenario(portfolio, scenarioModel, environment) logger.info { "Writing results to database" } @@ -373,6 +342,60 @@ class RunnerCli : CliktCommand(name = "runner") { } } } + + /** + * Convert the specified [topology] into an [EnvironmentReader] understood by Capelin. + */ + private fun convert(topology: Topology): EnvironmentReader { + val nodes = mutableListOf<MachineDef>() + val random = Random(0) + + val machines = topology.rooms.asSequence() + .flatMap { room -> + room.tiles.flatMap { tile -> + tile.rack?.machines?.map { machine -> tile.rack to machine } ?: emptyList() + } + } + for ((rack, machine) in machines) { + val clusterId = rack.id + val position = machine.position + + val processors = machine.cpus.flatMap { cpu -> + val cores = cpu.numberOfCores + val speed = cpu.clockRateMhz + // TODO Remove hard coding of vendor + val node = ProcessingNode("Intel", "amd64", cpu.name, cores) + List(cores) { coreId -> + ProcessingUnit(node, coreId, speed) + } + } + val memoryUnits = machine.memory.map { memory -> + MemoryUnit( + "Samsung", + memory.name, + memory.speedMbPerS, + memory.sizeMb.toLong() + ) + } + + val energyConsumptionW = machine.cpus.sumOf { it.energyConsumptionW } + + nodes.add( + MachineDef( + UUID(random.nextLong(), random.nextLong()), + "node-$clusterId-$position", + mapOf("cluster" to clusterId), + MachineModel(processors, memoryUnits), + LinearPowerModel(2 * energyConsumptionW, energyConsumptionW * 0.5) + ) + ) + } + + return object : EnvironmentReader { + override fun read(): List<MachineDef> = nodes + override fun close() {} + } + } } /** diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/ScenarioManager.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/ScenarioManager.kt new file mode 100644 index 00000000..4044cec9 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/ScenarioManager.kt @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.runner + +import org.opendc.web.client.ApiClient +import org.opendc.web.client.model.Job +import org.opendc.web.client.model.SimulationState + +/** + * Manages the queue of scenarios that need to be processed. + */ +public class ScenarioManager(private val client: ApiClient) { + /** + * Find the next job that the simulator needs to process. + */ + public suspend fun findNext(): Job? { + return client.getJobs().firstOrNull() + } + + /** + * Claim the simulation job with the specified id. + */ + public suspend fun claim(id: String): Boolean { + return client.updateJob(id, SimulationState.CLAIMED) + } + + /** + * Update the heartbeat of the specified scenario. + */ + public suspend fun heartbeat(id: String) { + client.updateJob(id, SimulationState.RUNNING) + } + + /** + * Mark the scenario as failed. + */ + public suspend fun fail(id: String) { + client.updateJob(id, SimulationState.FAILED) + } + + /** + * Persist the specified results. + */ + public suspend fun finish(id: String, results: List<WebExperimentMonitor.Result>) { + client.updateJob( + id, SimulationState.FINISHED, + mapOf( + "total_requested_burst" to results.map { it.totalRequestedBurst }, + "total_granted_burst" to results.map { it.totalGrantedBurst }, + "total_overcommitted_burst" to results.map { it.totalOvercommittedBurst }, + "total_interfered_burst" to results.map { it.totalInterferedBurst }, + "mean_cpu_usage" to results.map { it.meanCpuUsage }, + "mean_cpu_demand" to results.map { it.meanCpuDemand }, + "mean_num_deployed_images" to results.map { it.meanNumDeployedImages }, + "max_num_deployed_images" to results.map { it.maxNumDeployedImages }, + "total_power_draw" to results.map { it.totalPowerDraw }, + "total_failure_slices" to results.map { it.totalFailureSlices }, + "total_failure_vm_slices" to results.map { it.totalFailureVmSlices }, + "total_vms_submitted" to results.map { it.totalVmsSubmitted }, + "total_vms_queued" to results.map { it.totalVmsQueued }, + "total_vms_finished" to results.map { it.totalVmsFinished }, + "total_vms_failed" to results.map { it.totalVmsFailed } + ) + ) + } +} diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/WebExperimentMonitor.kt index c913f82f..d4445810 100644 --- a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/WebExperimentMonitor.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020 AtLarge Research + * Copyright (c) 2021 AtLarge Research * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -20,7 +20,7 @@ * SOFTWARE. */ -package org.opendc.runner.web +package org.opendc.web.runner import mu.KotlinLogging import org.opendc.compute.api.Server diff --git a/opendc-web/opendc-web-runner/src/main/resources/log4j2.xml b/opendc-web/opendc-web-runner/src/main/resources/log4j2.xml index 503bc5dc..ad99cc00 100644 --- a/opendc-web/opendc-web-runner/src/main/resources/log4j2.xml +++ b/opendc-web/opendc-web-runner/src/main/resources/log4j2.xml @@ -36,7 +36,7 @@ <AppenderRef ref="Console"/> <AppenderRef ref="Sentry"/> </Logger> - <Logger name="org.opendc.runner" level="info" additivity="false"> + <Logger name="org.opendc.web.runner" level="info" additivity="false"> <AppenderRef ref="Console"/> <AppenderRef ref="Sentry"/> </Logger> diff --git a/opendc-web/opendc-web-runner/src/test/kotlin/org/opendc/web/client/ApiClientTest.kt b/opendc-web/opendc-web-runner/src/test/kotlin/org/opendc/web/client/ApiClientTest.kt new file mode 100644 index 00000000..3a0730a6 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/test/kotlin/org/opendc/web/client/ApiClientTest.kt @@ -0,0 +1,264 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client + +import io.ktor.client.* +import io.ktor.client.engine.mock.* +import io.ktor.http.* +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test +import java.net.URI + +/** + * Test suite for the [ApiClient] class. + */ +class ApiClientTest { + /** + * The Ktor [HttpClient] instance. + */ + private val ktor = HttpClient(MockEngine) { + engine { + addHandler { request -> + when (request.url.fullPath) { + "/oauth/token" -> { + val responseHeaders = headersOf("Content-Type" to listOf(ContentType.Application.Json.toString())) + respond( + """ + { + "access_token": "eyJz93a...k4laUWw", + "token_type": "Bearer", + "expires_in": 86400 + } + """.trimIndent(), + headers = responseHeaders + ) + } + "/portfolios/5fda5daa97dca438e7cb0a4c" -> { + val responseHeaders = headersOf("Content-Type" to listOf(ContentType.Application.Json.toString())) + respond( + """ + { + "data": { + "_id": "string", + "projectId": "string", + "name": "string", + "scenarioIds": [ + "string" + ], + "targets": { + "enabledMetrics": [ + "string" + ], + "repeatsPerScenario": 0 + } + } + } + """.trimIndent(), + headers = responseHeaders + ) + } + "/portfolios/x" -> { + val responseHeaders = headersOf("Content-Type" to listOf(ContentType.Application.Json.toString())) + respond( + """ + { + "message": "Not Found" + } + """.trimIndent(), + headers = responseHeaders, status = HttpStatusCode.NotFound + ) + } + "/scenarios/5fda5db297dca438e7cb0a4d" -> { + val responseHeaders = headersOf("Content-Type" to listOf(ContentType.Application.Json.toString())) + respond( + """ + { + "data": { + "_id": "string", + "portfolioId": "string", + "name": "string", + "trace": { + "traceId": "string", + "loadSamplingFraction": 0 + }, + "topology": { + "topologyId": "string" + }, + "operational": { + "failuresEnabled": true, + "performanceInterferenceEnabled": true, + "schedulerName": "string" + } + } + } + """.trimIndent(), + headers = responseHeaders + ) + } + "/scenarios/x" -> { + val responseHeaders = headersOf("Content-Type" to listOf(ContentType.Application.Json.toString())) + respond( + """ + { + "message": "Not Found" + } + """.trimIndent(), + headers = responseHeaders, status = HttpStatusCode.NotFound + ) + } + "/topologies/5f9825a6cf6e4c24e380b86f" -> { + val responseHeaders = headersOf("Content-Type" to listOf(ContentType.Application.Json.toString())) + respond( + """ + { + "data": { + "_id": "string", + "projectId": "string", + "name": "string", + "rooms": [ + { + "_id": "string", + "name": "string", + "tiles": [ + { + "_id": "string", + "positionX": 0, + "positionY": 0, + "rack": { + "_id": "string", + "name": "string", + "capacity": 0, + "powerCapacityW": 0, + "machines": [ + { + "_id": "string", + "position": 0, + "cpus": [ + { + "_id": "string", + "name": "string", + "clockRateMhz": 0, + "numberOfCores": 0 + } + ], + "gpus": [ + { + "_id": "string", + "name": "string", + "clockRateMhz": 0, + "numberOfCores": 0 + } + ], + "memories": [ + { + "_id": "string", + "name": "string", + "speedMbPerS": 0, + "sizeMb": 0 + } + ], + "storages": [ + { + "_id": "string", + "name": "string", + "speedMbPerS": 0, + "sizeMb": 0 + } + ] + } + ] + } + } + ] + } + ] + } + } + """.trimIndent(), + headers = responseHeaders + ) + } + "/topologies/x" -> { + val responseHeaders = + headersOf("Content-Type" to listOf(ContentType.Application.Json.toString())) + respond( + """ + { + "message": "Not Found" + } + """.trimIndent(), + headers = responseHeaders, status = HttpStatusCode.NotFound + ) + } + else -> error("Unhandled ${request.url}") + } + } + } + } + + private val auth = AuthConfiguration("auth.opendc.org", "a", "b") + + @Test + fun testPortfolioExists(): Unit = runBlocking { + val client = ApiClient(URI("http://localhost:8081"), auth, client = ktor) + val portfolio = client.getPortfolio("5fda5daa97dca438e7cb0a4c") + assertNotNull(portfolio) + } + + @Test + fun testPortfolioDoesNotExists(): Unit = runBlocking { + val client = ApiClient(URI("http://localhost:8081"), auth, client = ktor) + val portfolio = client.getPortfolio("x") + assertNull(portfolio) + } + + @Test + fun testScenarioExists(): Unit = runBlocking { + val client = ApiClient(URI("http://localhost:8081"), auth, client = ktor) + val scenario = client.getScenario("5fda5db297dca438e7cb0a4d") + assertNotNull(scenario) + } + + @Test + fun testScenarioDoesNotExists(): Unit = runBlocking { + val client = ApiClient(URI("http://localhost:8081"), auth, client = ktor) + val scenario = client.getScenario("x") + assertNull(scenario) + } + + @Test + fun testTopologyExists(): Unit = runBlocking { + val client = ApiClient(URI("http://localhost:8081"), auth, client = ktor) + val topology = client.getTopology("5f9825a6cf6e4c24e380b86f") + assertNotNull(topology) + } + + @Test + fun testTopologyDoesNotExists(): Unit = runBlocking { + val client = ApiClient(URI("http://localhost:8081"), auth, client = ktor) + val topology = client.getTopology("x") + assertNull(topology) + } +} diff --git a/opendc-web/opendc-web-ui/src/api/topologies.js b/opendc-web/opendc-web-ui/src/api/topologies.js index c8744e6c..802be4bb 100644 --- a/opendc-web/opendc-web-ui/src/api/topologies.js +++ b/opendc-web/opendc-web-ui/src/api/topologies.js @@ -31,7 +31,8 @@ export function getTopology(auth, topologyId) { } export function updateTopology(auth, topology) { - return request(auth, `topologies/${topology._id}`, 'PUT', { topology }) + const { _id, ...data } = topology; + return request(auth, `topologies/${topology._id}`, 'PUT', { topology: data }) } export function deleteTopology(auth, topologyId) { |