From d4fec022c5e5ad38e7b5a488cb28e320ea1d6416 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 2 Jul 2021 13:16:45 +0200 Subject: api: Fix OpenAPI topology schema This change addresses some issues in the OpenAPI schema for the datacenter topology. --- opendc-web/opendc-web-api/static/schema.yml | 32 ++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/opendc-web/opendc-web-api/static/schema.yml b/opendc-web/opendc-web-api/static/schema.yml index 99e88095..6e0bddfd 100644 --- a/opendc-web/opendc-web-api/static/schema.yml +++ b/opendc-web/opendc-web-api/static/schema.yml @@ -1153,6 +1153,10 @@ components: object: type: object properties: + _id: + type: string + name: + type: string capacity: type: integer powerCapacityW: @@ -1162,52 +1166,70 @@ components: items: type: object properties: + _id: + type: string position: type: integer - cpuItems: + cpus: type: array items: type: object properties: + _id: + type: string name: type: string clockRateMhz: type: integer numberOfCores: type: integer - gpuItems: + energyConsumptionW: + type: integer + gpus: type: array items: type: object properties: + _id: + type: string name: type: string clockRateMhz: type: integer numberOfCores: type: integer - memoryItems: + energyConsumptionW: + type: integer + memories: type: array items: type: object properties: + _id: + type: string name: type: string speedMbPerS: type: integer sizeMb: type: integer - storageItems: + energyConsumptionW: + type: integer + storages: type: array items: - type: integer + type: object properties: + _id: + type: string name: type: string speedMbPerS: type: integer sizeMb: type: integer + energyConsumptionW: + type: integer Portfolio: type: object properties: -- cgit v1.2.3 From e2ec16a1a40f3ffc437378b4e22fda64f86fe284 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 2 Jul 2021 13:26:09 +0200 Subject: api: Pass audience during Swagger UI authentication This change updates the Swagger UI configuration to pass the Auth0 audience to the authorization URL in order to obtain a valid JWT token. --- opendc-web/opendc-web-api/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/opendc-web/opendc-web-api/app.py b/opendc-web/opendc-web-api/app.py index 5916046b..96a1ca7a 100755 --- a/opendc-web/opendc-web-api/app.py +++ b/opendc-web/opendc-web-api/app.py @@ -89,6 +89,7 @@ def setup_swagger(app): }, oauth_config={ 'clientId': os.environ.get("AUTH0_DOCS_CLIENT_ID", ""), + 'additionalQueryStringParams': {'audience': os.environ.get("AUTH0_AUDIENCE", "https://api.opendc.org/v2/")}, } ) app.register_blueprint(swaggerui_blueprint) -- cgit v1.2.3 From 45b73e4683cce35de79117c5b4a6919556d9644f Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 2 Jul 2021 14:26:23 +0200 Subject: api: Add stricter validation of input/output data This change adds stricter validation of data that enters and leaves the database. As a result, we clearly separate the database model from the data model that the REST API exports. --- opendc-web/opendc-web-api/opendc/api/portfolios.py | 9 +++---- opendc-web/opendc-web-api/opendc/api/prefabs.py | 19 ++++++++------- opendc-web/opendc-web-api/opendc/api/projects.py | 28 ++++++++++++---------- opendc-web/opendc-web-api/opendc/api/scenarios.py | 7 +++--- opendc-web/opendc-web-api/opendc/api/topologies.py | 10 ++++---- opendc-web/opendc-web-api/opendc/api/traces.py | 6 ++--- opendc-web/opendc-web-api/opendc/auth.py | 3 +-- opendc-web/opendc-web-api/opendc/database.py | 11 --------- opendc-web/opendc-web-api/opendc/exts.py | 4 +--- .../opendc-web-api/opendc/models/portfolio.py | 2 +- opendc-web/opendc-web-api/opendc/models/prefab.py | 3 ++- opendc-web/opendc-web-api/opendc/models/project.py | 13 ++++++++-- .../opendc-web-api/opendc/models/scenario.py | 24 ++++++++++++++++++- .../opendc-web-api/opendc/models/topology.py | 4 ++-- opendc-web/opendc-web-api/opendc/models/trace.py | 9 +++++++ opendc-web/opendc-web-ui/src/api/topologies.js | 3 ++- 16 files changed, 96 insertions(+), 59 deletions(-) diff --git a/opendc-web/opendc-web-api/opendc/api/portfolios.py b/opendc-web/opendc-web-api/opendc/api/portfolios.py index b07e9da5..84ec466c 100644 --- a/opendc-web/opendc-web-api/opendc/api/portfolios.py +++ b/opendc-web/opendc-web-api/opendc/api/portfolios.py @@ -44,7 +44,7 @@ class Portfolio(Resource): portfolio.check_exists() portfolio.check_user_access(current_user['sub'], False) - data = portfolio.obj + data = PortfolioSchema().dump(portfolio.obj) return {'data': data} def put(self, portfolio_id): @@ -63,7 +63,7 @@ class Portfolio(Resource): portfolio.set_property('targets.repeatsPerScenario', result['portfolio']['targets']['repeatsPerScenario']) portfolio.update() - data = portfolio.obj + data = PortfolioSchema().dump(portfolio.obj) return {'data': data} def delete(self, portfolio_id): @@ -84,7 +84,8 @@ class Portfolio(Resource): project.update() old_object = portfolio.delete() - return {'data': old_object} + data = PortfolioSchema().dump(old_object) + return {'data': data} class PutSchema(Schema): """ @@ -125,7 +126,7 @@ class PortfolioScenarios(Resource): portfolio.obj['scenarioIds'].append(scenario.get_id()) portfolio.update() - data = scenario.obj + data = ScenarioSchema().dump(scenario.obj) return {'data': data} class PostSchema(Schema): diff --git a/opendc-web/opendc-web-api/opendc/api/prefabs.py b/opendc-web/opendc-web-api/opendc/api/prefabs.py index 7bb17e7d..730546ba 100644 --- a/opendc-web/opendc-web-api/opendc/api/prefabs.py +++ b/opendc-web/opendc-web-api/opendc/api/prefabs.py @@ -24,7 +24,6 @@ from flask_restful import Resource from marshmallow import Schema, fields from opendc.models.prefab import Prefab as PrefabModel, PrefabSchema -from opendc.database import Database from opendc.exts import current_user, requires_auth, db @@ -56,14 +55,15 @@ class PrefabList(Resource): result = schema.load(request.json) prefab = PrefabModel(result['prefab']) - prefab.set_property('datetimeCreated', Database.datetime_to_string(datetime.now())) - prefab.set_property('datetimeLastEdited', Database.datetime_to_string(datetime.now())) + prefab.set_property('datetimeCreated', datetime.now()) + prefab.set_property('datetimeLastEdited', datetime.now()) user_id = current_user['sub'] prefab.set_property('authorId', user_id) prefab.insert() - return {'data': prefab.obj} + data = PrefabSchema().dump(prefab.obj) + return {'data': data} class PostSchema(Schema): """ @@ -83,7 +83,8 @@ class Prefab(Resource): prefab = PrefabModel.from_id(prefab_id) prefab.check_exists() prefab.check_user_access(current_user['sub']) - return {'data': prefab.obj} + data = PrefabSchema().dump(prefab.obj) + return {'data': data} def put(self, prefab_id): """Update a prefab's name and/or contents.""" @@ -97,10 +98,11 @@ class Prefab(Resource): prefab.set_property('name', result['prefab']['name']) prefab.set_property('rack', result['prefab']['rack']) - prefab.set_property('datetime_last_edited', Database.datetime_to_string(datetime.now())) + prefab.set_property('datetimeLastEdited', datetime.now()) prefab.update() - return {'data': prefab.obj} + data = PrefabSchema().dump(prefab.obj) + return {'data': data} def delete(self, prefab_id): """Delete this Prefab.""" @@ -111,7 +113,8 @@ class Prefab(Resource): old_object = prefab.delete() - return {'data': old_object} + data = PrefabSchema().dump(old_object) + return {'data': data} class PutSchema(Schema): """ diff --git a/opendc-web/opendc-web-api/opendc/api/projects.py b/opendc-web/opendc-web-api/opendc/api/projects.py index 8c44b680..05f02a84 100644 --- a/opendc-web/opendc-web-api/opendc/api/projects.py +++ b/opendc-web/opendc-web-api/opendc/api/projects.py @@ -27,7 +27,6 @@ from opendc.models.portfolio import Portfolio, PortfolioSchema from opendc.models.topology import Topology, TopologySchema from opendc.models.project import Project as ProjectModel, ProjectSchema from opendc.exts import current_user, requires_auth -from opendc.database import Database class ProjectList(Resource): @@ -40,7 +39,8 @@ class ProjectList(Resource): """Get the authorized projects of the user""" user_id = current_user['sub'] projects = ProjectModel.get_for_user(user_id) - return {'data': projects} + data = ProjectSchema().dump(projects, many=True) + return {'data': data} def post(self): """Create a new project, and return that new project.""" @@ -53,8 +53,8 @@ class ProjectList(Resource): topology.insert() project = ProjectModel(result['project']) - project.set_property('datetimeCreated', Database.datetime_to_string(datetime.now())) - project.set_property('datetimeLastEdited', Database.datetime_to_string(datetime.now())) + project.set_property('datetimeCreated', datetime.now()) + project.set_property('datetimeLastEdited', datetime.now()) project.set_property('topologyIds', [topology.get_id()]) project.set_property('portfolioIds', []) project.set_property('authorizations', [{'userId': user_id, 'level': 'OWN'}]) @@ -63,7 +63,8 @@ class ProjectList(Resource): topology.set_property('projectId', project.get_id()) topology.update() - return {'data': project.obj} + data = ProjectSchema().dump(project.obj) + return {'data': data} class Project(Resource): @@ -79,7 +80,8 @@ class Project(Resource): project.check_exists() project.check_user_access(current_user['sub'], False) - return {'data': project.obj} + data = ProjectSchema().dump(project.obj) + return {'data': data} def put(self, project_id): """Update a project's name.""" @@ -92,10 +94,11 @@ class Project(Resource): project.check_user_access(current_user['sub'], True) project.set_property('name', result['project']['name']) - project.set_property('datetimeLastEdited', Database.datetime_to_string(datetime.now())) + project.set_property('datetimeLastEdited', datetime.now()) project.update() - return {'data': project.obj} + data = ProjectSchema().dump(project.obj) + return {'data': data} def delete(self, project_id): """Delete this Project.""" @@ -113,8 +116,8 @@ class Project(Resource): portfolio.delete() old_object = project.delete() - - return {'data': old_object} + data = ProjectSchema().dump(old_object) + return {'data': data} class PutSchema(Schema): """ @@ -148,10 +151,11 @@ class ProjectTopologies(Resource): topology.insert() project.obj['topologyIds'].append(topology.get_id()) - project.set_property('datetimeLastEdited', Database.datetime_to_string(datetime.now())) + project.set_property('datetimeLastEdited', datetime.now()) project.update() - return {'data': topology.obj} + data = TopologySchema().dump(topology.obj) + return {'data': data} class PutSchema(Schema): """ diff --git a/opendc-web/opendc-web-api/opendc/api/scenarios.py b/opendc-web/opendc-web-api/opendc/api/scenarios.py index b566950a..234bdec1 100644 --- a/opendc-web/opendc-web-api/opendc/api/scenarios.py +++ b/opendc-web/opendc-web-api/opendc/api/scenarios.py @@ -38,7 +38,7 @@ class Scenario(Resource): scenario = ScenarioModel.from_id(scenario_id) scenario.check_exists() scenario.check_user_access(current_user['sub'], False) - data = scenario.obj + data = ScenarioSchema().dump(scenario.obj) return {'data': data} def put(self, scenario_id): @@ -54,7 +54,7 @@ class Scenario(Resource): scenario.set_property('name', result['scenario']['name']) scenario.update() - data = scenario.obj + data = ScenarioSchema().dump(scenario.obj) return {'data': data} def delete(self, scenario_id): @@ -72,7 +72,8 @@ class Scenario(Resource): portfolio.update() old_object = scenario.delete() - return {'data': old_object} + data = ScenarioSchema().dump(old_object) + return {'data': data} class PutSchema(Schema): """ diff --git a/opendc-web/opendc-web-api/opendc/api/topologies.py b/opendc-web/opendc-web-api/opendc/api/topologies.py index eedf049d..a2d3f41a 100644 --- a/opendc-web/opendc-web-api/opendc/api/topologies.py +++ b/opendc-web/opendc-web-api/opendc/api/topologies.py @@ -24,7 +24,6 @@ from flask import request from flask_restful import Resource from marshmallow import Schema, fields -from opendc.database import Database from opendc.models.project import Project from opendc.models.topology import Topology as TopologyModel, TopologySchema from opendc.exts import current_user, requires_auth @@ -43,7 +42,7 @@ class Topology(Resource): topology = TopologyModel.from_id(topology_id) topology.check_exists() topology.check_user_access(current_user['sub'], False) - data = topology.obj + data = TopologySchema().dump(topology.obj) return {'data': data} def put(self, topology_id): @@ -60,10 +59,10 @@ class Topology(Resource): topology.set_property('name', result['topology']['name']) topology.set_property('rooms', result['topology']['rooms']) - topology.set_property('datetimeLastEdited', Database.datetime_to_string(datetime.now())) + topology.set_property('datetimeLastEdited', datetime.now()) topology.update() - data = topology.obj + data = TopologySchema().dump(topology.obj) return {'data': data} def delete(self, topology_id): @@ -84,7 +83,8 @@ class Topology(Resource): project.update() old_object = topology.delete() - return {'data': old_object} + data = TopologySchema().dump(old_object) + return {'data': data} class PutSchema(Schema): """ diff --git a/opendc-web/opendc-web-api/opendc/api/traces.py b/opendc-web/opendc-web-api/opendc/api/traces.py index f685f00c..6be8c5e5 100644 --- a/opendc-web/opendc-web-api/opendc/api/traces.py +++ b/opendc-web/opendc-web-api/opendc/api/traces.py @@ -21,7 +21,7 @@ from flask_restful import Resource from opendc.exts import requires_auth -from opendc.models.trace import Trace as TraceModel +from opendc.models.trace import Trace as TraceModel, TraceSchema class TraceList(Resource): @@ -33,7 +33,7 @@ class TraceList(Resource): def get(self): """Get all available Traces.""" traces = TraceModel.get_all() - data = traces.obj + data = TraceSchema().dump(traces.obj, many=True) return {'data': data} @@ -47,5 +47,5 @@ class Trace(Resource): """Get trace information by identifier.""" trace = TraceModel.from_id(trace_id) trace.check_exists() - data = trace.obj + data = TraceSchema().dump(trace.obj) return {'data': data} diff --git a/opendc-web/opendc-web-api/opendc/auth.py b/opendc-web/opendc-web-api/opendc/auth.py index 1870f01c..6db06fb1 100644 --- a/opendc-web/opendc-web-api/opendc/auth.py +++ b/opendc-web/opendc-web-api/opendc/auth.py @@ -42,8 +42,7 @@ def get_token(): if parts[0].lower() != "bearer": raise AuthError({ "code": "invalid_header", - "description": "Authorization header must start with" - " Bearer" + "description": "Authorization header must start with Bearer" }, 401) if len(parts) == 1: raise AuthError({"code": "invalid_header", "description": "Token not found"}, 401) diff --git a/opendc-web/opendc-web-api/opendc/database.py b/opendc-web/opendc-web-api/opendc/database.py index f9a33b66..37fd1a4d 100644 --- a/opendc-web/opendc-web-api/opendc/database.py +++ b/opendc-web/opendc-web-api/opendc/database.py @@ -19,7 +19,6 @@ # SOFTWARE. import urllib.parse -from datetime import datetime from pymongo import MongoClient @@ -90,13 +89,3 @@ class Database: The query needs to be in json format, i.e.: `{'name': prefab_name}`. """ getattr(self.opendc_db, collection).delete_many(query) - - @staticmethod - def datetime_to_string(datetime_to_convert): - """Return a database-compatible string representation of the given datetime object.""" - return datetime_to_convert.strftime(DATETIME_STRING_FORMAT) - - @staticmethod - def string_to_datetime(string_to_convert): - """Return a datetime corresponding to the given string representation.""" - return datetime.strptime(string_to_convert, DATETIME_STRING_FORMAT) diff --git a/opendc-web/opendc-web-api/opendc/exts.py b/opendc-web/opendc-web-api/opendc/exts.py index f088a29c..d24f7197 100644 --- a/opendc-web/opendc-web-api/opendc/exts.py +++ b/opendc-web/opendc-web-api/opendc/exts.py @@ -34,8 +34,7 @@ def get_auth_context(): _auth_context = AuthContext( alg=AsymmetricJwtAlgorithm(jwks_url=f"https://{os.environ['AUTH0_DOMAIN']}/.well-known/jwks.json"), issuer=f"https://{os.environ['AUTH0_DOMAIN']}/", - audience=os.environ['AUTH0_AUDIENCE'] - ) + audience=os.environ['AUTH0_AUDIENCE']) g.auth_context = _auth_context return _auth_context @@ -46,7 +45,6 @@ auth_context = LocalProxy(get_auth_context) def requires_auth(f): """Decorator to determine if the Access Token is valid. """ - @wraps(f) def decorated(*args, **kwargs): token = get_token() diff --git a/opendc-web/opendc-web-api/opendc/models/portfolio.py b/opendc-web/opendc-web-api/opendc/models/portfolio.py index aff1d3f0..1643e23e 100644 --- a/opendc-web/opendc-web-api/opendc/models/portfolio.py +++ b/opendc-web/opendc-web-api/opendc/models/portfolio.py @@ -16,7 +16,7 @@ class PortfolioSchema(Schema): """ Schema representing a portfolio. """ - _id = fields.String() + _id = fields.String(dump_only=True) projectId = fields.String() name = fields.String(required=True) scenarioIds = fields.List(fields.String()) diff --git a/opendc-web/opendc-web-api/opendc/models/prefab.py b/opendc-web/opendc-web-api/opendc/models/prefab.py index d83ef4cb..5e4b81dc 100644 --- a/opendc-web/opendc-web-api/opendc/models/prefab.py +++ b/opendc-web/opendc-web-api/opendc/models/prefab.py @@ -9,7 +9,8 @@ class PrefabSchema(Schema): """ Schema for a Prefab. """ - _id = fields.String() + _id = fields.String(dump_only=True) + authorId = fields.String(dump_only=True) name = fields.String(required=True) datetimeCreated = fields.DateTime() datetimeLastEdited = fields.DateTime() diff --git a/opendc-web/opendc-web-api/opendc/models/project.py b/opendc-web/opendc-web-api/opendc/models/project.py index ee84c73e..f2b3b564 100644 --- a/opendc-web/opendc-web-api/opendc/models/project.py +++ b/opendc-web/opendc-web-api/opendc/models/project.py @@ -1,20 +1,29 @@ -from marshmallow import Schema, fields +from marshmallow import Schema, fields, validate from werkzeug.exceptions import Forbidden from opendc.models.model import Model from opendc.exts import db +class ProjectAuthorizations(Schema): + """ + Schema representing a project authorization. + """ + userId = fields.String(required=True) + level = fields.String(required=True, validate=validate.OneOf(["VIEW", "EDIT", "OWN"])) + + class ProjectSchema(Schema): """ Schema representing a Project. """ - _id = fields.String() + _id = fields.String(dump_only=True) name = fields.String(required=True) datetimeCreated = fields.DateTime() datetimeLastEdited = fields.DateTime() topologyIds = fields.List(fields.String()) portfolioIds = fields.List(fields.String()) + authorizations = fields.List(fields.Nested(ProjectAuthorizations)) class Project(Model): diff --git a/opendc-web/opendc-web-api/opendc/models/scenario.py b/opendc-web/opendc-web-api/opendc/models/scenario.py index 2911b1ae..658d790e 100644 --- a/opendc-web/opendc-web-api/opendc/models/scenario.py +++ b/opendc-web/opendc-web-api/opendc/models/scenario.py @@ -34,17 +34,39 @@ class OperationalSchema(Schema): schedulerName = fields.String() +class ResultSchema(Schema): + """ + Schema representing the simulation results. + """ + max_num_deployed_images = fields.List(fields.Number()) + max_cpu_demand = fields.List(fields.Number()) + max_cpu_usage = fields.List(fields.Number()) + mean_num_deployed_images = fields.List(fields.Number()) + total_failure_slices = fields.List(fields.Number()) + total_failure_vm_slices = fields.List(fields.Number()) + total_granted_burst = fields.List(fields.Number()) + total_interfered_burst = fields.List(fields.Number()) + total_overcommitted_burst = fields.List(fields.Number()) + total_power_draw = fields.List(fields.Number()) + total_requested_burst = fields.List(fields.Number()) + total_vms_failed = fields.List(fields.Number()) + total_vms_finished = fields.List(fields.Number()) + total_vms_queued = fields.List(fields.Number()) + total_vms_submitted = fields.List(fields.Number()) + + class ScenarioSchema(Schema): """ Schema representing a scenario. """ - _id = fields.String() + _id = fields.String(dump_only=True) portfolioId = fields.String() name = fields.String(required=True) simulation = fields.Nested(SimulationSchema) trace = fields.Nested(TraceSchema) topology = fields.Nested(TopologySchema) operational = fields.Nested(OperationalSchema) + results = fields.Nested(ResultSchema, dump_only=True) class Scenario(Model): diff --git a/opendc-web/opendc-web-api/opendc/models/topology.py b/opendc-web/opendc-web-api/opendc/models/topology.py index c6354ae6..71d2cade 100644 --- a/opendc-web/opendc-web-api/opendc/models/topology.py +++ b/opendc-web/opendc-web-api/opendc/models/topology.py @@ -72,8 +72,8 @@ class TopologySchema(Schema): """ Schema representing a datacenter topology. """ - _id = fields.String() - projectId = fields.String() + _id = fields.String(dump_only=True) + projectId = fields.String(dump_only=True) name = fields.String(required=True) rooms = fields.List(fields.Nested(RoomSchema), required=True) diff --git a/opendc-web/opendc-web-api/opendc/models/trace.py b/opendc-web/opendc-web-api/opendc/models/trace.py index 2f6e4926..69287f29 100644 --- a/opendc-web/opendc-web-api/opendc/models/trace.py +++ b/opendc-web/opendc-web-api/opendc/models/trace.py @@ -1,6 +1,15 @@ +from marshmallow import Schema, fields + from opendc.models.model import Model +class TraceSchema(Schema): + """Schema for a Trace.""" + _id = fields.String(dump_only=True) + name = fields.String() + type = fields.String() + + class Trace(Model): """Model representing a Trace.""" diff --git a/opendc-web/opendc-web-ui/src/api/topologies.js b/opendc-web/opendc-web-ui/src/api/topologies.js index c8744e6c..802be4bb 100644 --- a/opendc-web/opendc-web-ui/src/api/topologies.js +++ b/opendc-web/opendc-web-ui/src/api/topologies.js @@ -31,7 +31,8 @@ export function getTopology(auth, topologyId) { } export function updateTopology(auth, topology) { - return request(auth, `topologies/${topology._id}`, 'PUT', { topology }) + const { _id, ...data } = topology; + return request(auth, `topologies/${topology._id}`, 'PUT', { topology: data }) } export function deleteTopology(auth, topologyId) { -- cgit v1.2.3 From a2a5979bfb392565b55e489b6020aa391e782eb0 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 2 Jul 2021 16:14:52 +0200 Subject: api: Add endpoint for simulation jobs This change adds an API endpoint for simulation jobs which allows API consumers to manage simulation jobs without needing direct database access that is currently needed for the web runner. --- opendc-web/opendc-web-api/app.py | 3 + opendc-web/opendc-web-api/opendc/api/jobs.py | 105 ++++++++++++++ opendc-web/opendc-web-api/opendc/database.py | 8 +- .../opendc-web-api/opendc/models/scenario.py | 52 +++---- opendc-web/opendc-web-api/static/schema.yml | 151 +++++++++++++++++++-- opendc-web/opendc-web-api/tests/api/test_jobs.py | 139 +++++++++++++++++++ 6 files changed, 419 insertions(+), 39 deletions(-) create mode 100644 opendc-web/opendc-web-api/opendc/api/jobs.py create mode 100644 opendc-web/opendc-web-api/tests/api/test_jobs.py diff --git a/opendc-web/opendc-web-api/app.py b/opendc-web/opendc-web-api/app.py index 96a1ca7a..36c80b7a 100755 --- a/opendc-web/opendc-web-api/app.py +++ b/opendc-web/opendc-web-api/app.py @@ -10,6 +10,7 @@ from flask_restful import Api from flask_swagger_ui import get_swaggerui_blueprint from marshmallow import ValidationError +from opendc.api.jobs import JobList, Job from opendc.api.portfolios import Portfolio, PortfolioScenarios from opendc.api.prefabs import Prefab, PrefabList from opendc.api.projects import ProjectList, Project, ProjectTopologies, ProjectPortfolios @@ -60,6 +61,8 @@ def setup_api(app): api.add_resource(TraceList, '/traces/') api.add_resource(Trace, '/traces/') api.add_resource(SchedulerList, '/schedulers/') + api.add_resource(JobList, '/jobs/') + api.add_resource(Job, '/jobs/') @app.errorhandler(AuthError) def handle_auth_error(ex): diff --git a/opendc-web/opendc-web-api/opendc/api/jobs.py b/opendc-web/opendc-web-api/opendc/api/jobs.py new file mode 100644 index 00000000..5feaea16 --- /dev/null +++ b/opendc-web/opendc-web-api/opendc/api/jobs.py @@ -0,0 +1,105 @@ +# Copyright (c) 2021 AtLarge Research +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +from flask import request +from flask_restful import Resource +from marshmallow import fields, Schema, validate +from werkzeug.exceptions import BadRequest, Conflict + +from opendc.exts import requires_auth +from opendc.models.scenario import Scenario + + +def convert_to_job(scenario): + """Convert a scenario to a job. + """ + return JobSchema().dump({ + '_id': scenario['_id'], + 'scenarioId': scenario['_id'], + 'state': scenario['simulation']['state'], + 'heartbeat': scenario['simulation'].get('heartbeat', None), + 'results': scenario.get('results', {}) + }) + + +class JobSchema(Schema): + """ + Schema representing a simulation job. + """ + _id = fields.String(dump_only=True) + scenarioId = fields.String(dump_only=True) + state = fields.String(required=True, + validate=validate.OneOf(["QUEUED", "CLAIMED", "RUNNING", "FINISHED", "FAILED"])) + heartbeat = fields.DateTime() + results = fields.Dict() + + +class JobList(Resource): + """ + Resource representing the list of available jobs. + """ + method_decorators = [requires_auth] + + def get(self): + """Get all available jobs.""" + jobs = Scenario.get_jobs() + data = list(map(convert_to_job, jobs.obj)) + return {'data': data} + + +class Job(Resource): + """ + Resource representing a single job. + """ + method_decorators = [requires_auth] + + def get(self, job_id): + """Get the details of a single job.""" + job = Scenario.from_id(job_id) + job.check_exists() + data = convert_to_job(job.obj) + return {'data': data} + + def post(self, job_id): + """Update the details of a single job.""" + action = JobSchema(only=('state', 'results')).load(request.json) + + job = Scenario.from_id(job_id) + job.check_exists() + + old_state = job.obj['simulation']['state'] + new_state = action['state'] + + if old_state == new_state: + data = job.update_state(new_state) + elif (old_state, new_state) == ('QUEUED', 'CLAIMED'): + data = job.update_state('CLAIMED') + elif (old_state, new_state) == ('CLAIMED', 'RUNNING'): + data = job.update_state('RUNNING') + elif (old_state, new_state) == ('RUNNING', 'FINISHED'): + data = job.update_state('FINISHED', results=action.get('results', None)) + elif old_state in ('CLAIMED', 'RUNNING') and new_state == 'FAILED': + data = job.update_state('FAILED') + else: + raise BadRequest('Invalid state transition') + + if not data: + raise Conflict('State conflict') + + return {'data': convert_to_job(data)} diff --git a/opendc-web/opendc-web-api/opendc/database.py b/opendc-web/opendc-web-api/opendc/database.py index 37fd1a4d..dd6367f2 100644 --- a/opendc-web/opendc-web-api/opendc/database.py +++ b/opendc-web/opendc-web-api/opendc/database.py @@ -20,7 +20,7 @@ import urllib.parse -from pymongo import MongoClient +from pymongo import MongoClient, ReturnDocument DATETIME_STRING_FORMAT = '%Y-%m-%dT%H:%M:%S' CONNECTION_POOL = None @@ -76,6 +76,12 @@ class Database: """Updates an existing object.""" return getattr(self.opendc_db, collection).update({'_id': _id}, obj) + def fetch_and_update(self, query, update, collection): + """Updates an existing object.""" + return getattr(self.opendc_db, collection).find_one_and_update(query, + update, + return_document=ReturnDocument.AFTER) + def delete_one(self, query, collection): """Deletes one object matching the given query. diff --git a/opendc-web/opendc-web-api/opendc/models/scenario.py b/opendc-web/opendc-web-api/opendc/models/scenario.py index 658d790e..0fb6c453 100644 --- a/opendc-web/opendc-web-api/opendc/models/scenario.py +++ b/opendc-web/opendc-web-api/opendc/models/scenario.py @@ -1,15 +1,12 @@ +from datetime import datetime + from marshmallow import Schema, fields + +from opendc.exts import db from opendc.models.model import Model from opendc.models.portfolio import Portfolio -class SimulationSchema(Schema): - """ - Simulation details. - """ - state = fields.String() - - class TraceSchema(Schema): """ Schema for specifying the trace of a scenario. @@ -34,27 +31,6 @@ class OperationalSchema(Schema): schedulerName = fields.String() -class ResultSchema(Schema): - """ - Schema representing the simulation results. - """ - max_num_deployed_images = fields.List(fields.Number()) - max_cpu_demand = fields.List(fields.Number()) - max_cpu_usage = fields.List(fields.Number()) - mean_num_deployed_images = fields.List(fields.Number()) - total_failure_slices = fields.List(fields.Number()) - total_failure_vm_slices = fields.List(fields.Number()) - total_granted_burst = fields.List(fields.Number()) - total_interfered_burst = fields.List(fields.Number()) - total_overcommitted_burst = fields.List(fields.Number()) - total_power_draw = fields.List(fields.Number()) - total_requested_burst = fields.List(fields.Number()) - total_vms_failed = fields.List(fields.Number()) - total_vms_finished = fields.List(fields.Number()) - total_vms_queued = fields.List(fields.Number()) - total_vms_submitted = fields.List(fields.Number()) - - class ScenarioSchema(Schema): """ Schema representing a scenario. @@ -62,11 +38,9 @@ class ScenarioSchema(Schema): _id = fields.String(dump_only=True) portfolioId = fields.String() name = fields.String(required=True) - simulation = fields.Nested(SimulationSchema) trace = fields.Nested(TraceSchema) topology = fields.Nested(TopologySchema) operational = fields.Nested(OperationalSchema) - results = fields.Nested(ResultSchema, dump_only=True) class Scenario(Model): @@ -84,3 +58,21 @@ class Scenario(Model): """ portfolio = Portfolio.from_id(self.obj['portfolioId']) portfolio.check_user_access(user_id, edit_access) + + @classmethod + def get_jobs(cls): + """Obtain the scenarios that have been queued. + """ + return cls(db.fetch_all({'simulation.state': 'QUEUED'}, cls.collection_name)) + + def update_state(self, new_state, results=None): + """Atomically update the state of the Scenario. + """ + update = {'$set': {'simulation.state': new_state, 'simulation.heartbeat': datetime.now()}} + if results: + update['$set']['results'] = results + return db.fetch_and_update( + query={'_id': self.obj['_id'], 'simulation.state': self.obj['simulation']['state']}, + update=update, + collection=self.collection_name + ) diff --git a/opendc-web/opendc-web-api/static/schema.yml b/opendc-web/opendc-web-api/static/schema.yml index 6e0bddfd..5a8c6825 100644 --- a/opendc-web/opendc-web-api/static/schema.yml +++ b/opendc-web/opendc-web-api/static/schema.yml @@ -965,7 +965,7 @@ paths: application/json: schema: properties: - project: + prefab: $ref: "#/components/schemas/Prefab" description: Prefab's new properties. required: true @@ -1040,6 +1040,135 @@ paths: "application/json": schema: $ref: "#/components/schemas/NotFound" + /jobs: + get: + tags: + - jobs + description: Get all available jobs to run. + responses: + "200": + description: Successfully retrieved available jobs. + content: + "application/json": + schema: + type: object + required: + - data + properties: + data: + type: array + items: + $ref: "#/components/schemas/Job" + "401": + description: Unauthorized. + content: + "application/json": + schema: + $ref: "#/components/schemas/Unauthorized" + "/jobs/{jobId}": + get: + tags: + - jobs + description: Get this Job. + parameters: + - name: jobId + in: path + description: Job's ID. + required: true + schema: + type: string + responses: + "200": + description: Successfully retrieved Job. + content: + "application/json": + schema: + type: object + required: + - data + properties: + data: + $ref: "#/components/schemas/Job" + "401": + description: Unauthorized. + content: + "application/json": + schema: + $ref: "#/components/schemas/Unauthorized" + "403": + description: Forbidden from retrieving Job. + content: + "application/json": + schema: + $ref: "#/components/schemas/Forbidden" + "404": + description: Job not found. + content: + "application/json": + schema: + $ref: "#/components/schemas/NotFound" + post: + tags: + - jobs + description: Update this Job. + parameters: + - name: jobId + in: path + description: Job's ID. + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + properties: + job: + $ref: "#/components/schemas/Job" + description: Job's new properties. + required: true + responses: + "200": + description: Successfully updated Job. + content: + "application/json": + schema: + type: object + required: + - data + properties: + data: + $ref: "#/components/schemas/Job" + "400": + description: Missing or incorrectly typed parameter. + content: + "application/json": + schema: + $ref: "#/components/schemas/Invalid" + "401": + description: Unauthorized. + content: + "application/json": + schema: + $ref: "#/components/schemas/Unauthorized" + "403": + description: Forbidden from retrieving Job. + content: + "application/json": + schema: + $ref: "#/components/schemas/Forbidden" + "404": + description: Job not found. + content: + "application/json": + schema: + $ref: "#/components/schemas/NotFound" + "409": + description: State conflict. + content: + "application/json": + schema: + $ref: "#/components/schemas/Invalid" components: securitySchemes: auth0: @@ -1261,13 +1390,6 @@ components: type: string name: type: string - simulation: - type: object - properties: - state: - type: string - results: - type: object trace: type: object properties: @@ -1289,6 +1411,19 @@ components: type: boolean schedulerName: type: string + Job: + type: object + properties: + _id: + type: string + scenarioId: + type: string + state: + type: string + heartbeat: + type: string + results: + type: object Trace: type: object properties: diff --git a/opendc-web/opendc-web-api/tests/api/test_jobs.py b/opendc-web/opendc-web-api/tests/api/test_jobs.py new file mode 100644 index 00000000..2efe6933 --- /dev/null +++ b/opendc-web/opendc-web-api/tests/api/test_jobs.py @@ -0,0 +1,139 @@ +# Copyright (c) 2021 AtLarge Research +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# +from datetime import datetime + +from opendc.exts import db + +test_id = 24 * '1' +test_id_2 = 24 * '2' + + +def test_get_jobs(client, mocker): + mocker.patch.object(db, 'fetch_all', return_value=[ + {'_id': 'a', 'scenarioId': 'x', 'portfolioId': 'y', 'simulation': {'state': 'QUEUED'}} + ]) + res = client.get('/jobs/') + assert '200' in res.status + + +def test_get_job_non_existing(client, mocker): + mocker.patch.object(db, 'fetch_one', return_value=None) + assert '404' in client.get(f'/jobs/{test_id}').status + + +def test_get_job(client, mocker): + mocker.patch.object(db, 'fetch_one', return_value={ + '_id': 'a', 'scenarioId': 'x', 'portfolioId': 'y', 'simulation': {'state': 'QUEUED'} + }) + res = client.get(f'/jobs/{test_id}') + assert '200' in res.status + + +def test_update_job_nop(client, mocker): + mocker.patch.object(db, 'fetch_one', return_value={ + '_id': test_id, 'scenarioId': 'x', 'portfolioId': 'y', 'simulation': {'state': 'QUEUED'} + }) + update_mock = mocker.patch.object(db, 'fetch_and_update', return_value={ + '_id': test_id, 'scenarioId': 'x', 'portfolioId': 'y', + 'simulation': {'state': 'QUEUED', 'heartbeat': datetime.now()} + }) + res = client.post(f'/jobs/{test_id}', json={'state': 'QUEUED'}) + assert '200' in res.status + update_mock.assert_called_once() + + +def test_update_job_invalid_state(client, mocker): + mocker.patch.object(db, 'fetch_one', return_value={ + '_id': test_id, 'scenarioId': 'x', 'portfolioId': 'y', 'simulation': {'state': 'QUEUED'} + }) + res = client.post(f'/jobs/{test_id}', json={'state': 'FINISHED'}) + assert '400' in res.status + + +def test_update_job_claim(client, mocker): + mocker.patch.object(db, 'fetch_one', return_value={ + '_id': test_id, 'scenarioId': 'x', 'portfolioId': 'y', 'simulation': {'state': 'QUEUED'} + }) + update_mock = mocker.patch.object(db, 'fetch_and_update', return_value={ + '_id': test_id, 'scenarioId': 'x', 'portfolioId': 'y', + 'simulation': {'state': 'CLAIMED', 'heartbeat': datetime.now()} + }) + res = client.post(f'/jobs/{test_id}', json={'state': 'CLAIMED'}) + assert '200' in res.status + update_mock.assert_called_once() + + +def test_update_job_conflict(client, mocker): + mocker.patch.object(db, 'fetch_one', return_value={ + '_id': test_id, 'scenarioId': 'x', 'portfolioId': 'y', 'simulation': {'state': 'QUEUED'} + }) + update_mock = mocker.patch.object(db, 'fetch_and_update', return_value=None) + res = client.post(f'/jobs/{test_id}', json={'state': 'CLAIMED'}) + assert '409' in res.status + update_mock.assert_called_once() + + +def test_update_job_run(client, mocker): + mocker.patch.object(db, 'fetch_one', return_value={ + '_id': test_id, 'scenarioId': 'x', 'portfolioId': 'y', 'simulation': {'state': 'CLAIMED'} + }) + update_mock = mocker.patch.object(db, 'fetch_and_update', return_value={ + '_id': test_id, 'scenarioId': 'x', 'portfolioId': 'y', + 'simulation': {'state': 'RUNNING', 'heartbeat': datetime.now()} + }) + res = client.post(f'/jobs/{test_id}', json={'state': 'RUNNING'}) + assert '200' in res.status + update_mock.assert_called_once() + + +def test_update_job_finished(client, mocker): + mocker.patch.object(db, 'fetch_one', return_value={ + '_id': test_id, 'scenarioId': 'x', 'portfolioId': 'y', 'simulation': {'state': 'RUNNING'} + }) + update_mock = mocker.patch.object(db, 'fetch_and_update', return_value={ + '_id': test_id, 'scenarioId': 'x', 'portfolioId': 'y', + 'simulation': {'state': 'FINISHED', 'heartbeat': datetime.now()} + }) + res = client.post(f'/jobs/{test_id}', json={'state': 'FINISHED'}) + assert '200' in res.status + update_mock.assert_called_once() + + +def test_update_job_failed(client, mocker): + mocker.patch.object(db, 'fetch_one', return_value={ + '_id': test_id, 'scenarioId': 'x', 'portfolioId': 'y', 'simulation': {'state': 'RUNNING'} + }) + update_mock = mocker.patch.object(db, 'fetch_and_update', return_value={ + '_id': test_id, 'scenarioId': 'x', 'portfolioId': 'y', + 'simulation': {'state': 'FAILED', 'heartbeat': datetime.now()} + }) + res = client.post(f'/jobs/{test_id}', json={'state': 'FAILED'}) + assert '200' in res.status + update_mock.assert_called_once() -- cgit v1.2.3 From fa7ffd9d1594a5bc9dba4fc65af0a4100988341b Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 2 Jul 2021 16:47:40 +0200 Subject: api: Restrict API scopes This change adds support for restricting API scopes in the OpenDC API server. This is necessary to make a distinction between runners and regular users. --- opendc-web/opendc-web-api/conftest.py | 15 +++++++-- opendc-web/opendc-web-api/opendc/api/jobs.py | 6 ++-- opendc-web/opendc-web-api/opendc/api/portfolios.py | 7 +++-- opendc-web/opendc-web-api/opendc/api/scenarios.py | 8 +++-- opendc-web/opendc-web-api/opendc/api/topologies.py | 8 +++-- opendc-web/opendc-web-api/opendc/exts.py | 36 +++++++++++++++++++++- opendc-web/opendc-web-api/static/schema.yml | 1 + 7 files changed, 69 insertions(+), 12 deletions(-) diff --git a/opendc-web/opendc-web-api/conftest.py b/opendc-web/opendc-web-api/conftest.py index 430262f1..958a5894 100644 --- a/opendc-web/opendc-web-api/conftest.py +++ b/opendc-web/opendc-web-api/conftest.py @@ -8,7 +8,7 @@ from flask import _request_ctx_stack, g from opendc.database import Database -def decorator(f): +def requires_auth_mock(f): @wraps(f) def decorated_function(*args, **kwargs): _request_ctx_stack.top.current_user = {'sub': 'test'} @@ -16,13 +16,24 @@ def decorator(f): return decorated_function +def requires_scope_mock(required_scope): + def decorator(f): + @wraps(f) + def decorated_function(*args, **kwargs): + return f(*args, **kwargs) + return decorated_function + return decorator + + @pytest.fixture def client(): """Returns a Flask API client to interact with.""" # Disable authorization for test API endpoints from opendc import exts - exts.requires_auth = decorator + exts.requires_auth = requires_auth_mock + exts.requires_scope = requires_scope_mock + exts.has_scope = lambda x: False from app import create_app diff --git a/opendc-web/opendc-web-api/opendc/api/jobs.py b/opendc-web/opendc-web-api/opendc/api/jobs.py index 5feaea16..6fb0522b 100644 --- a/opendc-web/opendc-web-api/opendc/api/jobs.py +++ b/opendc-web/opendc-web-api/opendc/api/jobs.py @@ -22,7 +22,7 @@ from flask_restful import Resource from marshmallow import fields, Schema, validate from werkzeug.exceptions import BadRequest, Conflict -from opendc.exts import requires_auth +from opendc.exts import requires_auth, requires_scope from opendc.models.scenario import Scenario @@ -54,7 +54,7 @@ class JobList(Resource): """ Resource representing the list of available jobs. """ - method_decorators = [requires_auth] + method_decorators = [requires_auth, requires_scope('runner')] def get(self): """Get all available jobs.""" @@ -67,7 +67,7 @@ class Job(Resource): """ Resource representing a single job. """ - method_decorators = [requires_auth] + method_decorators = [requires_auth, requires_scope('runner')] def get(self, job_id): """Get the details of a single job.""" diff --git a/opendc-web/opendc-web-api/opendc/api/portfolios.py b/opendc-web/opendc-web-api/opendc/api/portfolios.py index 84ec466c..eea82289 100644 --- a/opendc-web/opendc-web-api/opendc/api/portfolios.py +++ b/opendc-web/opendc-web-api/opendc/api/portfolios.py @@ -22,7 +22,7 @@ from flask import request from flask_restful import Resource from marshmallow import Schema, fields -from opendc.exts import requires_auth, current_user +from opendc.exts import requires_auth, current_user, has_scope from opendc.models.portfolio import Portfolio as PortfolioModel, PortfolioSchema from opendc.models.project import Project from opendc.models.scenario import ScenarioSchema, Scenario @@ -42,7 +42,10 @@ class Portfolio(Resource): portfolio = PortfolioModel.from_id(portfolio_id) portfolio.check_exists() - portfolio.check_user_access(current_user['sub'], False) + + # Users with scope runner can access all portfolios + if not has_scope('runner'): + portfolio.check_user_access(current_user['sub'], False) data = PortfolioSchema().dump(portfolio.obj) return {'data': data} diff --git a/opendc-web/opendc-web-api/opendc/api/scenarios.py b/opendc-web/opendc-web-api/opendc/api/scenarios.py index 234bdec1..eacb0b49 100644 --- a/opendc-web/opendc-web-api/opendc/api/scenarios.py +++ b/opendc-web/opendc-web-api/opendc/api/scenarios.py @@ -24,7 +24,7 @@ from marshmallow import Schema, fields from opendc.models.scenario import Scenario as ScenarioModel, ScenarioSchema from opendc.models.portfolio import Portfolio -from opendc.exts import current_user, requires_auth +from opendc.exts import current_user, requires_auth, has_scope class Scenario(Resource): @@ -37,7 +37,11 @@ class Scenario(Resource): """Get scenario by identifier.""" scenario = ScenarioModel.from_id(scenario_id) scenario.check_exists() - scenario.check_user_access(current_user['sub'], False) + + # Users with scope runner can access all scenarios + if not has_scope('runner'): + scenario.check_user_access(current_user['sub'], False) + data = ScenarioSchema().dump(scenario.obj) return {'data': data} diff --git a/opendc-web/opendc-web-api/opendc/api/topologies.py b/opendc-web/opendc-web-api/opendc/api/topologies.py index a2d3f41a..c0b2e7ee 100644 --- a/opendc-web/opendc-web-api/opendc/api/topologies.py +++ b/opendc-web/opendc-web-api/opendc/api/topologies.py @@ -26,7 +26,7 @@ from marshmallow import Schema, fields from opendc.models.project import Project from opendc.models.topology import Topology as TopologyModel, TopologySchema -from opendc.exts import current_user, requires_auth +from opendc.exts import current_user, requires_auth, has_scope class Topology(Resource): @@ -41,7 +41,11 @@ class Topology(Resource): """ topology = TopologyModel.from_id(topology_id) topology.check_exists() - topology.check_user_access(current_user['sub'], False) + + # Users with scope runner can access all topologies + if not has_scope('runner'): + topology.check_user_access(current_user['sub'], False) + data = TopologySchema().dump(topology.obj) return {'data': data} diff --git a/opendc-web/opendc-web-api/opendc/exts.py b/opendc-web/opendc-web-api/opendc/exts.py index d24f7197..17dacd5e 100644 --- a/opendc-web/opendc-web-api/opendc/exts.py +++ b/opendc-web/opendc-web-api/opendc/exts.py @@ -2,10 +2,11 @@ import os from functools import wraps from flask import g, _request_ctx_stack +from jose import jwt from werkzeug.local import LocalProxy from opendc.database import Database -from opendc.auth import AuthContext, AsymmetricJwtAlgorithm, get_token +from opendc.auth import AuthContext, AsymmetricJwtAlgorithm, get_token, AuthError def get_db(): @@ -56,3 +57,36 @@ def requires_auth(f): current_user = LocalProxy(lambda: getattr(_request_ctx_stack.top, 'current_user', None)) + + +def has_scope(required_scope): + """Determines if the required scope is present in the Access Token + Args: + required_scope (str): The scope required to access the resource + """ + token = get_token() + unverified_claims = jwt.get_unverified_claims(token) + if unverified_claims.get("scope"): + token_scopes = unverified_claims["scope"].split() + for token_scope in token_scopes: + if token_scope == required_scope: + return True + return False + + +def requires_scope(required_scope): + """Determines if the required scope is present in the Access Token + Args: + required_scope (str): The scope required to access the resource + """ + def decorator(f): + @wraps(f) + def decorated(*args, **kwargs): + if not has_scope(required_scope): + raise AuthError({ + "code": "Unauthorized", + "description": "You don't have access to this resource" + }, 403) + return f(*args, **kwargs) + return decorated + return decorator diff --git a/opendc-web/opendc-web-api/static/schema.yml b/opendc-web/opendc-web-api/static/schema.yml index 5a8c6825..6a07ae52 100644 --- a/opendc-web/opendc-web-api/static/schema.yml +++ b/opendc-web/opendc-web-api/static/schema.yml @@ -1180,6 +1180,7 @@ components: tokenUrl: https://opendc.eu.auth0.com/oauth/token scopes: openid: Grants access to user_id + runner: Grants access to runner jobs schemas: Unauthorized: type: object -- cgit v1.2.3 From 6752b6d50faab447b3edc13bddf14f53401392f1 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 2 Jul 2021 17:52:12 +0200 Subject: runner: Use public API for scheduling simulation jobs This change updates the web runner to not require direct database access for scheduling simulation jobs. Instead, the runner polls the public REST API for available jobs and reports its results through there. --- Dockerfile | 4 +- docker-compose.yml | 9 +- gradle/libs.versions.toml | 10 +- opendc-web/opendc-web-runner/build.gradle.kts | 14 +- .../src/main/kotlin/org/opendc/runner/web/Main.kt | 381 ------------------- .../org/opendc/runner/web/ScenarioManager.kt | 115 ------ .../kotlin/org/opendc/runner/web/TopologyParser.kt | 126 ------- .../org/opendc/runner/web/WebExperimentMonitor.kt | 189 ---------- .../main/kotlin/org/opendc/web/client/ApiClient.kt | 179 +++++++++ .../main/kotlin/org/opendc/web/client/ApiResult.kt | 43 +++ .../org/opendc/web/client/AuthConfiguration.kt | 32 ++ .../main/kotlin/org/opendc/web/client/model/Job.kt | 38 ++ .../kotlin/org/opendc/web/client/model/Machine.kt | 42 +++ .../org/opendc/web/client/model/MemoryUnit.kt | 37 ++ .../web/client/model/OperationalPhenomena.kt | 32 ++ .../org/opendc/web/client/model/Portfolio.kt | 38 ++ .../opendc/web/client/model/PortfolioTargets.kt | 28 ++ .../org/opendc/web/client/model/ProcessingUnit.kt | 37 ++ .../kotlin/org/opendc/web/client/model/Rack.kt | 39 ++ .../kotlin/org/opendc/web/client/model/Room.kt | 37 ++ .../kotlin/org/opendc/web/client/model/RoomTile.kt | 38 ++ .../kotlin/org/opendc/web/client/model/Scenario.kt | 39 ++ .../opendc/web/client/model/ScenarioTopology.kt | 28 ++ .../org/opendc/web/client/model/ScenarioTrace.kt | 28 ++ .../org/opendc/web/client/model/SimulationState.kt | 30 ++ .../kotlin/org/opendc/web/client/model/Topology.kt | 38 ++ .../src/main/kotlin/org/opendc/web/runner/Main.kt | 404 +++++++++++++++++++++ .../org/opendc/web/runner/ScenarioManager.kt | 86 +++++ .../org/opendc/web/runner/WebExperimentMonitor.kt | 189 ++++++++++ .../src/main/resources/log4j2.xml | 2 +- .../kotlin/org/opendc/web/client/ApiClientTest.kt | 264 ++++++++++++++ 31 files changed, 1750 insertions(+), 826 deletions(-) delete mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/Main.kt delete mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/ScenarioManager.kt delete mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/TopologyParser.kt delete mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/ApiClient.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/ApiResult.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/AuthConfiguration.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Job.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Machine.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/MemoryUnit.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/OperationalPhenomena.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Portfolio.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/PortfolioTargets.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ProcessingUnit.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Rack.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Room.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/RoomTile.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Scenario.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ScenarioTopology.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ScenarioTrace.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/SimulationState.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Topology.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/Main.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/ScenarioManager.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/WebExperimentMonitor.kt create mode 100644 opendc-web/opendc-web-runner/src/test/kotlin/org/opendc/web/client/ApiClientTest.kt diff --git a/Dockerfile b/Dockerfile index 9fc64b41..bb9a0d0d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM openjdk:15-slim +FROM openjdk:16-slim MAINTAINER OpenDC Maintainers # Obtain (cache) Gradle wrapper @@ -11,7 +11,7 @@ RUN ./gradlew --version COPY ./ /app/ RUN ./gradlew --no-daemon :installDist -FROM openjdk:15-slim +FROM openjdk:16-slim COPY --from=0 /app/build/install /opt/ COPY --from=0 /app/traces /opt/opendc/traces WORKDIR /opt/opendc diff --git a/docker-compose.yml b/docker-compose.yml index b6246613..b5d8cb98 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,10 +47,11 @@ services: source: ./traces target: /app/traces environment: - - OPENDC_DB - - OPENDC_DB_USERNAME - - OPENDC_DB_PASSWORD - - OPENDC_DB_HOST=mongo + - OPENDC_API_URL=${OPENDC_API_BASE_URL} + - AUTH0_DOMAIN=${OPENDC_AUTH0_DOMAIN} + - AUTH0_AUDIENCE=${OPENDC_AUTH0_AUDIENCE} + - AUTH0_CLIENT_ID=${OPENDC_AUTH0_CLIENT_ID_RUNNER} + - AUTH0_CLIENT_SECRET=${OPENDC_AUTH0_CLIENT_SECRET_RUNNER} - SENTRY_DSN=${OPENDC_SIMULATOR_SENTRY_DSN} - SENTRY_ENVIRONMENT diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 5c0f9ce3..8553787d 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -6,6 +6,8 @@ log4j = "2.14.1" opentelemetry-main = "1.2.0" opentelemetry-metrics = "1.2.0-alpha" hadoop = "3.3.0" +ktor = "1.6.1" +jackson = "2.12.3" [libraries] kotlinx-coroutines = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version = "1.5.0" } @@ -35,7 +37,8 @@ clikt = { module = "com.github.ajalt.clikt:clikt", version = "3.2.0" } progressbar = { module = "me.tongfei:progressbar", version = "0.9.0" } # Format -jackson-module-kotlin = { module = "com.fasterxml.jackson.module:jackson-module-kotlin", version = "2.12.3" } +jackson-module-kotlin = { module = "com.fasterxml.jackson.module:jackson-module-kotlin", version.ref = "jackson" } +jackson-datatype-jsr310 = { module = "com.fasterxml.jackson.datatype:jackson-datatype-jsr310", version.ref = "jackson" } parquet = { module = "org.apache.parquet:parquet-avro", version = "1.12.0" } yaml = { module = "org.yaml:snakeyaml", version = "1.28" } config = { module = "com.typesafe:config", version = "1.4.1" } @@ -44,7 +47,10 @@ config = { module = "com.typesafe:config", version = "1.4.1" } kotlinx-benchmark-runtime-jvm = { module = "org.jetbrains.kotlinx:kotlinx-benchmark-runtime-jvm", version = "0.3.1" } # Other -mongodb = { module = "org.mongodb:mongodb-driver-sync", version = "4.2.3" } classgraph = { module = "io.github.classgraph:classgraph", version = "4.8.108" } hadoop-common = { module = "org.apache.hadoop:hadoop-common", version.ref = "hadoop" } hadoop-mapreduce-client-core = { module = "org.apache.hadoop:hadoop-mapreduce-client-core", version.ref = "hadoop" } +ktor-client-cio = { module = "io.ktor:ktor-client-cio", version.ref = "ktor" } +ktor-client-auth = { module = "io.ktor:ktor-client-auth", version.ref = "ktor" } +ktor-client-jackson = { module = "io.ktor:ktor-client-jackson", version.ref = "ktor" } +ktor-client-mock = { module = "io.ktor:ktor-client-mock", version.ref = "ktor" } diff --git a/opendc-web/opendc-web-runner/build.gradle.kts b/opendc-web/opendc-web-runner/build.gradle.kts index f2b2ba23..1f705b79 100644 --- a/opendc-web/opendc-web-runner/build.gradle.kts +++ b/opendc-web/opendc-web-runner/build.gradle.kts @@ -25,11 +25,12 @@ description = "Experiment runner for OpenDC" /* Build configuration */ plugins { `kotlin-conventions` + `testing-conventions` application } application { - mainClass.set("org.opendc.runner.web.MainKt") + mainClass.set("org.opendc.web.runner.MainKt") } dependencies { @@ -42,12 +43,13 @@ dependencies { implementation(libs.kotlin.logging) implementation(libs.clikt) - implementation(libs.jackson.module.kotlin) { - exclude(group = "org.jetbrains.kotlin", module = "kotlin-reflect") - } - implementation(kotlin("reflect")) implementation(libs.sentry.log4j2) - implementation(libs.mongodb) + implementation(libs.ktor.client.cio) + implementation(libs.ktor.client.auth) + implementation(libs.ktor.client.jackson) + implementation(libs.jackson.datatype.jsr310) runtimeOnly(libs.log4j.slf4j) + + testImplementation(libs.ktor.client.mock) } diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/Main.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/Main.kt deleted file mode 100644 index d0b97d90..00000000 --- a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/Main.kt +++ /dev/null @@ -1,381 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.runner.web - -import com.github.ajalt.clikt.core.CliktCommand -import com.github.ajalt.clikt.parameters.options.* -import com.github.ajalt.clikt.parameters.types.file -import com.github.ajalt.clikt.parameters.types.int -import com.github.ajalt.clikt.parameters.types.long -import com.mongodb.MongoClientSettings -import com.mongodb.MongoCredential -import com.mongodb.ServerAddress -import com.mongodb.client.MongoClients -import com.mongodb.client.MongoDatabase -import com.mongodb.client.model.Filters -import io.opentelemetry.api.metrics.MeterProvider -import io.opentelemetry.sdk.metrics.SdkMeterProvider -import io.opentelemetry.sdk.metrics.export.MetricProducer -import kotlinx.coroutines.* -import kotlinx.coroutines.channels.Channel -import mu.KotlinLogging -import org.bson.Document -import org.bson.types.ObjectId -import org.opendc.compute.service.scheduler.FilterScheduler -import org.opendc.compute.service.scheduler.filters.ComputeCapabilitiesFilter -import org.opendc.compute.service.scheduler.filters.ComputeFilter -import org.opendc.compute.service.scheduler.weights.* -import org.opendc.experiments.capelin.* -import org.opendc.experiments.capelin.model.Workload -import org.opendc.experiments.capelin.trace.ParquetTraceReader -import org.opendc.experiments.capelin.trace.PerformanceInterferenceReader -import org.opendc.experiments.capelin.trace.RawParquetTraceReader -import org.opendc.format.environment.EnvironmentReader -import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel -import org.opendc.simulator.core.runBlockingSimulation -import org.opendc.telemetry.sdk.toOtelClock -import java.io.File -import kotlin.random.Random -import kotlin.random.asJavaRandom - -private val logger = KotlinLogging.logger {} - -/** - * Represents the CLI command for starting the OpenDC web runner. - */ -@OptIn(ExperimentalCoroutinesApi::class) -class RunnerCli : CliktCommand(name = "runner") { - /** - * The name of the database to use. - */ - private val mongoDb by option( - "--mongo-db", - help = "name of the database to use", - envvar = "OPENDC_DB" - ) - .default("opendc") - - /** - * The database host to connect to. - */ - private val mongoHost by option( - "--mongo-host", - help = "database host to connect to", - envvar = "OPENDC_DB_HOST" - ) - .default("localhost") - - /** - * The database port to connect to. - */ - private val mongoPort by option( - "--mongo-port", - help = "database port to connect to", - envvar = "OPENDC_DB_PORT" - ) - .int() - .default(27017) - - /** - * The database user to connect with. - */ - private val mongoUser by option( - "--mongo-user", - help = "database user to connect with", - envvar = "OPENDC_DB_USER" - ) - .default("opendc") - - /** - * The database password to connect with. - */ - private val mongoPassword by option( - "--mongo-password", - help = "database password to connect with", - envvar = "OPENDC_DB_PASSWORD" - ) - .convert { it.toCharArray() } - .required() - - /** - * The path to the traces directory. - */ - private val tracePath by option( - "--traces", - help = "path to the directory containing the traces", - envvar = "OPENDC_TRACES" - ) - .file(canBeFile = false) - .defaultLazy { File("traces/") } - - /** - * The maximum duration of a single experiment run. - */ - private val runTimeout by option( - "--run-timeout", - help = "maximum duration of experiment in seconds", - envvar = "OPENDC_RUN_TIMEOUT" - ) - .long() - .default(60 * 3) // Experiment may run for a maximum of three minutes - - /** - * Connect to the user-specified database. - */ - private fun createDatabase(): MongoDatabase { - val credential = MongoCredential.createScramSha1Credential( - mongoUser, - mongoDb, - mongoPassword - ) - - val settings = MongoClientSettings.builder() - .credential(credential) - .applyToClusterSettings { it.hosts(listOf(ServerAddress(mongoHost, mongoPort))) } - .build() - val client = MongoClients.create(settings) - return client.getDatabase(mongoDb) - } - - /** - * Run a single scenario. - */ - private suspend fun runScenario(portfolio: Document, scenario: Document, topologyParser: TopologyParser): List { - val id = scenario.getObjectId("_id") - - logger.info { "Constructing performance interference model" } - - val traceDir = File( - tracePath, - scenario.getEmbedded(listOf("trace", "traceId"), String::class.java) - ) - val traceReader = RawParquetTraceReader(traceDir) - val interferenceGroups = let { - val path = File(traceDir, "performance-interference-model.json") - val operational = scenario.get("operational", Document::class.java) - val enabled = operational.getBoolean("performanceInterferenceEnabled") - - if (!enabled || !path.exists()) { - return@let null - } - - PerformanceInterferenceReader(path.inputStream()).use { reader -> reader.read() } - } - - val targets = portfolio.get("targets", Document::class.java) - val topologyId = scenario.getEmbedded(listOf("topology", "topologyId"), ObjectId::class.java) - val environment = topologyParser.read(topologyId) - - val results = (0 until targets.getInteger("repeatsPerScenario")).map { repeat -> - logger.info { "Starting repeat $repeat" } - withTimeout(runTimeout * 1000) { - val interferenceModel = interferenceGroups?.let { VmInterferenceModel(it, Random(repeat.toLong()).asJavaRandom()) } - runRepeat(scenario, repeat, environment, traceReader, interferenceModel) - } - } - - logger.info { "Finished simulation for scenario $id" } - - return results - } - - /** - * Run a single repeat. - */ - private suspend fun runRepeat( - scenario: Document, - repeat: Int, - environment: EnvironmentReader, - traceReader: RawParquetTraceReader, - interferenceModel: VmInterferenceModel? - ): WebExperimentMonitor.Result { - val monitor = WebExperimentMonitor() - - try { - runBlockingSimulation { - val seed = repeat - val traceDocument = scenario.get("trace", Document::class.java) - val workloadName = traceDocument.getString("traceId") - val workloadFraction = traceDocument.get("loadSamplingFraction", Number::class.java).toDouble() - - val seeder = Random(seed) - - val chan = Channel(Channel.CONFLATED) - - val meterProvider: MeterProvider = SdkMeterProvider - .builder() - .setClock(clock.toOtelClock()) - .build() - val metricProducer = meterProvider as MetricProducer - - val operational = scenario.get("operational", Document::class.java) - val allocationPolicy = - when (val policyName = operational.getString("schedulerName")) { - "mem" -> FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(MemoryWeigher() to -1.0) - ) - "mem-inv" -> FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(MemoryWeigher() to 1.0) - ) - "core-mem" -> FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(CoreMemoryWeigher() to -1.0) - ) - "core-mem-inv" -> FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(CoreMemoryWeigher() to 1.0) - ) - "active-servers" -> FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(ProvisionedCoresWeigher() to -1.0) - ) - "active-servers-inv" -> FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(InstanceCountWeigher() to 1.0) - ) - "provisioned-cores" -> FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(ProvisionedCoresWeigher() to -1.0) - ) - "provisioned-cores-inv" -> FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(ProvisionedCoresWeigher() to 1.0) - ) - "random" -> FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(RandomWeigher(java.util.Random(seeder.nextLong())) to 1.0) - ) - else -> throw IllegalArgumentException("Unknown policy $policyName") - } - - val trace = ParquetTraceReader( - listOf(traceReader), - Workload(workloadName, workloadFraction), - seed - ) - val failureFrequency = if (operational.getBoolean("failuresEnabled", false)) 24.0 * 7 else 0.0 - - withComputeService(clock, meterProvider, environment, allocationPolicy, interferenceModel) { scheduler -> - val failureDomain = if (failureFrequency > 0) { - logger.debug { "ENABLING failures" } - createFailureDomain( - this, - clock, - seeder.nextInt(), - failureFrequency, - scheduler, - chan - ) - } else { - null - } - - withMonitor(monitor, clock, meterProvider as MetricProducer, scheduler) { - processTrace( - clock, - trace, - scheduler, - chan, - monitor - ) - } - - failureDomain?.cancel() - } - - val monitorResults = collectMetrics(metricProducer) - logger.debug { "Finish SUBMIT=${monitorResults.submittedVms} FAIL=${monitorResults.unscheduledVms} QUEUE=${monitorResults.queuedVms} RUNNING=${monitorResults.runningVms}" } - } - } catch (cause: Throwable) { - logger.warn(cause) { "Experiment failed" } - } - - return monitor.getResult() - } - - private val POLL_INTERVAL = 5000L // ms = 5 s - private val HEARTBEAT_INTERVAL = 60000L // ms = 1 min - - override fun run(): Unit = runBlocking(Dispatchers.Default) { - logger.info { "Starting OpenDC web runner" } - logger.info { "Connecting to MongoDB instance" } - val database = createDatabase() - val manager = ScenarioManager(database.getCollection("scenarios")) - val portfolios = database.getCollection("portfolios") - val topologies = database.getCollection("topologies") - val topologyParser = TopologyParser(topologies) - - logger.info { "Watching for queued scenarios" } - - while (true) { - val scenario = manager.findNext() - - if (scenario == null) { - delay(POLL_INTERVAL) - continue - } - - val id = scenario.getObjectId("_id") - - logger.info { "Found queued scenario $id: attempting to claim" } - - if (!manager.claim(id)) { - logger.info { "Failed to claim scenario" } - continue - } - - coroutineScope { - // Launch heartbeat process - val heartbeat = launch { - while (true) { - delay(HEARTBEAT_INTERVAL) - manager.heartbeat(id) - } - } - - try { - val portfolio = portfolios.find(Filters.eq("_id", scenario.getObjectId("portfolioId"))).first()!! - val results = runScenario(portfolio, scenario, topologyParser) - - logger.info { "Writing results to database" } - - manager.finish(id, results) - - logger.info { "Successfully finished scenario $id" } - } catch (e: Exception) { - logger.error(e) { "Scenario failed to finish" } - manager.fail(id) - } finally { - heartbeat.cancel() - } - } - } - } -} - -/** - * Main entry point of the runner. - */ -fun main(args: Array): Unit = RunnerCli().main(args) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/ScenarioManager.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/ScenarioManager.kt deleted file mode 100644 index a3907051..00000000 --- a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/ScenarioManager.kt +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.runner.web - -import com.mongodb.client.MongoCollection -import com.mongodb.client.model.Filters -import com.mongodb.client.model.Updates -import org.bson.Document -import org.bson.types.ObjectId -import java.time.Instant - -/** - * Manages the queue of scenarios that need to be processed. - */ -public class ScenarioManager(private val collection: MongoCollection) { - /** - * Find the next scenario that the simulator needs to process. - */ - public fun findNext(): Document? { - return collection - .find(Filters.eq("simulation.state", "QUEUED")) - .first() - } - - /** - * Claim the scenario in the database with the specified id. - */ - public fun claim(id: ObjectId): Boolean { - val res = collection.findOneAndUpdate( - Filters.and( - Filters.eq("_id", id), - Filters.eq("simulation.state", "QUEUED") - ), - Updates.combine( - Updates.set("simulation.state", "RUNNING"), - Updates.set("simulation.heartbeat", Instant.now()) - ) - ) - return res != null - } - - /** - * Update the heartbeat of the specified scenario. - */ - public fun heartbeat(id: ObjectId) { - collection.findOneAndUpdate( - Filters.and( - Filters.eq("_id", id), - Filters.eq("simulation.state", "RUNNING") - ), - Updates.set("simulation.heartbeat", Instant.now()) - ) - } - - /** - * Mark the scenario as failed. - */ - public fun fail(id: ObjectId) { - collection.findOneAndUpdate( - Filters.eq("_id", id), - Updates.combine( - Updates.set("simulation.state", "FAILED"), - Updates.set("simulation.heartbeat", Instant.now()) - ) - ) - } - - /** - * Persist the specified results. - */ - public fun finish(id: ObjectId, results: List) { - collection.findOneAndUpdate( - Filters.eq("_id", id), - Updates.combine( - Updates.set("simulation.state", "FINISHED"), - Updates.unset("simulation.time"), - Updates.set("results.total_requested_burst", results.map { it.totalRequestedBurst }), - Updates.set("results.total_granted_burst", results.map { it.totalGrantedBurst }), - Updates.set("results.total_overcommitted_burst", results.map { it.totalOvercommittedBurst }), - Updates.set("results.total_interfered_burst", results.map { it.totalInterferedBurst }), - Updates.set("results.mean_cpu_usage", results.map { it.meanCpuUsage }), - Updates.set("results.mean_cpu_demand", results.map { it.meanCpuDemand }), - Updates.set("results.mean_num_deployed_images", results.map { it.meanNumDeployedImages }), - Updates.set("results.max_num_deployed_images", results.map { it.maxNumDeployedImages }), - Updates.set("results.total_power_draw", results.map { it.totalPowerDraw }), - Updates.set("results.total_failure_slices", results.map { it.totalFailureSlices }), - Updates.set("results.total_failure_vm_slices", results.map { it.totalFailureVmSlices }), - Updates.set("results.total_vms_submitted", results.map { it.totalVmsSubmitted }), - Updates.set("results.total_vms_queued", results.map { it.totalVmsQueued }), - Updates.set("results.total_vms_finished", results.map { it.totalVmsFinished }), - Updates.set("results.total_vms_failed", results.map { it.totalVmsFailed }) - ) - ) - } -} diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/TopologyParser.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/TopologyParser.kt deleted file mode 100644 index 2135ee1d..00000000 --- a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/TopologyParser.kt +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.runner.web - -import com.mongodb.client.AggregateIterable -import com.mongodb.client.MongoCollection -import com.mongodb.client.model.Aggregates -import com.mongodb.client.model.Field -import com.mongodb.client.model.Filters -import com.mongodb.client.model.Projections -import org.bson.Document -import org.bson.types.ObjectId -import org.opendc.format.environment.EnvironmentReader -import org.opendc.format.environment.MachineDef -import org.opendc.simulator.compute.model.MachineModel -import org.opendc.simulator.compute.model.MemoryUnit -import org.opendc.simulator.compute.model.ProcessingNode -import org.opendc.simulator.compute.model.ProcessingUnit -import org.opendc.simulator.compute.power.LinearPowerModel -import java.util.* - -/** - * A helper class that converts the MongoDB topology into an OpenDC environment. - */ -public class TopologyParser(private val collection: MongoCollection) { - - /** - * Parse the topology from the specified [id]. - */ - public fun read(id: ObjectId): EnvironmentReader { - val nodes = mutableListOf() - val random = Random(0) - - for (machine in fetchMachines(id)) { - val clusterId = machine.get("rack_id").toString() - val position = machine.getInteger("position") - - val processors = machine.getList("cpus", Document::class.java).flatMap { cpu -> - val cores = cpu.getInteger("numberOfCores") - val speed = cpu.get("clockRateMhz", Number::class.java).toDouble() - // TODO Remove hardcoding of vendor - val node = ProcessingNode("Intel", "amd64", cpu.getString("name"), cores) - List(cores) { coreId -> - ProcessingUnit(node, coreId, speed) - } - } - val memoryUnits = machine.getList("memories", Document::class.java).map { memory -> - MemoryUnit( - "Samsung", - memory.getString("name"), - memory.get("speedMbPerS", Number::class.java).toDouble(), - memory.get("sizeMb", Number::class.java).toLong() - ) - } - - val energyConsumptionW = machine.getList("cpus", Document::class.java).sumOf { it.getInteger("energyConsumptionW") }.toDouble() - - nodes.add( - MachineDef( - UUID(random.nextLong(), random.nextLong()), - "node-$clusterId-$position", - mapOf("cluster" to clusterId), - MachineModel(processors, memoryUnits), - LinearPowerModel(2 * energyConsumptionW, energyConsumptionW * 0.5) - ) - ) - } - - return object : EnvironmentReader { - override fun read(): List = nodes - override fun close() {} - } - } - - /** - * Fetch the metadata of the topology. - */ - private fun fetchName(id: ObjectId): String { - return collection.aggregate( - listOf( - Aggregates.match(Filters.eq("_id", id)), - Aggregates.project(Projections.include("name")) - ) - ) - .first()!! - .getString("name") - } - - /** - * Fetch a topology from the database with the specified [id]. - */ - private fun fetchMachines(id: ObjectId): AggregateIterable { - return collection.aggregate( - listOf( - Aggregates.match(Filters.eq("_id", id)), - Aggregates.project(Projections.fields(Document("racks", "\$rooms.tiles.rack"))), - Aggregates.unwind("\$racks"), - Aggregates.unwind("\$racks"), - Aggregates.replaceRoot("\$racks"), - Aggregates.addFields(Field("machines.rack_id", "\$_id")), - Aggregates.unwind("\$machines"), - Aggregates.replaceRoot("\$machines") - ) - ) - } -} diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt deleted file mode 100644 index c913f82f..00000000 --- a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.runner.web - -import mu.KotlinLogging -import org.opendc.compute.api.Server -import org.opendc.compute.api.ServerState -import org.opendc.compute.service.driver.Host -import org.opendc.compute.service.driver.HostState -import org.opendc.experiments.capelin.monitor.ExperimentMonitor -import org.opendc.experiments.capelin.telemetry.HostEvent -import kotlin.math.max - -/** - * An [ExperimentMonitor] that tracks the aggregate metrics for each repeat. - */ -public class WebExperimentMonitor : ExperimentMonitor { - private val logger = KotlinLogging.logger {} - - override fun reportVmStateChange(time: Long, server: Server, newState: ServerState) {} - - override fun reportHostStateChange(time: Long, host: Host, newState: HostState) { - logger.debug { "Host ${host.uid} changed state $newState [$time]" } - } - - override fun reportHostSlice( - time: Long, - requestedBurst: Long, - grantedBurst: Long, - overcommissionedBurst: Long, - interferedBurst: Long, - cpuUsage: Double, - cpuDemand: Double, - powerDraw: Double, - numberOfDeployedImages: Int, - host: Host, - ) { - processHostEvent( - HostEvent( - time, - 5 * 60 * 1000L, - host, - numberOfDeployedImages, - requestedBurst, - grantedBurst, - overcommissionedBurst, - interferedBurst, - cpuUsage, - cpuDemand, - powerDraw, - host.model.cpuCount - ) - ) - } - - private var hostAggregateMetrics: AggregateHostMetrics = AggregateHostMetrics() - private val hostMetrics: MutableMap = mutableMapOf() - - private fun processHostEvent(event: HostEvent) { - val slices = event.duration / SLICE_LENGTH - - hostAggregateMetrics = AggregateHostMetrics( - hostAggregateMetrics.totalRequestedBurst + event.requestedBurst, - hostAggregateMetrics.totalGrantedBurst + event.grantedBurst, - hostAggregateMetrics.totalOvercommittedBurst + event.overcommissionedBurst, - hostAggregateMetrics.totalInterferedBurst + event.interferedBurst, - hostAggregateMetrics.totalPowerDraw + (event.duration * event.powerDraw) / 3600, - hostAggregateMetrics.totalFailureSlices + if (event.host.state != HostState.UP) slices else 0, - hostAggregateMetrics.totalFailureVmSlices + if (event.host.state != HostState.UP) event.vmCount * slices else 0 - ) - - hostMetrics.compute(event.host) { _, prev -> - HostMetrics( - (event.cpuUsage.takeIf { event.host.state == HostState.UP } ?: 0.0) + (prev?.cpuUsage ?: 0.0), - (event.cpuDemand.takeIf { event.host.state == HostState.UP } ?: 0.0) + (prev?.cpuDemand ?: 0.0), - event.vmCount + (prev?.vmCount ?: 0), - 1 + (prev?.count ?: 0) - ) - } - } - - private val SLICE_LENGTH: Long = 5 * 60 * 1000 - - public data class AggregateHostMetrics( - val totalRequestedBurst: Long = 0, - val totalGrantedBurst: Long = 0, - val totalOvercommittedBurst: Long = 0, - val totalInterferedBurst: Long = 0, - val totalPowerDraw: Double = 0.0, - val totalFailureSlices: Long = 0, - val totalFailureVmSlices: Long = 0, - ) - - public data class HostMetrics( - val cpuUsage: Double, - val cpuDemand: Double, - val vmCount: Long, - val count: Long - ) - - private var provisionerMetrics: AggregateProvisionerMetrics = AggregateProvisionerMetrics() - - override fun reportProvisionerMetrics( - time: Long, - totalHostCount: Int, - availableHostCount: Int, - totalVmCount: Int, - activeVmCount: Int, - inactiveVmCount: Int, - waitingVmCount: Int, - failedVmCount: Int - ) { - provisionerMetrics = AggregateProvisionerMetrics( - max(totalVmCount, provisionerMetrics.vmTotalCount), - max(waitingVmCount, provisionerMetrics.vmWaitingCount), - max(activeVmCount, provisionerMetrics.vmActiveCount), - max(inactiveVmCount, provisionerMetrics.vmInactiveCount), - max(failedVmCount, provisionerMetrics.vmFailedCount), - ) - } - - public data class AggregateProvisionerMetrics( - val vmTotalCount: Int = 0, - val vmWaitingCount: Int = 0, - val vmActiveCount: Int = 0, - val vmInactiveCount: Int = 0, - val vmFailedCount: Int = 0 - ) - - override fun close() {} - - public fun getResult(): Result { - return Result( - hostAggregateMetrics.totalRequestedBurst, - hostAggregateMetrics.totalGrantedBurst, - hostAggregateMetrics.totalOvercommittedBurst, - hostAggregateMetrics.totalInterferedBurst, - hostMetrics.map { it.value.cpuUsage / it.value.count }.average(), - hostMetrics.map { it.value.cpuDemand / it.value.count }.average(), - hostMetrics.map { it.value.vmCount.toDouble() / it.value.count }.average(), - hostMetrics.map { it.value.vmCount.toDouble() / it.value.count }.maxOrNull() ?: 0.0, - hostAggregateMetrics.totalPowerDraw, - hostAggregateMetrics.totalFailureSlices, - hostAggregateMetrics.totalFailureVmSlices, - provisionerMetrics.vmTotalCount, - provisionerMetrics.vmWaitingCount, - provisionerMetrics.vmInactiveCount, - provisionerMetrics.vmFailedCount, - ) - } - - public data class Result( - public val totalRequestedBurst: Long, - public val totalGrantedBurst: Long, - public val totalOvercommittedBurst: Long, - public val totalInterferedBurst: Long, - public val meanCpuUsage: Double, - public val meanCpuDemand: Double, - public val meanNumDeployedImages: Double, - public val maxNumDeployedImages: Double, - public val totalPowerDraw: Double, - public val totalFailureSlices: Long, - public val totalFailureVmSlices: Long, - public val totalVmsSubmitted: Int, - public val totalVmsQueued: Int, - public val totalVmsFinished: Int, - public val totalVmsFailed: Int - ) -} diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/ApiClient.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/ApiClient.kt new file mode 100644 index 00000000..9f2656c4 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/ApiClient.kt @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client + +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import io.ktor.client.* +import io.ktor.client.features.auth.* +import io.ktor.client.features.auth.providers.* +import io.ktor.client.features.json.* +import io.ktor.client.request.* +import io.ktor.client.statement.* +import io.ktor.http.* +import org.opendc.web.client.model.* +import java.net.URI + +/** + * Client implementation for the OpenDC REST API (version 2). + * + * @param baseUrl The base url of the API. + * @param auth The authentication configuration for the client. + * @param client The HTTP client to use. + */ +public class ApiClient( + private val baseUrl: URI, + private val auth: AuthConfiguration, + private val audience: String = "https://api.opendc.org/v2/", + client: HttpClient = HttpClient {} +) : AutoCloseable { + /** + * The Ktor [HttpClient] that is used to communicate with the REST API. + */ + private val client = client.config { + install(JsonFeature) { + serializer = JacksonSerializer { + registerModule(JavaTimeModule()) + } + } + install(Auth) { + bearer { + loadTokens { requestToken() } + refreshTokens { requestToken() } + } + } + expectSuccess = false + } + + /** + * Retrieve the topology with the specified [id]. + */ + public suspend fun getPortfolio(id: String): Portfolio? { + val url = URLBuilder(Url(baseUrl)) + .path("portfolios", id) + .build() + return when (val result = client.get>(url)) { + is ApiResult.Success -> result.data + else -> null + } + } + + /** + * Retrieve the scenario with the specified [id]. + */ + public suspend fun getScenario(id: String): Scenario? { + val url = URLBuilder(Url(baseUrl)) + .path("scenarios", id) + .build() + return when (val result = client.get>(url)) { + is ApiResult.Success -> result.data + else -> null + } + } + + /** + * Retrieve the topology with the specified [id]. + */ + public suspend fun getTopology(id: String): Topology? { + val url = URLBuilder(Url(baseUrl)) + .path("topologies", id) + .build() + return when (val result = client.get>(url)) { + is ApiResult.Success -> result.data + else -> null + } + } + + /** + * Retrieve the available jobs. + */ + public suspend fun getJobs(): List { + val url = URLBuilder(Url(baseUrl)) + .path("jobs") + .build() + return when (val result = client.get>>(url)) { + is ApiResult.Success -> result.data + else -> emptyList() + } + } + + /** + * Update the specified job. + * + * @param id The identifier of the job. + * @param state The new state of the job. + * @param results The results of the job. + */ + public suspend fun updateJob(id: String, state: SimulationState, results: Map = emptyMap()): Boolean { + val url = URLBuilder(Url(baseUrl)) + .path("jobs", id) + .build() + + data class Request( + val state: SimulationState, + val results: Map + ) + + val res = client.post { + url(url) + contentType(ContentType.Application.Json) + body = Request(state, results) + } + return res.status.isSuccess() + } + + /** + * Request the auth token for the API. + */ + private suspend fun requestToken(): BearerTokens { + data class Request( + val audience: String, + @JsonProperty("grant_type") + val grantType: String, + @JsonProperty("client_id") + val clientId: String, + @JsonProperty("client_secret") + val clientSecret: String + ) + + data class Response( + @JsonProperty("access_token") + val accessToken: String, + @JsonProperty("token_type") + val tokenType: String, + val scope: String = "", + @JsonProperty("expires_in") + val expiresIn: Long + ) + + val result = client.post { + url(Url("https://${auth.domain}/oauth/token")) + contentType(ContentType.Application.Json) + body = Request(audience, "client_credentials", auth.clientId, auth.clientSecret) + } + + return BearerTokens(result.accessToken, "") + } + + override fun close() = client.close() +} diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/ApiResult.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/ApiResult.kt new file mode 100644 index 00000000..a3df01c5 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/ApiResult.kt @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client + +import com.fasterxml.jackson.annotation.JsonSubTypes +import com.fasterxml.jackson.annotation.JsonTypeInfo + +/** + * Generic response model for the OpenDC API. + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.DEDUCTION) +@JsonSubTypes(JsonSubTypes.Type(ApiResult.Success::class), JsonSubTypes.Type(ApiResult.Failure::class)) +public sealed class ApiResult { + /** + * A response indicating everything is okay. + */ + public data class Success(val data: T) : ApiResult() + + /** + * A response indicating a failure. + */ + public data class Failure(val message: String, val errors: List = emptyList()) : ApiResult() +} diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/AuthConfiguration.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/AuthConfiguration.kt new file mode 100644 index 00000000..5dbf2f59 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/AuthConfiguration.kt @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client + +/** + * The authentication configuration for the API client. + */ +public data class AuthConfiguration( + val domain: String, + val clientId: String, + val clientSecret: String +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Job.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Job.kt new file mode 100644 index 00000000..eeb65e49 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Job.kt @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonProperty +import java.time.LocalDateTime + +/** + * A description of a simulation job. + */ +public data class Job( + @JsonProperty("_id") + val id: String, + val scenarioId: String, + val state: SimulationState, + val heartbeat: LocalDateTime, + val results: Map +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Machine.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Machine.kt new file mode 100644 index 00000000..c6757c5c --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Machine.kt @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * A machine in a rack. + */ +@JsonIgnoreProperties("id_legacy") +public data class Machine( + @JsonProperty("_id") + val id: String, + val position: Int, + val cpus: List = emptyList(), + val gpus: List = emptyList(), + @JsonProperty("memories") + val memory: List = emptyList(), + @JsonProperty("storages") + val storage: List = emptyList() +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/MemoryUnit.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/MemoryUnit.kt new file mode 100644 index 00000000..11e794e8 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/MemoryUnit.kt @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * A memory unit in a system. + */ +public data class MemoryUnit( + @JsonProperty("_id") + val id: String, + val name: String, + val speedMbPerS: Double, + val sizeMb: Double, + val energyConsumptionW: Double +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/OperationalPhenomena.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/OperationalPhenomena.kt new file mode 100644 index 00000000..ef5b4902 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/OperationalPhenomena.kt @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +/** + * Object describing the enabled operational phenomena for a scenario. + */ +public data class OperationalPhenomena( + val failuresEnabled: Boolean, + val performanceInterferenceEnabled: Boolean, + val schedulerName: String +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Portfolio.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Portfolio.kt new file mode 100644 index 00000000..6904920b --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Portfolio.kt @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * A portfolio in OpenDC. + */ +public data class Portfolio( + @JsonProperty("_id") + val id: String, + val projectId: String, + val name: String, + @JsonProperty("scenarioIds") + val scenarios: Set, + val targets: PortfolioTargets +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/PortfolioTargets.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/PortfolioTargets.kt new file mode 100644 index 00000000..07c11c19 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/PortfolioTargets.kt @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +/** + * The targets of a portfolio. + */ +public data class PortfolioTargets(val enabledMetrics: Set, val repeatsPerScenario: Int) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ProcessingUnit.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ProcessingUnit.kt new file mode 100644 index 00000000..449b5c43 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ProcessingUnit.kt @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * A CPU model. + */ +public data class ProcessingUnit( + @JsonProperty("_id") + val id: String, + val name: String, + val clockRateMhz: Double, + val numberOfCores: Int, + val energyConsumptionW: Double +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Rack.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Rack.kt new file mode 100644 index 00000000..a0464388 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Rack.kt @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * A rack in a datacenter. + */ +@JsonIgnoreProperties("id_legacy") +public class Rack( + @JsonProperty("_id") + val id: String, + val name: String, + val capacity: Int, + val powerCapacityW: Double, + val machines: List +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Room.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Room.kt new file mode 100644 index 00000000..e961d6db --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Room.kt @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * A room in a datacenter. + */ +@JsonIgnoreProperties("id_legacy") +public data class Room( + @JsonProperty("_id") + val id: String, + val name: String, + val tiles: Set +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/RoomTile.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/RoomTile.kt new file mode 100644 index 00000000..3bee3204 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/RoomTile.kt @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * A room tile. + */ +@JsonIgnoreProperties("id_legacy") +public data class RoomTile( + @JsonProperty("_id") + val id: String, + val positionX: Double, + val positionY: Double, + val rack: Rack? = null +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Scenario.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Scenario.kt new file mode 100644 index 00000000..851ff980 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Scenario.kt @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * A simulation scenario. + */ +public data class Scenario( + @JsonProperty("_id") + val id: String, + val portfolioId: String, + val name: String, + val trace: ScenarioTrace, + val topology: ScenarioTopology, + @JsonProperty("operational") + val operationalPhenomena: OperationalPhenomena +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ScenarioTopology.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ScenarioTopology.kt new file mode 100644 index 00000000..2b90f7ef --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ScenarioTopology.kt @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +/** + * The topology details for a scenario. + */ +public data class ScenarioTopology(val topologyId: String) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ScenarioTrace.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ScenarioTrace.kt new file mode 100644 index 00000000..adff6d97 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ScenarioTrace.kt @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +/** + * The trace details of a scenario. + */ +public data class ScenarioTrace(val traceId: String, val loadSamplingFraction: Double) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/SimulationState.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/SimulationState.kt new file mode 100644 index 00000000..2eadd747 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/SimulationState.kt @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +/** + * The state of a simulation job. + */ +public enum class SimulationState { + QUEUED, CLAIMED, RUNNING, FINISHED, FAILED +} diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Topology.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Topology.kt new file mode 100644 index 00000000..b59aba42 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Topology.kt @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * Model for an OpenDC topology. + */ +@JsonIgnoreProperties("id_legacy", "datacenter_id_legacy", "datetimeLastUpdated", "datetimeLastEdited") +public data class Topology( + @JsonProperty("_id") + val id: String, + val projectId: String, + val name: String, + val rooms: Set, +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/Main.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/Main.kt new file mode 100644 index 00000000..5b5ef802 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/Main.kt @@ -0,0 +1,404 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.runner + +import com.github.ajalt.clikt.core.CliktCommand +import com.github.ajalt.clikt.parameters.options.* +import com.github.ajalt.clikt.parameters.types.file +import com.github.ajalt.clikt.parameters.types.long +import io.opentelemetry.api.metrics.MeterProvider +import io.opentelemetry.sdk.metrics.SdkMeterProvider +import io.opentelemetry.sdk.metrics.export.MetricProducer +import kotlinx.coroutines.* +import kotlinx.coroutines.channels.Channel +import mu.KotlinLogging +import org.opendc.compute.service.scheduler.FilterScheduler +import org.opendc.compute.service.scheduler.filters.ComputeCapabilitiesFilter +import org.opendc.compute.service.scheduler.filters.ComputeFilter +import org.opendc.compute.service.scheduler.weights.* +import org.opendc.experiments.capelin.* +import org.opendc.experiments.capelin.model.Workload +import org.opendc.experiments.capelin.trace.ParquetTraceReader +import org.opendc.experiments.capelin.trace.PerformanceInterferenceReader +import org.opendc.experiments.capelin.trace.RawParquetTraceReader +import org.opendc.format.environment.EnvironmentReader +import org.opendc.format.environment.MachineDef +import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel +import org.opendc.simulator.compute.model.MachineModel +import org.opendc.simulator.compute.model.MemoryUnit +import org.opendc.simulator.compute.model.ProcessingNode +import org.opendc.simulator.compute.model.ProcessingUnit +import org.opendc.simulator.compute.power.LinearPowerModel +import org.opendc.simulator.core.runBlockingSimulation +import org.opendc.telemetry.sdk.toOtelClock +import org.opendc.web.client.ApiClient +import org.opendc.web.client.AuthConfiguration +import org.opendc.web.client.model.Scenario +import org.opendc.web.client.model.Topology +import java.io.File +import java.net.URI +import java.util.* +import kotlin.random.Random +import kotlin.random.asJavaRandom +import org.opendc.web.client.model.Portfolio as ClientPortfolio + +private val logger = KotlinLogging.logger {} + +/** + * Represents the CLI command for starting the OpenDC web runner. + */ +class RunnerCli : CliktCommand(name = "runner") { + /** + * The URL to the OpenDC API. + */ + private val apiUrl by option( + "--api-url", + help = "url to the OpenDC API", + envvar = "OPENDC_API_URL" + ) + .convert { URI(it) } + .default(URI("https://api.opendc.org/v2")) + + /** + * The auth domain to use. + */ + private val authDomain by option( + "--auth-domain", + help = "auth domain of the OpenDC API", + envvar = "AUTH0_DOMAIN" + ) + .required() + + /** + * The auth client ID to use. + */ + private val authClientId by option( + "--auth-id", + help = "auth client id of the OpenDC API", + envvar = "AUTH0_CLIENT_ID" + ) + .required() + + /** + * The auth client secret to use. + */ + private val authClientSecret by option( + "--auth-secret", + help = "auth client secret of the OpenDC API", + envvar = "AUTH0_CLIENT_SECRET" + ) + .required() + + /** + * The path to the traces directory. + */ + private val tracePath by option( + "--traces", + help = "path to the directory containing the traces", + envvar = "OPENDC_TRACES" + ) + .file(canBeFile = false) + .defaultLazy { File("traces/") } + + /** + * The maximum duration of a single experiment run. + */ + private val runTimeout by option( + "--run-timeout", + help = "maximum duration of experiment in seconds", + envvar = "OPENDC_RUN_TIMEOUT" + ) + .long() + .default(60L * 3) // Experiment may run for a maximum of three minutes + + /** + * Run a single scenario. + */ + private suspend fun runScenario(portfolio: ClientPortfolio, scenario: Scenario, environment: EnvironmentReader): List { + val id = scenario.id + + logger.info { "Constructing performance interference model" } + + val traceDir = File( + tracePath, + scenario.trace.traceId + ) + val traceReader = RawParquetTraceReader(traceDir) + val interferenceGroups = let { + val path = File(traceDir, "performance-interference-model.json") + val operational = scenario.operationalPhenomena + val enabled = operational.performanceInterferenceEnabled + + if (!enabled || !path.exists()) { + return@let null + } + + PerformanceInterferenceReader(path.inputStream()).use { reader -> reader.read() } + } + + val targets = portfolio.targets + val results = (0 until targets.repeatsPerScenario).map { repeat -> + logger.info { "Starting repeat $repeat" } + withTimeout(runTimeout * 1000) { + val interferenceModel = interferenceGroups?.let { VmInterferenceModel(it, Random(repeat.toLong()).asJavaRandom()) } + runRepeat(scenario, repeat, environment, traceReader, interferenceModel) + } + } + + logger.info { "Finished simulation for scenario $id" } + + return results + } + + /** + * Run a single repeat. + */ + private suspend fun runRepeat( + scenario: Scenario, + repeat: Int, + environment: EnvironmentReader, + traceReader: RawParquetTraceReader, + interferenceModel: VmInterferenceModel? + ): WebExperimentMonitor.Result { + val monitor = WebExperimentMonitor() + + try { + runBlockingSimulation { + val seed = repeat + val workloadName = scenario.trace.traceId + val workloadFraction = scenario.trace.loadSamplingFraction + + val seeder = Random(seed) + + val chan = Channel(Channel.CONFLATED) + + val meterProvider: MeterProvider = SdkMeterProvider + .builder() + .setClock(clock.toOtelClock()) + .build() + val metricProducer = meterProvider as MetricProducer + + val operational = scenario.operationalPhenomena + val allocationPolicy = + when (val policyName = operational.schedulerName) { + "mem" -> FilterScheduler( + filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), + weighers = listOf(MemoryWeigher() to -1.0) + ) + "mem-inv" -> FilterScheduler( + filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), + weighers = listOf(MemoryWeigher() to 1.0) + ) + "core-mem" -> FilterScheduler( + filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), + weighers = listOf(CoreMemoryWeigher() to -1.0) + ) + "core-mem-inv" -> FilterScheduler( + filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), + weighers = listOf(CoreMemoryWeigher() to 1.0) + ) + "active-servers" -> FilterScheduler( + filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), + weighers = listOf(ProvisionedCoresWeigher() to -1.0) + ) + "active-servers-inv" -> FilterScheduler( + filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), + weighers = listOf(InstanceCountWeigher() to 1.0) + ) + "provisioned-cores" -> FilterScheduler( + filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), + weighers = listOf(ProvisionedCoresWeigher() to -1.0) + ) + "provisioned-cores-inv" -> FilterScheduler( + filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), + weighers = listOf(ProvisionedCoresWeigher() to 1.0) + ) + "random" -> FilterScheduler( + filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), + weighers = listOf(RandomWeigher(java.util.Random(seeder.nextLong())) to 1.0) + ) + else -> throw IllegalArgumentException("Unknown policy $policyName") + } + + val trace = ParquetTraceReader( + listOf(traceReader), + Workload(workloadName, workloadFraction), + seed + ) + val failureFrequency = if (operational.failuresEnabled) 24.0 * 7 else 0.0 + + withComputeService(clock, meterProvider, environment, allocationPolicy, interferenceModel) { scheduler -> + val failureDomain = if (failureFrequency > 0) { + logger.debug { "ENABLING failures" } + createFailureDomain( + this, + clock, + seeder.nextInt(), + failureFrequency, + scheduler, + chan + ) + } else { + null + } + + withMonitor(monitor, clock, meterProvider as MetricProducer, scheduler) { + processTrace( + clock, + trace, + scheduler, + chan, + monitor + ) + } + + failureDomain?.cancel() + } + + val monitorResults = collectMetrics(metricProducer) + logger.debug { "Finish SUBMIT=${monitorResults.submittedVms} FAIL=${monitorResults.unscheduledVms} QUEUE=${monitorResults.queuedVms} RUNNING=${monitorResults.runningVms}" } + } + } catch (cause: Throwable) { + logger.warn(cause) { "Experiment failed" } + } + + return monitor.getResult() + } + + private val POLL_INTERVAL = 30000L // ms = 30 s + private val HEARTBEAT_INTERVAL = 60000L // ms = 1 min + + override fun run(): Unit = runBlocking(Dispatchers.Default) { + logger.info { "Starting OpenDC web runner" } + + val client = ApiClient(baseUrl = apiUrl, AuthConfiguration(authDomain, authClientId, authClientSecret)) + val manager = ScenarioManager(client) + + logger.info { "Watching for queued scenarios" } + + while (true) { + val scenario = manager.findNext() + + if (scenario == null) { + delay(POLL_INTERVAL) + continue + } + + val id = scenario.id + + logger.info { "Found queued scenario $id: attempting to claim" } + + if (!manager.claim(id)) { + logger.info { "Failed to claim scenario" } + continue + } + + coroutineScope { + // Launch heartbeat process + val heartbeat = launch { + while (true) { + manager.heartbeat(id) + delay(HEARTBEAT_INTERVAL) + } + } + + try { + val scenarioModel = client.getScenario(id)!! + val portfolio = client.getPortfolio(scenarioModel.portfolioId)!! + val environment = convert(client.getTopology(scenarioModel.topology.topologyId)!!) + val results = runScenario(portfolio, scenarioModel, environment) + + logger.info { "Writing results to database" } + + manager.finish(id, results) + + logger.info { "Successfully finished scenario $id" } + } catch (e: Exception) { + logger.error(e) { "Scenario failed to finish" } + manager.fail(id) + } finally { + heartbeat.cancel() + } + } + } + } + + /** + * Convert the specified [topology] into an [EnvironmentReader] understood by Capelin. + */ + private fun convert(topology: Topology): EnvironmentReader { + val nodes = mutableListOf() + val random = Random(0) + + val machines = topology.rooms.asSequence() + .flatMap { room -> + room.tiles.flatMap { tile -> + tile.rack?.machines?.map { machine -> tile.rack to machine } ?: emptyList() + } + } + for ((rack, machine) in machines) { + val clusterId = rack.id + val position = machine.position + + val processors = machine.cpus.flatMap { cpu -> + val cores = cpu.numberOfCores + val speed = cpu.clockRateMhz + // TODO Remove hard coding of vendor + val node = ProcessingNode("Intel", "amd64", cpu.name, cores) + List(cores) { coreId -> + ProcessingUnit(node, coreId, speed) + } + } + val memoryUnits = machine.memory.map { memory -> + MemoryUnit( + "Samsung", + memory.name, + memory.speedMbPerS, + memory.sizeMb.toLong() + ) + } + + val energyConsumptionW = machine.cpus.sumOf { it.energyConsumptionW } + + nodes.add( + MachineDef( + UUID(random.nextLong(), random.nextLong()), + "node-$clusterId-$position", + mapOf("cluster" to clusterId), + MachineModel(processors, memoryUnits), + LinearPowerModel(2 * energyConsumptionW, energyConsumptionW * 0.5) + ) + ) + } + + return object : EnvironmentReader { + override fun read(): List = nodes + override fun close() {} + } + } +} + +/** + * Main entry point of the runner. + */ +fun main(args: Array): Unit = RunnerCli().main(args) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/ScenarioManager.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/ScenarioManager.kt new file mode 100644 index 00000000..4044cec9 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/ScenarioManager.kt @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.runner + +import org.opendc.web.client.ApiClient +import org.opendc.web.client.model.Job +import org.opendc.web.client.model.SimulationState + +/** + * Manages the queue of scenarios that need to be processed. + */ +public class ScenarioManager(private val client: ApiClient) { + /** + * Find the next job that the simulator needs to process. + */ + public suspend fun findNext(): Job? { + return client.getJobs().firstOrNull() + } + + /** + * Claim the simulation job with the specified id. + */ + public suspend fun claim(id: String): Boolean { + return client.updateJob(id, SimulationState.CLAIMED) + } + + /** + * Update the heartbeat of the specified scenario. + */ + public suspend fun heartbeat(id: String) { + client.updateJob(id, SimulationState.RUNNING) + } + + /** + * Mark the scenario as failed. + */ + public suspend fun fail(id: String) { + client.updateJob(id, SimulationState.FAILED) + } + + /** + * Persist the specified results. + */ + public suspend fun finish(id: String, results: List) { + client.updateJob( + id, SimulationState.FINISHED, + mapOf( + "total_requested_burst" to results.map { it.totalRequestedBurst }, + "total_granted_burst" to results.map { it.totalGrantedBurst }, + "total_overcommitted_burst" to results.map { it.totalOvercommittedBurst }, + "total_interfered_burst" to results.map { it.totalInterferedBurst }, + "mean_cpu_usage" to results.map { it.meanCpuUsage }, + "mean_cpu_demand" to results.map { it.meanCpuDemand }, + "mean_num_deployed_images" to results.map { it.meanNumDeployedImages }, + "max_num_deployed_images" to results.map { it.maxNumDeployedImages }, + "total_power_draw" to results.map { it.totalPowerDraw }, + "total_failure_slices" to results.map { it.totalFailureSlices }, + "total_failure_vm_slices" to results.map { it.totalFailureVmSlices }, + "total_vms_submitted" to results.map { it.totalVmsSubmitted }, + "total_vms_queued" to results.map { it.totalVmsQueued }, + "total_vms_finished" to results.map { it.totalVmsFinished }, + "total_vms_failed" to results.map { it.totalVmsFailed } + ) + ) + } +} diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/WebExperimentMonitor.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/WebExperimentMonitor.kt new file mode 100644 index 00000000..d4445810 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/WebExperimentMonitor.kt @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.runner + +import mu.KotlinLogging +import org.opendc.compute.api.Server +import org.opendc.compute.api.ServerState +import org.opendc.compute.service.driver.Host +import org.opendc.compute.service.driver.HostState +import org.opendc.experiments.capelin.monitor.ExperimentMonitor +import org.opendc.experiments.capelin.telemetry.HostEvent +import kotlin.math.max + +/** + * An [ExperimentMonitor] that tracks the aggregate metrics for each repeat. + */ +public class WebExperimentMonitor : ExperimentMonitor { + private val logger = KotlinLogging.logger {} + + override fun reportVmStateChange(time: Long, server: Server, newState: ServerState) {} + + override fun reportHostStateChange(time: Long, host: Host, newState: HostState) { + logger.debug { "Host ${host.uid} changed state $newState [$time]" } + } + + override fun reportHostSlice( + time: Long, + requestedBurst: Long, + grantedBurst: Long, + overcommissionedBurst: Long, + interferedBurst: Long, + cpuUsage: Double, + cpuDemand: Double, + powerDraw: Double, + numberOfDeployedImages: Int, + host: Host, + ) { + processHostEvent( + HostEvent( + time, + 5 * 60 * 1000L, + host, + numberOfDeployedImages, + requestedBurst, + grantedBurst, + overcommissionedBurst, + interferedBurst, + cpuUsage, + cpuDemand, + powerDraw, + host.model.cpuCount + ) + ) + } + + private var hostAggregateMetrics: AggregateHostMetrics = AggregateHostMetrics() + private val hostMetrics: MutableMap = mutableMapOf() + + private fun processHostEvent(event: HostEvent) { + val slices = event.duration / SLICE_LENGTH + + hostAggregateMetrics = AggregateHostMetrics( + hostAggregateMetrics.totalRequestedBurst + event.requestedBurst, + hostAggregateMetrics.totalGrantedBurst + event.grantedBurst, + hostAggregateMetrics.totalOvercommittedBurst + event.overcommissionedBurst, + hostAggregateMetrics.totalInterferedBurst + event.interferedBurst, + hostAggregateMetrics.totalPowerDraw + (event.duration * event.powerDraw) / 3600, + hostAggregateMetrics.totalFailureSlices + if (event.host.state != HostState.UP) slices else 0, + hostAggregateMetrics.totalFailureVmSlices + if (event.host.state != HostState.UP) event.vmCount * slices else 0 + ) + + hostMetrics.compute(event.host) { _, prev -> + HostMetrics( + (event.cpuUsage.takeIf { event.host.state == HostState.UP } ?: 0.0) + (prev?.cpuUsage ?: 0.0), + (event.cpuDemand.takeIf { event.host.state == HostState.UP } ?: 0.0) + (prev?.cpuDemand ?: 0.0), + event.vmCount + (prev?.vmCount ?: 0), + 1 + (prev?.count ?: 0) + ) + } + } + + private val SLICE_LENGTH: Long = 5 * 60 * 1000 + + public data class AggregateHostMetrics( + val totalRequestedBurst: Long = 0, + val totalGrantedBurst: Long = 0, + val totalOvercommittedBurst: Long = 0, + val totalInterferedBurst: Long = 0, + val totalPowerDraw: Double = 0.0, + val totalFailureSlices: Long = 0, + val totalFailureVmSlices: Long = 0, + ) + + public data class HostMetrics( + val cpuUsage: Double, + val cpuDemand: Double, + val vmCount: Long, + val count: Long + ) + + private var provisionerMetrics: AggregateProvisionerMetrics = AggregateProvisionerMetrics() + + override fun reportProvisionerMetrics( + time: Long, + totalHostCount: Int, + availableHostCount: Int, + totalVmCount: Int, + activeVmCount: Int, + inactiveVmCount: Int, + waitingVmCount: Int, + failedVmCount: Int + ) { + provisionerMetrics = AggregateProvisionerMetrics( + max(totalVmCount, provisionerMetrics.vmTotalCount), + max(waitingVmCount, provisionerMetrics.vmWaitingCount), + max(activeVmCount, provisionerMetrics.vmActiveCount), + max(inactiveVmCount, provisionerMetrics.vmInactiveCount), + max(failedVmCount, provisionerMetrics.vmFailedCount), + ) + } + + public data class AggregateProvisionerMetrics( + val vmTotalCount: Int = 0, + val vmWaitingCount: Int = 0, + val vmActiveCount: Int = 0, + val vmInactiveCount: Int = 0, + val vmFailedCount: Int = 0 + ) + + override fun close() {} + + public fun getResult(): Result { + return Result( + hostAggregateMetrics.totalRequestedBurst, + hostAggregateMetrics.totalGrantedBurst, + hostAggregateMetrics.totalOvercommittedBurst, + hostAggregateMetrics.totalInterferedBurst, + hostMetrics.map { it.value.cpuUsage / it.value.count }.average(), + hostMetrics.map { it.value.cpuDemand / it.value.count }.average(), + hostMetrics.map { it.value.vmCount.toDouble() / it.value.count }.average(), + hostMetrics.map { it.value.vmCount.toDouble() / it.value.count }.maxOrNull() ?: 0.0, + hostAggregateMetrics.totalPowerDraw, + hostAggregateMetrics.totalFailureSlices, + hostAggregateMetrics.totalFailureVmSlices, + provisionerMetrics.vmTotalCount, + provisionerMetrics.vmWaitingCount, + provisionerMetrics.vmInactiveCount, + provisionerMetrics.vmFailedCount, + ) + } + + public data class Result( + public val totalRequestedBurst: Long, + public val totalGrantedBurst: Long, + public val totalOvercommittedBurst: Long, + public val totalInterferedBurst: Long, + public val meanCpuUsage: Double, + public val meanCpuDemand: Double, + public val meanNumDeployedImages: Double, + public val maxNumDeployedImages: Double, + public val totalPowerDraw: Double, + public val totalFailureSlices: Long, + public val totalFailureVmSlices: Long, + public val totalVmsSubmitted: Int, + public val totalVmsQueued: Int, + public val totalVmsFinished: Int, + public val totalVmsFailed: Int + ) +} diff --git a/opendc-web/opendc-web-runner/src/main/resources/log4j2.xml b/opendc-web/opendc-web-runner/src/main/resources/log4j2.xml index 503bc5dc..ad99cc00 100644 --- a/opendc-web/opendc-web-runner/src/main/resources/log4j2.xml +++ b/opendc-web/opendc-web-runner/src/main/resources/log4j2.xml @@ -36,7 +36,7 @@ - + diff --git a/opendc-web/opendc-web-runner/src/test/kotlin/org/opendc/web/client/ApiClientTest.kt b/opendc-web/opendc-web-runner/src/test/kotlin/org/opendc/web/client/ApiClientTest.kt new file mode 100644 index 00000000..3a0730a6 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/test/kotlin/org/opendc/web/client/ApiClientTest.kt @@ -0,0 +1,264 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client + +import io.ktor.client.* +import io.ktor.client.engine.mock.* +import io.ktor.http.* +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test +import java.net.URI + +/** + * Test suite for the [ApiClient] class. + */ +class ApiClientTest { + /** + * The Ktor [HttpClient] instance. + */ + private val ktor = HttpClient(MockEngine) { + engine { + addHandler { request -> + when (request.url.fullPath) { + "/oauth/token" -> { + val responseHeaders = headersOf("Content-Type" to listOf(ContentType.Application.Json.toString())) + respond( + """ + { + "access_token": "eyJz93a...k4laUWw", + "token_type": "Bearer", + "expires_in": 86400 + } + """.trimIndent(), + headers = responseHeaders + ) + } + "/portfolios/5fda5daa97dca438e7cb0a4c" -> { + val responseHeaders = headersOf("Content-Type" to listOf(ContentType.Application.Json.toString())) + respond( + """ + { + "data": { + "_id": "string", + "projectId": "string", + "name": "string", + "scenarioIds": [ + "string" + ], + "targets": { + "enabledMetrics": [ + "string" + ], + "repeatsPerScenario": 0 + } + } + } + """.trimIndent(), + headers = responseHeaders + ) + } + "/portfolios/x" -> { + val responseHeaders = headersOf("Content-Type" to listOf(ContentType.Application.Json.toString())) + respond( + """ + { + "message": "Not Found" + } + """.trimIndent(), + headers = responseHeaders, status = HttpStatusCode.NotFound + ) + } + "/scenarios/5fda5db297dca438e7cb0a4d" -> { + val responseHeaders = headersOf("Content-Type" to listOf(ContentType.Application.Json.toString())) + respond( + """ + { + "data": { + "_id": "string", + "portfolioId": "string", + "name": "string", + "trace": { + "traceId": "string", + "loadSamplingFraction": 0 + }, + "topology": { + "topologyId": "string" + }, + "operational": { + "failuresEnabled": true, + "performanceInterferenceEnabled": true, + "schedulerName": "string" + } + } + } + """.trimIndent(), + headers = responseHeaders + ) + } + "/scenarios/x" -> { + val responseHeaders = headersOf("Content-Type" to listOf(ContentType.Application.Json.toString())) + respond( + """ + { + "message": "Not Found" + } + """.trimIndent(), + headers = responseHeaders, status = HttpStatusCode.NotFound + ) + } + "/topologies/5f9825a6cf6e4c24e380b86f" -> { + val responseHeaders = headersOf("Content-Type" to listOf(ContentType.Application.Json.toString())) + respond( + """ + { + "data": { + "_id": "string", + "projectId": "string", + "name": "string", + "rooms": [ + { + "_id": "string", + "name": "string", + "tiles": [ + { + "_id": "string", + "positionX": 0, + "positionY": 0, + "rack": { + "_id": "string", + "name": "string", + "capacity": 0, + "powerCapacityW": 0, + "machines": [ + { + "_id": "string", + "position": 0, + "cpus": [ + { + "_id": "string", + "name": "string", + "clockRateMhz": 0, + "numberOfCores": 0 + } + ], + "gpus": [ + { + "_id": "string", + "name": "string", + "clockRateMhz": 0, + "numberOfCores": 0 + } + ], + "memories": [ + { + "_id": "string", + "name": "string", + "speedMbPerS": 0, + "sizeMb": 0 + } + ], + "storages": [ + { + "_id": "string", + "name": "string", + "speedMbPerS": 0, + "sizeMb": 0 + } + ] + } + ] + } + } + ] + } + ] + } + } + """.trimIndent(), + headers = responseHeaders + ) + } + "/topologies/x" -> { + val responseHeaders = + headersOf("Content-Type" to listOf(ContentType.Application.Json.toString())) + respond( + """ + { + "message": "Not Found" + } + """.trimIndent(), + headers = responseHeaders, status = HttpStatusCode.NotFound + ) + } + else -> error("Unhandled ${request.url}") + } + } + } + } + + private val auth = AuthConfiguration("auth.opendc.org", "a", "b") + + @Test + fun testPortfolioExists(): Unit = runBlocking { + val client = ApiClient(URI("http://localhost:8081"), auth, client = ktor) + val portfolio = client.getPortfolio("5fda5daa97dca438e7cb0a4c") + assertNotNull(portfolio) + } + + @Test + fun testPortfolioDoesNotExists(): Unit = runBlocking { + val client = ApiClient(URI("http://localhost:8081"), auth, client = ktor) + val portfolio = client.getPortfolio("x") + assertNull(portfolio) + } + + @Test + fun testScenarioExists(): Unit = runBlocking { + val client = ApiClient(URI("http://localhost:8081"), auth, client = ktor) + val scenario = client.getScenario("5fda5db297dca438e7cb0a4d") + assertNotNull(scenario) + } + + @Test + fun testScenarioDoesNotExists(): Unit = runBlocking { + val client = ApiClient(URI("http://localhost:8081"), auth, client = ktor) + val scenario = client.getScenario("x") + assertNull(scenario) + } + + @Test + fun testTopologyExists(): Unit = runBlocking { + val client = ApiClient(URI("http://localhost:8081"), auth, client = ktor) + val topology = client.getTopology("5f9825a6cf6e4c24e380b86f") + assertNotNull(topology) + } + + @Test + fun testTopologyDoesNotExists(): Unit = runBlocking { + val client = ApiClient(URI("http://localhost:8081"), auth, client = ktor) + val topology = client.getTopology("x") + assertNull(topology) + } +} -- cgit v1.2.3