From 05d2318538eba71ac0555dc5ec146499d9cb0592 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 14 May 2021 16:50:23 +0200 Subject: api: Remove user handling from OpenDC API server This change removes any of the user handling and endpoints from the OpenDC API server. The API server does not need to store user information other than an identifier in the database. --- .../opendc-web-api/opendc/models/portfolio.py | 17 ++++------ opendc-web/opendc-web-api/opendc/models/prefab.py | 19 +++--------- opendc-web/opendc-web-api/opendc/models/project.py | 29 ++++++++--------- .../opendc-web-api/opendc/models/scenario.py | 16 +++------- .../opendc-web-api/opendc/models/topology.py | 20 ++++-------- opendc-web/opendc-web-api/opendc/models/user.py | 36 ---------------------- 6 files changed, 34 insertions(+), 103 deletions(-) delete mode 100644 opendc-web/opendc-web-api/opendc/models/user.py (limited to 'opendc-web/opendc-web-api/opendc/models') diff --git a/opendc-web/opendc-web-api/opendc/models/portfolio.py b/opendc-web/opendc-web-api/opendc/models/portfolio.py index 32961b63..8e3f2a52 100644 --- a/opendc-web/opendc-web-api/opendc/models/portfolio.py +++ b/opendc-web/opendc-web-api/opendc/models/portfolio.py @@ -1,7 +1,5 @@ +from opendc.models.project import Project from opendc.models.model import Model -from opendc.models.user import User -from opendc.util.exceptions import ClientError -from opendc.util.rest import Response class Portfolio(Model): @@ -9,16 +7,13 @@ class Portfolio(Model): collection_name = 'portfolios' - def check_user_access(self, google_id, edit_access): - """Raises an error if the user with given [google_id] has insufficient access. + def check_user_access(self, user_id, edit_access): + """Raises an error if the user with given [user_id] has insufficient access. Checks access on the parent project. - :param google_id: The Google ID of the user. + :param user_id: The User ID of the user. :param edit_access: True when edit access should be checked, otherwise view access. """ - user = User.from_google_id(google_id) - authorizations = list( - filter(lambda x: str(x['projectId']) == str(self.obj['projectId']), user.obj['authorizations'])) - if len(authorizations) == 0 or (edit_access and authorizations[0]['authorizationLevel'] == 'VIEW'): - raise ClientError(Response(403, 'Forbidden from retrieving/editing portfolio.')) + project = Project.from_id(self.obj['projectId']) + project.check_user_access(user_id, edit_access) diff --git a/opendc-web/opendc-web-api/opendc/models/prefab.py b/opendc-web/opendc-web-api/opendc/models/prefab.py index edf1d4c4..05356358 100644 --- a/opendc-web/opendc-web-api/opendc/models/prefab.py +++ b/opendc-web/opendc-web-api/opendc/models/prefab.py @@ -1,5 +1,4 @@ from opendc.models.model import Model -from opendc.models.user import User from opendc.util.exceptions import ClientError from opendc.util.rest import Response @@ -9,20 +8,10 @@ class Prefab(Model): collection_name = 'prefabs' - def check_user_access(self, google_id): - """Raises an error if the user with given [google_id] has insufficient access to view this prefab. + def check_user_access(self, user_id): + """Raises an error if the user with given [user_id] has insufficient access to view this prefab. - :param google_id: The Google ID of the user. + :param user_id: The Google ID of the user. """ - user = User.from_google_id(google_id) - - # TODO(Jacob) add special handling for OpenDC-provided prefabs - - #try: - - print(self.obj) - if self.obj['authorId'] != user.get_id() and self.obj['visibility'] == "private": + if self.obj['authorId'] != user_id and self.obj['visibility'] == "private": raise ClientError(Response(403, "Forbidden from retrieving prefab.")) - #except KeyError: - # OpenDC-authored objects don't necessarily have an authorId - # return diff --git a/opendc-web/opendc-web-api/opendc/models/project.py b/opendc-web/opendc-web-api/opendc/models/project.py index b57e9f77..2b3fd5f4 100644 --- a/opendc-web/opendc-web-api/opendc/models/project.py +++ b/opendc-web/opendc-web-api/opendc/models/project.py @@ -1,5 +1,4 @@ from opendc.models.model import Model -from opendc.models.user import User from opendc.util.database import DB from opendc.util.exceptions import ClientError from opendc.util.rest import Response @@ -10,22 +9,20 @@ class Project(Model): collection_name = 'projects' - def check_user_access(self, google_id, edit_access): - """Raises an error if the user with given [google_id] has insufficient access. + def check_user_access(self, user_id, edit_access): + """Raises an error if the user with given [user_id] has insufficient access. - :param google_id: The Google ID of the user. + :param user_id: The User ID of the user. :param edit_access: True when edit access should be checked, otherwise view access. """ - user = User.from_google_id(google_id) - authorizations = list(filter(lambda x: str(x['projectId']) == str(self.get_id()), - user.obj['authorizations'])) - if len(authorizations) == 0 or (edit_access and authorizations[0]['authorizationLevel'] == 'VIEW'): - raise ClientError(Response(403, "Forbidden from retrieving project.")) + for authorization in self.obj['authorizations']: + if user_id == authorization['userId'] and authorization['authorizationLevel'] != 'VIEW' or not edit_access: + return + raise ClientError(Response(403, "Forbidden from retrieving project.")) - def get_all_authorizations(self): - """Get all user IDs having access to this project.""" - return [ - str(user['_id']) for user in DB.fetch_all({'authorizations': { - 'projectId': self.obj['_id'] - }}, User.collection_name) - ] + @classmethod + def get_for_user(cls, user_id): + """Get all projects for the specified user id.""" + return DB.fetch_all({'authorizations': { + 'userId': user_id + }}, Project.collection_name) diff --git a/opendc-web/opendc-web-api/opendc/models/scenario.py b/opendc-web/opendc-web-api/opendc/models/scenario.py index 8d53e408..3dfde012 100644 --- a/opendc-web/opendc-web-api/opendc/models/scenario.py +++ b/opendc-web/opendc-web-api/opendc/models/scenario.py @@ -1,8 +1,5 @@ from opendc.models.model import Model from opendc.models.portfolio import Portfolio -from opendc.models.user import User -from opendc.util.exceptions import ClientError -from opendc.util.rest import Response class Scenario(Model): @@ -10,17 +7,14 @@ class Scenario(Model): collection_name = 'scenarios' - def check_user_access(self, google_id, edit_access): - """Raises an error if the user with given [google_id] has insufficient access. + def check_user_access(self, user_id, edit_access): + """Raises an error if the user with given [user_id] has insufficient access. Checks access on the parent project. - :param google_id: The Google ID of the user. + :param user_id: The User ID of the user. :param edit_access: True when edit access should be checked, otherwise view access. """ portfolio = Portfolio.from_id(self.obj['portfolioId']) - user = User.from_google_id(google_id) - authorizations = list( - filter(lambda x: str(x['projectId']) == str(portfolio.obj['projectId']), user.obj['authorizations'])) - if len(authorizations) == 0 or (edit_access and authorizations[0]['authorizationLevel'] == 'VIEW'): - raise ClientError(Response(403, 'Forbidden from retrieving/editing scenario.')) + print(portfolio.obj) + portfolio.check_user_access(user_id, edit_access) diff --git a/opendc-web/opendc-web-api/opendc/models/topology.py b/opendc-web/opendc-web-api/opendc/models/topology.py index cb4c4bab..3ebec16d 100644 --- a/opendc-web/opendc-web-api/opendc/models/topology.py +++ b/opendc-web/opendc-web-api/opendc/models/topology.py @@ -1,7 +1,5 @@ +from opendc.models.project import Project from opendc.models.model import Model -from opendc.models.user import User -from opendc.util.exceptions import ClientError -from opendc.util.rest import Response class Topology(Model): @@ -9,19 +7,13 @@ class Topology(Model): collection_name = 'topologies' - def check_user_access(self, google_id, edit_access): - """Raises an error if the user with given [google_id] has insufficient access. + def check_user_access(self, user_id, edit_access): + """Raises an error if the user with given [user_id] has insufficient access. Checks access on the parent project. - :param google_id: The Google ID of the user. + :param user_id: The User ID of the user. :param edit_access: True when edit access should be checked, otherwise view access. """ - user = User.from_google_id(google_id) - if 'projectId' not in self.obj: - raise ClientError(Response(400, 'Missing projectId in topology.')) - - authorizations = list( - filter(lambda x: str(x['projectId']) == str(self.obj['projectId']), user.obj['authorizations'])) - if len(authorizations) == 0 or (edit_access and authorizations[0]['authorizationLevel'] == 'VIEW'): - raise ClientError(Response(403, 'Forbidden from retrieving topology.')) + project = Project.from_id(self.obj['projectId']) + project.check_user_access(user_id, edit_access) diff --git a/opendc-web/opendc-web-api/opendc/models/user.py b/opendc-web/opendc-web-api/opendc/models/user.py deleted file mode 100644 index 8e8ff945..00000000 --- a/opendc-web/opendc-web-api/opendc/models/user.py +++ /dev/null @@ -1,36 +0,0 @@ -from opendc.models.model import Model -from opendc.util.database import DB -from opendc.util.exceptions import ClientError -from opendc.util.rest import Response - - -class User(Model): - """Model representing a User.""" - - collection_name = 'users' - - @classmethod - def from_email(cls, email): - """Fetches the user with given email from the collection.""" - return User(DB.fetch_one({'email': email}, User.collection_name)) - - @classmethod - def from_google_id(cls, google_id): - """Fetches the user with given Google ID from the collection.""" - return User(DB.fetch_one({'googleId': google_id}, User.collection_name)) - - def check_correct_user(self, request_google_id): - """Raises an error if a user tries to modify another user. - - :param request_google_id: - """ - if request_google_id is not None and self.obj['googleId'] != request_google_id: - raise ClientError(Response(403, f'Forbidden from editing user with ID {self.obj["_id"]}.')) - - def check_already_exists(self): - """Checks if the user already exists in the database.""" - - existing_user = DB.fetch_one({'googleId': self.obj['googleId']}, self.collection_name) - - if existing_user is not None: - raise ClientError(Response(409, 'User already exists.')) -- cgit v1.2.3 From 2281d3265423d01e60f8cc088de5a5730bb8a910 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Sat, 15 May 2021 13:09:06 +0200 Subject: api: Migrate to Flask Restful This change updates the API to use Flask Restful instead of our own in-house REST library. This change reduces the maintenance effort and allows us to drastically simplify the API implementation needed for the OpenDC v2 API. --- opendc-web/opendc-web-api/opendc/models/model.py | 19 +++--- .../opendc-web-api/opendc/models/portfolio.py | 21 ++++++ opendc-web/opendc-web-api/opendc/models/prefab.py | 23 +++++-- opendc-web/opendc-web-api/opendc/models/project.py | 27 +++++--- .../opendc-web-api/opendc/models/scenario.py | 46 ++++++++++++- .../opendc-web-api/opendc/models/topology.py | 76 ++++++++++++++++++++++ 6 files changed, 187 insertions(+), 25 deletions(-) (limited to 'opendc-web/opendc-web-api/opendc/models') diff --git a/opendc-web/opendc-web-api/opendc/models/model.py b/opendc-web/opendc-web-api/opendc/models/model.py index f9dfc9ad..28299453 100644 --- a/opendc-web/opendc-web-api/opendc/models/model.py +++ b/opendc-web/opendc-web-api/opendc/models/model.py @@ -1,8 +1,7 @@ from bson.objectid import ObjectId +from werkzeug.exceptions import NotFound -from opendc.util.database import DB -from opendc.util.exceptions import ClientError -from opendc.util.rest import Response +from opendc.exts import db class Model: @@ -15,15 +14,13 @@ class Model: """Fetches the document with given ID from the collection.""" if isinstance(_id, str) and len(_id) == 24: _id = ObjectId(_id) - elif not isinstance(_id, ObjectId): - return cls(None) - return cls(DB.fetch_one({'_id': _id}, cls.collection_name)) + return cls(db.fetch_one({'_id': _id}, cls.collection_name)) @classmethod def get_all(cls): """Fetches all documents from the collection.""" - return cls(DB.fetch_all({}, cls.collection_name)) + return cls(db.fetch_all({}, cls.collection_name)) def __init__(self, obj): self.obj = obj @@ -35,7 +32,7 @@ class Model: def check_exists(self): """Raises an error if the enclosed object does not exist.""" if self.obj is None: - raise ClientError(Response(404, 'Not found.')) + raise NotFound('Entity not found.') def set_property(self, key, value): """Sets the given property on the enclosed object, with support for simple nested access.""" @@ -48,11 +45,11 @@ class Model: def insert(self): """Inserts the enclosed object and generates a UUID for it.""" self.obj['_id'] = ObjectId() - DB.insert(self.obj, self.collection_name) + db.insert(self.obj, self.collection_name) def update(self): """Updates the enclosed object and updates the internal reference to the newly inserted object.""" - DB.update(self.get_id(), self.obj, self.collection_name) + db.update(self.get_id(), self.obj, self.collection_name) def delete(self): """Deletes the enclosed object in the database, if it existed.""" @@ -60,5 +57,5 @@ class Model: return None old_object = self.obj.copy() - DB.delete_one({'_id': self.get_id()}, self.collection_name) + db.delete_one({'_id': self.get_id()}, self.collection_name) return old_object diff --git a/opendc-web/opendc-web-api/opendc/models/portfolio.py b/opendc-web/opendc-web-api/opendc/models/portfolio.py index 8e3f2a52..aff1d3f0 100644 --- a/opendc-web/opendc-web-api/opendc/models/portfolio.py +++ b/opendc-web/opendc-web-api/opendc/models/portfolio.py @@ -1,7 +1,28 @@ +from marshmallow import Schema, fields + from opendc.models.project import Project from opendc.models.model import Model +class TargetSchema(Schema): + """ + Schema representing a target. + """ + enabledMetrics = fields.List(fields.String()) + repeatsPerScenario = fields.Integer(required=True) + + +class PortfolioSchema(Schema): + """ + Schema representing a portfolio. + """ + _id = fields.String() + projectId = fields.String() + name = fields.String(required=True) + scenarioIds = fields.List(fields.String()) + targets = fields.Nested(TargetSchema) + + class Portfolio(Model): """Model representing a Portfolio.""" diff --git a/opendc-web/opendc-web-api/opendc/models/prefab.py b/opendc-web/opendc-web-api/opendc/models/prefab.py index 05356358..d83ef4cb 100644 --- a/opendc-web/opendc-web-api/opendc/models/prefab.py +++ b/opendc-web/opendc-web-api/opendc/models/prefab.py @@ -1,17 +1,30 @@ +from marshmallow import Schema, fields +from werkzeug.exceptions import Forbidden + +from opendc.models.topology import ObjectSchema from opendc.models.model import Model -from opendc.util.exceptions import ClientError -from opendc.util.rest import Response + + +class PrefabSchema(Schema): + """ + Schema for a Prefab. + """ + _id = fields.String() + name = fields.String(required=True) + datetimeCreated = fields.DateTime() + datetimeLastEdited = fields.DateTime() + rack = fields.Nested(ObjectSchema) class Prefab(Model): - """Model representing a Project.""" + """Model representing a Prefab.""" collection_name = 'prefabs' def check_user_access(self, user_id): """Raises an error if the user with given [user_id] has insufficient access to view this prefab. - :param user_id: The Google ID of the user. + :param user_id: The user ID of the user. """ if self.obj['authorId'] != user_id and self.obj['visibility'] == "private": - raise ClientError(Response(403, "Forbidden from retrieving prefab.")) + raise Forbidden("Forbidden from retrieving prefab.") diff --git a/opendc-web/opendc-web-api/opendc/models/project.py b/opendc-web/opendc-web-api/opendc/models/project.py index 2b3fd5f4..ee84c73e 100644 --- a/opendc-web/opendc-web-api/opendc/models/project.py +++ b/opendc-web/opendc-web-api/opendc/models/project.py @@ -1,7 +1,20 @@ +from marshmallow import Schema, fields +from werkzeug.exceptions import Forbidden + from opendc.models.model import Model -from opendc.util.database import DB -from opendc.util.exceptions import ClientError -from opendc.util.rest import Response +from opendc.exts import db + + +class ProjectSchema(Schema): + """ + Schema representing a Project. + """ + _id = fields.String() + name = fields.String(required=True) + datetimeCreated = fields.DateTime() + datetimeLastEdited = fields.DateTime() + topologyIds = fields.List(fields.String()) + portfolioIds = fields.List(fields.String()) class Project(Model): @@ -16,13 +29,11 @@ class Project(Model): :param edit_access: True when edit access should be checked, otherwise view access. """ for authorization in self.obj['authorizations']: - if user_id == authorization['userId'] and authorization['authorizationLevel'] != 'VIEW' or not edit_access: + if user_id == authorization['userId'] and authorization['level'] != 'VIEW' or not edit_access: return - raise ClientError(Response(403, "Forbidden from retrieving project.")) + raise Forbidden("Forbidden from retrieving project.") @classmethod def get_for_user(cls, user_id): """Get all projects for the specified user id.""" - return DB.fetch_all({'authorizations': { - 'userId': user_id - }}, Project.collection_name) + return db.fetch_all({'authorizations.userId': user_id}, Project.collection_name) diff --git a/opendc-web/opendc-web-api/opendc/models/scenario.py b/opendc-web/opendc-web-api/opendc/models/scenario.py index 3dfde012..2911b1ae 100644 --- a/opendc-web/opendc-web-api/opendc/models/scenario.py +++ b/opendc-web/opendc-web-api/opendc/models/scenario.py @@ -1,7 +1,52 @@ +from marshmallow import Schema, fields from opendc.models.model import Model from opendc.models.portfolio import Portfolio +class SimulationSchema(Schema): + """ + Simulation details. + """ + state = fields.String() + + +class TraceSchema(Schema): + """ + Schema for specifying the trace of a scenario. + """ + traceId = fields.String() + loadSamplingFraction = fields.Float() + + +class TopologySchema(Schema): + """ + Schema for topology specification for a scenario. + """ + topologyId = fields.String() + + +class OperationalSchema(Schema): + """ + Schema for the operational phenomena for a scenario. + """ + failuresEnabled = fields.Boolean() + performanceInterferenceEnabled = fields.Boolean() + schedulerName = fields.String() + + +class ScenarioSchema(Schema): + """ + Schema representing a scenario. + """ + _id = fields.String() + portfolioId = fields.String() + name = fields.String(required=True) + simulation = fields.Nested(SimulationSchema) + trace = fields.Nested(TraceSchema) + topology = fields.Nested(TopologySchema) + operational = fields.Nested(OperationalSchema) + + class Scenario(Model): """Model representing a Scenario.""" @@ -16,5 +61,4 @@ class Scenario(Model): :param edit_access: True when edit access should be checked, otherwise view access. """ portfolio = Portfolio.from_id(self.obj['portfolioId']) - print(portfolio.obj) portfolio.check_user_access(user_id, edit_access) diff --git a/opendc-web/opendc-web-api/opendc/models/topology.py b/opendc-web/opendc-web-api/opendc/models/topology.py index 3ebec16d..c6354ae6 100644 --- a/opendc-web/opendc-web-api/opendc/models/topology.py +++ b/opendc-web/opendc-web-api/opendc/models/topology.py @@ -1,7 +1,83 @@ +from marshmallow import Schema, fields + from opendc.models.project import Project from opendc.models.model import Model +class MemorySchema(Schema): + """ + Schema representing a memory unit. + """ + _id = fields.String() + name = fields.String() + speedMbPerS = fields.Integer() + sizeMb = fields.Integer() + energyConsumptionW = fields.Integer() + + +class PuSchema(Schema): + """ + Schema representing a processing unit. + """ + _id = fields.String() + name = fields.String() + clockRateMhz = fields.Integer() + numberOfCores = fields.Integer() + energyConsumptionW = fields.Integer() + + +class MachineSchema(Schema): + """ + Schema representing a machine. + """ + _id = fields.String() + position = fields.Integer() + cpus = fields.List(fields.Nested(PuSchema)) + gpus = fields.List(fields.Nested(PuSchema)) + memories = fields.List(fields.Nested(MemorySchema)) + storages = fields.List(fields.Nested(MemorySchema)) + + +class ObjectSchema(Schema): + """ + Schema representing a room object. + """ + _id = fields.String() + name = fields.String() + capacity = fields.Integer() + powerCapacityW = fields.Integer() + machines = fields.List(fields.Nested(MachineSchema)) + + +class TileSchema(Schema): + """ + Schema representing a room tile. + """ + _id = fields.String() + positionX = fields.Integer() + positionY = fields.Integer() + rack = fields.Nested(ObjectSchema) + + +class RoomSchema(Schema): + """ + Schema representing a room. + """ + _id = fields.String() + name = fields.String(required=True) + tiles = fields.List(fields.Nested(TileSchema), required=True) + + +class TopologySchema(Schema): + """ + Schema representing a datacenter topology. + """ + _id = fields.String() + projectId = fields.String() + name = fields.String(required=True) + rooms = fields.List(fields.Nested(RoomSchema), required=True) + + class Topology(Model): """Model representing a Project.""" -- cgit v1.2.3 From 45b73e4683cce35de79117c5b4a6919556d9644f Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 2 Jul 2021 14:26:23 +0200 Subject: api: Add stricter validation of input/output data This change adds stricter validation of data that enters and leaves the database. As a result, we clearly separate the database model from the data model that the REST API exports. --- .../opendc-web-api/opendc/models/portfolio.py | 2 +- opendc-web/opendc-web-api/opendc/models/prefab.py | 3 ++- opendc-web/opendc-web-api/opendc/models/project.py | 13 ++++++++++-- .../opendc-web-api/opendc/models/scenario.py | 24 +++++++++++++++++++++- .../opendc-web-api/opendc/models/topology.py | 4 ++-- opendc-web/opendc-web-api/opendc/models/trace.py | 9 ++++++++ 6 files changed, 48 insertions(+), 7 deletions(-) (limited to 'opendc-web/opendc-web-api/opendc/models') diff --git a/opendc-web/opendc-web-api/opendc/models/portfolio.py b/opendc-web/opendc-web-api/opendc/models/portfolio.py index aff1d3f0..1643e23e 100644 --- a/opendc-web/opendc-web-api/opendc/models/portfolio.py +++ b/opendc-web/opendc-web-api/opendc/models/portfolio.py @@ -16,7 +16,7 @@ class PortfolioSchema(Schema): """ Schema representing a portfolio. """ - _id = fields.String() + _id = fields.String(dump_only=True) projectId = fields.String() name = fields.String(required=True) scenarioIds = fields.List(fields.String()) diff --git a/opendc-web/opendc-web-api/opendc/models/prefab.py b/opendc-web/opendc-web-api/opendc/models/prefab.py index d83ef4cb..5e4b81dc 100644 --- a/opendc-web/opendc-web-api/opendc/models/prefab.py +++ b/opendc-web/opendc-web-api/opendc/models/prefab.py @@ -9,7 +9,8 @@ class PrefabSchema(Schema): """ Schema for a Prefab. """ - _id = fields.String() + _id = fields.String(dump_only=True) + authorId = fields.String(dump_only=True) name = fields.String(required=True) datetimeCreated = fields.DateTime() datetimeLastEdited = fields.DateTime() diff --git a/opendc-web/opendc-web-api/opendc/models/project.py b/opendc-web/opendc-web-api/opendc/models/project.py index ee84c73e..f2b3b564 100644 --- a/opendc-web/opendc-web-api/opendc/models/project.py +++ b/opendc-web/opendc-web-api/opendc/models/project.py @@ -1,20 +1,29 @@ -from marshmallow import Schema, fields +from marshmallow import Schema, fields, validate from werkzeug.exceptions import Forbidden from opendc.models.model import Model from opendc.exts import db +class ProjectAuthorizations(Schema): + """ + Schema representing a project authorization. + """ + userId = fields.String(required=True) + level = fields.String(required=True, validate=validate.OneOf(["VIEW", "EDIT", "OWN"])) + + class ProjectSchema(Schema): """ Schema representing a Project. """ - _id = fields.String() + _id = fields.String(dump_only=True) name = fields.String(required=True) datetimeCreated = fields.DateTime() datetimeLastEdited = fields.DateTime() topologyIds = fields.List(fields.String()) portfolioIds = fields.List(fields.String()) + authorizations = fields.List(fields.Nested(ProjectAuthorizations)) class Project(Model): diff --git a/opendc-web/opendc-web-api/opendc/models/scenario.py b/opendc-web/opendc-web-api/opendc/models/scenario.py index 2911b1ae..658d790e 100644 --- a/opendc-web/opendc-web-api/opendc/models/scenario.py +++ b/opendc-web/opendc-web-api/opendc/models/scenario.py @@ -34,17 +34,39 @@ class OperationalSchema(Schema): schedulerName = fields.String() +class ResultSchema(Schema): + """ + Schema representing the simulation results. + """ + max_num_deployed_images = fields.List(fields.Number()) + max_cpu_demand = fields.List(fields.Number()) + max_cpu_usage = fields.List(fields.Number()) + mean_num_deployed_images = fields.List(fields.Number()) + total_failure_slices = fields.List(fields.Number()) + total_failure_vm_slices = fields.List(fields.Number()) + total_granted_burst = fields.List(fields.Number()) + total_interfered_burst = fields.List(fields.Number()) + total_overcommitted_burst = fields.List(fields.Number()) + total_power_draw = fields.List(fields.Number()) + total_requested_burst = fields.List(fields.Number()) + total_vms_failed = fields.List(fields.Number()) + total_vms_finished = fields.List(fields.Number()) + total_vms_queued = fields.List(fields.Number()) + total_vms_submitted = fields.List(fields.Number()) + + class ScenarioSchema(Schema): """ Schema representing a scenario. """ - _id = fields.String() + _id = fields.String(dump_only=True) portfolioId = fields.String() name = fields.String(required=True) simulation = fields.Nested(SimulationSchema) trace = fields.Nested(TraceSchema) topology = fields.Nested(TopologySchema) operational = fields.Nested(OperationalSchema) + results = fields.Nested(ResultSchema, dump_only=True) class Scenario(Model): diff --git a/opendc-web/opendc-web-api/opendc/models/topology.py b/opendc-web/opendc-web-api/opendc/models/topology.py index c6354ae6..71d2cade 100644 --- a/opendc-web/opendc-web-api/opendc/models/topology.py +++ b/opendc-web/opendc-web-api/opendc/models/topology.py @@ -72,8 +72,8 @@ class TopologySchema(Schema): """ Schema representing a datacenter topology. """ - _id = fields.String() - projectId = fields.String() + _id = fields.String(dump_only=True) + projectId = fields.String(dump_only=True) name = fields.String(required=True) rooms = fields.List(fields.Nested(RoomSchema), required=True) diff --git a/opendc-web/opendc-web-api/opendc/models/trace.py b/opendc-web/opendc-web-api/opendc/models/trace.py index 2f6e4926..69287f29 100644 --- a/opendc-web/opendc-web-api/opendc/models/trace.py +++ b/opendc-web/opendc-web-api/opendc/models/trace.py @@ -1,6 +1,15 @@ +from marshmallow import Schema, fields + from opendc.models.model import Model +class TraceSchema(Schema): + """Schema for a Trace.""" + _id = fields.String(dump_only=True) + name = fields.String() + type = fields.String() + + class Trace(Model): """Model representing a Trace.""" -- cgit v1.2.3 From a2a5979bfb392565b55e489b6020aa391e782eb0 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 2 Jul 2021 16:14:52 +0200 Subject: api: Add endpoint for simulation jobs This change adds an API endpoint for simulation jobs which allows API consumers to manage simulation jobs without needing direct database access that is currently needed for the web runner. --- .../opendc-web-api/opendc/models/scenario.py | 52 +++++++++------------- 1 file changed, 22 insertions(+), 30 deletions(-) (limited to 'opendc-web/opendc-web-api/opendc/models') diff --git a/opendc-web/opendc-web-api/opendc/models/scenario.py b/opendc-web/opendc-web-api/opendc/models/scenario.py index 658d790e..0fb6c453 100644 --- a/opendc-web/opendc-web-api/opendc/models/scenario.py +++ b/opendc-web/opendc-web-api/opendc/models/scenario.py @@ -1,15 +1,12 @@ +from datetime import datetime + from marshmallow import Schema, fields + +from opendc.exts import db from opendc.models.model import Model from opendc.models.portfolio import Portfolio -class SimulationSchema(Schema): - """ - Simulation details. - """ - state = fields.String() - - class TraceSchema(Schema): """ Schema for specifying the trace of a scenario. @@ -34,27 +31,6 @@ class OperationalSchema(Schema): schedulerName = fields.String() -class ResultSchema(Schema): - """ - Schema representing the simulation results. - """ - max_num_deployed_images = fields.List(fields.Number()) - max_cpu_demand = fields.List(fields.Number()) - max_cpu_usage = fields.List(fields.Number()) - mean_num_deployed_images = fields.List(fields.Number()) - total_failure_slices = fields.List(fields.Number()) - total_failure_vm_slices = fields.List(fields.Number()) - total_granted_burst = fields.List(fields.Number()) - total_interfered_burst = fields.List(fields.Number()) - total_overcommitted_burst = fields.List(fields.Number()) - total_power_draw = fields.List(fields.Number()) - total_requested_burst = fields.List(fields.Number()) - total_vms_failed = fields.List(fields.Number()) - total_vms_finished = fields.List(fields.Number()) - total_vms_queued = fields.List(fields.Number()) - total_vms_submitted = fields.List(fields.Number()) - - class ScenarioSchema(Schema): """ Schema representing a scenario. @@ -62,11 +38,9 @@ class ScenarioSchema(Schema): _id = fields.String(dump_only=True) portfolioId = fields.String() name = fields.String(required=True) - simulation = fields.Nested(SimulationSchema) trace = fields.Nested(TraceSchema) topology = fields.Nested(TopologySchema) operational = fields.Nested(OperationalSchema) - results = fields.Nested(ResultSchema, dump_only=True) class Scenario(Model): @@ -84,3 +58,21 @@ class Scenario(Model): """ portfolio = Portfolio.from_id(self.obj['portfolioId']) portfolio.check_user_access(user_id, edit_access) + + @classmethod + def get_jobs(cls): + """Obtain the scenarios that have been queued. + """ + return cls(db.fetch_all({'simulation.state': 'QUEUED'}, cls.collection_name)) + + def update_state(self, new_state, results=None): + """Atomically update the state of the Scenario. + """ + update = {'$set': {'simulation.state': new_state, 'simulation.heartbeat': datetime.now()}} + if results: + update['$set']['results'] = results + return db.fetch_and_update( + query={'_id': self.obj['_id'], 'simulation.state': self.obj['simulation']['state']}, + update=update, + collection=self.collection_name + ) -- cgit v1.2.3 From 5ec19973eb3d23046d874b097275857a58c23082 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 7 Jul 2021 20:45:06 +0200 Subject: api: Add endpoints for accessing project relations This change adds additional endpoints to the REST API to access the project relations, the portfolios and topologies that belong to a project. --- opendc-web/opendc-web-api/opendc/models/portfolio.py | 7 +++++++ opendc-web/opendc-web-api/opendc/models/scenario.py | 6 ++++++ opendc-web/opendc-web-api/opendc/models/topology.py | 7 +++++++ 3 files changed, 20 insertions(+) (limited to 'opendc-web/opendc-web-api/opendc/models') diff --git a/opendc-web/opendc-web-api/opendc/models/portfolio.py b/opendc-web/opendc-web-api/opendc/models/portfolio.py index 1643e23e..eb016947 100644 --- a/opendc-web/opendc-web-api/opendc/models/portfolio.py +++ b/opendc-web/opendc-web-api/opendc/models/portfolio.py @@ -1,5 +1,7 @@ +from bson import ObjectId from marshmallow import Schema, fields +from opendc.exts import db from opendc.models.project import Project from opendc.models.model import Model @@ -38,3 +40,8 @@ class Portfolio(Model): """ project = Project.from_id(self.obj['projectId']) project.check_user_access(user_id, edit_access) + + @classmethod + def get_for_project(cls, project_id): + """Get all portfolios for the specified project id.""" + return db.fetch_all({'projectId': ObjectId(project_id)}, cls.collection_name) diff --git a/opendc-web/opendc-web-api/opendc/models/scenario.py b/opendc-web/opendc-web-api/opendc/models/scenario.py index 0fb6c453..63160448 100644 --- a/opendc-web/opendc-web-api/opendc/models/scenario.py +++ b/opendc-web/opendc-web-api/opendc/models/scenario.py @@ -1,5 +1,6 @@ from datetime import datetime +from bson import ObjectId from marshmallow import Schema, fields from opendc.exts import db @@ -65,6 +66,11 @@ class Scenario(Model): """ return cls(db.fetch_all({'simulation.state': 'QUEUED'}, cls.collection_name)) + @classmethod + def get_for_portfolio(cls, portfolio_id): + """Get all scenarios for the specified portfolio id.""" + return db.fetch_all({'portfolioId': ObjectId(portfolio_id)}, cls.collection_name) + def update_state(self, new_state, results=None): """Atomically update the state of the Scenario. """ diff --git a/opendc-web/opendc-web-api/opendc/models/topology.py b/opendc-web/opendc-web-api/opendc/models/topology.py index 71d2cade..7d2349ab 100644 --- a/opendc-web/opendc-web-api/opendc/models/topology.py +++ b/opendc-web/opendc-web-api/opendc/models/topology.py @@ -1,5 +1,7 @@ +from bson import ObjectId from marshmallow import Schema, fields +from opendc.exts import db from opendc.models.project import Project from opendc.models.model import Model @@ -93,3 +95,8 @@ class Topology(Model): """ project = Project.from_id(self.obj['projectId']) project.check_user_access(user_id, edit_access) + + @classmethod + def get_for_project(cls, project_id): + """Get all topologies for the specified project id.""" + return db.fetch_all({'projectId': ObjectId(project_id)}, cls.collection_name) -- cgit v1.2.3 From 1157cc3c56c0f8d36be277bd1ea633f03dd7abbf Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 8 Jul 2021 11:45:06 +0200 Subject: api: Re-expose simulation results from scenario endpoint This change updates the OpenDC API to re-expose the simulation results as they are necessary for the frontend to display the results. --- opendc-web/opendc-web-api/opendc/models/scenario.py | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'opendc-web/opendc-web-api/opendc/models') diff --git a/opendc-web/opendc-web-api/opendc/models/scenario.py b/opendc-web/opendc-web-api/opendc/models/scenario.py index 63160448..47771e06 100644 --- a/opendc-web/opendc-web-api/opendc/models/scenario.py +++ b/opendc-web/opendc-web-api/opendc/models/scenario.py @@ -8,6 +8,13 @@ from opendc.models.model import Model from opendc.models.portfolio import Portfolio +class SimulationSchema(Schema): + """ + Simulation details. + """ + state = fields.String() + + class TraceSchema(Schema): """ Schema for specifying the trace of a scenario. @@ -42,6 +49,8 @@ class ScenarioSchema(Schema): trace = fields.Nested(TraceSchema) topology = fields.Nested(TopologySchema) operational = fields.Nested(OperationalSchema) + simulation = fields.Nested(SimulationSchema, dump_only=True) + results = fields.Dict(dump_only=True) class Scenario(Model): -- cgit v1.2.3 From 29196842447d841d2e21462adcfc8c2ed1d851ad Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 8 Jul 2021 13:15:28 +0200 Subject: ui: Simplify normalization of topology This change updates the OpenDC frontend to use the normalizr library for normalizing the user topology. --- opendc-web/opendc-web-api/opendc/models/topology.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'opendc-web/opendc-web-api/opendc/models') diff --git a/opendc-web/opendc-web-api/opendc/models/topology.py b/opendc-web/opendc-web-api/opendc/models/topology.py index 7d2349ab..592f82c5 100644 --- a/opendc-web/opendc-web-api/opendc/models/topology.py +++ b/opendc-web/opendc-web-api/opendc/models/topology.py @@ -75,7 +75,7 @@ class TopologySchema(Schema): Schema representing a datacenter topology. """ _id = fields.String(dump_only=True) - projectId = fields.String(dump_only=True) + projectId = fields.String() name = fields.String(required=True) rooms = fields.List(fields.Nested(RoomSchema), required=True) -- cgit v1.2.3 From e200dbfdc076ac6263c9ac6f9dabdcc475f01d6e Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Sat, 10 Jul 2021 12:41:00 +0200 Subject: fix(ui): Relax topology schema requirements This change fixes an issue where the topology generated by the frontend was not accepted by the API server. --- opendc-web/opendc-web-api/opendc/models/topology.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'opendc-web/opendc-web-api/opendc/models') diff --git a/opendc-web/opendc-web-api/opendc/models/topology.py b/opendc-web/opendc-web-api/opendc/models/topology.py index 592f82c5..44994818 100644 --- a/opendc-web/opendc-web-api/opendc/models/topology.py +++ b/opendc-web/opendc-web-api/opendc/models/topology.py @@ -38,6 +38,7 @@ class MachineSchema(Schema): gpus = fields.List(fields.Nested(PuSchema)) memories = fields.List(fields.Nested(MemorySchema)) storages = fields.List(fields.Nested(MemorySchema)) + rackId = fields.String() class ObjectSchema(Schema): @@ -49,6 +50,7 @@ class ObjectSchema(Schema): capacity = fields.Integer() powerCapacityW = fields.Integer() machines = fields.List(fields.Nested(MachineSchema)) + tileId = fields.String() class TileSchema(Schema): @@ -56,9 +58,11 @@ class TileSchema(Schema): Schema representing a room tile. """ _id = fields.String() + topologyId = fields.String() positionX = fields.Integer() positionY = fields.Integer() rack = fields.Nested(ObjectSchema) + roomId = fields.String() class RoomSchema(Schema): @@ -67,6 +71,7 @@ class RoomSchema(Schema): """ _id = fields.String() name = fields.String(required=True) + topologyId = fields.String() tiles = fields.List(fields.Nested(TileSchema), required=True) @@ -78,6 +83,7 @@ class TopologySchema(Schema): projectId = fields.String() name = fields.String(required=True) rooms = fields.List(fields.Nested(RoomSchema), required=True) + datetimeLastEdited = fields.DateTime() class Topology(Model): -- cgit v1.2.3