merge: Integrate v2.1 progress into public repository

This pull request integrates the changes planned for the v2.1 release of
OpenDC into the public Github repository in order to sync the progress
of both repositories.

8 files changed, 283 insertions, 122 deletions

diff --git a/opendc-web/opendc-web-api/opendc/models/model.py b/opendc-web/opendc-web-api/opendc/models/model.py
index f9dfc9ad..28299453 100644
--- a/opendc-web/opendc-web-api/opendc/models/model.py
+++ b/opendc-web/opendc-web-api/opendc/models/model.py
@@ -1,8 +1,7 @@
 from bson.objectid import ObjectId
+from werkzeug.exceptions import NotFound
 
-from opendc.util.database import DB
-from opendc.util.exceptions import ClientError
-from opendc.util.rest import Response
+from opendc.exts import db
 
 
 class Model:
@@ -15,15 +14,13 @@ class Model:
         """Fetches the document with given ID from the collection."""
         if isinstance(_id, str) and len(_id) == 24:
             _id = ObjectId(_id)
-        elif not isinstance(_id, ObjectId):
-            return cls(None)
 
-        return cls(DB.fetch_one({'_id': _id}, cls.collection_name))
+        return cls(db.fetch_one({'_id': _id}, cls.collection_name))
 
     @classmethod
     def get_all(cls):
         """Fetches all documents from the collection."""
-        return cls(DB.fetch_all({}, cls.collection_name))
+        return cls(db.fetch_all({}, cls.collection_name))
 
     def __init__(self, obj):
         self.obj = obj
@@ -35,7 +32,7 @@ class Model:
     def check_exists(self):
         """Raises an error if the enclosed object does not exist."""
         if self.obj is None:
-            raise ClientError(Response(404, 'Not found.'))
+            raise NotFound('Entity not found.')
 
     def set_property(self, key, value):
         """Sets the given property on the enclosed object, with support for simple nested access."""
@@ -48,11 +45,11 @@ class Model:
     def insert(self):
         """Inserts the enclosed object and generates a UUID for it."""
         self.obj['_id'] = ObjectId()
-        DB.insert(self.obj, self.collection_name)
+        db.insert(self.obj, self.collection_name)
 
     def update(self):
         """Updates the enclosed object and updates the internal reference to the newly inserted object."""
-        DB.update(self.get_id(), self.obj, self.collection_name)
+        db.update(self.get_id(), self.obj, self.collection_name)
 
     def delete(self):
         """Deletes the enclosed object in the database, if it existed."""
@@ -60,5 +57,5 @@ class Model:
             return None
 
         old_object = self.obj.copy()
-        DB.delete_one({'_id': self.get_id()}, self.collection_name)
+        db.delete_one({'_id': self.get_id()}, self.collection_name)
         return old_object
diff --git a/opendc-web/opendc-web-api/opendc/models/portfolio.py b/opendc-web/opendc-web-api/opendc/models/portfolio.py
index 32961b63..eb016947 100644
--- a/opendc-web/opendc-web-api/opendc/models/portfolio.py
+++ b/opendc-web/opendc-web-api/opendc/models/portfolio.py
@@ -1,7 +1,28 @@
+from bson import ObjectId
+from marshmallow import Schema, fields
+
+from opendc.exts import db
+from opendc.models.project import Project
 from opendc.models.model import Model
-from opendc.models.user import User
-from opendc.util.exceptions import ClientError
-from opendc.util.rest import Response
+
+
+class TargetSchema(Schema):
+    """
+    Schema representing a target.
+    """
+    enabledMetrics = fields.List(fields.String())
+    repeatsPerScenario = fields.Integer(required=True)
+
+
+class PortfolioSchema(Schema):
+    """
+    Schema representing a portfolio.
+    """
+    _id = fields.String(dump_only=True)
+    projectId = fields.String()
+    name = fields.String(required=True)
+    scenarioIds = fields.List(fields.String())
+    targets = fields.Nested(TargetSchema)
 
 
 class Portfolio(Model):
@@ -9,16 +30,18 @@ class Portfolio(Model):
 
     collection_name = 'portfolios'
 
-    def check_user_access(self, google_id, edit_access):
-        """Raises an error if the user with given [google_id] has insufficient access.
+    def check_user_access(self, user_id, edit_access):
+        """Raises an error if the user with given [user_id] has insufficient access.
 
         Checks access on the parent project.
 
-        :param google_id: The Google ID of the user.
+        :param user_id: The User ID of the user.
         :param edit_access: True when edit access should be checked, otherwise view access.
         """
-        user = User.from_google_id(google_id)
-        authorizations = list(
-            filter(lambda x: str(x['projectId']) == str(self.obj['projectId']), user.obj['authorizations']))
-        if len(authorizations) == 0 or (edit_access and authorizations[0]['authorizationLevel'] == 'VIEW'):
-            raise ClientError(Response(403, 'Forbidden from retrieving/editing portfolio.'))
+        project = Project.from_id(self.obj['projectId'])
+        project.check_user_access(user_id, edit_access)
+
+    @classmethod
+    def get_for_project(cls, project_id):
+        """Get all portfolios for the specified project id."""
+        return db.fetch_all({'projectId': ObjectId(project_id)}, cls.collection_name)
diff --git a/opendc-web/opendc-web-api/opendc/models/prefab.py b/opendc-web/opendc-web-api/opendc/models/prefab.py
index edf1d4c4..5e4b81dc 100644
--- a/opendc-web/opendc-web-api/opendc/models/prefab.py
+++ b/opendc-web/opendc-web-api/opendc/models/prefab.py
@@ -1,28 +1,31 @@
+from marshmallow import Schema, fields
+from werkzeug.exceptions import Forbidden
+
+from opendc.models.topology import ObjectSchema
 from opendc.models.model import Model
-from opendc.models.user import User
-from opendc.util.exceptions import ClientError
-from opendc.util.rest import Response
+
+
+class PrefabSchema(Schema):
+    """
+    Schema for a Prefab.
+    """
+    _id = fields.String(dump_only=True)
+    authorId = fields.String(dump_only=True)
+    name = fields.String(required=True)
+    datetimeCreated = fields.DateTime()
+    datetimeLastEdited = fields.DateTime()
+    rack = fields.Nested(ObjectSchema)
 
 
 class Prefab(Model):
-    """Model representing a Project."""
+    """Model representing a Prefab."""
 
     collection_name = 'prefabs'
 
-    def check_user_access(self, google_id):
-        """Raises an error if the user with given [google_id] has insufficient access to view this prefab.
+    def check_user_access(self, user_id):
+        """Raises an error if the user with given [user_id] has insufficient access to view this prefab.
 
-        :param google_id: The Google ID of the user.
+        :param user_id: The user ID of the user.
         """
-        user = User.from_google_id(google_id)
-
-        # TODO(Jacob) add special handling for OpenDC-provided prefabs
-
-        #try:
-
-        print(self.obj)
-        if self.obj['authorId'] != user.get_id() and self.obj['visibility'] == "private":
-            raise ClientError(Response(403, "Forbidden from retrieving prefab."))
-        #except KeyError:
-            # OpenDC-authored objects don't necessarily have an authorId
-        #    return
+        if self.obj['authorId'] != user_id and self.obj['visibility'] == "private":
+            raise Forbidden("Forbidden from retrieving prefab.")
diff --git a/opendc-web/opendc-web-api/opendc/models/project.py b/opendc-web/opendc-web-api/opendc/models/project.py
index b57e9f77..f2b3b564 100644
--- a/opendc-web/opendc-web-api/opendc/models/project.py
+++ b/opendc-web/opendc-web-api/opendc/models/project.py
@@ -1,8 +1,29 @@
+from marshmallow import Schema, fields, validate
+from werkzeug.exceptions import Forbidden
+
 from opendc.models.model import Model
-from opendc.models.user import User
-from opendc.util.database import DB
-from opendc.util.exceptions import ClientError
-from opendc.util.rest import Response
+from opendc.exts import db
+
+
+class ProjectAuthorizations(Schema):
+    """
+    Schema representing a project authorization.
+    """
+    userId = fields.String(required=True)
+    level = fields.String(required=True, validate=validate.OneOf(["VIEW", "EDIT", "OWN"]))
+
+
+class ProjectSchema(Schema):
+    """
+    Schema representing a Project.
+    """
+    _id = fields.String(dump_only=True)
+    name = fields.String(required=True)
+    datetimeCreated = fields.DateTime()
+    datetimeLastEdited = fields.DateTime()
+    topologyIds = fields.List(fields.String())
+    portfolioIds = fields.List(fields.String())
+    authorizations = fields.List(fields.Nested(ProjectAuthorizations))
 
 
 class Project(Model):
@@ -10,22 +31,18 @@ class Project(Model):
 
     collection_name = 'projects'
 
-    def check_user_access(self, google_id, edit_access):
-        """Raises an error if the user with given [google_id] has insufficient access.
+    def check_user_access(self, user_id, edit_access):
+        """Raises an error if the user with given [user_id] has insufficient access.
 
-        :param google_id: The Google ID of the user.
+        :param user_id: The User ID of the user.
         :param edit_access: True when edit access should be checked, otherwise view access.
         """
-        user = User.from_google_id(google_id)
-        authorizations = list(filter(lambda x: str(x['projectId']) == str(self.get_id()),
-                                     user.obj['authorizations']))
-        if len(authorizations) == 0 or (edit_access and authorizations[0]['authorizationLevel'] == 'VIEW'):
-            raise ClientError(Response(403, "Forbidden from retrieving project."))
-
-    def get_all_authorizations(self):
-        """Get all user IDs having access to this project."""
-        return [
-            str(user['_id']) for user in DB.fetch_all({'authorizations': {
-                'projectId': self.obj['_id']
-            }}, User.collection_name)
-        ]
+        for authorization in self.obj['authorizations']:
+            if user_id == authorization['userId'] and authorization['level'] != 'VIEW' or not edit_access:
+                return
+        raise Forbidden("Forbidden from retrieving project.")
+
+    @classmethod
+    def get_for_user(cls, user_id):
+        """Get all projects for the specified user id."""
+        return db.fetch_all({'authorizations.userId': user_id}, Project.collection_name)
diff --git a/opendc-web/opendc-web-api/opendc/models/scenario.py b/opendc-web/opendc-web-api/opendc/models/scenario.py
index 8d53e408..47771e06 100644
--- a/opendc-web/opendc-web-api/opendc/models/scenario.py
+++ b/opendc-web/opendc-web-api/opendc/models/scenario.py
@@ -1,8 +1,56 @@
+from datetime import datetime
+
+from bson import ObjectId
+from marshmallow import Schema, fields
+
+from opendc.exts import db
 from opendc.models.model import Model
 from opendc.models.portfolio import Portfolio
-from opendc.models.user import User
-from opendc.util.exceptions import ClientError
-from opendc.util.rest import Response
+
+
+class SimulationSchema(Schema):
+    """
+    Simulation details.
+    """
+    state = fields.String()
+
+
+class TraceSchema(Schema):
+    """
+    Schema for specifying the trace of a scenario.
+    """
+    traceId = fields.String()
+    loadSamplingFraction = fields.Float()
+
+
+class TopologySchema(Schema):
+    """
+    Schema for topology specification for a scenario.
+    """
+    topologyId = fields.String()
+
+
+class OperationalSchema(Schema):
+    """
+    Schema for the operational phenomena for a scenario.
+    """
+    failuresEnabled = fields.Boolean()
+    performanceInterferenceEnabled = fields.Boolean()
+    schedulerName = fields.String()
+
+
+class ScenarioSchema(Schema):
+    """
+    Schema representing a scenario.
+    """
+    _id = fields.String(dump_only=True)
+    portfolioId = fields.String()
+    name = fields.String(required=True)
+    trace = fields.Nested(TraceSchema)
+    topology = fields.Nested(TopologySchema)
+    operational = fields.Nested(OperationalSchema)
+    simulation = fields.Nested(SimulationSchema, dump_only=True)
+    results = fields.Dict(dump_only=True)
 
 
 class Scenario(Model):
@@ -10,17 +58,36 @@ class Scenario(Model):
 
     collection_name = 'scenarios'
 
-    def check_user_access(self, google_id, edit_access):
-        """Raises an error if the user with given [google_id] has insufficient access.
+    def check_user_access(self, user_id, edit_access):
+        """Raises an error if the user with given [user_id] has insufficient access.
 
         Checks access on the parent project.
 
-        :param google_id: The Google ID of the user.
+        :param user_id: The User ID of the user.
         :param edit_access: True when edit access should be checked, otherwise view access.
         """
         portfolio = Portfolio.from_id(self.obj['portfolioId'])
-        user = User.from_google_id(google_id)
-        authorizations = list(
-            filter(lambda x: str(x['projectId']) == str(portfolio.obj['projectId']), user.obj['authorizations']))
-        if len(authorizations) == 0 or (edit_access and authorizations[0]['authorizationLevel'] == 'VIEW'):
-            raise ClientError(Response(403, 'Forbidden from retrieving/editing scenario.'))
+        portfolio.check_user_access(user_id, edit_access)
+
+    @classmethod
+    def get_jobs(cls):
+        """Obtain the scenarios that have been queued.
+        """
+        return cls(db.fetch_all({'simulation.state': 'QUEUED'}, cls.collection_name))
+
+    @classmethod
+    def get_for_portfolio(cls, portfolio_id):
+        """Get all scenarios for the specified portfolio id."""
+        return db.fetch_all({'portfolioId': ObjectId(portfolio_id)}, cls.collection_name)
+
+    def update_state(self, new_state, results=None):
+        """Atomically update the state of the Scenario.
+        """
+        update = {'$set': {'simulation.state': new_state, 'simulation.heartbeat': datetime.now()}}
+        if results:
+            update['$set']['results'] = results
+        return db.fetch_and_update(
+            query={'_id': self.obj['_id'], 'simulation.state': self.obj['simulation']['state']},
+            update=update,
+            collection=self.collection_name
+        )
diff --git a/opendc-web/opendc-web-api/opendc/models/topology.py b/opendc-web/opendc-web-api/opendc/models/topology.py
index cb4c4bab..44994818 100644
--- a/opendc-web/opendc-web-api/opendc/models/topology.py
+++ b/opendc-web/opendc-web-api/opendc/models/topology.py
@@ -1,7 +1,89 @@
+from bson import ObjectId
+from marshmallow import Schema, fields
+
+from opendc.exts import db
+from opendc.models.project import Project
 from opendc.models.model import Model
-from opendc.models.user import User
-from opendc.util.exceptions import ClientError
-from opendc.util.rest import Response
+
+
+class MemorySchema(Schema):
+    """
+    Schema representing a memory unit.
+    """
+    _id = fields.String()
+    name = fields.String()
+    speedMbPerS = fields.Integer()
+    sizeMb = fields.Integer()
+    energyConsumptionW = fields.Integer()
+
+
+class PuSchema(Schema):
+    """
+    Schema representing a processing unit.
+    """
+    _id = fields.String()
+    name = fields.String()
+    clockRateMhz = fields.Integer()
+    numberOfCores = fields.Integer()
+    energyConsumptionW = fields.Integer()
+
+
+class MachineSchema(Schema):
+    """
+    Schema representing a machine.
+    """
+    _id = fields.String()
+    position = fields.Integer()
+    cpus = fields.List(fields.Nested(PuSchema))
+    gpus = fields.List(fields.Nested(PuSchema))
+    memories = fields.List(fields.Nested(MemorySchema))
+    storages = fields.List(fields.Nested(MemorySchema))
+    rackId = fields.String()
+
+
+class ObjectSchema(Schema):
+    """
+    Schema representing a room object.
+    """
+    _id = fields.String()
+    name = fields.String()
+    capacity = fields.Integer()
+    powerCapacityW = fields.Integer()
+    machines = fields.List(fields.Nested(MachineSchema))
+    tileId = fields.String()
+
+
+class TileSchema(Schema):
+    """
+    Schema representing a room tile.
+    """
+    _id = fields.String()
+    topologyId = fields.String()
+    positionX = fields.Integer()
+    positionY = fields.Integer()
+    rack = fields.Nested(ObjectSchema)
+    roomId = fields.String()
+
+
+class RoomSchema(Schema):
+    """
+    Schema representing a room.
+    """
+    _id = fields.String()
+    name = fields.String(required=True)
+    topologyId = fields.String()
+    tiles = fields.List(fields.Nested(TileSchema), required=True)
+
+
+class TopologySchema(Schema):
+    """
+    Schema representing a datacenter topology.
+    """
+    _id = fields.String(dump_only=True)
+    projectId = fields.String()
+    name = fields.String(required=True)
+    rooms = fields.List(fields.Nested(RoomSchema), required=True)
+    datetimeLastEdited = fields.DateTime()
 
 
 class Topology(Model):
@@ -9,19 +91,18 @@ class Topology(Model):
 
     collection_name = 'topologies'
 
-    def check_user_access(self, google_id, edit_access):
-        """Raises an error if the user with given [google_id] has insufficient access.
+    def check_user_access(self, user_id, edit_access):
+        """Raises an error if the user with given [user_id] has insufficient access.
 
         Checks access on the parent project.
 
-        :param google_id: The Google ID of the user.
+        :param user_id: The User ID of the user.
         :param edit_access: True when edit access should be checked, otherwise view access.
         """
-        user = User.from_google_id(google_id)
-        if 'projectId' not in self.obj:
-            raise ClientError(Response(400, 'Missing projectId in topology.'))
-
-        authorizations = list(
-            filter(lambda x: str(x['projectId']) == str(self.obj['projectId']), user.obj['authorizations']))
-        if len(authorizations) == 0 or (edit_access and authorizations[0]['authorizationLevel'] == 'VIEW'):
-            raise ClientError(Response(403, 'Forbidden from retrieving topology.'))
+        project = Project.from_id(self.obj['projectId'])
+        project.check_user_access(user_id, edit_access)
+
+    @classmethod
+    def get_for_project(cls, project_id):
+        """Get all topologies for the specified project id."""
+        return db.fetch_all({'projectId': ObjectId(project_id)}, cls.collection_name)
diff --git a/opendc-web/opendc-web-api/opendc/models/trace.py b/opendc-web/opendc-web-api/opendc/models/trace.py
index 2f6e4926..69287f29 100644
--- a/opendc-web/opendc-web-api/opendc/models/trace.py
+++ b/opendc-web/opendc-web-api/opendc/models/trace.py
@@ -1,6 +1,15 @@
+from marshmallow import Schema, fields
+
 from opendc.models.model import Model
 
 
+class TraceSchema(Schema):
+    """Schema for a Trace."""
+    _id = fields.String(dump_only=True)
+    name = fields.String()
+    type = fields.String()
+
+
 class Trace(Model):
     """Model representing a Trace."""
 
diff --git a/opendc-web/opendc-web-api/opendc/models/user.py b/opendc-web/opendc-web-api/opendc/models/user.py
deleted file mode 100644
index 8e8ff945..00000000
--- a/opendc-web/opendc-web-api/opendc/models/user.py
+++ /dev/null
@@ -1,36 +0,0 @@
-from opendc.models.model import Model
-from opendc.util.database import DB
-from opendc.util.exceptions import ClientError
-from opendc.util.rest import Response
-
-
-class User(Model):
-    """Model representing a User."""
-
-    collection_name = 'users'
-
-    @classmethod
-    def from_email(cls, email):
-        """Fetches the user with given email from the collection."""
-        return User(DB.fetch_one({'email': email}, User.collection_name))
-
-    @classmethod
-    def from_google_id(cls, google_id):
-        """Fetches the user with given Google ID from the collection."""
-        return User(DB.fetch_one({'googleId': google_id}, User.collection_name))
-
-    def check_correct_user(self, request_google_id):
-        """Raises an error if a user tries to modify another user.
-
-        :param request_google_id:
-        """
-        if request_google_id is not None and self.obj['googleId'] != request_google_id:
-            raise ClientError(Response(403, f'Forbidden from editing user with ID {self.obj["_id"]}.'))
-
-    def check_already_exists(self):
-        """Checks if the user already exists in the database."""
-
-        existing_user = DB.fetch_one({'googleId': self.obj['googleId']}, self.collection_name)
-
-        if existing_user is not None:
-            raise ClientError(Response(409, 'User already exists.'))