From 17327a642738e0500f9a007b32a46bb4f426f881 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 14 May 2021 12:43:47 +0200 Subject: api: Remove Socket.IO endpoint from public API This change removes the Socket.IO endpoint from the public API now that we have switched to the REST API instead. This decreases the possible exposure to vulnerabilities as well as the maintenance burden. --- opendc-web/opendc-web-api/app.py | 173 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100755 opendc-web/opendc-web-api/app.py (limited to 'opendc-web/opendc-web-api/app.py') diff --git a/opendc-web/opendc-web-api/app.py b/opendc-web/opendc-web-api/app.py new file mode 100755 index 00000000..7a687678 --- /dev/null +++ b/opendc-web/opendc-web-api/app.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +import json +import os +import sys +import traceback +import urllib.request + +from dotenv import load_dotenv +from flask import Flask, request, jsonify +from flask_compress import Compress +from flask_cors import CORS +from oauth2client import client, crypt + +from opendc.models.user import User +from opendc.util import rest, path_parser, database +from opendc.util.exceptions import AuthorizationTokenError, RequestInitializationError +from opendc.util.json import JSONEncoder + +load_dotenv() + +TEST_MODE = "OPENDC_FLASK_TESTING" in os.environ + +# Setup Sentry if DSN is specified +if 'SENTRY_DSN' in os.environ: + import sentry_sdk + from sentry_sdk.integrations.flask import FlaskIntegration + + sentry_sdk.init( + integrations=[FlaskIntegration()], + traces_sample_rate=0.1 + ) + +# Set up database if not testing +if not TEST_MODE: + database.DB.initialize_database( + user=os.environ['OPENDC_DB_USERNAME'], + password=os.environ['OPENDC_DB_PASSWORD'], + database=os.environ['OPENDC_DB'], + host=os.environ.get('OPENDC_DB_HOST', 'localhost')) + +# Set up the core app +app = Flask("opendc") +app.testing = TEST_MODE +app.config['SECRET_KEY'] = os.environ['OPENDC_FLASK_SECRET'] +app.json_encoder = JSONEncoder + +# Set up CORS support +CORS(app) + +compress = Compress() +compress.init_app(app) + +API_VERSIONS = {'v2'} + + +@app.route('/tokensignin', methods=['POST']) +def sign_in(): + """Authenticate a user with Google sign in""" + + try: + token = request.form['idtoken'] + except KeyError: + return 'No idtoken provided', 401 + + try: + idinfo = client.verify_id_token(token, os.environ['OPENDC_OAUTH_CLIENT_ID']) + + if idinfo['aud'] != os.environ['OPENDC_OAUTH_CLIENT_ID']: + raise crypt.AppIdentityError('Unrecognized client.') + + if idinfo['iss'] not in ['accounts.google.com', 'https://accounts.google.com']: + raise crypt.AppIdentityError('Wrong issuer.') + except ValueError: + url = "https://www.googleapis.com/oauth2/v3/tokeninfo?id_token={}".format(token) + req = urllib.request.Request(url) + response = urllib.request.urlopen(url=req, timeout=30) + res = response.read() + idinfo = json.loads(res) + except crypt.AppIdentityError as e: + return 'Did not successfully authenticate' + + user = User.from_google_id(idinfo['sub']) + + data = {'isNewUser': user.obj is None} + + if user.obj is not None: + data['userId'] = user.get_id() + + return jsonify(**data) + + +@app.route('//', methods=['GET', 'POST', 'PUT', 'DELETE']) +def api_call(version, endpoint_path): + """Call an API endpoint directly over HTTP.""" + + # Check whether given version is valid + if version not in API_VERSIONS: + return jsonify(error='API version not found'), 404 + + # Get path and parameters + (path, path_parameters) = path_parser.parse(version, endpoint_path) + + query_parameters = request.args.to_dict() + for param in query_parameters: + try: + query_parameters[param] = int(query_parameters[param]) + except: + pass + + try: + body_parameters = json.loads(request.get_data()) + except: + body_parameters = {} + + # Create and call request + (req, response) = _process_message({ + 'id': 0, + 'method': request.method, + 'parameters': { + 'body': body_parameters, + 'path': path_parameters, + 'query': query_parameters + }, + 'path': path, + 'token': request.headers.get('auth-token') + }) + + print( + f'HTTP:\t{req.method} to `/{req.path}` resulted in {response.status["code"]}: {response.status["description"]}') + sys.stdout.flush() + + flask_response = jsonify(json.loads(response.to_JSON())) + flask_response.status_code = response.status['code'] + return flask_response + + +def _process_message(message): + """Process a request message and return the response.""" + + try: + req = rest.Request(message) + res = req.process() + + return req, res + + except AuthorizationTokenError: + res = rest.Response(401, 'Authorization error') + res.id = message['id'] + + except RequestInitializationError as e: + res = rest.Response(400, str(e)) + res.id = message['id'] + + if not 'method' in message: + message['method'] = 'UNSPECIFIED' + if not 'path' in message: + message['path'] = 'UNSPECIFIED' + + except Exception: + res = rest.Response(500, 'Internal server error') + if 'id' in message: + res.id = message['id'] + traceback.print_exc() + + req = rest.Request() + req.method = message['method'] + req.path = message['path'] + + return req, res + + +if __name__ == '__main__': + app.run() -- cgit v1.2.3 From 0c6ccca5fac44ab40671627fd3181e9b138672fa Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 14 May 2021 15:17:49 +0200 Subject: api: Migrate to Auth0 for API authorization This change updates the OpenDC API to use Auth0 for API authorization. This removes the hard dependency on Google for logging into OpenDC and simplifies implementation as we do not have to store user information anymore, other than the user identifier. --- opendc-web/opendc-web-api/app.py | 47 +++++++++------------------------------- 1 file changed, 10 insertions(+), 37 deletions(-) (limited to 'opendc-web/opendc-web-api/app.py') diff --git a/opendc-web/opendc-web-api/app.py b/opendc-web/opendc-web-api/app.py index 7a687678..ee4b3d32 100755 --- a/opendc-web/opendc-web-api/app.py +++ b/opendc-web/opendc-web-api/app.py @@ -3,16 +3,14 @@ import json import os import sys import traceback -import urllib.request from dotenv import load_dotenv from flask import Flask, request, jsonify from flask_compress import Compress from flask_cors import CORS -from oauth2client import client, crypt -from opendc.models.user import User from opendc.util import rest, path_parser, database +from opendc.util.auth import AuthError, AuthManager, AsymmetricJwtAlgorithm from opendc.util.exceptions import AuthorizationTokenError, RequestInitializationError from opendc.util.json import JSONEncoder @@ -50,46 +48,21 @@ CORS(app) compress = Compress() compress.init_app(app) -API_VERSIONS = {'v2'} - - -@app.route('/tokensignin', methods=['POST']) -def sign_in(): - """Authenticate a user with Google sign in""" +auth = AuthManager(AsymmetricJwtAlgorithm(jwks_url=f"https://{os.environ['AUTH0_DOMAIN']}/.well-known/jwks.json"), + issuer=f"https://{os.environ['AUTH0_DOMAIN']}/", audience=os.environ['AUTH0_AUDIENCE']) - try: - token = request.form['idtoken'] - except KeyError: - return 'No idtoken provided', 401 - - try: - idinfo = client.verify_id_token(token, os.environ['OPENDC_OAUTH_CLIENT_ID']) - - if idinfo['aud'] != os.environ['OPENDC_OAUTH_CLIENT_ID']: - raise crypt.AppIdentityError('Unrecognized client.') - - if idinfo['iss'] not in ['accounts.google.com', 'https://accounts.google.com']: - raise crypt.AppIdentityError('Wrong issuer.') - except ValueError: - url = "https://www.googleapis.com/oauth2/v3/tokeninfo?id_token={}".format(token) - req = urllib.request.Request(url) - response = urllib.request.urlopen(url=req, timeout=30) - res = response.read() - idinfo = json.loads(res) - except crypt.AppIdentityError as e: - return 'Did not successfully authenticate' - - user = User.from_google_id(idinfo['sub']) - - data = {'isNewUser': user.obj is None} +API_VERSIONS = {'v2'} - if user.obj is not None: - data['userId'] = user.get_id() - return jsonify(**data) +@app.errorhandler(AuthError) +def handle_auth_error(ex): + response = jsonify(ex.error) + response.status_code = ex.status_code + return response @app.route('//', methods=['GET', 'POST', 'PUT', 'DELETE']) +@auth.require def api_call(version, endpoint_path): """Call an API endpoint directly over HTTP.""" -- cgit v1.2.3 From 2281d3265423d01e60f8cc088de5a5730bb8a910 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Sat, 15 May 2021 13:09:06 +0200 Subject: api: Migrate to Flask Restful This change updates the API to use Flask Restful instead of our own in-house REST library. This change reduces the maintenance effort and allows us to drastically simplify the API implementation needed for the OpenDC v2 API. --- opendc-web/opendc-web-api/app.py | 179 +++++++++++++++------------------------ 1 file changed, 67 insertions(+), 112 deletions(-) (limited to 'opendc-web/opendc-web-api/app.py') diff --git a/opendc-web/opendc-web-api/app.py b/opendc-web/opendc-web-api/app.py index ee4b3d32..5041457f 100755 --- a/opendc-web/opendc-web-api/app.py +++ b/opendc-web/opendc-web-api/app.py @@ -1,25 +1,35 @@ #!/usr/bin/env python3 -import json import os -import sys -import traceback from dotenv import load_dotenv -from flask import Flask, request, jsonify +from flask import Flask, jsonify from flask_compress import Compress from flask_cors import CORS +from flask_restful import Api +from marshmallow import ValidationError -from opendc.util import rest, path_parser, database -from opendc.util.auth import AuthError, AuthManager, AsymmetricJwtAlgorithm -from opendc.util.exceptions import AuthorizationTokenError, RequestInitializationError -from opendc.util.json import JSONEncoder +from opendc.api.portfolios import Portfolio, PortfolioScenarios +from opendc.api.prefabs import Prefab, PrefabList +from opendc.api.projects import ProjectList, Project, ProjectTopologies, ProjectPortfolios +from opendc.api.scenarios import Scenario +from opendc.api.schedulers import SchedulerList +from opendc.api.topologies import Topology +from opendc.api.traces import TraceList, Trace +from opendc.auth import AuthError +from opendc.util import JSONEncoder + +# Load environmental variables from dotenv file load_dotenv() -TEST_MODE = "OPENDC_FLASK_TESTING" in os.environ -# Setup Sentry if DSN is specified -if 'SENTRY_DSN' in os.environ: +def setup_sentry(): + """ + Setup the Sentry integration for Flask if a DSN is supplied via the environmental variables. + """ + if 'SENTRY_DSN' not in os.environ: + return + import sentry_sdk from sentry_sdk.integrations.flask import FlaskIntegration @@ -28,119 +38,64 @@ if 'SENTRY_DSN' in os.environ: traces_sample_rate=0.1 ) -# Set up database if not testing -if not TEST_MODE: - database.DB.initialize_database( - user=os.environ['OPENDC_DB_USERNAME'], - password=os.environ['OPENDC_DB_PASSWORD'], - database=os.environ['OPENDC_DB'], - host=os.environ.get('OPENDC_DB_HOST', 'localhost')) - -# Set up the core app -app = Flask("opendc") -app.testing = TEST_MODE -app.config['SECRET_KEY'] = os.environ['OPENDC_FLASK_SECRET'] -app.json_encoder = JSONEncoder - -# Set up CORS support -CORS(app) - -compress = Compress() -compress.init_app(app) - -auth = AuthManager(AsymmetricJwtAlgorithm(jwks_url=f"https://{os.environ['AUTH0_DOMAIN']}/.well-known/jwks.json"), - issuer=f"https://{os.environ['AUTH0_DOMAIN']}/", audience=os.environ['AUTH0_AUDIENCE']) - -API_VERSIONS = {'v2'} - - -@app.errorhandler(AuthError) -def handle_auth_error(ex): - response = jsonify(ex.error) - response.status_code = ex.status_code - return response - - -@app.route('//', methods=['GET', 'POST', 'PUT', 'DELETE']) -@auth.require -def api_call(version, endpoint_path): - """Call an API endpoint directly over HTTP.""" - - # Check whether given version is valid - if version not in API_VERSIONS: - return jsonify(error='API version not found'), 404 - - # Get path and parameters - (path, path_parameters) = path_parser.parse(version, endpoint_path) - - query_parameters = request.args.to_dict() - for param in query_parameters: - try: - query_parameters[param] = int(query_parameters[param]) - except: - pass - - try: - body_parameters = json.loads(request.get_data()) - except: - body_parameters = {} - - # Create and call request - (req, response) = _process_message({ - 'id': 0, - 'method': request.method, - 'parameters': { - 'body': body_parameters, - 'path': path_parameters, - 'query': query_parameters - }, - 'path': path, - 'token': request.headers.get('auth-token') - }) - print( - f'HTTP:\t{req.method} to `/{req.path}` resulted in {response.status["code"]}: {response.status["description"]}') - sys.stdout.flush() +def setup_api(app): + """ + Setup the API interface. + """ + api = Api(app) + # Map to ('string', 'ObjectId') passing type and format + api.add_resource(ProjectList, '/projects/') + api.add_resource(Project, '/projects/') + api.add_resource(ProjectTopologies, '/projects//topologies') + api.add_resource(ProjectPortfolios, '/projects//portfolios') + api.add_resource(Topology, '/topologies/') + api.add_resource(PrefabList, '/prefabs/') + api.add_resource(Prefab, '/prefabs/') + api.add_resource(Portfolio, '/portfolios/') + api.add_resource(PortfolioScenarios, '/portfolios//scenarios') + api.add_resource(Scenario, '/scenarios/') + api.add_resource(TraceList, '/traces/') + api.add_resource(Trace, '/traces/') + api.add_resource(SchedulerList, '/schedulers/') - flask_response = jsonify(json.loads(response.to_JSON())) - flask_response.status_code = response.status['code'] - return flask_response + @app.errorhandler(AuthError) + def handle_auth_error(ex): + response = jsonify(ex.error) + response.status_code = ex.status_code + return response + @app.errorhandler(ValidationError) + def handle_validation_error(ex): + return {'message': 'Input validation failed', 'errors': ex.messages}, 400 -def _process_message(message): - """Process a request message and return the response.""" + return api - try: - req = rest.Request(message) - res = req.process() - return req, res +def create_app(testing=False): + app = Flask(__name__) + app.config['TESTING'] = testing + app.config['SECRET_KEY'] = os.environ['OPENDC_FLASK_SECRET'] + app.config['RESTFUL_JSON'] = {'cls': JSONEncoder} + app.json_encoder = JSONEncoder - except AuthorizationTokenError: - res = rest.Response(401, 'Authorization error') - res.id = message['id'] + # Setup Sentry if DSN is specified + setup_sentry() - except RequestInitializationError as e: - res = rest.Response(400, str(e)) - res.id = message['id'] + # Set up CORS support + CORS(app) - if not 'method' in message: - message['method'] = 'UNSPECIFIED' - if not 'path' in message: - message['path'] = 'UNSPECIFIED' + # Setup compression + compress = Compress() + compress.init_app(app) - except Exception: - res = rest.Response(500, 'Internal server error') - if 'id' in message: - res.id = message['id'] - traceback.print_exc() + # Setup API + setup_api(app) - req = rest.Request() - req.method = message['method'] - req.path = message['path'] + return app - return req, res +application = create_app(testing="OPENDC_FLASK_TESTING" in os.environ) if __name__ == '__main__': - app.run() + application.run() -- cgit v1.2.3 From 5c710d329b16efb947a6d25793f6a0f7865f3df1 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Sun, 16 May 2021 11:43:03 +0200 Subject: api: Add Swagger UI for API documentation This change adds Swagger UI to the REST API endpoint in order to experiment with the API endpoints interactively. It also serves as the documentation for the API endpoints. --- opendc-web/opendc-web-api/app.py | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) (limited to 'opendc-web/opendc-web-api/app.py') diff --git a/opendc-web/opendc-web-api/app.py b/opendc-web/opendc-web-api/app.py index 5041457f..c05e56b5 100755 --- a/opendc-web/opendc-web-api/app.py +++ b/opendc-web/opendc-web-api/app.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 +import mimetypes import os from dotenv import load_dotenv -from flask import Flask, jsonify +from flask import Flask, jsonify, redirect from flask_compress import Compress from flask_cors import CORS from flask_restful import Api +from flask_swagger_ui import get_swaggerui_blueprint from marshmallow import ValidationError from opendc.api.portfolios import Portfolio, PortfolioScenarios @@ -72,13 +74,33 @@ def setup_api(app): return api +def setup_swagger(app): + """ + Setup Swagger UI + """ + SWAGGER_URL = '/docs' + API_URL = '../schema.yml' + + swaggerui_blueprint = get_swaggerui_blueprint( + SWAGGER_URL, + API_URL, + config={ + 'app_name': "OpenDC API v2" + } + ) + app.register_blueprint(swaggerui_blueprint) + + def create_app(testing=False): - app = Flask(__name__) + app = Flask(__name__, static_url_path='/') app.config['TESTING'] = testing app.config['SECRET_KEY'] = os.environ['OPENDC_FLASK_SECRET'] app.config['RESTFUL_JSON'] = {'cls': JSONEncoder} app.json_encoder = JSONEncoder + # Define YAML content type + mimetypes.add_type('text/yaml', '.yml') + # Setup Sentry if DSN is specified setup_sentry() @@ -89,8 +111,15 @@ def create_app(testing=False): compress = Compress() compress.init_app(app) - # Setup API setup_api(app) + setup_swagger(app) + + @app.route('/') + def index(): + """ + Redirect the user to the API documentation if it accesses the API root. + """ + return redirect('docs/') return app -- cgit v1.2.3 From 1f2e8460a730ffc96ad45a68d7cd8e45f67bfd7a Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Mon, 17 May 2021 12:21:16 +0200 Subject: api: Add support for pre-specified Swagger OAuth2 client id This change adds support for specifying the OAuth2 client identifier for Swagger API docs authentication. This allows users to experiment with the API documentation without needing to create an Auth0 account themselves. --- opendc-web/opendc-web-api/app.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'opendc-web/opendc-web-api/app.py') diff --git a/opendc-web/opendc-web-api/app.py b/opendc-web/opendc-web-api/app.py index c05e56b5..5916046b 100755 --- a/opendc-web/opendc-web-api/app.py +++ b/opendc-web/opendc-web-api/app.py @@ -86,6 +86,9 @@ def setup_swagger(app): API_URL, config={ 'app_name': "OpenDC API v2" + }, + oauth_config={ + 'clientId': os.environ.get("AUTH0_DOCS_CLIENT_ID", ""), } ) app.register_blueprint(swaggerui_blueprint) -- cgit v1.2.3