From 17327a642738e0500f9a007b32a46bb4f426f881 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 14 May 2021 12:43:47 +0200 Subject: api: Remove Socket.IO endpoint from public API This change removes the Socket.IO endpoint from the public API now that we have switched to the REST API instead. This decreases the possible exposure to vulnerabilities as well as the maintenance burden. --- opendc-web/opendc-web-api/README.md | 65 ++++--- opendc-web/opendc-web-api/app.py | 173 +++++++++++++++++++ opendc-web/opendc-web-api/conftest.py | 6 +- .../opendc-web-api/docs/component-diagram.png | Bin 0 -> 90161 bytes opendc-web/opendc-web-api/main.py | 188 --------------------- .../opendc-web-server-component-diagram.png | Bin 90161 -> 0 bytes opendc-web/opendc-web-api/requirements.txt | 2 - opendc-web/opendc-web-api/static/index.html | 22 --- 8 files changed, 219 insertions(+), 237 deletions(-) create mode 100755 opendc-web/opendc-web-api/app.py create mode 100644 opendc-web/opendc-web-api/docs/component-diagram.png delete mode 100755 opendc-web/opendc-web-api/main.py delete mode 100644 opendc-web/opendc-web-api/misc/artwork/opendc-web-server-component-diagram.png delete mode 100644 opendc-web/opendc-web-api/static/index.html (limited to 'opendc-web') diff --git a/opendc-web/opendc-web-api/README.md b/opendc-web/opendc-web-api/README.md index 4932f823..e1d83daf 100644 --- a/opendc-web/opendc-web-api/README.md +++ b/opendc-web/opendc-web-api/README.md @@ -9,15 +9,19 @@
-The OpenDC web server is the bridge between OpenDC's frontend and database. It is built with Flask/SocketIO in Python and implements the OpenAPI-compliant [OpenDC API specification](../../opendc-api-spec.yml). +The OpenDC web server is the bridge between OpenDC's frontend and database. It is built with Flask/SocketIO in Python +and implements the OpenAPI-compliant [OpenDC API specification](../../opendc-api-spec.yml). -This document explains a high-level view of the web server architecture ([jump](#architecture)), and describes how to set up the web server for local development ([jump](#setup-for-local-development)). +This document explains a high-level view of the web server architecture ([jump](#architecture)), and describes how to +set up the web server for local development ([jump](#setup-for-local-development)). ## Architecture -The following diagram shows a high-level view of the architecture of the OpenDC web server. Squared-off colored boxes indicate packages (colors become more saturated as packages are nested); rounded-off boxes indicate individual components; dotted lines indicate control flow; and solid lines indicate data flow. +The following diagram shows a high-level view of the architecture of the OpenDC web server. Squared-off colored boxes +indicate packages (colors become more saturated as packages are nested); rounded-off boxes indicate individual +components; dotted lines indicate control flow; and solid lines indicate data flow. -![OpenDC Web Server Component Diagram](misc/artwork/opendc-web-server-component-diagram.png) +![OpenDC Web Server Component Diagram](docs/component-diagram.png) The OpenDC API is implemented by the `Main Server Loop`, which is the only component in the base package. @@ -25,74 +29,91 @@ The OpenDC API is implemented by the `Main Server Loop`, which is the only compo The `Util` package handles several miscellaneous tasks: -* `Database API`: Wraps database access functionality used by `Models` to read themselves from/write themselves into the database. +* `Database API`: Wraps database access functionality used by `Models` to read themselves from/write themselves into the + database. * `Exceptions`: Holds definitions for exceptions used throughout the web server. * `Parameter Checker`: Recursively checks whether required `Request` parameters are present and correctly typed. -* `REST`: Parses SocketIO and HTTP messages into `Request` objects, and calls the appropriate `API` endpoint to get a `Response` object to return to the `Main Server Loop`. +* `REST`: Parses HTTP messages into `Request` objects, and calls the appropriate `API` endpoint to get + a `Response` object to return to the `Main Server Loop`. ### API Package -The `API` package contains the logic for the HTTP methods in each API endpoint. Packages are structured to mirror the API: the code for the endpoint `GET api/projects`, for example, would be located at the `endpoint.py` inside the `projects` package (so at `api/projects/endpoint.py`). +The `API` package contains the logic for the HTTP methods in each API endpoint. Packages are structured to mirror the +API: the code for the endpoint `GET api/projects`, for example, would be located at the `endpoint.py` inside +the `projects` package (so at `api/projects/endpoint.py`). -An `endpoint.py` file contains methods for each HTTP method it supports, which takes a request as input (such as `def GET(request):`). Typically, such a method checks whether the parameters were passed correctly (using the `Parameter Checker`); fetches some model from the database; checks whether the data exists and is accessible by the user who made the request; possibly modifies this data and writes it back to the database; and returns a JSON representation of the model. +An `endpoint.py` file contains methods for each HTTP method it supports, which takes a request as input (such +as `def GET(request):`). Typically, such a method checks whether the parameters were passed correctly (using +the `Parameter Checker`); fetches some model from the database; checks whether the data exists and is accessible by the +user who made the request; possibly modifies this data and writes it back to the database; and returns a JSON +representation of the model. -The `REST` component dynamically imports the appropriate method from the appropriate `endpoint`, according to request it receives, and executes it. +The `REST` component dynamically imports the appropriate method from the appropriate `endpoint`, according to request it +receives, and executes it. ### Models Package -The `models` package contains the logic for mapping Python objects to their database representations. This involves an abstract `model` which has generic CRUD operations. Extensions of `model`, such as a `User` or `Project`, specify some more specific operations and their collection metadata. +The `models` package contains the logic for mapping Python objects to their database representations. This involves an +abstract `model` which has generic CRUD operations. Extensions of `model`, such as a `User` or `Project`, specify some +more specific operations and their collection metadata. `Endpoint`s import these `models` and use them to execute requests. ## Setup for Local Development -The following steps will guide you through setting up the OpenDC web server locally for development. To test individual endpoints, edit `static/index.html`. +The following steps will guide you through setting up the OpenDC web server locally for development. ### Local Setup #### Install requirements -Make sure you have Python 3.7+ installed (if not, get it [here](https://www.python.org/)), as well as pip (if not, get it [here](https://pip.pypa.io/en/stable/installing/)). Then run the following to install the requirements. +Make sure you have Python 3.7+ installed (if not, get it [here](https://www.python.org/)), as well as pip (if not, get +it [here](https://pip.pypa.io/en/stable/installing/)). Then run the following to install the requirements. ```bash pip install -r requirements.txt ``` -The web server also requires a running MongoDB instance. We recommend setting this up through docker, by running `docker-compose build` and `docker-compose up` in the [`mongodb` directory](../database) of the main OpenDC repository. +The web server also requires a running MongoDB instance. We recommend setting this up through docker, by +running `docker-compose build` and `docker-compose up` in the [`mongodb` directory](../../database) of the main OpenDC +repository. #### Get and configure the code -Clone OpenDC and follow the [instructions in the main repository](../) to set up a Google OAuth ID and environment variables. +Clone OpenDC and follow the [instructions in the main repository](../../) to set up a Google OAuth ID and environment +variables. **Important:** Be sure to set up environment variables according to those instructions, in a `.env` file. -If you want to test REST calls manually, add your own `OAUTH_CLIENT_ID` in `content=` on line `2` in `api/static/index.html`. - #### Set up the database -You can selectively run only the database services from the standard OpenDC `docker-compose` setup (in the root directory): +You can selectively run only the database services from the standard OpenDC `docker-compose` setup (in the root +directory): ```bash docker-compose build mongo mongo-express docker-compose up mongo mongo-express ``` -This will set you up with a running MongoDB instance and a visual inspection tool running on [localhost:8082](http://localhost:8082), with which you can view and manipulate the database. Add the simulator images to the command lists above if you want to test simulation capabilities, as well. +This will set you up with a running MongoDB instance and a visual inspection tool running +on [localhost:8082](http://localhost:8082), with which you can view and manipulate the database. Add the simulator +images to the command lists above if you want to test simulation capabilities, as well. ### Local Development Run the server. ```bash -cd api -python main.py +python3 -m flask run --port 8081 ``` -When editing the web server code, restart the server (`CTRL` + `c` followed by `python main.py` in the console running the server) to see the result of your changes. +When editing the web server code, restart the server (`CTRL` + `c` followed by `python app.py` in the console running +the server) to see the result of your changes. #### Code Style -To format all files, run `format.sh` in this directory. The script uses `yapf` internally to format everything automatically. +To format all files, run `format.sh` in this directory. The script uses `yapf` internally to format everything +automatically. To check if code style is up to modern standards, run `check.sh` in this directory. The script uses `pylint` internally. diff --git a/opendc-web/opendc-web-api/app.py b/opendc-web/opendc-web-api/app.py new file mode 100755 index 00000000..7a687678 --- /dev/null +++ b/opendc-web/opendc-web-api/app.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +import json +import os +import sys +import traceback +import urllib.request + +from dotenv import load_dotenv +from flask import Flask, request, jsonify +from flask_compress import Compress +from flask_cors import CORS +from oauth2client import client, crypt + +from opendc.models.user import User +from opendc.util import rest, path_parser, database +from opendc.util.exceptions import AuthorizationTokenError, RequestInitializationError +from opendc.util.json import JSONEncoder + +load_dotenv() + +TEST_MODE = "OPENDC_FLASK_TESTING" in os.environ + +# Setup Sentry if DSN is specified +if 'SENTRY_DSN' in os.environ: + import sentry_sdk + from sentry_sdk.integrations.flask import FlaskIntegration + + sentry_sdk.init( + integrations=[FlaskIntegration()], + traces_sample_rate=0.1 + ) + +# Set up database if not testing +if not TEST_MODE: + database.DB.initialize_database( + user=os.environ['OPENDC_DB_USERNAME'], + password=os.environ['OPENDC_DB_PASSWORD'], + database=os.environ['OPENDC_DB'], + host=os.environ.get('OPENDC_DB_HOST', 'localhost')) + +# Set up the core app +app = Flask("opendc") +app.testing = TEST_MODE +app.config['SECRET_KEY'] = os.environ['OPENDC_FLASK_SECRET'] +app.json_encoder = JSONEncoder + +# Set up CORS support +CORS(app) + +compress = Compress() +compress.init_app(app) + +API_VERSIONS = {'v2'} + + +@app.route('/tokensignin', methods=['POST']) +def sign_in(): + """Authenticate a user with Google sign in""" + + try: + token = request.form['idtoken'] + except KeyError: + return 'No idtoken provided', 401 + + try: + idinfo = client.verify_id_token(token, os.environ['OPENDC_OAUTH_CLIENT_ID']) + + if idinfo['aud'] != os.environ['OPENDC_OAUTH_CLIENT_ID']: + raise crypt.AppIdentityError('Unrecognized client.') + + if idinfo['iss'] not in ['accounts.google.com', 'https://accounts.google.com']: + raise crypt.AppIdentityError('Wrong issuer.') + except ValueError: + url = "https://www.googleapis.com/oauth2/v3/tokeninfo?id_token={}".format(token) + req = urllib.request.Request(url) + response = urllib.request.urlopen(url=req, timeout=30) + res = response.read() + idinfo = json.loads(res) + except crypt.AppIdentityError as e: + return 'Did not successfully authenticate' + + user = User.from_google_id(idinfo['sub']) + + data = {'isNewUser': user.obj is None} + + if user.obj is not None: + data['userId'] = user.get_id() + + return jsonify(**data) + + +@app.route('//', methods=['GET', 'POST', 'PUT', 'DELETE']) +def api_call(version, endpoint_path): + """Call an API endpoint directly over HTTP.""" + + # Check whether given version is valid + if version not in API_VERSIONS: + return jsonify(error='API version not found'), 404 + + # Get path and parameters + (path, path_parameters) = path_parser.parse(version, endpoint_path) + + query_parameters = request.args.to_dict() + for param in query_parameters: + try: + query_parameters[param] = int(query_parameters[param]) + except: + pass + + try: + body_parameters = json.loads(request.get_data()) + except: + body_parameters = {} + + # Create and call request + (req, response) = _process_message({ + 'id': 0, + 'method': request.method, + 'parameters': { + 'body': body_parameters, + 'path': path_parameters, + 'query': query_parameters + }, + 'path': path, + 'token': request.headers.get('auth-token') + }) + + print( + f'HTTP:\t{req.method} to `/{req.path}` resulted in {response.status["code"]}: {response.status["description"]}') + sys.stdout.flush() + + flask_response = jsonify(json.loads(response.to_JSON())) + flask_response.status_code = response.status['code'] + return flask_response + + +def _process_message(message): + """Process a request message and return the response.""" + + try: + req = rest.Request(message) + res = req.process() + + return req, res + + except AuthorizationTokenError: + res = rest.Response(401, 'Authorization error') + res.id = message['id'] + + except RequestInitializationError as e: + res = rest.Response(400, str(e)) + res.id = message['id'] + + if not 'method' in message: + message['method'] = 'UNSPECIFIED' + if not 'path' in message: + message['path'] = 'UNSPECIFIED' + + except Exception: + res = rest.Response(500, 'Internal server error') + if 'id' in message: + res.id = message['id'] + traceback.print_exc() + + req = rest.Request() + req.method = message['method'] + req.path = message['path'] + + return req, res + + +if __name__ == '__main__': + app.run() diff --git a/opendc-web/opendc-web-api/conftest.py b/opendc-web/opendc-web-api/conftest.py index 1f4831b8..8bb55ccc 100644 --- a/opendc-web/opendc-web-api/conftest.py +++ b/opendc-web/opendc-web-api/conftest.py @@ -3,13 +3,13 @@ Configuration file for all unit tests. """ import pytest -from main import FLASK_CORE_APP +from app import app @pytest.fixture def client(): """Returns a Flask API client to interact with.""" - FLASK_CORE_APP.config['TESTING'] = True + app.config['TESTING'] = True - with FLASK_CORE_APP.test_client() as client: + with app.test_client() as client: yield client diff --git a/opendc-web/opendc-web-api/docs/component-diagram.png b/opendc-web/opendc-web-api/docs/component-diagram.png new file mode 100644 index 00000000..91b26006 Binary files /dev/null and b/opendc-web/opendc-web-api/docs/component-diagram.png differ diff --git a/opendc-web/opendc-web-api/main.py b/opendc-web/opendc-web-api/main.py deleted file mode 100755 index 5c6dac31..00000000 --- a/opendc-web/opendc-web-api/main.py +++ /dev/null @@ -1,188 +0,0 @@ -#!/usr/bin/env python3 -import json -import os -import sys -import traceback -import urllib.request - -import flask_socketio -from dotenv import load_dotenv -from flask import Flask, request, jsonify -from flask_compress import Compress -from flask_cors import CORS -from oauth2client import client, crypt - -from opendc.models.user import User -from opendc.util import rest, path_parser, database -from opendc.util.exceptions import AuthorizationTokenError, RequestInitializationError -from opendc.util.json import JSONEncoder - -load_dotenv() - -TEST_MODE = "OPENDC_FLASK_TESTING" in os.environ - -# Setup Sentry if DSN is specified -if 'SENTRY_DSN' in os.environ: - import sentry_sdk - from sentry_sdk.integrations.flask import FlaskIntegration - - sentry_sdk.init( - integrations=[FlaskIntegration()], - traces_sample_rate=0.1 - ) - -# Set up database if not testing -if not TEST_MODE: - database.DB.initialize_database( - user=os.environ['OPENDC_DB_USERNAME'], - password=os.environ['OPENDC_DB_PASSWORD'], - database=os.environ['OPENDC_DB'], - host=os.environ.get('OPENDC_DB_HOST', 'localhost')) - -# Set up the core app -FLASK_CORE_APP = Flask(__name__) -FLASK_CORE_APP.testing = TEST_MODE -FLASK_CORE_APP.config['SECRET_KEY'] = os.environ['OPENDC_FLASK_SECRET'] -FLASK_CORE_APP.json_encoder = JSONEncoder - -# Set up CORS support -CORS(FLASK_CORE_APP) - -compress = Compress() -compress.init_app(FLASK_CORE_APP) - -SOCKET_IO_CORE = flask_socketio.SocketIO(FLASK_CORE_APP, cors_allowed_origins="*") - -API_VERSIONS = {'v2'} - - -@FLASK_CORE_APP.route('/tokensignin', methods=['POST']) -def sign_in(): - """Authenticate a user with Google sign in""" - - try: - token = request.form['idtoken'] - except KeyError: - return 'No idtoken provided', 401 - - try: - idinfo = client.verify_id_token(token, os.environ['OPENDC_OAUTH_CLIENT_ID']) - - if idinfo['aud'] != os.environ['OPENDC_OAUTH_CLIENT_ID']: - raise crypt.AppIdentityError('Unrecognized client.') - - if idinfo['iss'] not in ['accounts.google.com', 'https://accounts.google.com']: - raise crypt.AppIdentityError('Wrong issuer.') - except ValueError: - url = "https://www.googleapis.com/oauth2/v3/tokeninfo?id_token={}".format(token) - req = urllib.request.Request(url) - response = urllib.request.urlopen(url=req, timeout=30) - res = response.read() - idinfo = json.loads(res) - except crypt.AppIdentityError as e: - return 'Did not successfully authenticate' - - user = User.from_google_id(idinfo['sub']) - - data = {'isNewUser': user.obj is None} - - if user.obj is not None: - data['userId'] = user.get_id() - - return jsonify(**data) - - -@FLASK_CORE_APP.route('//', methods=['GET', 'POST', 'PUT', 'DELETE']) -def api_call(version, endpoint_path): - """Call an API endpoint directly over HTTP.""" - - # Check whether given version is valid - if version not in API_VERSIONS: - return jsonify(error='API version not found'), 404 - - # Get path and parameters - (path, path_parameters) = path_parser.parse(version, endpoint_path) - - query_parameters = request.args.to_dict() - for param in query_parameters: - try: - query_parameters[param] = int(query_parameters[param]) - except: - pass - - try: - body_parameters = json.loads(request.get_data()) - except: - body_parameters = {} - - # Create and call request - (req, response) = _process_message({ - 'id': 0, - 'method': request.method, - 'parameters': { - 'body': body_parameters, - 'path': path_parameters, - 'query': query_parameters - }, - 'path': path, - 'token': request.headers.get('auth-token') - }) - - print( - f'HTTP:\t{req.method} to `/{req.path}` resulted in {response.status["code"]}: {response.status["description"]}') - sys.stdout.flush() - - flask_response = jsonify(json.loads(response.to_JSON())) - flask_response.status_code = response.status['code'] - return flask_response - - -@SOCKET_IO_CORE.on('request') -def receive_message(message): - """"Receive a SocketIO request""" - (req, res) = _process_message(message) - - print(f'Socket: {req.method} to `/{req.path}` resulted in {res.status["code"]}: {res.status["description"]}') - sys.stdout.flush() - - flask_socketio.emit('response', res.to_JSON(), json=True) - - -def _process_message(message): - """Process a request message and return the response.""" - - try: - req = rest.Request(message) - res = req.process() - - return req, res - - except AuthorizationTokenError: - res = rest.Response(401, 'Authorization error') - res.id = message['id'] - - except RequestInitializationError as e: - res = rest.Response(400, str(e)) - res.id = message['id'] - - if not 'method' in message: - message['method'] = 'UNSPECIFIED' - if not 'path' in message: - message['path'] = 'UNSPECIFIED' - - except Exception: - res = rest.Response(500, 'Internal server error') - if 'id' in message: - res.id = message['id'] - traceback.print_exc() - - req = rest.Request() - req.method = message['method'] - req.path = message['path'] - - return req, res - - -if __name__ == '__main__': - print("Web server started on 8081") - SOCKET_IO_CORE.run(FLASK_CORE_APP, host='0.0.0.0', port=8081, use_reloader=False) diff --git a/opendc-web/opendc-web-api/misc/artwork/opendc-web-server-component-diagram.png b/opendc-web/opendc-web-api/misc/artwork/opendc-web-server-component-diagram.png deleted file mode 100644 index 91b26006..00000000 Binary files a/opendc-web/opendc-web-api/misc/artwork/opendc-web-server-component-diagram.png and /dev/null differ diff --git a/opendc-web/opendc-web-api/requirements.txt b/opendc-web/opendc-web-api/requirements.txt index 146f1717..555ba751 100644 --- a/opendc-web/opendc-web-api/requirements.txt +++ b/opendc-web/opendc-web-api/requirements.txt @@ -33,8 +33,6 @@ pytest-cov==2.11.1 pytest-env==0.6.2 pytest-mock==3.2.0 python-dotenv==0.14.0 -python-engineio==3.13.2 -python-socketio==4.6.0 rsa==4.7 sentry-sdk==0.19.2 six==1.15.0 diff --git a/opendc-web/opendc-web-api/static/index.html b/opendc-web/opendc-web-api/static/index.html deleted file mode 100644 index ac78cbfb..00000000 --- a/opendc-web/opendc-web-api/static/index.html +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - -
-Sign out - -

Your auth token:

-

Loading...

\ No newline at end of file -- cgit v1.2.3