From 4c347eb501a16eaa2a54933ed1ec8fa26521cc88 Mon Sep 17 00:00:00 2001 From: jc0b Date: Thu, 23 Apr 2020 14:59:48 +0200 Subject: Added basic mongodb framework --- docker-compose.yml | 22 ++++++++++++++ mongodb/Dockerfile | 6 ++++ mongodb/mongo-init-opendc-db.sh | 12 ++++++++ mongodb/mongo-init.js | 67 +++++++++++++++++++++++++++++++++++++++++ mongodb/mongo-opendc-schema.sh | 7 +++++ 5 files changed, 114 insertions(+) create mode 100644 mongodb/Dockerfile create mode 100644 mongodb/mongo-init-opendc-db.sh create mode 100644 mongodb/mongo-init.js create mode 100644 mongodb/mongo-opendc-schema.sh diff --git a/docker-compose.yml b/docker-compose.yml index a89b7260..2bbc0d9e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,3 +43,25 @@ services: # uncomment in production # volumes: # - "/data/mariadb:/var/lib/mysql" + mongo: + build: + context: ./mongodb + restart: on-failure + environment: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: rootpassword + MONGO_INITDB_DATABASE: admin + OPENDC_DB: opendc + OPENDC_DB_USERNAME: opendc + OPENDC_DB_PASSWORD: opendcpassword + ports: + - 27017:27017 + + mongo-express: + image: mongo-express + restart: on-failure + ports: + - 8082:8081 + environment: + ME_CONFIG_MONGODB_ADMINUSERNAME: root + ME_CONFIG_MONGODB_ADMINPASSWORD: rootpassword \ No newline at end of file diff --git a/mongodb/Dockerfile b/mongodb/Dockerfile new file mode 100644 index 00000000..e8dd98bc --- /dev/null +++ b/mongodb/Dockerfile @@ -0,0 +1,6 @@ +FROM mongo:4.2.5 +MAINTAINER Jacob Burley + +# Import init script +ADD mongo-init-opendc-db.sh /docker-entrypoint-initdb.d +ADD mongo-opendc-schema.sh /docker-entrypoint-initdb.d \ No newline at end of file diff --git a/mongodb/mongo-init-opendc-db.sh b/mongodb/mongo-init-opendc-db.sh new file mode 100644 index 00000000..55e9ec68 --- /dev/null +++ b/mongodb/mongo-init-opendc-db.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +echo 'Creating opendc user and db' + +mongo opendc \ + --host localhost \ + --port 27017 \ + -u $MONGO_INITDB_ROOT_USERNAME \ + -p $MONGO_INITDB_ROOT_PASSWORD \ + --authenticationDatabase admin \ + --eval "db.createUser({user: 'opendc', pwd: 'opendcpassword', roles:[{role:'dbOwner', db: 'opendc'}]});" +$MONGO_CMD = "mongo opendc --host localhost --port 27017 -u $MONGO_INITDB_ROOT_USERNAME -p $MONGO_INITDB_ROOT_PASSWORD --authenticationDatabase admin" \ No newline at end of file diff --git a/mongodb/mongo-init.js b/mongodb/mongo-init.js new file mode 100644 index 00000000..562bda6f --- /dev/null +++ b/mongodb/mongo-init.js @@ -0,0 +1,67 @@ +db.auth('root', 'rootpassword') + +let error = true + +db.createUser( + { + user: "admin", + pwd: "adminpassword", + roles: [ + { + role: "readWrite", + db: "admin" + } + ] + } +) + +db = db.getSiblingDB('opendc') + +db.createUser( + { + user: "opendc", + pwd: "opendcpassword", + roles: [ + { + role: "readWrite", + db: "opendc" + } + ] + } +); + +db.createCollection(users) +db.createCollection(authorizations) +db.createCollection(authorization_levels) +db.createCollection(simulations) +db.createCollection(experiments) +db.createCollection(paths) +db.createCollection(sections) +db.createCollection(schedulers) +db.createCollection(traces) +db.createCollection(jobs) +db.createCollection(tasks) +db.createCollection(task_dependencies) +db.createCollection(task_states) +db.createCollection(machine_states) +db.createCollection(datacenters) +db.createCollection(rooms) +db.createCollection(room_types) +db.createCollection(tiles) +db.createCollection(objects) +db.createCollection(object_types) +db.createCollection(allowed_objects) +db.createCollection(psus) +db.createCollection(cooling_items) +db.createCollection(racks) +db.createCollection(machines) +db.createCollection(machine_tags) +db.createCollection(failure_models) +db.createCollection(cpus) +db.createCollection(machine_cpus) +db.createCollection(gpus) +db.createCollection(machine_gpus) +db.createCollection(memories) +db.createCollection(machine_memories) +db.createCollection(storages) +db.createCollection(machine_storages) \ No newline at end of file diff --git a/mongodb/mongo-opendc-schema.sh b/mongodb/mongo-opendc-schema.sh new file mode 100644 index 00000000..7fd9d10f --- /dev/null +++ b/mongodb/mongo-opendc-schema.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +echo 'Creating opendc db schema...' + +$MONGO_CMD = "mongo opendc --host localhost --port 27017 -u $OPENDC_DB_USERNAME -p $OPENDC_DB_PASSWORD --authenticationDatabase opendc" + +eval $MONGO_COMMAND --eval "db.createCollection('environments'); db.createCollection('rooms'); db.createCollection('datacenters');" \ No newline at end of file -- cgit v1.2.3 From 9663374aa7f086b8a0aa9e0f254f0487f5ce09b4 Mon Sep 17 00:00:00 2001 From: jc0b Date: Mon, 11 May 2020 20:43:03 +0200 Subject: Added mongodb dockerfile, removed tests being added to sql db --- database/Dockerfile | 2 +- docker-compose.yml | 8 ++++++- mongodb/Dockerfile | 3 +-- mongodb/mongo-init-opendc-db.sh | 50 +++++++++++++++++++++++++++++++++++++---- mongodb/mongo-opendc-schema.sh | 7 ------ 5 files changed, 55 insertions(+), 15 deletions(-) delete mode 100644 mongodb/mongo-opendc-schema.sh diff --git a/database/Dockerfile b/database/Dockerfile index 0e933b40..e30aed51 100644 --- a/database/Dockerfile +++ b/database/Dockerfile @@ -5,4 +5,4 @@ MAINTAINER Fabian Mastenbroek ADD schema.sql /docker-entrypoint-initdb.d # Add test data into database -ADD test.sql /docker-entrypoint-initdb.d +#ADD test.sql /docker-entrypoint-initdb.d diff --git a/docker-compose.yml b/docker-compose.yml index 2bbc0d9e..4331a5a8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,6 +56,8 @@ services: OPENDC_DB_PASSWORD: opendcpassword ports: - 27017:27017 + #volumes: + # - mongo-volume:/data/db mongo-express: image: mongo-express @@ -64,4 +66,8 @@ services: - 8082:8081 environment: ME_CONFIG_MONGODB_ADMINUSERNAME: root - ME_CONFIG_MONGODB_ADMINPASSWORD: rootpassword \ No newline at end of file + ME_CONFIG_MONGODB_ADMINPASSWORD: rootpassword + +volumes: + mongo-volume: + external: false \ No newline at end of file diff --git a/mongodb/Dockerfile b/mongodb/Dockerfile index e8dd98bc..b4eb9dd1 100644 --- a/mongodb/Dockerfile +++ b/mongodb/Dockerfile @@ -2,5 +2,4 @@ FROM mongo:4.2.5 MAINTAINER Jacob Burley # Import init script -ADD mongo-init-opendc-db.sh /docker-entrypoint-initdb.d -ADD mongo-opendc-schema.sh /docker-entrypoint-initdb.d \ No newline at end of file +ADD mongo-init-opendc-db.sh /docker-entrypoint-initdb.d \ No newline at end of file diff --git a/mongodb/mongo-init-opendc-db.sh b/mongodb/mongo-init-opendc-db.sh index 55e9ec68..ed33e955 100644 --- a/mongodb/mongo-init-opendc-db.sh +++ b/mongodb/mongo-init-opendc-db.sh @@ -2,11 +2,53 @@ echo 'Creating opendc user and db' -mongo opendc \ - --host localhost \ +mongo opendc --host localhost \ --port 27017 \ -u $MONGO_INITDB_ROOT_USERNAME \ -p $MONGO_INITDB_ROOT_PASSWORD \ --authenticationDatabase admin \ - --eval "db.createUser({user: 'opendc', pwd: 'opendcpassword', roles:[{role:'dbOwner', db: 'opendc'}]});" -$MONGO_CMD = "mongo opendc --host localhost --port 27017 -u $MONGO_INITDB_ROOT_USERNAME -p $MONGO_INITDB_ROOT_PASSWORD --authenticationDatabase admin" \ No newline at end of file + --eval "db.createUser({user: '$OPENDC_DB_USERNAME', pwd: '$OPENDC_DB_PASSWORD', roles:[{role:'dbOwner', db: '$OPENDC_DB'}]});" +MONGO_ROOT_CMD="mongo $OPENDC_DB --host localhost --port 27017 -u $MONGO_INITDB_ROOT_USERNAME -p $MONGO_INITDB_ROOT_PASSWORD --authenticationDatabase admin" + +#echo 'Creating opendc db schema...' +MONGO_CMD="mongo $OPENDC_DB -u $OPENDC_DB_USERNAME -p $OPENDC_DB_PASSWORD --authenticationDatabase $OPENDC_DB" +$MONGO_CMD --eval 'db.createCollection("environments", { + validator: { + $jsonSchema: { + bsonType: "object", + required: ["name"], + properties: { + name: { + bsonType: "string", + description: "The name of the environment i.e. Production, or Compute Cluster" + }, + datacenters: { + bsonType: "object", + required: ["name, location, length, width, height"], + properties: { + name: { + bsonType: "string", + description: "The name of the datacenter i.e. eu-west-1, or Science Building" + }, + location: { + bsonType: "string", + description: "The location of the datacenter i.e. Frankfurt, or De Boelelaan 1105" + }, + length: { + bsonType: "double", + description: "The physical length of the datacenter, in centimetres" + }, + width: { + bsonType: "double", + description: "The physical width of the datacenter, in centimetres" + }, + height: { + bsonType: "double", + description: "The physical height of the datacenter, in centimetres" + } + } + } + } + } + } +});' \ No newline at end of file diff --git a/mongodb/mongo-opendc-schema.sh b/mongodb/mongo-opendc-schema.sh deleted file mode 100644 index 7fd9d10f..00000000 --- a/mongodb/mongo-opendc-schema.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -echo 'Creating opendc db schema...' - -$MONGO_CMD = "mongo opendc --host localhost --port 27017 -u $OPENDC_DB_USERNAME -p $OPENDC_DB_PASSWORD --authenticationDatabase opendc" - -eval $MONGO_COMMAND --eval "db.createCollection('environments'); db.createCollection('rooms'); db.createCollection('datacenters');" \ No newline at end of file -- cgit v1.2.3 From 3e4233c0d8b8d76b0f407341560ed84de38323ec Mon Sep 17 00:00:00 2001 From: jc0b Date: Sun, 31 May 2020 00:14:50 +0200 Subject: Prefabs progress --- .gitignore | 2 + mongodb/docker-compose.yml | 30 +++++++ mongodb/mongo-init-opendc-db.sh | 173 ++++++++++++++++++++++++++++++---------- mongodb/prefab.py | 110 +++++++++++++++++++++++++ mongodb/prefabs.py | 124 ++++++++++++++++++++++++++++ 5 files changed, 399 insertions(+), 40 deletions(-) create mode 100644 mongodb/docker-compose.yml create mode 100755 mongodb/prefab.py create mode 100755 mongodb/prefabs.py diff --git a/.gitignore b/.gitignore index b20ef8ef..b53549f4 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ .idea/ keys.json + +mongodb/opendc_testing/* diff --git a/mongodb/docker-compose.yml b/mongodb/docker-compose.yml new file mode 100644 index 00000000..0253cd7d --- /dev/null +++ b/mongodb/docker-compose.yml @@ -0,0 +1,30 @@ +version: "3" +services: + mongo: + build: + context: ./ + restart: on-failure + environment: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: rootpassword + MONGO_INITDB_DATABASE: admin + OPENDC_DB: opendc + OPENDC_DB_USERNAME: opendc + OPENDC_DB_PASSWORD: opendcpassword + ports: + - 27017:27017 + #volumes: + # - mongo-volume:/data/db + + mongo-express: + image: mongo-express + restart: on-failure + ports: + - 8082:8081 + environment: + ME_CONFIG_MONGODB_ADMINUSERNAME: root + ME_CONFIG_MONGODB_ADMINPASSWORD: rootpassword + +volumes: + mongo-volume: + external: false \ No newline at end of file diff --git a/mongodb/mongo-init-opendc-db.sh b/mongodb/mongo-init-opendc-db.sh index ed33e955..965698e7 100644 --- a/mongodb/mongo-init-opendc-db.sh +++ b/mongodb/mongo-init-opendc-db.sh @@ -12,43 +12,136 @@ MONGO_ROOT_CMD="mongo $OPENDC_DB --host localhost --port 27017 -u $MONGO_INITDB_ #echo 'Creating opendc db schema...' MONGO_CMD="mongo $OPENDC_DB -u $OPENDC_DB_USERNAME -p $OPENDC_DB_PASSWORD --authenticationDatabase $OPENDC_DB" -$MONGO_CMD --eval 'db.createCollection("environments", { - validator: { - $jsonSchema: { - bsonType: "object", - required: ["name"], - properties: { - name: { - bsonType: "string", - description: "The name of the environment i.e. Production, or Compute Cluster" - }, - datacenters: { - bsonType: "object", - required: ["name, location, length, width, height"], - properties: { - name: { - bsonType: "string", - description: "The name of the datacenter i.e. eu-west-1, or Science Building" - }, - location: { - bsonType: "string", - description: "The location of the datacenter i.e. Frankfurt, or De Boelelaan 1105" - }, - length: { - bsonType: "double", - description: "The physical length of the datacenter, in centimetres" - }, - width: { - bsonType: "double", - description: "The physical width of the datacenter, in centimetres" - }, - height: { - bsonType: "double", - description: "The physical height of the datacenter, in centimetres" - } - } - } - } - } - } -});' \ No newline at end of file +$MONGO_CMD --eval 'db.createCollection("prefabs");' + +$MONGO_CMD --eval 'db.prefabs.insertOne( + { + "type": "rack", + "name": "test_rack3", + "size": 42, + "depth": 42, + "author": "Jacob Burley", + "visibility": "public", + "children": [ + { + "type": "switch", + "ports": 48, + "power_draw": 150, + "psus": 1, + "size": 1 + }, + { + "type": "chassis", + "size": 4, + "children": [ + { + "type": "mainboard", + "sockets": 1, + "dimm_slots": 4, + "nics": 1, + "pcie_slots": 2, + "children": [ + { + "type": "CPU", + "coreCount": 4, + "SMT": true, + "base_clk": 3.5, + "boost_clk": 3.9, + "brand": "Intel", + "SKU": "i7-3770K", + "socket": "LGA1155", + "TDP": 77 + }, + { + "type": "DDR3", + "capacity": 4096, + "memfreq": 1333, + "ecc": false + }, + { + "type": "DDR3", + "capacity": 4096, + "memfreq": 1333, + "ecc": false + }, + { + "type": "DDR3", + "capacity": 4096, + "memfreq": 1333, + "ecc": false + }, + { + "type": "DDR3", + "capacity": 4096, + "memfreq": 1333, + "ecc": false + }, + { + "type": "GPU", + "VRAM": 8192, + "coreCount": 2304, + "brand": "AMD", + "technologies": "OpenCL", + "pcie_gen": "3x16", + "tdp": 169, + "slots": 2 + } + ] + }, + { + "type": "PSU", + "wattage": 550, + "ac": true + }, + { + "type": "disk", + "size": 2000, + "interface": "SATA", + "media": "flash", + "form_factor": 2.5 + } + ] + } + ] + });' + +# $MONGO_CMD --eval 'db.createCollection("prefabs", { +# validator: { +# $jsonSchema: { +# bsonType: "object", +# required: ["name"], +# properties: { +# name: { +# bsonType: "string", +# description: "The name of the environment i.e. Production, or Compute Cluster" +# }, +# datacenters: { +# bsonType: "object", +# required: ["name, location, length, width, height"], +# properties: { +# name: { +# bsonType: "string", +# description: "The name of the datacenter i.e. eu-west-1, or Science Building" +# }, +# location: { +# bsonType: "string", +# description: "The location of the datacenter i.e. Frankfurt, or De Boelelaan 1105" +# }, +# length: { +# bsonType: "double", +# description: "The physical length of the datacenter, in centimetres" +# }, +# width: { +# bsonType: "double", +# description: "The physical width of the datacenter, in centimetres" +# }, +# height: { +# bsonType: "double", +# description: "The physical height of the datacenter, in centimetres" +# } +# } +# } +# } +# } +# } +# });' \ No newline at end of file diff --git a/mongodb/prefab.py b/mongodb/prefab.py new file mode 100755 index 00000000..1ce355fd --- /dev/null +++ b/mongodb/prefab.py @@ -0,0 +1,110 @@ +#!/Users/jacobburley/thesis-src/opendc/mongodb/opendc_testing/bin/python3 +#Change shebang to /usr/bin/python3 before using with docker +# encoding: utf-8 +""" +prefab + +CLI frontend for viewing, modifying and creating prefabs in OpenDC. + +""" +import sys +import prefabs + +def usage(): + print("Usage: prefab add : imports a prefab from YML or JSON") + print(" list: lists all (public) prefabs") + print(" export [json|yaml]: exports the specified prefab to the specified filetype (with JSON used by default)") + +def interactive(): #interactive CLI mode: recommended + print("OpenDC Prefab CLI") + running = True + while(exit): + print(">", end=" ") + try: + command = input() + command = command.split() + except EOFError as e: + print("exit") + print("bye!") + exit() + except KeyboardInterrupt as KI: + print("\nbye!") + exit() + if(len(command) >= 1): + if(command[0] == "exit"): + print("bye!") + exit() + elif(command[0] == "list"): # decrypt + prefabs.list() + elif(command[0] == "help"): # decrypt + usage() + elif(command[0] == "add"): + if(len(command) == 3): + prefabs.add(command[1], command[2]) + else: + prefabs.add(command[1], None) + elif(command[0] == "clone"): + if(len(command) == 3): + prefabs.clone(command[1], command[2]) + else: + prefabs.clone(command[1], None) + elif(command[0] == "export"): + #print(sys.argv[2]) + prefabs.export(command[1], "json") + elif(command[0] == "remove"): + print("WARNING: Doing so will permanently remove the specified prefab. \nThis action CANNOT be undone. Please type the name of the prefab to confirm deletion.") + confirm = input() + if confirm == command[1]: + prefabs.remove(command[1]) + print(f'Prefab {command[1]} has been removed.') + else: + print("Confirmation failed. The prefab has not been removed.") + else: + print("prefabs: try 'help' for more information\n") + else: + print("prefabs: try 'help' for more information\n") + + +def main(): + if(len(sys.argv) >= 2): + if(sys.argv[1] == "list"): # decrypt + prefabs.list() + exit() + #elif(sys.argv[1] == "-e"): # encrypt + # encrypt(sys.argv[2], sys.argv[3], sys.argv[4]) + #elif(sys.argv[1] == "-v"): # verify + # verify(sys.argv[2], sys.argv[3], sys.argv[4]) + elif(sys.argv[1] == "help"): # decrypt + usage() + exit() + elif(sys.argv[1] == "add"): + if(sys.argv[3]): + prefabs.add(sys.argv[2], sys.argv[3]) + else: + prefabs.add(sys.argv[2]) + exit() + elif(sys.argv[1] == "export"): + #print(sys.argv[2]) + prefabs.export(sys.argv[2], "json") + exit() + elif(sys.argv[1] == "remove"): + print("WARNING: Doing so will permanently remove the specified prefab. \nThis action CANNOT be undone. Please type the name of the prefab to confirm deletion.") + confirm = input() + if confirm == sys.argv[2]: + prefabs.remove(sys.argv[2]) + print(f'Prefab {sys.argv[2]} has been removed.') + else: + print("Confirmation failed. The prefab has not been removed.") + exit() + else: + print("prefabs: try 'prefabs help' for more information\n") + elif(len(sys.argv) == 1): + interactive() + + else: + # print "Incorrect number of arguments!\n" + print("prefabs: try 'prefabs help' for more information\n") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/mongodb/prefabs.py b/mongodb/prefabs.py new file mode 100755 index 00000000..f6f46cbc --- /dev/null +++ b/mongodb/prefabs.py @@ -0,0 +1,124 @@ +#!/Users/jacobburley/thesis-src/opendc/mongodb/opendc_testing/bin/python3 +#Change shebang to /usr/bin/python3 before using with docker +# encoding: utf-8 +""" +prefabs + +Python Library for interacting with mongoDB prefabs collection. + +""" +import urllib.parse +import pprint +import sys +import os +import json +import re +import ujson +#import pyyaml + +from pymongo import MongoClient +from bson.json_util import loads, dumps, RELAXED_JSON_OPTIONS, CANONICAL_JSON_OPTIONS + +#mongodb_opendc_db = os.environ['OPENDC_DB'] +#mongodb_opendc_user = os.environ['OPENDC_DB_USERNAME'] +#mongodb_opendc_password = os.environ['OPENDC_DB_PASSWORD'] + +#if mongodb_opendc_db == None or mongodb_opendc_user == None or mongodb_opendc_password == None: +# print("One or more environment variables are not set correctly. \nYou may experience issues connecting to the mongodb database.") + +user = urllib.parse.quote_plus('opendc') #TODO: replace this with environment variable +password = urllib.parse.quote_plus('opendcpassword') #TODO: same as above +database = urllib.parse.quote_plus('opendc') + +client = MongoClient('mongodb://%s:%s@localhost/default_db?authSource=%s' % (user, password, database)) +opendcdb = client.opendc +prefabs_collection = opendcdb.prefabs + + +def add(prefab_file, name): + if(re.match(r"\w+(\\\ \w*)*\.json", prefab_file)): + try: + with open(prefab_file, "r") as json_file: + json_prefab = json.load(json_file) + #print(json_prefab) + if name != None: + json_prefab["name"] = name + try: + prefab_id = prefabs_collection.insert(json_prefab) + except ConnectionFailure: + print("ERROR: Could not connect to the mongoDB database.") + except DuplicateKeyError: + print("ERROR: A prefab with the same unique ID already exists in the database. \nPlease remove the '_id' before trying again.\nYour prefab has not been imported.") + except: + print("ERROR: A general error has occurred. Your prefab has not been imported.") + if prefab_id != None: + if name != None: + print(f'Prefab "{name}" has been imported successfully.') + else: + print(f'Prefab "{prefab_file}" has been imported successfully.') + except FileNotFoundError: + print(f"ERROR: {prefab_file} could not be found in the specified path. No prefabs have been imported.") + elif(re.match(r"\w+(\\\ \w*)*\.yml", prefab_file)): + print("expecting a yaml file here") + #yaml + else: + print("The filetype provided is an unsupported filetype.") + #unsupported filetype + +def clone(prefab_name, new_name): + bson = prefabs_collection.find_one({'name': prefab_name}) + json_string = dumps(bson) #convert BSON representation to JSON + chosen_prefab = json.loads(json_string) #load as a JSON object + + chosen_prefab.pop("_id") # clean out our _id field from the export: mongo will generate a new one if this is imported back in + + if new_name != None: + chosen_prefab["name"] = new_name + try: + prefab_id = prefabs_collection.insert_one(chosen_prefab) + except ConnectionFailure: + print("ERROR: Could not connect to the mongoDB database.") + except: + print("ERROR: A general error has occurred. Your selected prefab has not been cloned.") + if prefab_id != None: + if new_name != None: + print(f'Prefab "{prefab_name}" has been cloned successfully as {new_name}.') + else: + print(f'Prefab "{prefab_name}" has been cloned successfully.') + +def export(prefab_name, type): + bson = prefabs_collection.find_one({'name': prefab_name}) + json_string = dumps(bson) #convert BSON representation to JSON + chosen_prefab = json.loads(json_string) #load as a JSON object + + chosen_prefab.pop("_id") # clean out our _id field from the export: mongo will generate a new one if this is imported back in + + with open(f'{prefab_name}.json', 'w', encoding='utf8') as f: + json.dump(chosen_prefab, f, ensure_ascii=False, indent=4) + print(f'Prefab {prefab_name} written to {os.getcwd()}/{prefab_name}.json.') + #pprint.pprint(json_string) + #pprint.pprint(json.loads(str(json_string))) + +def list(): + #TODO: why does it output in single quotations? + cursor = prefabs_collection.find() + prefabs = [] + for record in cursor: + #pprint.pprint(record) + #print(record) + json_string = dumps(record, json_options=RELAXED_JSON_OPTIONS) ##pymongo retrieves BSON objects, which need to be converted to json for pythons json module + prefabs.append(json.loads(json_string)) + + #print(f'There are {str(len(prefabs))} prefabs in the database. They are:') + print("Name Author") + for prefab in prefabs: + if(prefab['visibility'] == "private"): + continue + print(f"{prefab['name']} {prefab['author']}") + #pprint.pprint(prefab) + + +def remove(prefab_name): + prefabs_collection.delete_one({'name': prefab_name}) + + -- cgit v1.2.3 From 3ad2c696003294d8f49fac14d2dc56e81426c3c3 Mon Sep 17 00:00:00 2001 From: jc0b Date: Tue, 9 Jun 2020 14:19:54 +0200 Subject: progress so far (for remote) --- Dockerfile | 14 ++++++--- build/configure.sh | 20 ++++++------- docker-compose.yml | 15 ++++++---- mongodb/mongo-init-opendc-db.sh | 65 ++++++++++++++++------------------------- 4 files changed, 54 insertions(+), 60 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8a8323b6..0132b40e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,18 @@ FROM node:14.2.0 MAINTAINER Sacheendra Talluri +# Adding the mongodb repo and installing the client +RUN wget -qO - https://www.mongodb.org/static/pgp/server-4.2.asc | apt-key add - \ + && echo "deb http://repo.mongodb.org/apt/debian stretch/mongodb-org/4.2 main" | tee /etc/apt/sources.list.d/mongodb-org-4.2.list \ + && apt-get update \ + && apt-get install -y mongodb-org + # Installing python and web-server dependencies RUN echo "deb http://ftp.debian.org/debian stretch main" >> /etc/apt/sources.list \ && apt-get update \ - && apt-get install -y python python-pip yarn git sed mysql-client \ - && pip install oauth2client eventlet flask-socketio flask-compress mysql-connector-python-rf \ - && pip install --upgrade pyasn1-modules \ + && apt-get install -y python3 python3-pip yarn git sed mysql-client pymongo \ + && pip3 install oauth2client eventlet flask-socketio flask-compress mysql-connector-python-rf \ + && pip3 install --upgrade pyasn1-modules \ && rm -rf /var/lib/apt/lists/* # Copy OpenDC directory @@ -24,4 +30,4 @@ RUN chmod 555 /opendc/build/configure.sh \ # Set working directory WORKDIR /opendc -CMD ["sh", "-c", "./build/configure.sh && python2.7 opendc-web-server/main.py keys.json"] +CMD ["sh", "-c", "./build/configure.sh && python3 opendc-web-server/main.py keys.json"] diff --git a/build/configure.sh b/build/configure.sh index eb7324d2..fcb33b1a 100755 --- a/build/configure.sh +++ b/build/configure.sh @@ -22,20 +22,20 @@ done echo "MariaDB available" -NUM_TABLES=$(eval "$MYSQL_COMMAND -B --disable-column-names -e \"SELECT count(*) FROM information_schema.tables WHERE table_schema='$MYSQL_DATABASE';\"") +#NUM_TABLES=$(eval "$MYSQL_COMMAND -B --disable-column-names -e \"SELECT count(*) FROM information_schema.tables WHERE table_schema='$MYSQL_DATABASE';\"") # Check if database is empty -if [ "$NUM_TABLES" -eq 0 ]; then - eval $MYSQL_COMMAND "$MYSQL_DATABASE" < ./database/schema.sql - eval $MYSQL_COMMAND "$MYSQL_DATABASE" < ./database/test.sql -fi +#if [ "$NUM_TABLES" -eq 0 ]; then +# eval $MYSQL_COMMAND "$MYSQL_DATABASE" < ./database/schema.sql +# eval $MYSQL_COMMAND "$MYSQL_DATABASE" < ./database/test.sql +#fi # Writing databse config values to keys.json cat keys.json | python -c "import os, sys, json; ks = json.load(sys.stdin); \ - ks['MYSQL_HOST'] = 'mariadb'; \ - ks['MYSQL_PORT'] = '3306'; \ - ks['MYSQL_DATABASE'] = os.environ['MYSQL_DATABASE']; \ - ks['MYSQL_USER'] = os.environ['MYSQL_USER']; \ - ks['MYSQL_PASSWORD'] = os.environ['MYSQL_PASSWORD']; \ + ks['MONGODB_HOST'] = 'mongo'; \ + ks['MONGODB_PORT'] = '27017'; \ + ks['MONGODB_DATABASE'] = os.environ['MONGO_DB']; \ + ks['MYSQL_USER'] = os.environ['MONGO_DB_USER']; \ + ks['MYSQL_PASSWORD'] = os.environ['MONGO_DB_PASSWORD']; \ print json.dumps(ks, indent=4)" > new_keys.json mv new_keys.json keys.json diff --git a/docker-compose.yml b/docker-compose.yml index 4331a5a8..52c43ee7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,9 @@ services: - MYSQL_DATABASE=opendc - MYSQL_USER=opendc - MYSQL_PASSWORD=opendcpassword + - MONGO_DB=opendc + - MONGO_DB_USERNAME=opendc + - MONGO_DB_PASSWORD=opendcpassword simulator: build: context: ./opendc-simulator @@ -48,12 +51,12 @@ services: context: ./mongodb restart: on-failure environment: - MONGO_INITDB_ROOT_USERNAME: root - MONGO_INITDB_ROOT_PASSWORD: rootpassword - MONGO_INITDB_DATABASE: admin - OPENDC_DB: opendc - OPENDC_DB_USERNAME: opendc - OPENDC_DB_PASSWORD: opendcpassword + - MONGO_INITDB_ROOT_USERNAME=root + - MONGO_INITDB_ROOT_PASSWORD=rootpassword + - MONGO_INITDB_DATABASE=admin + - OPENDC_DB=opendc + - OPENDC_DB_USERNAME=opendc + - OPENDC_DB_PASSWORD=opendcpassword ports: - 27017:27017 #volumes: diff --git a/mongodb/mongo-init-opendc-db.sh b/mongodb/mongo-init-opendc-db.sh index 965698e7..38a8d85d 100644 --- a/mongodb/mongo-init-opendc-db.sh +++ b/mongodb/mongo-init-opendc-db.sh @@ -105,43 +105,28 @@ $MONGO_CMD --eval 'db.prefabs.insertOne( ] });' -# $MONGO_CMD --eval 'db.createCollection("prefabs", { -# validator: { -# $jsonSchema: { -# bsonType: "object", -# required: ["name"], -# properties: { -# name: { -# bsonType: "string", -# description: "The name of the environment i.e. Production, or Compute Cluster" -# }, -# datacenters: { -# bsonType: "object", -# required: ["name, location, length, width, height"], -# properties: { -# name: { -# bsonType: "string", -# description: "The name of the datacenter i.e. eu-west-1, or Science Building" -# }, -# location: { -# bsonType: "string", -# description: "The location of the datacenter i.e. Frankfurt, or De Boelelaan 1105" -# }, -# length: { -# bsonType: "double", -# description: "The physical length of the datacenter, in centimetres" -# }, -# width: { -# bsonType: "double", -# description: "The physical width of the datacenter, in centimetres" -# }, -# height: { -# bsonType: "double", -# description: "The physical height of the datacenter, in centimetres" -# } -# } -# } -# } -# } -# } -# });' \ No newline at end of file +$MONGO_CMD --eval 'db.createCollection("topologies");' + +$MONGO_CMD --eval 'db.createCollection("users");' + +$MONGO_CMD --eval 'db.users.insertOne( + { + "google_id": "23483578932789231", + "email": "jorgos.andreadis@gmail.com", + "given_name": "Jorgos", + "family_name": "Andreadis", + "authorizations": [ + { + "simulation_id": 1, + "authorization_level": "OWN" + }, + { + "simulation_id": 2, + "authorization_level": "READ" + }, + { + "simulation_id": 3, + "authorization_level": "READWRITE" + } + ] + });' \ No newline at end of file -- cgit v1.2.3