diff --git a/benchmark_generator/benchmark_generator/code_composer.py b/benchmark_generator/benchmark_generator/code_composer.py new file mode 100644 index 00000000..8799eb37 --- /dev/null +++ b/benchmark_generator/benchmark_generator/code_composer.py @@ -0,0 +1,92 @@ +import os +import uuid + +def load_benchmark_code(benchmark_name, language="python"): + current_dir = os.getcwd() + path_to_code = os.path.join(current_dir, benchmark_name, language, "function.py" if language == "python" else "function.js") + + if os.path.exists(path_to_code): + try: + with open(path_to_code, "r") as source_file: + source_code = source_file.read() + [_, after_test] = source_code.split("#test") + [_, after_import] = after_test.split("#import") + [import_part, after_function] = after_import.split("#function") + [function_part, run_part] = after_function.split("#run") + + return { + "import": import_part, + "function": function_part, + "run": run_part + } + except Exception as e: + print(f"Error loading code from {path_to_code}: {e}") + + print(f"Path {path_to_code} does not exist") + return { + "import": "", + "function": "", + "run": "" + } + +def intend(body): + return "\n".join(["\t" + line for line in body.splitlines()]) + +def generate_huge_dict(number_of_elements): + return { + str(uuid.uuid1()) + "-" + str(i): str(uuid.uuid1()) for i in range(number_of_elements) + } + +def generate_python_handler(config, code_maps): + code = "\ndef handler(event):\n" + handler_function = "result = {}\n" + + for number, (benchmark_name, benchmark_config) in enumerate(config): + handler_function += f"\nnumber = {number}\n" + handler_function += f"config = {benchmark_config}\n" + handler_function += code_maps[benchmark_name]["run"] + if benchmark_name == "artificial_code": + number_of_elements = benchmark_config.get("number_of_elements", 0) + handler_function += f"artificial_dict{number} = {generate_huge_dict(number_of_elements)}\n" + + handler_function += """\nreturn {'result': result }\n""" + + code += intend(handler_function) + return code + +def generate_async_nodejs_handler(config, code_maps): + code = "\nexports.handler = async function(event) {\n" + handler_function = """var result = {};\nawait (async () => { return [result, 0] })()""" + + for number, (benchmark_name, benchmark_config) in enumerate(config): + handler_function += ".then(async ([result, number]) => {\n" + inner_function = f"var config = {benchmark_config};\n" + inner_function += code_maps[benchmark_name]["run"] + "\n" + inner_function += "return [result, number + 1]\n" + handler_function += intend(inner_function) + handler_function += "\n})\n" + if benchmark_name == "artificial_code": + number_of_elements = benchmark_config.get("number_of_elements", 0) + handler_function += f"var artificial_dict{number} = {generate_huge_dict(number_of_elements)};\n" + + handler_function += """\nreturn {'result': result }\n}""" + + code += intend(handler_function) + return code + +def compose(config, language): + benchmarks_list = {benchmark for (benchmark, _) in config.items()} + + code_maps = { + benchmark_name: load_benchmark_code(benchmark_name, language) for benchmark_name in benchmarks_list + } + + code = "\n".join(code_map["import"] for code_map in code_maps.values()) + code += "\n".join(code_map["function"] for code_map in code_maps.values()) + + if language == "python": + return code + generate_python_handler(config.items(), code_maps) + elif language == "async_nodejs": + return code + generate_async_nodejs_handler(config.items(), code_maps) + else: + return "" diff --git a/benchmark_generator/benchmark_generator/disc/async_nodejs/function.js b/benchmark_generator/benchmark_generator/disc/async_nodejs/function.js new file mode 100644 index 00000000..d0b03dfe --- /dev/null +++ b/benchmark_generator/benchmark_generator/disc/async_nodejs/function.js @@ -0,0 +1,41 @@ +//#test +var result = {}; +var config = { + "block_size": 1024 * 1024 * 128 +}; +var number = 0; +//#import +var fs = require('fs'); +var uuid = require('uuid'); +var uuidv1 = uuid.v1; +//#function +function generate_data_disc(block_size) { + return Array(block_size + 1).fill('x').join(''); // Generate string of specified block size +} + +async function testDisc(block_size) { + try { + var data = generate_data_disc(block_size); // Generate data based on block size + var path = "/tmp/serverless-benchmark-test-file.json"; + var t0 = new Date().getTime(); // Get current time in milliseconds + fs.writeFileSync(path, data); // Write data to file + var t1 = new Date().getTime(); // Get current time in milliseconds + await fs.promises.readFile(path); // Read file asynchronously + var t2 = new Date().getTime(); // Get current time in milliseconds + + return { + "write_time": t1 - t0, + "read_time": t2 - t1, + "bytes": block_size + }; + } catch (error) { + return { "error": error.toString() }; + } +} + +//#run +var block_size = config.block_size; +testDisc(block_size).then(returnJson => { + result[number] = returnJson; + console.log(result); // Output the result +}); diff --git a/benchmark_generator/benchmark_generator/disc/async_nodejs/package.json b/benchmark_generator/benchmark_generator/disc/async_nodejs/package.json new file mode 100644 index 00000000..2c4ed0f7 --- /dev/null +++ b/benchmark_generator/benchmark_generator/disc/async_nodejs/package.json @@ -0,0 +1,13 @@ +{ + "name": "disc", + "version": "1.0.0", + "description": "", + "author": "", + "license": "", + "dependencies": { + "uuid": "8.2.0" + }, + "devDependencies": { + } + } + \ No newline at end of file diff --git a/benchmark_generator/benchmark_generator/disc/python/function.py b/benchmark_generator/benchmark_generator/disc/python/function.py new file mode 100644 index 00000000..77fd3df3 --- /dev/null +++ b/benchmark_generator/benchmark_generator/disc/python/function.py @@ -0,0 +1,36 @@ +import numpy as np +import time +import uuid +import os + +def test_disc(block_size, file_name): + a = np.ones(int(block_size / 4), dtype=np.dtype("int32")) * 2 # Create array of specified block size + t0 = time.perf_counter() # Start time for write operation + np.save(file_name, a) # Save array to disk + t1 = time.perf_counter() # End time for write operation + + t2 = time.perf_counter() # Start time for read operation + loaded_array = np.load(file_name) # Load array from disk + t3 = time.perf_counter() # End time for read operation + + write_time = t1 - t0 + read_time = t3 - t2 + return { + "block_size": block_size, + "write_time": write_time, + "read_time": read_time + } + +# Test configuration +config = { + "block_size": 1024 * 1024 * 128 # 128 MB +} + +result = {} +number = 0 + +block_size = config.get("block_size", 1024 * 100) # Default 100 KB +file_name = "/tmp/sebs.npy" + +result[str(number)] = test_disc(block_size, file_name) +print(result) diff --git a/benchmark_generator/benchmark_generator/generator.py b/benchmark_generator/benchmark_generator/generator.py new file mode 100644 index 00000000..e8776825 --- /dev/null +++ b/benchmark_generator/benchmark_generator/generator.py @@ -0,0 +1,59 @@ +import sys +import json +import code_composer +import requirements_composer +import input_composer +import os + +def read_config_file(file_path): + try: + with open(file_path, 'r') as config_file: + return json.load(config_file) + except FileNotFoundError: + print(f"Error: File {file_path} not found.") + sys.exit(1) + except json.JSONDecodeError: + print(f"Error: Failed to parse JSON from {file_path}.") + sys.exit(1) + +def write_to_file(file_path, content): + try: + with open(file_path, 'w+') as file: + file.write(content) + except Exception as e: + print(f"Error writing to {file_path}: {e}") + sys.exit(1) + +def generate_and_write_code(config, language, path_to_benchmark): + if language == "python": + write_to_file(os.path.join(path_to_benchmark, "function.py"), code_composer.compose(config, "python")) + write_to_file(os.path.join(path_to_benchmark, "requirements.txt"), requirements_composer.compose(config, "python")) + elif language == "async_nodejs": + write_to_file(os.path.join(path_to_benchmark, "function.js"), code_composer.compose(config, "async_nodejs")) + write_to_file(os.path.join(path_to_benchmark, "package.json"), requirements_composer.compose(config, "async_nodejs")) + +if len(sys.argv) < 2: + print("Missing argument: path to config") + sys.exit(1) + +total_config = read_config_file(sys.argv[1]) + +config = total_config["config"] +language = total_config["language"] + +path_to_benchmark_base = "./../benchmarks/600.generated/620.generated" + +if language == "python": + path_to_benchmark = os.path.join(path_to_benchmark_base, "python") + if not os.path.exists(path_to_benchmark): + os.makedirs(path_to_benchmark) +elif language == "async_nodejs": + path_to_benchmark = os.path.join(path_to_benchmark_base, "nodejs") + if not os.path.exists(path_to_benchmark): + os.makedirs(path_to_benchmark) + +generate_and_write_code(config, language, path_to_benchmark) + +# Ensure path_to_benchmark is initialized before calling input_composer.compose +if "path_to_benchmark" in locals(): + write_to_file(os.path.join(path_to_benchmark, "../input.py"), input_composer.compose(config)) diff --git a/benchmark_generator/benchmark_generator/input_composer.py b/benchmark_generator/benchmark_generator/input_composer.py new file mode 100644 index 00000000..70b8aeb6 --- /dev/null +++ b/benchmark_generator/benchmark_generator/input_composer.py @@ -0,0 +1,45 @@ +import uuid + +def generate_input_dict(benchmark_config): + """Generate input dictionary based on input_size in benchmark_config.""" + input_dict = {} + if "input_size" in benchmark_config: + for _ in range(int(benchmark_config["input_size"])): + input_dict[str(uuid.uuid1())] = 100 + return input_dict + +def generate_buckets_count_code(benchmarks_list): + """Generate buckets_count function code.""" + if "storage" in benchmarks_list: + return """def buckets_count(): + return (0, 1)\n""" + else: + return """def buckets_count(): + return (0, 0)\n""" + +def generate_generate_input_code(benchmarks_list): + """Generate generate_input function code.""" + if "storage" in benchmarks_list: + return """def generate_input(data_dir, size, input_buckets, output_buckets, upload_func): + input_dict = {'bucket': {}} + input_dict['bucket']['output'] = output_buckets[0] + return input_dict """ + else: + return """def generate_input(data_dir, size, input_buckets, output_buckets, upload_func): + return input_dict """ + +def generate_code(config): + benchmarks_list = {benchmark for (benchmark, _) in config.items()} + + input_dict = {} + for benchmark, benchmark_config in config.items(): + if benchmark == "function_input": + input_dict = generate_input_dict(benchmark_config) + break + + code = "" + code += "input_dict = " + str(input_dict) + "\n" + code += generate_buckets_count_code(benchmarks_list) + code += generate_generate_input_code(benchmarks_list) + + return code diff --git a/benchmark_generator/benchmark_generator/memory/async_nodejs/function.js b/benchmark_generator/benchmark_generator/memory/async_nodejs/function.js new file mode 100644 index 00000000..c79e3abd --- /dev/null +++ b/benchmark_generator/benchmark_generator/memory/async_nodejs/function.js @@ -0,0 +1,25 @@ +//#test +var config = { + "size_in_bytes": 10485760 +}; +var result = {}; +var number = 0; +//#import +var math = require('mathjs'); +//#function +const testMemory = async (size) => { + var t0 = new Date().getTime(); // Get current time in milliseconds + var a = math.ones([Math.floor(size / 8)]); // Ensure size is an integer + var t1 = new Date().getTime(); // Get current time in milliseconds + + return { + "time": t1 - t0, + "size_in_bytes": size + }; +}; +//#run +var array_size_in_bytes = config["size_in_bytes"]; +testMemory(array_size_in_bytes).then(returnJson => { + result[number] = returnJson; + console.log(result); // Output the result +}); diff --git a/benchmark_generator/benchmark_generator/memory/async_nodejs/package.json b/benchmark_generator/benchmark_generator/memory/async_nodejs/package.json new file mode 100644 index 00000000..b46a7b4a --- /dev/null +++ b/benchmark_generator/benchmark_generator/memory/async_nodejs/package.json @@ -0,0 +1,12 @@ +{ + "name": "sleep", + "version": "1.0.0", + "description": "", + "author": "", + "license": "", + "dependencies": { + "mathjs": "7.0.2" + }, + "devDependencies": { + } +} diff --git a/benchmark_generator/benchmark_generator/memory/python/function.py b/benchmark_generator/benchmark_generator/memory/python/function.py new file mode 100644 index 00000000..fa6b24fa --- /dev/null +++ b/benchmark_generator/benchmark_generator/memory/python/function.py @@ -0,0 +1,24 @@ +import numpy as np +import time + +def allocate(size_in_bytes): + t0 = time.perf_counter() + arr = np.ones(int(size_in_bytes/4), dtype=np.dtype("int32")) + t1 = time.perf_counter() + + return { + "time": t1 - t0, + "size_in_bytes": size_in_bytes + } + +config = { + "size_in_bytes": 1024 * 1024 +} + +result = {} +number = 0 + +size_of_allocated_memory = config.get("size_in_bytes", 1024 * 1024) # Default 1 MB +result[str(number)] = allocate(size_of_allocated_memory) + +print(result) diff --git a/benchmark_generator/benchmark_generator/memory/python/requirements.txt b/benchmark_generator/benchmark_generator/memory/python/requirements.txt new file mode 100644 index 00000000..683f00cc --- /dev/null +++ b/benchmark_generator/benchmark_generator/memory/python/requirements.txt @@ -0,0 +1 @@ +numpy==1.18.5 \ No newline at end of file diff --git a/benchmark_generator/benchmark_generator/requirements_composer.py b/benchmark_generator/benchmark_generator/requirements_composer.py new file mode 100644 index 00000000..e15ea474 --- /dev/null +++ b/benchmark_generator/benchmark_generator/requirements_composer.py @@ -0,0 +1,84 @@ +import os +import json + +def read_file_content(file_path): + """Read the content of a file.""" + try: + with open(file_path, "r") as file: + return file.read() + except FileNotFoundError: + print(f"Error: File {file_path} not found.") + return "" + except Exception as e: + print(f"Error reading {file_path}: {e}") + return "" + +def load_benchmark_requirements(benchmark_name): + """Load requirements for a specific benchmark.""" + current_dir = os.getcwd() + path_to_requirements = os.path.join(current_dir, benchmark_name, "python", "requirements.txt") + + if os.path.exists(path_to_requirements) and os.path.isfile(path_to_requirements): + return read_file_content(path_to_requirements) + else: + print(f"Path to {path_to_requirements} does not exist.") + return "" + +def prepare_python_file(config): + """Prepare requirements for Python benchmarks.""" + benchmarks_list = {benchmark for (benchmark, benchmark_config) in config.items()} + + requirements_for_all_benchmarks = "" + for benchmark_name in benchmarks_list: + requirements_for_all_benchmarks += "\n" + load_benchmark_requirements(benchmark_name) + + return requirements_for_all_benchmarks.strip() + +def load_benchmark_dependencies(benchmark_name, language): + """Load dependencies for a specific benchmark.""" + current_dir = os.getcwd() + path_to_dependencies = os.path.join(current_dir, benchmark_name, language, "package.json") + + if os.path.exists(path_to_dependencies) and os.path.isfile(path_to_dependencies): + try: + with open(path_to_dependencies, "r") as json_file: + package_json = json.load(json_file) + return (package_json.get("dependencies", {}), package_json.get("devDependencies", {})) + except json.JSONDecodeError: + print(f"Error: Failed to decode JSON from {path_to_dependencies}.") + return ({}, {}) + else: + print(f"Path to {path_to_dependencies} does not exist.") + return ({}, {}) + +def prepare_nodejs_file(config): + """Prepare dependencies for Node.js benchmarks.""" + benchmarks_list = {benchmark for (benchmark, benchmark_config) in config.items()} + + dependencies_list = [load_benchmark_dependencies(benchmark_name, "nodejs") for benchmark_name in benchmarks_list] + + dependencies = {} + dev_dependencies = {} + for dependency, dev_dependency in dependencies_list: + dependencies.update(dependency) + dev_dependencies.update(dev_dependency) + + return json.dumps({ + "name": "generated_benchmark", + "version": "1.0.0", + "description": "", + "author": "", + "license": "", + "dependencies": dependencies, + "devDependencies": dev_dependencies + }) + +def compose(config, language): + """Compose the dependencies based on the language.""" + if language == "python": + return prepare_python_file(config) + elif language == "async_nodejs": + return prepare_nodejs_file(config) + else: + print(f"Unsupported language: {language}") + return "" diff --git a/benchmark_generator/benchmark_generator/storage/async_nodejs/function.js b/benchmark_generator/benchmark_generator/storage/async_nodejs/function.js new file mode 100644 index 00000000..9d73c417 --- /dev/null +++ b/benchmark_generator/benchmark_generator/storage/async_nodejs/function.js @@ -0,0 +1,76 @@ +//#test +var result = {}; +var config = { + "entries_number": 1000 +}; +var number = 0; +var event = {}; +//#import +var storage = require('./storage'); +var uuid = require('uuid'); +var uuidv1 = uuid.v1; +var { Readable } = require("stream") +//#function +function generate_data(entries_number) { + var dictToFill = {}; + for(var i = 0;i < entries_number;i++) { + dictToFill[uuidv1()] = uuidv1() + } + return dictToFill +} +function streamToPromise(stream) { + return new Promise(function(resolve, reject) { + stream.on("close", () => { + resolve(); + }); + stream.on("error", reject); + }) + } +async function testBucketStorage(dataAsDict, bucket_config) { + var [client, bucket] = bucket_config; + var dataAsString = JSON.stringify(dataAsDict); + var inputStream = Readable.from(dataAsString); + var result = {}; + var t0 = new Date() + var [writeStream, uploadPromise, storageKey] = client.uploadStream(bucket, "serverless-benchmark-data.json") + inputStream.pipe(writeStream) + await uploadPromise.then(async () => { + var t1 = new Date() + read_promise = client.downloadStream(bucket, storageKey) + await read_promise.then(async (stream) => { + await (streamToPromise(stream).then((any) => { + var t2 = new Date(); + result = { + "uploaded_to_bucket_bytes": dataAsString.length, + "upload_time": t1 - t0, + "downloaded_from_bucket_bytes": dataAsString.length, + "download_time": t2 - t1, + "key": storageKey + } + })) + }) + }) + return result; +} +async function testStorage(entries_number, bucket_config, storage_type) { + try { + var data = generate_data(entries_number); + if(storage_type == "bucket") { + var res = {} + await testBucketStorage(data, bucket_config).then((resJson) => res = resJson) + return res; + } + return { "error": "unknown storage"} + } catch (error) { + return { "error": error.toString() } + } +}; +//#run +var output_bucket = event.bucket.output; +var entries_number = config.entries_number; +let client = new storage.storage(); +var bucket_config = [client, output_bucket]; +await testStorage(entries_number, bucket_config, "bucket").then(returnJson => { + result[number] = returnJson; + } +); \ No newline at end of file diff --git a/benchmark_generator/benchmark_generator/storage/async_nodejs/package.json b/benchmark_generator/benchmark_generator/storage/async_nodejs/package.json new file mode 100644 index 00000000..171182df --- /dev/null +++ b/benchmark_generator/benchmark_generator/storage/async_nodejs/package.json @@ -0,0 +1,12 @@ +{ + "name": "sleep", + "version": "1.0.0", + "description": "", + "author": "", + "license": "", + "dependencies": { + "uuid": "8.2.0" + }, + "devDependencies": { + } +} diff --git a/benchmark_generator/benchmark_generator/storage/python/function.py b/benchmark_generator/benchmark_generator/storage/python/function.py new file mode 100644 index 00000000..e5ab55b4 --- /dev/null +++ b/benchmark_generator/benchmark_generator/storage/python/function.py @@ -0,0 +1,69 @@ +#test +result = {} +config = { + "entries_number": 1000 +} +number = 0 +event = {} + +#import +import storage +import uuid +import time +import traceback +import io + +#function +def generate_data(entries_number): + dict_to_fill = {} + for i in range(entries_number): + dict_to_fill[str(uuid.uuid1())] = str(uuid.uuid1()) + return dict_to_fill + +def upload_to_bucket(config, bytes_buffer): + (client, output_bucket) = config + try: + key_name = client.upload_stream(output_bucket, "sebs_test.sth", bytes_buffer) + except Exception as inst: + key_name = str(inst) + "\n" + traceback.format_exc() + return key_name + +def download_from_bucket(config, file_key): + (client, output_bucket) = config + buffer = client.download_stream(output_bucket, file_key) + downloaded_size = len(buffer.tobytes()) + return downloaded_size + +def test_bucket_like(config, dict_to_upload): + string_to_upload = str(dict_to_upload) + bytes_to_upload = str.encode(string_to_upload) + buffer_to_upload = io.BytesIO(bytes_to_upload) + t0 = time.perf_counter() + key = upload_to_bucket(config, buffer_to_upload) + t1 = time.perf_counter() + downloaded_bytes = download_from_bucket(config, key) + t2 = time.perf_counter() + return { + "uploaded_to_bucket_bytes": len(bytes_to_upload), + "upload_time": t1 - t0, + "downloaded_from_bucket_bytes": downloaded_bytes, + "download_time": t2 - t1, + "key": key + } + +def test_storage(dict_to_upload, config, storage_type="bucket"): + if storage_type == "bucket": + return test_bucket_like(config, dict_to_upload) + # elif other storage types: + # return ... + else: + return {} + +#run +output_bucket = event.get('bucket').get('output') +entries_number = config.get("entries_number", 10) +client = storage.storage.get_instance() +dict_to_upload = generate_data(entries_number) +bucket_config = (client, output_bucket) +result[str(number)] = test_storage(dict_to_upload, bucket_config) +print(result)