diff --git a/cadCAD/distroduce/__init__.py b/cadCAD/distroduce/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cadCAD/distroduce/configuration/__init__.py b/cadCAD/distroduce/configuration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cadCAD/distroduce/configuration/kakfa.py b/cadCAD/distroduce/configuration/kakfa.py new file mode 100644 index 0000000..258232f --- /dev/null +++ b/cadCAD/distroduce/configuration/kakfa.py @@ -0,0 +1,30 @@ +from datetime import datetime + +# ToDo: Model Async communication here +# ToDo: Configuration of Input +def configure_producer(msg, config): + from kafka import KafkaProducer + def send_messages(events): + producer = KafkaProducer(**config) + + start_timestamp = datetime.now() + for event in range(events): + producer.send('test', msg(event)).get() + delta = datetime.now() - start_timestamp + + return start_timestamp, delta.total_seconds() + + return send_messages + +# def action(step): +# step += 1 +# percent = step / 10000 +# print(str(datetime.now()) + " - " + str(percent) + ": " + str(step)) + +# def configure_consumer(config, elemental_action: function): +# from kafka import KafkaConsumer +# def receive_messages(): +# consumer = KafkaConsumer(**config) +# return [elemental_action(i) for i, message in enumerate(consumer)] +# +# return receive_messages \ No newline at end of file diff --git a/cadCAD/distroduce/executor/__init__.py b/cadCAD/distroduce/executor/__init__.py new file mode 100644 index 0000000..a1ade4e --- /dev/null +++ b/cadCAD/distroduce/executor/__init__.py @@ -0,0 +1 @@ +from cadCAD.distroduce.executor.spark.jobs import distributed_produce \ No newline at end of file diff --git a/cadCAD/distroduce/executor/spark/jobs/__init__.py b/cadCAD/distroduce/executor/spark/jobs/__init__.py new file mode 100644 index 0000000..b22f7e4 --- /dev/null +++ b/cadCAD/distroduce/executor/spark/jobs/__init__.py @@ -0,0 +1,34 @@ +from cadCAD.distroduce.configuration.kakfa import configure_producer +from pyspark.context import SparkContext + +ascii_art = r''' + ___ _ __ _ __ __ __ + / _ \ (_)___ / /_ ____ (_)/ / __ __ / /_ ___ ___/ / + / // // /(_-