diff --git a/distroduce/bash/build_source.sh b/distroduce/bash/build_source.sh index 522bbca..0d33994 100644 --- a/distroduce/bash/build_source.sh +++ b/distroduce/bash/build_source.sh @@ -1,4 +1,3 @@ #!/bin/bash pip3 install -r requirements.txt -#zip -rq dist/distroduce.zip distroduce/ zip -rq distroduce/dist/distroduce.zip distroduce/ \ No newline at end of file diff --git a/distroduce/bash/distroduce.sh b/distroduce/bash/distroduce.sh index 50929af..8cd762a 100644 --- a/distroduce/bash/distroduce.sh +++ b/distroduce/bash/distroduce.sh @@ -7,11 +7,11 @@ yes | sudo python3 -m pip install cadCAD-0.0.2-py3-none-any.whl # check for master node -PRIVATE_IP=localhost IS_MASTER=false if grep -i isMaster /mnt/var/lib/info/instance.json | grep -i true; then IS_MASTER=true + PRIVATE_IP=`hostname -I | xargs` wget https://raw.githubusercontent.com/JEJodesty/cadCAD/dev/distroduce/dist/distroduce.zip wget https://raw.githubusercontent.com/JEJodesty/cadCAD/dev/distroduce/messaging_sim.py sudo sed -i -e '$a\export PYSPARK_PYTHON=/usr/bin/python3' /etc/spark/conf/spark-env.sh @@ -20,7 +20,5 @@ then cd kafka_2.12-2.3.0 bin/zookeeper-server-start.sh config/zookeeper.properties & bin/kafka-server-start.sh config/server.properties & - bin/kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 --topic test -# PRIVATE_IP=`hostname -I | xargs` -# bin/kafka-topics.sh --create --bootstrap-server ${PRIVATE_IP}:9092 --replication-factor 1 --partitions 1 --topic test + bin/kafka-topics.sh --create --bootstrap-server ${PRIVATE_IP}:9092 --replication-factor 1 --partitions 1 --topic test fi diff --git a/distroduce/bash/spark_submit.sh b/distroduce/bash/spark_submit.sh index b9bd344..88f1be7 100644 --- a/distroduce/bash/spark_submit.sh +++ b/distroduce/bash/spark_submit.sh @@ -1,3 +1,3 @@ #!/bin/bash sudo sed -i -e '$a\export PYSPARK_PYTHON=/usr/bin/python3' /etc/spark/conf/spark-env.sh -spark-submit --master yarn --py-files distroduce.zip messaging_sim.py \ No newline at end of file +spark-submit --master yarn --py-files distroduce.zip messaging_sim.py $PRIVATE_IP \ No newline at end of file diff --git a/distroduce/messaging_sim.py b/distroduce/messaging_sim.py index 68ad8bc..05d55e2 100644 --- a/distroduce/messaging_sim.py +++ b/distroduce/messaging_sim.py @@ -1,3 +1,4 @@ +import sys import pandas as pd from datetime import datetime from tabulate import tabulate @@ -77,7 +78,7 @@ if __name__ == "__main__": ) # parmeterize localhost, PRIVATE_IP=`hostname -I | xargs` - kafkaConfig = {'send_topic': 'test', 'producer_config': {'bootstrap_servers': '10.0.0.7:9092', 'acks': 'all'}} + kafkaConfig = {'send_topic': 'test', 'producer_config': {'bootstrap_servers': f'{sys.argv[1]}:9092', 'acks': 'all'}} dist_proc_ctx = ExecutionContext(context=exec_mode.dist_proc, method=distributed_produce, kafka_config=kafkaConfig) run = Executor(exec_context=dist_proc_ctx, configs=configs, spark_context=sc)