jeffemmett
/
cadCAD
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixed bootstrap & parameterized private_ip

This commit is contained in:
Joshua E. Jodesty 2019-10-02 11:32:45 -04:00
parent 5f25cb95fb
commit 8897c1f973
4 changed files with 5 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
distroduce/bash/build_source.sh
View File

@ -1,4 +1,3 @@
#!/bin/bash
pip3 install -r requirements.txt
#zip -rq dist/distroduce.zip distroduce/
zip -rq distroduce/dist/distroduce.zip distroduce/

6
distroduce/bash/distroduce.sh
View File

@ -7,11 +7,11 @@ yes | sudo python3 -m pip install cadCAD-0.0.2-py3-none-any.whl
# check for master node
PRIVATE_IP=localhost
IS_MASTER=false
if grep -i isMaster /mnt/var/lib/info/instance.json | grep -i true;
then
IS_MASTER=true
PRIVATE_IP=`hostname -I | xargs`
wget https://raw.githubusercontent.com/JEJodesty/cadCAD/dev/distroduce/dist/distroduce.zip
wget https://raw.githubusercontent.com/JEJodesty/cadCAD/dev/distroduce/messaging_sim.py
sudo sed -i -e '$a\export PYSPARK_PYTHON=/usr/bin/python3' /etc/spark/conf/spark-env.sh
@ -20,7 +20,5 @@ then
cd kafka_2.12-2.3.0
bin/zookeeper-server-start.sh config/zookeeper.properties &
bin/kafka-server-start.sh config/server.properties &
bin/kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 --topic test
# PRIVATE_IP=`hostname -I | xargs`
# bin/kafka-topics.sh --create --bootstrap-server ${PRIVATE_IP}:9092 --replication-factor 1 --partitions 1 --topic test
bin/kafka-topics.sh --create --bootstrap-server ${PRIVATE_IP}:9092 --replication-factor 1 --partitions 1 --topic test
fi

2
distroduce/bash/spark_submit.sh
View File

@ -1,3 +1,3 @@
#!/bin/bash
sudo sed -i -e '$a\export PYSPARK_PYTHON=/usr/bin/python3' /etc/spark/conf/spark-env.sh
spark-submit --master yarn --py-files distroduce.zip messaging_sim.py
spark-submit --master yarn --py-files distroduce.zip messaging_sim.py $PRIVATE_IP

3
distroduce/messaging_sim.py
View File

@ -1,3 +1,4 @@
import sys
import pandas as pd
from datetime import datetime
from tabulate import tabulate
@ -77,7 +78,7 @@ if __name__ == "__main__":
)
# parmeterize localhost, PRIVATE_IP=`hostname -I | xargs`
kafkaConfig = {'send_topic': 'test', 'producer_config': {'bootstrap_servers': '10.0.0.7:9092', 'acks': 'all'}}
kafkaConfig = {'send_topic': 'test', 'producer_config': {'bootstrap_servers': f'{sys.argv[1]}:9092', 'acks': 'all'}}
dist_proc_ctx = ExecutionContext(context=exec_mode.dist_proc, method=distributed_produce, kafka_config=kafkaConfig)
run = Executor(exec_context=dist_proc_ctx, configs=configs, spark_context=sc)