Skip to content

Commit 55577a5

Browse files
author
Anton Kirillov
committed
eval of custom bash command for SPARK_PUBLIC_DNS
1 parent 929c348 commit 55577a5

File tree

9 files changed

+72
-34
lines changed

9 files changed

+72
-34
lines changed

‎README.md

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@ Submitting Spark job to Chronos to be executed on a regular manner:
3232
"image": "datastrophic/mesos-spark:mesos-1.1.0-spark-2.1.0",
3333
"network": "HOST"
3434
},
35-
"cpus": "0.5",
36-
"mem": "1024",
35+
"cpus": "1",
36+
"mem": "2048",
3737
"fetch": [],
3838
"command": "spark-submit --master mesos://master.mesos:5050 --conf spark.mesos.executor.docker.image=datastrophic/mesos-spark:mesos-1.1.0-spark-2.1.0 --class org.apache.spark.examples.SparkPi /spark/examples/jars/spark-examples_2.11-2.1.0.jar 250"
3939
}'
@@ -48,7 +48,7 @@ From any of Mesos agent nodes with Docker installed:
4848
-e CHRONOS_ZK_HOSTS=<zookeeper_1>:2181,<zookeeper_2>:2181,<zookeeper_3>:2181 \
4949
datastrophic/chronos:mesos-1.1.0-chronos-3.0.1
5050

51-
Submitting to Marathon:
51+
Submitting to Marathon (port should be adjusted to be in resource offers port range):
5252

5353
curl -XPOST 'http://marathon.mesos:8090/v2/apps' -H 'Content-Type: application/json' -d '{
5454
"id": "chronos",
@@ -66,7 +66,40 @@ Submitting to Marathon:
6666
},
6767
"ports": [4400],
6868
"cpus": 1,
69-
"mem": 512,
69+
"mem": 2048,
70+
"instances": 1,
71+
"constraints": [["hostname", "UNIQUE"]]
72+
}'
73+
74+
## Running Zeppelin on Mesos
75+
From any of Mesos agent nodes with Docker installed (assuming S3 access is needed for a job):
76+
77+
docker run -ti \
78+
--net=host -p 4400:4400 \
79+
-e CHRONOS_HTTP_PORT=4400 \
80+
-e CHRONOS_MASTER=zk://<zookeeper_1>:2181,<zookeeper_2>:2181,<zookeeper_3>:2181/mesos \
81+
-e CHRONOS_ZK_HOSTS=<zookeeper_1>:2181,<zookeeper_2>:2181,<zookeeper_3>:2181 \
82+
datastrophic/chronos:mesos-1.1.0-chronos-3.0.1
83+
84+
Submitting to Marathon (port should be adjusted to be in resource offers port range):
85+
86+
curl -XPOST 'http://marathon.mesos:8090/v2/apps' -H 'Content-Type: application/json' -d '{
87+
"id": "chronos",
88+
"container": {
89+
"type": "DOCKER",
90+
"docker": {
91+
"network": "HOST",
92+
"image": "datastrophic/chronos:mesos-1.1.0-chronos-3.0.1"
93+
}
94+
},
95+
"env": {
96+
"CHRONOS_HTTP_PORT":"4400",
97+
"CHRONOS_MASTER":"zk://<zookeeper_1>:2181,<zookeeper_2>:2181,<zookeeper_3>:2181/mesos",
98+
"CHRONOS_ZK_HOSTS":"<zookeeper_1>:2181,<zookeeper_2>:2181,<zookeeper_3>:2181"
99+
},
100+
"ports": [4400],
101+
"cpus": 1,
102+
"mem": 2048,
70103
"instances": 1,
71104
"constraints": [["hostname", "UNIQUE"]]
72105
}'

‎bin/build-all.sh

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ marathon_tag=1.3.6
77
chronos_tag=3.0.1
88
kafka_tag=0.10.0.0
99
spark_tag=2.1.0
10+
zeppelin_tag=0.7.0
1011

1112
dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
1213
home="$(dirname "$dir")"
@@ -28,11 +29,13 @@ docker build --tag ${prefix}/marathon:${marathon_tag} ${hom
2829

2930
if [ "$tag" = "latest" ]
3031
then
31-
docker build --tag ${prefix}/chronos:${tag} ${home}/chronos
32-
docker build --tag ${prefix}/mesos-kafka:${tag} ${home}/mesos-kafka
33-
docker build --tag ${prefix}/mesos-spark:${tag} ${home}/mesos-spark
32+
docker build --tag ${prefix}/chronos:${tag} ${home}/chronos
33+
docker build --tag ${prefix}/mesos-kafka:${tag} ${home}/mesos-kafka
34+
docker build --tag ${prefix}/mesos-spark:${tag} ${home}/mesos-spark
35+
docker build --tag ${prefix}/mesos-spark-zeppelin:${tag} ${home}/mesos-spark
3436
else
35-
docker build --tag ${prefix}/chronos:mesos-${tag}-chronos-${chronos_tag} ${home}/chronos
36-
docker build --tag ${prefix}/mesos-kafka:mesos-${tag}-kafka-${kafka_tag} ${home}/mesos-kafka
37-
docker build --tag ${prefix}/mesos-spark:mesos-${tag}-spark-${spark_tag} ${home}/mesos-spark
37+
docker build --tag ${prefix}/chronos:mesos-${tag}-chronos-${chronos_tag} ${home}/chronos
38+
docker build --tag ${prefix}/mesos-kafka:mesos-${tag}-kafka-${kafka_tag} ${home}/mesos-kafka
39+
docker build --tag ${prefix}/mesos-spark:mesos-${tag}-spark-${spark_tag} ${home}/mesos-spark
40+
docker build --tag ${prefix}/mesos-spark:mesos-${tag}-spark-${spark_tag}-zeppelin-${zeppelin_tag} ${home}/mesos-spark-zeppelin
3841
fi

‎mesos-slave/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ RUN apt-get update && apt-get install -y curl apt-transport-https ca-certificate
55
apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D && \
66
echo deb https://apt.dockerproject.org/repo ubuntu-xenial main > /etc/apt/sources.list.d/docker.list && \
77
apt-get update && \
8-
apt-get -y install curl docker-engine
8+
apt-get -y install docker-engine
99

1010
ENV MESOS_CONTAINERIZERS docker,mesos
1111
ENV MESOS_EXECUTOR_REGISTRATION_TIMEOUT 5mins

‎mesos-spark-zeppelin/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ RUN mkdir -p /zeppelin && \
66
curl -Ls -o /zeppelin/${ZEPPELIN_BINARY} http://apache.mirrors.spacedump.net/zeppelin/zeppelin-0.7.0/${ZEPPELIN_BINARY} && \
77
tar -xzf /zeppelin/${ZEPPELIN_BINARY} -C /zeppelin --strip-components 1
88

9-
EXPOSE 8080 4040
9+
EXPOSE 8080
1010

1111
WORKDIR /zeppelin
1212

‎mesos-spark-zeppelin/entrypoint.sh

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,7 @@
11
#!/bin/bash
2-
#This script is work in progress and will be changed in future
3-
for k in `env | grep ^SPARK_ | cut -d= -f1`; do
4-
eval v=\$$k
5-
CMD="$CMD `echo $k | cut -d_ -f2- | tr '[:upper:]' '[:lower:]' | tr '_' '.'`=$v"
6-
done
72

8-
echo "properties found:"
9-
echo $CMD
3+
#this script is created in parent image (mesos-spark) and performs spark-env variables parsing (e.g. SPARK_PUBLIC_DNS)
4+
${SPARK_HOME}/bootstrap.sh
105

116
if [ $# -gt 0 ]; then
127
exec "$@"

‎mesos-spark/Dockerfile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@ RUN apt-get update && apt-get install -y curl && \
1010
ENV SPARK_HOME /spark
1111
ENV PATH=$PATH:$SPARK_HOME/bin
1212

13-
COPY spark-defaults.conf /spark/conf
14-
COPY entrypoint.sh /
13+
COPY spark-defaults.conf ${SPARK_HOME}/conf
14+
COPY bootstrap.sh ${SPARK_HOME}
1515

1616
EXPOSE 4040
1717

‎mesos-spark/boostrap.sh

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/bin/bash
2+
#This script is work in progress and will be changed in future
3+
DEFAULT_SPARK_DNS=$(hostname -i)
4+
5+
if [ -z "$SPARK_PUBLIC_DNS_EVAL" ]; then
6+
if (DEFAULT_SPARK_DNS=$($SPARK_PUBLIC_DNS_EVAL))
7+
then echo "Using SPARK_PUBLIC_DNS=$DEFAULT_SPARK_DNS"
8+
fi
9+
fi
10+
11+
SPARK_PUBLIC_DNS=${SPARK_PUBLIC_DNS:-${DEFAULT_SPARK_DNS:-"127.0.0.1"}}
12+
13+
echo > ${SPARK_HOME}/conf/spark-env.sh
14+
echo "SPARK_PUBLIC_DNS=$SPARK_PUBLIC_DNS" >> ${SPARK_HOME}/conf/spark-env.sh
15+
16+
echo "Contents of ${SPARK_HOME}/conf/spark-env.sh"
17+
cat ${SPARK_HOME}/conf/spark-env.sh

‎mesos-spark/entrypoint.sh

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,5 @@
11
#!/bin/bash
22

3-
CURRENT_IP=$(hostname -i)
3+
${SPARK_HOME}/bootstrap.sh
44

5-
mv ${SPARK_HOME}/conf/spark-env.sh.template ${SPARK_HOME}/conf/spark-env.sh
6-
7-
SPARK_LOCAL_IP=${SPARK_LOCAL_IP:-${CURRENT_IP:-"127.0.0.1"}}
8-
SPARK_PUBLIC_DNS=${SPARK_PUBLIC_DNS:-${CURRENT_IP:-"127.0.0.1"}}
9-
10-
echo "SPARK_LOCAL_IP=$SPARK_LOCAL_IP" >> ${SPARK_HOME}/conf/spark-env.sh
11-
echo "SPARK_PUBLIC_DNS=$SPARK_PUBLIC_DNS" >> ${SPARK_HOME}/conf/spark-env.sh
12-
13-
cat ${SPARK_HOME}/conf/spark-env.sh
14-
15-
exec "$@"
5+
exec "$@"

‎mesos/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ ARG MESOS_VERSION=1.1.0-2.0.107.ubuntu1604
55
RUN echo "deb http://repos.mesosphere.com/ubuntu xenial main" > /etc/apt/sources.list.d/mesosphere.list && \
66
apt-key adv --keyserver keyserver.ubuntu.com --recv E56151BF && \
77
apt-get -y update && \
8-
apt-get -y install mesos=${MESOS_VERSION} && \
8+
apt-get -y install curl mesos=${MESOS_VERSION} && \
99
apt-get clean && rm -rf /var/lib/apt/lists/*
1010

1111
ENV MESOS_WORK_DIR /tmp/mesos

0 commit comments

Comments
 (0)