Skip to content

Commit 75887ab

Browse files
author
Anton Kirillov
committed
zeppelin added, drill added
1 parent 2051c16 commit 75887ab

File tree

12 files changed

+473
-16
lines changed

12 files changed

+473
-16
lines changed

README.md

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,34 @@ Submitting to Marathon (port should be adjusted to be in resource offers port ra
104104
"constraints": [["hostname", "UNIQUE"]]
105105
}'
106106

107+
## Running Apache Drill in Marathon
108+
109+
curl -XPOST 'http://marathon.mesos:8090/v2/apps' -H 'Content-Type: application/json' -d '{
110+
"id": "drill",
111+
"container": {
112+
"type": "DOCKER",
113+
"docker": {
114+
"network": "HOST",
115+
"image": "datastrophic/apache-drill:drill-1.10"
116+
}
117+
},
118+
"env": {
119+
"DRILL_HEAP":"8G",
120+
"DRILL_MAX_DIRECT_MEMORY":"12G",
121+
"ZK_SERVERS":"phisical_zk_address:2181",
122+
"CLUSTER_ID":"drillbeatz"
123+
"AWS_ACCESS_KEY_ID":"{{ lookup('env','AWS_ACCESS_KEY_ID') }}",
124+
"AWS_SECRET_ACCESS_KEY":"{{ lookup('env','AWS_SECRET_ACCESS_KEY') }}"
125+
},
126+
"8": 1,
127+
"mem": 28000,
128+
"instances": 10,
129+
"constraints": [["hostname", "UNIQUE"]]
130+
}'
131+
132+
133+
134+
107135
## Running Mesos locally
108136
This setup is more for development and educational purposes and hits its limits when it comes to running docker containers via Marathon.
109137

apache-drill/Dockerfile

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
FROM datastrophic/mesos-java:1.1.0
2+
3+
ARG DRILL_BINARY=apache-drill-1.10.0.tar.gz
4+
ENV DRILL_HOME /apache-drill
5+
ENV DRILL_JAVA_LIB_PATH ${DRILL_HOME}/native
6+
7+
RUN apt-get update && apt-get install -y curl && \
8+
mkdir -p ${DRILL_HOME} ${DRILL_JAVA_LIB_PATH} && \
9+
curl -o ${DRILL_HOME}/${DRILL_BINARY} http://apache.mirrors.hoobly.com/drill/drill-1.10.0/${DRILL_BINARY} && \
10+
tar -xzf ${DRILL_HOME}/${DRILL_BINARY} -C ${DRILL_HOME} --strip-components 1
11+
12+
ENV PATH=$PATH:$DRILL_HOME/bin
13+
14+
COPY conf/drill-override.conf ${DRILL_HOME}/conf
15+
COPY conf/core-site.xml ${DRILL_HOME}/conf
16+
COPY conf/drill-env.sh ${DRILL_HOME}/conf
17+
18+
COPY bootstrap.sh ${DRILL_HOME}
19+
COPY entrypoint.sh /
20+
21+
#native auth library
22+
COPY libjpam.so ${DRILL_JAVA_LIB_PATH}
23+
24+
EXPOSE 8047 31010 31011 31012
25+
26+
ENTRYPOINT ["/entrypoint.sh"]

apache-drill/bootstrap.sh

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/bash
2+
sed -i -e "s/CLUSTER_ID/${CLUSTER_ID}/" /apache-drill/conf/drill-override.conf
3+
sed -i -e "s/ZK_SERVERS/${ZK_SERVERS}/" /apache-drill/conf/drill-override.conf
4+
5+
sed -i -e "s/AWS_ACCESS_KEY_ID/${AWS_ACCESS_KEY_ID}/" /apache-drill/conf/core-site.xml
6+
sed -i -e "s/AWS_SECRET_ACCESS_KEY/${AWS_SECRET_ACCESS_KEY}/" /apache-drill/conf/core-site.xml

apache-drill/conf/core-site.xml

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
<?xml version="1.0" encoding="UTF-8" ?>
2+
<!--
3+
Licensed to the Apache Software Foundation (ASF) under one or more
4+
contributor license agreements. See the NOTICE file distributed with
5+
this work for additional information regarding copyright ownership.
6+
The ASF licenses this file to You under the Apache License, Version 2.0
7+
(the "License"); you may not use this file except in compliance with
8+
the License. You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
-->
18+
19+
<configuration>
20+
21+
<property>
22+
<name>fs.s3a.access.key</name>
23+
<value>AWS_ACCESS_KEY_ID</value>
24+
</property>
25+
26+
<property>
27+
<name>fs.s3a.secret.key</name>
28+
<value>AWS_SECRET_ACCESS_KEY</value>
29+
</property>
30+
31+
<property>
32+
<name>fs.s3a.connection.maximum</name>
33+
<value>100</value>
34+
</property>
35+
36+
</configuration>

apache-drill/conf/drill-env.sh

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#-----------------------------------------------------------------------------
16+
17+
# This file provides a variety of site-specific settings to control Drill
18+
# launch settings. These are settings required when launching the Drillbit
19+
# or sqlline processes using Java. Some settings are for both, some for one
20+
# or the other.
21+
#
22+
# Variables may be set in one of four places:
23+
#
24+
# Environment (per run)
25+
# drill-env.sh (this file, per site)
26+
# distrib-env.sh (per distribution)
27+
# drill-config.sh (Drill defaults)
28+
#
29+
# Properties "inherit" from items lower on the list, and may be "overridden" by items
30+
# higher on the list. In the environment, just set the variable:
31+
#
32+
# export FOO=value
33+
#
34+
# To support inheritance from the environment, you must set values as shown below:
35+
#
36+
# export FOO=${FOO:-"value"}
37+
#
38+
# or a more specialized form.
39+
40+
# Amount of heap memory for the Drillbit process. Values are those supported by
41+
# the Java -Xms option. The default is 4G.
42+
43+
export DRILL_HEAP=${DRILL_HEAP:-"4G"}
44+
45+
# Maximum amount of direct memory to allocate to the Drillbit in the format
46+
# supported by -XX:MaxDirectMemorySize. Default is 8G.
47+
48+
export DRILL_MAX_DIRECT_MEMORY=${DRILL_MAX_DIRECT_MEMORY:-"8G"}
49+
50+
# Value for the JVM -XX:MaxPermSize option for the Drillbit. Default is 512M.
51+
52+
#export DRILLBIT_MAX_PERM=${DRILLBIT_MAX_PERM:-"512M"}
53+
54+
# Native library path passed to Java. Note: use this form instead
55+
# of the old form of DRILLBIT_JAVA_OPTS="-Djava.library.path=<dir>"
56+
# The old form is not compatible with Drill-on-YARN.
57+
58+
#export DRILL_JAVA_LIB_PATH=
59+
60+
# Value for the code cache size for the Drillbit. Because the Drillbit generates
61+
# code, it benefits from a large cache. Default is 1G.
62+
63+
#export DRILLBIT_CODE_CACHE_SIZE=${DRILLBIT_CODE_CACHE_SIZE:-"1G"}
64+
65+
# Provide a customized host name for when the default mechanism is not accurate
66+
67+
#export DRILL_HOST_NAME=`hostname`
68+
69+
# Base name for Drill log files. Files are named ${DRILL_LOG_NAME}.out, etc.
70+
71+
# DRILL_LOG_NAME="drillbit"
72+
73+
# Location to place Drill logs. Set to $DRILL_HOME/log by default.
74+
75+
#export DRILL_LOG_DIR=${DRILL_LOG_DIR:-$DRILL_HOME/conf}
76+
77+
# Location to place the Drillbit pid file when running as a daemon using
78+
# drillbit.sh start.
79+
# Set to $DRILL_HOME by default.
80+
81+
#export DRILL_PID_DIR=${DRILL_PID_DIR:-$DRILL_HOME}
82+
83+
# Custom JVM arguments to pass to the both the Drillbit and sqlline. Typically
84+
# used to override system properties as shown below. Empty by default.
85+
86+
#export DRILL_JAVA_OPTS="$DRILL_JAVA_OPTS -Dproperty=value"
87+
88+
# As above, but only for the Drillbit. Empty by default.
89+
90+
#export DRILLBIT_JAVA_OPTS="$DRILLBIT_JAVA_OPTS -Dproperty=value"
91+
92+
# Process priority (niceness) for the Drillbit when running as a daemon.
93+
# Defaults to 0.
94+
95+
#export DRILL_NICENESS=${DRILL_NICENESS:-0}
96+
97+
# Custom class path for Drill. In general, you should put your custom libraries into
98+
# your site directory's jars subfolder ($DRILL_HOME/conf/jars by default, but can be
99+
# customized with DRILL_CONF_DIR or the --config argument. But, if you must reference
100+
# jar files in other locations, you can add them here. These jars are added to the
101+
# Drill classpath after all Drill-provided jars. Empty by default.
102+
103+
# custom="/your/path/here:/your/second/path"
104+
# if [ -z "$DRILL_CLASSPATH" ]; then
105+
# export DRILL_CLASSPATH=${DRILL_CLASSPATH:$custom}
106+
# else
107+
# export DRILL_CLASSPATH="$custom"
108+
# fi
109+
110+
# Extension classpath for things like HADOOP, HBase and so on. Set as above.
111+
112+
# EXTN_CLASSPATH=...
113+
114+
# Note that one environment variable can't be set here: DRILL_CONF_DIR.
115+
# That variable tells Drill the location of this file, so this file can't
116+
# set it. Instead, you can set it in the environment, or using the
117+
# --config option of drillbit.sh or sqlline.
118+
119+
#-----------------------------------------------------------------------------
120+
# The following are "advanced" options seldom used except when diagnosing
121+
# complex issues.
122+
#
123+
# The prefix class path appears before any Drill-provided classpath entries.
124+
# Use it to override Drill jars with specialized versions.
125+
126+
#export DRILL_CLASSPATH_PREFIX=...
127+
128+
# Enable garbage collection logging in the Drillbit. Logging goes to
129+
# $DRILL_LOG_DIR/drillbit.gc. A value of 1 enables logging, all other values
130+
# (including the default unset value) disables logging.
131+
132+
#export SERVER_LOG_GC=${SERVER_LOG_GC:-1}
133+
134+
# JVM options when running the sqlline Drill client. For example, adjust the
135+
# JVM heap memory here. These are used ONLY in non-embedded mode; these
136+
# are client-only settings. (The Drillbit settings are used when Drill
137+
# is embedded.)
138+
139+
#export SQLLINE_JAVA_OPTS="-XX:MaxPermSize=512M"
140+
141+
# Arguments passed to sqlline (the Drill shell) at all times: whether
142+
# Drill is embedded in Sqlline or not.
143+
144+
#export DRILL_SHELL_JAVA_OPTS="..."
145+
146+
# Location Drill should use for temporary files, such as downloaded dynamic UDFs jars.
147+
# Set to "/tmp" by default.
148+
#
149+
# export DRILL_TMP_DIR="..."

0 commit comments

Comments
 (0)