Skip to content
This repository was archived by the owner on Jun 19, 2025. It is now read-only.

Commit 1385cf6

Browse files
lissyxlissyx
authored
Merge pull request #1582 from lissyx/update-dockerfile
Adding TaskCluster testing of docker build
2 parents 8a5a000 + 1eb176b commit 1385cf6

File tree

3 files changed

+107
-22
lines changed

3 files changed

+107
-22
lines changed

‎Dockerfile‎

Lines changed: 42 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -33,18 +33,19 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
3333
pkg-config \
3434
libsox-dev
3535

36-
36+
# Install NCCL 2.2
37+
RUN apt-get install -qq -y --allow-downgrades --allow-change-held-packages libnccl2=2.2.13-1+cuda9.0 libnccl-dev=2.2.13-1+cuda9.0
3738

3839

3940
# Install Bazel
4041
RUN apt-get install -y openjdk-8-jdk
4142
# Use bazel 0.11.1 cause newer bazel fails to compile TensorFlow (https://github.com/tensorflow/tensorflow/issues/18450#issuecomment-381380000)
4243
RUN apt-get install -y --no-install-recommends bash-completion g++ zlib1g-dev
43-
RUN curl -LO "https://github.com/bazelbuild/bazel/releases/download/0.11.1/bazel_0.11.1-linux-x86_64.deb"
44+
RUN curl -LO "https://github.com/bazelbuild/bazel/releases/download/0.15.2/bazel_0.15.2-linux-x86_64.deb"
4445
RUN dpkg -i bazel_*.deb
4546

4647
# Install CUDA CLI Tools
47-
RUN apt-get install -y cuda-command-line-tools-9-0
48+
RUN apt-get install -qq -y cuda-command-line-tools-9-0
4849

4950
# Install pip
5051
RUN wget https://bootstrap.pypa.io/get-pip.py && \
@@ -61,7 +62,7 @@ RUN wget https://bootstrap.pypa.io/get-pip.py && \
6162
# Clone TensoFlow from Mozilla repo
6263
RUN git clone https://github.com/mozilla/tensorflow/
6364
WORKDIR /tensorflow
64-
RUN git checkout r1.6
65+
RUN git checkout r1.11
6566

6667

6768
# GPU Environment Setup
@@ -70,9 +71,11 @@ ENV CUDA_TOOLKIT_PATH /usr/local/cuda
7071
ENV CUDA_PKG_VERSION 9-0=9.0.176-1
7172
ENV CUDA_VERSION 9.0.176
7273
ENV TF_CUDA_VERSION 9.0
73-
ENV TF_CUDNN_VERSION 7.2.1
74+
ENV TF_CUDNN_VERSION 7.3.0
7475
ENV CUDNN_INSTALL_PATH /usr/lib/x86_64-linux-gnu/
7576
ENV TF_CUDA_COMPUTE_CAPABILITIES 6.0
77+
ENV TF_NCCL_VERSION 2.2.13
78+
# ENV NCCL_INSTALL_PATH /usr/lib/x86_64-linux-gnu/
7679

7780
# Common Environment Setup
7881
ENV TF_BUILD_CONTAINER_TYPE GPU
@@ -92,6 +95,14 @@ ENV TF_NEED_OPENCL 0
9295
ENV TF_CUDA_CLANG 0
9396
ENV TF_NEED_MKL 0
9497
ENV TF_ENABLE_XLA 0
98+
ENV TF_NEED_AWS 0
99+
ENV TF_NEED_KAFKA 0
100+
ENV TF_NEED_NGRAPH 0
101+
ENV TF_DOWNLOAD_CLANG 0
102+
ENV TF_NEED_TENSORRT 0
103+
ENV TF_NEED_GDR 0
104+
ENV TF_NEED_VERBS 0
105+
ENV TF_NEED_OPENCL_SYCL 0
95106
ENV PYTHON_BIN_PATH /usr/bin/python2.7
96107
ENV PYTHON_LIB_PATH /usr/lib/python2.7/dist-packages
97108

@@ -112,8 +123,12 @@ RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \
112123
>>/etc/bazel.bazelrc
113124

114125
# Put cuda libraries to where they are expected to be
115-
RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1
116-
RUN cp /usr/include/cudnn.h /usr/local/cuda/include/cudnn.h
126+
RUN mkdir /usr/local/cuda/lib && \
127+
ln -s /usr/lib/x86_64-linux-gnu/libnccl.so.2 /usr/local/cuda/lib/libnccl.so.2 && \
128+
ln -s /usr/include/nccl.h /usr/local/cuda/include/nccl.h && \
129+
ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 && \
130+
ln -s /usr/include/cudnn.h /usr/local/cuda/include/cudnn.h
131+
117132

118133
# Set library paths
119134
ENV LD_LIBRARY_PATH $LD_LIBRARY_PATH:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/lib/x86_64-linux-gnu/:/usr/local/cuda/lib64/stubs/
@@ -140,6 +155,9 @@ RUN ln -s /DeepSpeech/native_client /tensorflow
140155

141156
WORKDIR /tensorflow
142157

158+
# Fix for not found script https://github.com/tensorflow/tensorflow/issues/471
159+
RUN ./configure
160+
143161
# Using CPU optimizations:
144162
# -mtune=generic -march=x86-64 -msse -msse2 -msse3 -msse4.1 -msse4.2 -mavx.
145163
# Adding --config=cuda flag to build using CUDA.
@@ -153,24 +171,27 @@ RUN bazel build -c opt --copt=-O3 --copt="-D_GLIBCXX_USE_CXX11_ABI=0" --copt=-mt
153171
# Build DeepSpeech
154172
RUN bazel build --config=monolithic --config=cuda -c opt --copt=-O3 --copt="-D_GLIBCXX_USE_CXX11_ABI=0" --copt=-mtune=generic --copt=-march=x86-64 --copt=-msse --copt=-msse2 --copt=-msse3 --copt=-msse4.1 --copt=-msse4.2 --copt=-mavx --copt=-fvisibility=hidden //native_client:libdeepspeech.so //native_client:generate_trie --verbose_failures --action_env=LD_LIBRARY_PATH=${LD_LIBRARY_PATH}
155173

156-
157-
# Build TF pip package
158-
RUN bazel build --config=opt --config=cuda --copt="-D_GLIBCXX_USE_CXX11_ABI=0" --copt=-mtune=generic --copt=-march=x86-64 --copt=-msse --copt=-msse2 --copt=-msse3 --copt=-msse4.1 --copt=-msse4.2 --copt=-mavx //tensorflow/tools/pip_package:build_pip_package --verbose_failures --action_env=LD_LIBRARY_PATH=${LD_LIBRARY_PATH}
159-
160-
# Fix for not found script https://github.com/tensorflow/tensorflow/issues/471
161-
RUN ./configure
162-
163-
# Build wheel
164-
RUN bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg
165-
166-
# Install tensorflow from our custom wheel
167-
RUN pip install /tmp/tensorflow_pkg/*.whl
174+
###
175+
### Using TensorFlow upstream should work
176+
###
177+
# # Build TF pip package
178+
# RUN bazel build --config=opt --config=cuda --copt="-D_GLIBCXX_USE_CXX11_ABI=0" --copt=-mtune=generic --copt=-march=x86-64 --copt=-msse --copt=-msse2 --copt=-msse3 --copt=-msse4.1 --copt=-msse4.2 --copt=-mavx //tensorflow/tools/pip_package:build_pip_package --verbose_failures --action_env=LD_LIBRARY_PATH=${LD_LIBRARY_PATH}
179+
#
180+
# # Build wheel
181+
# RUN bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg
182+
#
183+
# # Install tensorflow from our custom wheel
184+
# RUN pip install /tmp/tensorflow_pkg/*.whl
168185

169186
# Copy built libs to /DeepSpeech/native_client
170187
RUN cp /tensorflow/bazel-bin/native_client/libctc_decoder_with_kenlm.so /DeepSpeech/native_client/ \
171188
&& cp /tensorflow/bazel-bin/native_client/generate_trie /DeepSpeech/native_client/ \
172189
&& cp /tensorflow/bazel-bin/native_client/libdeepspeech.so /DeepSpeech/native_client/
173-
190+
191+
# Install TensorFlow
192+
WORKDIR /DeepSpeech/
193+
RUN pip install tensorflow-gpu==1.11.0rc2
194+
174195

175196
# Make DeepSpeech and install Python bindings
176197
ENV TFDIR /tensorflow
@@ -197,6 +218,5 @@ RUN rm -rf kenlm \
197218
&& cmake .. \
198219
&& make -j 4
199220

200-
201221
# Done
202-
WORKDIR /DeepSpeech
222+
WORKDIR /DeepSpeech

‎taskcluster/docker-build-base.tyml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
$if: '(event.event != "push") && (event.event != "tag")'
2+
then:
3+
taskId: ${taskcluster.taskId}
4+
provisionerId: ${taskcluster.docker.provisionerId}
5+
workerType: ${taskcluster.docker.workerType}
6+
taskGroupId: ${taskcluster.taskGroupId}
7+
schedulerId: ${taskcluster.schedulerId}
8+
created: { $fromNow: '0 sec' }
9+
deadline: { $fromNow: '1 day' }
10+
expires: { $fromNow: '7 days' }
11+
12+
extra:
13+
github:
14+
{ $eval: taskcluster.github_events.pull_request }
15+
16+
routes:
17+
- "notify.irc-channel.${notifications.irc}.on-exception"
18+
- "notify.irc-channel.${notifications.irc}.on-failed"
19+
20+
scopes: [
21+
"queue:route:notify.irc-channel.*"
22+
]
23+
24+
payload:
25+
maxRunTime: { $eval: to_int(build.maxRunTime) }
26+
image: "ubuntu:14.04"
27+
features:
28+
dind: true
29+
30+
env:
31+
DOCKER_API_VERSION: 1.18
32+
33+
command:
34+
- "/bin/bash"
35+
- "--login"
36+
- "-cxe"
37+
- $let:
38+
dockerfile: { $eval: strip(str(build.dockerfile)) }
39+
in: >
40+
${aptEc2Mirrors} &&
41+
apt-get -qq update && apt-get -qq -y install git wget pkg-config apt-transport-https ca-certificates curl software-properties-common &&
42+
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - &&
43+
add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" &&
44+
apt-get -qq update && apt-get -qq -y install docker-ce && mkdir -p /opt/deepspeech &&
45+
git clone --quiet ${event.head.repo.url} /opt/deepspeech && cd /opt/deepspeech && git checkout --quiet ${event.head.sha} &&
46+
curl -fsSL "https://raw.githubusercontent.com/${event.head.user.login}/${event.head.repo.name}/${event.head.sha}/${dockerfile}" > ${dockerfile} &&
47+
docker build .
48+
49+
artifacts:
50+
"public":
51+
type: "directory"
52+
path: "/tmp/artifacts/"
53+
expires: { $fromNow: '7 days' }
54+
55+
metadata:
56+
name: ${build.metadata.name}
57+
description: ${build.metadata.description}
58+
owner: ${event.head.user.email}
59+
source: ${event.head.repo.url}

‎taskcluster/docker-image-build.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
build:
2+
template_file: docker-build-base.tyml
3+
dockerfile: "Dockerfile"
4+
metadata:
5+
name: "DeepSpeech Docker build"
6+
description: "Testing |docker build| of DeepSpeech"

0 commit comments

Comments
 (0)