scanner-research · swjz · Aug 16, 2017 · Aug 17, 2017 · Aug 18, 2017 · Aug 18, 2017
diff --git a/.Doxyfile b/.Doxyfile
@@ -811,7 +811,7 @@ RECURSIVE              = YES
 # Note that relative paths are relative to the directory from which doxygen is
 # run.
 
-EXCLUDE                = ./scanner/main.cpp
+EXCLUDE                = ./scanner/main.cpp ./python/scannerpy/build
 
 # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
 # directories that are symbolic links (a Unix file system feature) are excluded
@@ -838,7 +838,7 @@ EXCLUDE_PATTERNS       =
 # Note that the wildcards are matched against the file with absolute path, so to
 # exclude all test directories use the pattern */test/*
 
-EXCLUDE_SYMBOLS        =
+EXCLUDE_SYMBOLS        = scanner::internal
 
 # The EXAMPLE_PATH tag can be used to specify one or more files or directories
 # that contain example code fragments that are included (see the \include

diff --git a/.clang-format b/.clang-format
@@ -41,7 +41,7 @@ BreakStringLiterals: true
 ColumnLimit:     80
 CommentPragmas:  '^ IWYU pragma:'
 ConstructorInitializerAllOnOneLineOrOnePerLine: true
-ConstructorInitializerIndentWidth: 4
+ConstructorInitializerIndentWidth: 2
 ContinuationIndentWidth: 4
 Cpp11BracedListStyle: true
 DerivePointerAlignment: false
@@ -89,6 +89,6 @@ SpacesInCStyleCastParentheses: false
 SpacesInParentheses: false
 SpacesInSquareBrackets: false
 Standard:        Auto
-TabWidth:        8
+TabWidth:        2
 UseTab:          Never
 ...
diff --git a/.gdbinit b/.gdbinit
@@ -0,0 +1 @@
+handle SIG40 nostop noprint
diff --git a/.gitignore b/.gitignore
@@ -2,15 +2,20 @@
 # similar but **not quite** the same. If you add a file here, make sure that
 # the Docker build still works.
 
+dependencies.txt
+nvidia-docker-compose.yml
 .cache
 build
 **/*.pyc
 **/*.trace
 thirdparty
+!thidparty/resources
 !thirdparty/CMakeLists.txt
+!docker/**/thirdparty
 python/*.egg-info
-
+docker/**/deps.sh
 **/*.mp4
 **/*.mkv
 
-python/scannerpy/include
+python/scannerpy/include
+dist
diff --git a/.scanner.example.toml b/.scanner.example.toml
@@ -11,4 +11,6 @@ scanner_path = "/opt/scanner"
 #     bucket = "gcs-bucket"
 
 [network]
-    master = "localhost:5001"
+    master = "localhost"
+    master_port = "5001"
+    worker_port = "5002"
diff --git a/.travis.yml b/.travis.yml
@@ -5,17 +5,27 @@ language: cpp
 services:
 - docker
 
+addons:
+  apt:
+    packages:
+      - docker-ce
+
 env:
   global:
     - DOCKER_REPO=scannerresearch/scanner
     - [email protected]
     - DOCKER_USER=wcrichto
-    - secure: "Jf7f8UvpJ8yRA/2nnxaRb9jGrzWZvbRVRxzQtmcx+ltBpflmLFbXB6o4sySGETn5YrJe1g4VNWvkFBZIdLY0n+v4AQ/D1eMWtJT+kZLHihVGcxB3G7SUSOyEUaN1DzdHbaRW4VCSXQJWgK48amxlwnNfXtbNkSpiqIifDURBeHK1UT9+w1FKuTDZmEXv+ubDKjXvv1DuPdTtnolItDfrdtKLh5iXgTRvgXFanmNvydIcHqfz+72qJZJRCdTwUrA8FPq7jKNndftX9y2/dKuA/Xd/cP818iR57Z2PJXeR1c6ZmQ0RBTIZ8pdwG6bfix+48a5+aWlPJ3friA0G2b8psdZ/rQMxBbEClTI+BfnuJnYaIW8JREqmOBACBL7/4eo/Jifv1+vl+VhWHSicS6mWg2khCNAMPWj3cIA/bZrRlTmNlC06GkUgnHar/HhovGrowbZCGOxN0CNWWcuCGeLYfvPgPiI4JRU8eq0VqVhhqXZyPuS/RT1FoqZ4mMfNZG+RSIKHU56g5M4elakp/bZdu9lRoS4hr9VFpvPaLorOTOXPa3J+JxmB9OIP+NLxMIGjUhYnSYfTJ9/+Opxg2lWUOdqXzPZsyPyI+4RuIyyPh1w6jN/xD3EqQC55u5JlFlJ1WuNoyhx16QcKu1Eytxd2JxU6LutMw9WgQET53Q3EOVk=" # DOCKER_PASS
+    - secure: "DAwB02uem8/aIRNfHPknzqZCCmwDlJoQq6r3qE1CyUqkM9VMu6mQa/RBYMBwp3Upny6oI6yBlYJqqVr1rjZzI4by0j0IVhuhQPp2G5DJ/mVZ5PQSJ0bxJtCzm29hHo8hi+EgIk7r7YTcHOr3SRmplf0vM0WFuJplW5fJ6J/RbJSPPpnlTOIgp0Ry9B4eowLhi4b//TbwmtzEX6j4yKxWnDTJ6cV9xRVN1FoNd15PvpiBkOZ3dVGWtdUZm9K3Q3IumeyuemAoTSuKU3r3aqMSamdLf7pmYjpzTrYDT9CMbbEVjzr8M3Y5wFyjQsIdFKzYM/0jRBlUbKaLMhz7a/9vlkplBI954ufOqHiGAx3Cdk4jfkJ3hyucCQqYuQwHxWMmru3lV6jIZf+rYn6UBomHeNN1AIiikZ1EjvNDwY8iwGZPJbcZ42gn2mTaUVxsmwMD757AalTOzZo94+pdFJDjWY6y6kPQlZbdj8AT14bBHf8x8zF8EsZrh/WQMcRgzEU3BcjDE3RIqNgK38Sv1OzqNVTYy57PXbEMKMqfpAQx8FTcx/7NXBRrStrQ24pRZJA+wvkUefpyWZnayyMPvttzB4106GCOWAjSdJEyYQTnZMrXMRX4Qf+NHx6KHlQcExxpww/hSvewsh0JRrvMol5HKXJaZqnF6ZTqxeQdLUDdn3g=" # DOCKER_PASS
     - secure: "mJDNPmfRd3cyFNIhRg6TQo8ow5hOS+FC+DX6MLpM6Giv2nL1KkJD0fgnLqgiZeEKDOFHvGpGswGjYTAzNWYeocsneby88w58sNbueUE6PT74UqsbzrmvMhKh2xtEERkJFz5gKvdwc9a4nAvH5ejJW7OZ5OvSscR2vRBl4tUwTuA3czWbod6NwBC/sUrf0jTZhNgZWWWT1j2SLt379Q52Xmd5+ixAaSibctf8hCEMzE/lNsWprR1gO7H89eyLVRQc8VQh8msVtgo7QD/aapr+w96GDWWUAm6z5iNe8gLGZB9v4BtTm82XOV0iefNDfT0fwIhHrM6vqIdiiuMeYMteErT3rF55h0o8hJrP57lwT5u7hU/yxUhNTvtCdJTo8THJihXHbOT7LjxaF20SlJzjARSGZq9mYHeBgv9sGej2jF80iSn+xTAbalVFbc4hKumELnn/D7mGWRTjsGGTbTqNXvTQvxm/QjXmNusav5vKeRFwjbPGfb0iDSfhlZ7BHkR50OyskZ/R/69WjIFS9gBE2pvp4FlVpndKWkkkd4TVSgmnKCXxnQdeNS8We8NodvTBlTcvHFub8j2izKTZRU7FBXRgo9d+Gu+43nSRvjJUgwg/6D6Bou6boVHJo8ib9xzakVjojbJxehatj+dadpuVkG3tjULejPNg/qnjlJc10Pg=" # PYPI_PASS
+  matrix:
+    - BUILD_TYPE=cpu
+    - BUILD_TYPE=gpu
 
 install:
-- sudo apt-get install -y doxygen graphviz python-pip
-- pip install doxypypy twine
+- sudo apt-get update
+- sudo apt-get install -y doxygen graphviz
+- sudo pip install requests[security] --upgrade
+- sudo pip install doxypypy twine Sphinx
 
 script: ./scripts/travis-build.sh
 

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -19,8 +19,6 @@ project(Scanner)
 ###### Config options #####
 option(BUILD_CUDA "" ON)
 option(BUILD_TESTS "" ON)
-option(BUILD_SERVER "" OFF)
-option(BUILD_EXAMPLES "" ON)
 option(ENABLE_PROFILING "" OFF)
 
 if (BUILD_TESTS)
@@ -66,97 +64,7 @@ if (ENABLE_PROFILING)
   add_definitions(-DSCANNER_PROFILING)
 endif()
 
-###### Optional Dependencies #######
-if (BUILD_CUDA)
-  find_package(CUDA REQUIRED)
-  add_definitions(-DHAVE_CUDA)
-  include_directories(${CUDA_INCLUDE_DIRS})
-  if(COMPILER_SUPPORTS_CXX1Y)
-    set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -std=c++11")
-  endif()
-endif()
-
-if (BUILD_CUDA)
-  add_library(scanner_halide scanner/engine/halide_context.cpp)
-endif()
-
-###### Required Dependencies #######
-find_package(SaneProtobuf REQUIRED)
-find_package(FFmpeg REQUIRED)
-find_package(LibLZMA REQUIRED)
-find_package(OpenSSL REQUIRED)
-find_package(BZip2 REQUIRED)
-find_package(Boost COMPONENTS thread program_options regex python REQUIRED)
-find_package(GFlags REQUIRED)
-find_package(Glog REQUIRED)
-find_package(GoogleTest REQUIRED)
-find_package(CURL REQUIRED)
-find_package(Iconv REQUIRED)
-find_package(Storehouse REQUIRED CONFIG
-  PATHS "thirdparty/build/bin/storehouse")
-find_package(TinyToml REQUIRED)
-find_package(PythonLibs 2.7 EXACT REQUIRED)
-
-set(GTEST_INCLUDE_DIRS
-  "${THIRDPARTY_OUTPUT_PATH}/googletest/include")
-set(GTEST_LIBRARIES
-  "${THIRDPARTY_OUTPUT_PATH}/googletest/lib/libgtest.a")
-set(GTEST_LIB_MAIN
-  "${THIRDPARTY_OUTPUT_PATH}/googletest/lib/libgtest_main.a")
-
-set(SCANNER_LIBRARIES
-  "${PROTOBUF_LIBRARY}"
-  "${STOREHOUSE_LIBRARIES}"
-  "${FFMPEG_LIBRARIES}"
-  "-L/opt/ffmpeg-3.2.2/lib"
-  "-lswscale"
-  "${LIBLZMA_LIBRARIES}"
-  "${OPENSSL_LIBRARIES}"
-  "${BZIP2_LIBRARIES}"
-  "${PROXYGEN_LIBRARIES}"
-  "${FOLLY_LIBRARIES}"
-  "${Boost_LIBRARIES}"
-  "${GFLAGS_LIBRARIES}"
-  "${GLOG_LIBRARIES}"
-  "${CURL_LIBRARIES}"
-  "${ICONV_LIBRARIES}"
-  "${SCANNER_LIBRARIES}"
-  "${PYTHON_LIBRARIES}"
-  "-ljpeg"
-  "-lz"
-  "-ldl"
-  "-lgrpc++_unsecure -lgrpc -lgpr")
-
-include_directories(
-  "."
-  "${CMAKE_CURRENT_BINARY_DIR}" # for protobuf generated files
-  "${PROTOBUF_INCLUDE_DIRS}"
-  "${FFMPEG_INCLUDE_DIR}"
-  "${TINYTOML_INCLUDE_DIR}"
-  "${STOREHOUSE_INCLUDE_DIRS}"
-  "${OPENSSL_INCLUDE_DIR}"
-  "${Boost_INCLUDE_DIRS}"
-  "${GLOG_INCLUDE_DIRS}"
-  "${LIBLZMA_INCLUDE_DIRS}"
-  "${PYTHON_INCLUDE_DIRS}")
-
-if (BUILD_TESTS)
-  include_directories("${GTEST_INCLUDE_DIRS}")
-endif()
-
-if (BUILD_CUDA)
-  list(APPEND SCANNER_LIBRARIES
-    util_cuda
-    "${CUDA_LIBRARIES}"
-    "/usr/lib/x86_64-linux-gnu/libnvcuvid.so"
-    "-lcuda")
-endif()
-
-if (APPLE)
-  include_directories(
-    "/System/Library/Frameworks/Accelerate.framework/Versions/Current/Frameworks/vecLib.framework/Versions/Current/Headers/")
-elseif()
-endif()
+include(cmake/Dependencies.cmake)
 
 ###### Project code #######
 set(PROTO_FILES
@@ -200,6 +108,7 @@ add_library(scanner SHARED
   $<TARGET_OBJECTS:engine>
   $<TARGET_OBJECTS:video>
   $<TARGET_OBJECTS:util>
+  scanner/util/halide_context.cpp
   ${PROTO_SRCS}
   ${GRPC_PROTO_SRCS}
   ${STRUCK_SOURCES}
@@ -236,10 +145,6 @@ endforeach()
 
 add_subdirectory(stdlib)
 
-if (BUILD_EXAMPLES)
-  add_subdirectory(examples)
-endif()
-
 if (BUILD_TESTS)
   add_subdirectory(tests)
 endif()
diff --git a/Dockerfile b/Dockerfile
diff --git a/INSTALL.md b/INSTALL.md
@@ -0,0 +1,118 @@
+# Building Scanner
+
+*NOTE*: The following build instructions have only been tested on Ubuntu 16.04.
+
+There are five major steps to build and install Scanner:
+1 Install apt-get dependencies
+2 Install python pip dependencies
+3 Run deps.sh to download and install external dependencies
+4 Build Scanner
+5 Install scannerpy python package
+
+Scanner depends on the following *major* dependencies:
+
+* Python == 2.7
+* boost >= 1.63.0
+* ffmpeg >= 3.3.1
+* opencv >= 3.2.0
+* protobuf == 3.4.0
+* grpc == 1.7.2
+* caffe >= rc5 OR intel-caffe >= 1.0.6
+
+Scanner optionally requires:
+* CUDA >= 8.0
+
+Scanner provides a dependency script `deps.sh` to automatically install any or
+all of the *major* dependencies if they are not already installed. Each of these
+dependencies has a set of required system-level packages. If you need to install
+all or most of of these dependencies, run the 'All dependencies' apt-get command
+below. If you only need to install a few, we also provide apt-get commands for
+each package.
+
+## 1. apt-get Dependencies
+
+All dependencies
+```bash
+apt-get install \
+  build-essential \
+  cmake git libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev \
+  libswscale-dev unzip llvm clang libc++-dev libgflags-dev libgtest-dev \
+  libssl-dev libcurl3-dev liblzma-dev libeigen3-dev  \
+  libgoogle-glog-dev libatlas-base-dev libsuitesparse-dev libgflags-dev \
+  libx264-dev libopenjpeg-dev libxvidcore-dev \
+  libpng-dev libjpeg-dev libbz2-dev git python-pip wget \
+  libleveldb-dev libsnappy-dev libhdf5-serial-dev liblmdb-dev python-dev \
+  python-tk autoconf autogen libtool libtbb-dev libopenblas-dev \
+  liblapacke-dev swig yasm python2.7 cpio \
+  automake libass-dev libfreetype6-dev libsdl2-dev libtheora-dev libtool \
+  libva-dev libvdpau-dev libvorbis-dev libxcb1-dev libxcb-shm0-dev \
+  libxcb-xfixes0-dev mercurial pkg-config texinfo wget zlib1g-dev \
+  curl unzip
+```
+
+For gRPC
+```bash
+apt-get install \
+  build-essential autoconf automake libtool curl make g++ \
+  unzip clang libc++-dev libgflags-dev libgtest-dev unzip
+```
+
+For OpenCV
+```bash
+apt-get install \
+  build-essential cmake git libgtk2.0-dev pkg-config \
+  python-dev libtbb2 libtbb-dev libjpeg-dev libpng-dev libtiff-dev \
+  libjasper-dev libdc1394-22-dev
+```
+
+For FFMPEG
+```bash
+apt-get -y install \
+  build-essential autoconf automake build-essential cmake git libass-dev \
+  libfreetype6-dev \ libsdl2-dev libtheora-dev libtool libva-dev libvdpau-dev \
+  libvorbis-dev libxcb1-dev libxcb-shm0-dev libxcb-xfixes0-dev mercurial \
+  pkg-config texinfo wget zlib1g-dev
+```
+
+## 2. Python Pip Dependencies
+
+Scanner depends on several python packages installable via pip. From the 
+top-level directory, run:
+
+```bash
+pip install -r requirements.txt
+```
+
+## 3. Run deps.sh
+
+To install or specify where your *major* dependencies are, from the top-level
+directory run:
+
+```bash
+bash ./deps.sh
+```
+
+This script will query you for each major
+dependency and install those that are not already installed. By default,
+it will install the dependencies to a local directory inside the scanner repo
+(it will not install system-wide).
+
+## 4. Build Scanner
+
+Run the following commands from the top-level directory:
+```bash
+mkdir build
+cd build
+cmake ..
+make -j
+```
+
+## 5. Install scannerpy python package
+
+Run the following commands from the top-level directory:
+```bash
+python python/setup.py bdist_wheel
+pip install dist/scannerpy-0.1.13-py2-none-any.whl
+```
+
+Congratulations! You've installed the scannerpy package.