tools

Author: phi-dbq

bin/totgen.sh

@@ -162,7 +162,10 @@ exec "${_spark_shell}" \
      --master "local[4]" \
      --conf spark.app.name="[drgscl]::spark-shell" \
      --conf spark.eventLog.enabled=false \
+     --conf spark.driver.memory=10g \
+     --conf spark.executor.memory=10g \
      --jars "${_submit_jars}" \
+     --verbose \
      $@
 
 _EXEC_SCRIPT_EOF_

build.sbt

@@ -1,7 +1,7 @@
 // Your sbt build file. Guides on how to write one can be found at
 // http://www.scala-sbt.org/0.13/docs/index.html
 
-val sparkVer = sys.props.getOrElse("spark.version", "2.1.1")
+val sparkVer = sys.props.getOrElse("spark.version", "2.2.0")
 val sparkBranch = sparkVer.substring(0, 3)
 val defaultScalaVer = sparkBranch match {
   case "2.0" => "2.11.8"
@@ -19,7 +19,7 @@ scalaVersion := scalaVer
 spName := "databricks/spark-deep-learning"
 
 // Don't forget to set the version
-version := s"0.1.0-spark$sparkBranch"
+version := s"0.2.0-spark$sparkBranch"
 
 // All Spark Packages need a license
 licenses := Seq("Apache-2.0" -> url("http://opensource.org/licenses/Apache-2.0"))

linter.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+_bsd_="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+if [[ $# -gt 1 ]]; then
+	target_files=(${@})
+else
+	target_files=($(git diff --name-only upstream/master HEAD))
+fi
+
+echo "${target_files[@]}"
+pushd "${_bsd_}"
+exec prospector --profile ${_bsd_}/prospector.yaml "${target_files[@]}"
+popd

project/GenClasspathPlugin.scala

@@ -0,0 +1,78 @@
+package sbtgenclasspath
+
+import sbt._, Keys._
+import sbtsparkpackage.SparkPackagePlugin.autoImport._
+import libdeps.LibVers._
+
+object GenClasspathPlugin extends sbt.AutoPlugin {
+
+  object autoImport {
+
+    lazy val genClasspath = taskKey[Unit]("Build runnable script with classpath")
+    lazy val extraSparkSubmitModules = settingKey[Seq[ModuleID]]("Additional spark submit jar dependencies")
+
+    lazy val genClasspathSettings: Seq[Def.Setting[_]] = Seq(
+      
+      extraSparkSubmitModules := Seq.empty[ModuleID],
+
+      genClasspath := {
+        import java.io.PrintWriter
+        
+        val sbtPathRoot = baseDirectory.value / ".sbt.paths"
+        sbtPathRoot.mkdirs()
+
+        def writeClasspath(cpType: String)(R: => String): Unit = {
+          val fout = new PrintWriter((sbtPathRoot / s"SBT_${cpType}_CLASSPATH").toString)
+          println(s"Building ${cpType} classpath for current project")
+          try fout.write(R) finally fout.close()
+        }
+
+        writeClasspath("RUNTIME") {
+          (fullClasspath in Runtime).value.files.map(_.toString).mkString(":")
+        }
+
+        writeClasspath("SPARK_PACKAGE") {
+          import scala.util.matching.Regex
+          val patt = s"(.+?)/(.+?):(.+?)(-s_${scalaMajorVer})?".r
+          val pkgs = (spDependencies.value).map { _ match {
+            case patt(orgName, pkgName, pkgVer, stem, _*) =>
+              if (null != stem) {
+                println(s"org ${orgName}, pkg ${pkgName}, ver ${pkgVer}, ${stem}")
+                s"${pkgName}-${pkgVer}${stem}.jar"
+              } else {
+                println(s"org ${orgName}, pkg ${pkgName}, ver ${pkgVer}")
+                s"${pkgName}-${pkgVer}.jar"
+              }
+          }}.toSet
+
+          // TODO: not knowing the proper way, I just fall back to Regex
+          val extraSpModIds = (extraSparkSubmitModules in Compile).value.flatMap { mod =>
+            //"com.typesafe.scala-logging:scala-logging-api:2.1.2"
+            // scala-logging-api_2.11-2.1.2.jar
+            val patt = s"(.+?):(.+?):(.+?)".r
+            mod.toString match {
+              case patt(orgName, pkgName, pkgVer) => 
+                Seq(s"${pkgName}_${scalaMajorVer}-${pkgVer}.jar", s"${pkgName}-${pkgVer}.jar")
+            }
+          }.toSet
+
+          (fullClasspath in Compile).value.files.filter { cpFile =>
+            val cpName = cpFile.getName
+            println(cpName)
+            (pkgs contains cpName) || (extraSpModIds contains cpName)
+          }.map(_.toString).mkString(":")
+        }
+      }
+    )
+  }
+  import autoImport._
+  
+  override def requires = sbt.plugins.JvmPlugin
+
+  // This plugin is automatically enabled for projects which are JvmPlugin.
+  override def trigger = allRequirements
+
+  // a group of settings that are automatically added to projects.
+  override val projectSettings =
+    inConfig(Compile)(genClasspathSettings) ++ inConfig(Test)(genClasspathSettings)
+}

project/LibDeps.scala

@@ -0,0 +1,30 @@
+package libdeps
+
+/**
+ ======================================================
+ * Build parameters
+ ======================================================
+ */
+object LibVers {
+
+  lazy val sparkVer = sys.props.getOrElse("spark.version", "2.1.1")
+  lazy val sparkBranch = sparkVer.substring(0, 3)
+  lazy val defaultScalaVer = sparkBranch match {
+    case "2.0" => "2.11.11"
+    case "2.1" => "2.11.11"
+    case "2.2" => "2.11.11"
+    case _ => throw new IllegalArgumentException(s"Unsupported Spark version: $sparkVer.")
+  }
+
+  lazy val scalaVer = sys.props.getOrElse("scala.version", defaultScalaVer)
+  lazy val scalaMajorVer = scalaVer.substring(0, scalaVer.indexOf(".", scalaVer.indexOf(".") + 1))
+
+  lazy val protobufVer = "3.3.1"  // use protocol buffer, will be shaded
+  lazy val tensorflowVer = "1.2.1"
+  lazy val ammVer = "0.9.9"
+ 
+  lazy val defaultScalaTestVer = scalaVer match {
+    case s if s.startsWith("2.10") => "2.0"
+    case s if s.startsWith("2.11") => "2.2.6" // scalatest_2.11 does not have 2.0 published
+  }
+}

prospector.yaml

@@ -0,0 +1,39 @@
+strictness: high
+test-warnings: True
+doc-warnings: false
+
+ignore-paths:
+  - docs
+  - spark-warehouse
+  - cover
+
+max-line-length: 100
+
+pep8:
+  run: true
+  disable: 
+    - N802
+    - N803
+    - N806
+    - E302
+
+pylint:
+  run: true
+  disable:
+    - too-many-instance-attributes
+    - cyclic-import
+    - len-as-condition
+    - invalid-name
+    - no-else-return
+    - no-self-use
+    - import-error
+    - protected-access
+    - reimported
+
+mccabe:
+  disable:
+    - MC0001
+
+pyroma:
+  run: true
+