Removed deprecated DB mode

The DB mode is no longer first choice for large-scale evaluations.
Use the json mode to process the resulting data.

Author: Erik Derr

README.md

@@ -99,10 +99,9 @@ Analysis results can be written in different formats.
 <ol>
     <li> the JSON format (-j switch), creates subfolders in the specified directory following the app package, i.e. *com.foo* will create *com/foo* subfolders.
         This is useful when coping with a large number of apps. For detailed information about the information stored, please refer to the <a href="https://github.com/reddr/LibScout/wiki#json-output-format-specification">JSON output specification</a>.</li>
-    <li> the <b>serialization</b> option (-s switch) writes stat files per app to disk that can be used with the database module to create a SQLite database (the DB structure can be found in class
-    <b>de.infsec.tpl.stats.SQLStats</b></li>
+    <li> the <b>serialization</b> option (-s switch) writes stat files per app to disk (deprecated)</li>
 </ol>
-<pre>java -jar LibScout.jar -o match -p <i>path_to_profiles</i> [-a <i>android_sdk_jar</i>] [-u] [-j <i>json_dir</i>] [-s <i>stats_dir</i>] [-m] [-d <i>log_dir</i>] <i>path_to_app(s)</i>  </pre>
+<pre>java -jar LibScout.jar -o match -p <i>path_to_profiles</i> [-a <i>android_sdk_jar</i>] [-u] [-j <i>json_dir</i>] [-m] [-d <i>log_dir</i>] <i>path_to_app(s)</i>  </pre>
 
 ### Library API compatibility analysis (-o lib_api_analysis)
 
@@ -122,12 +121,7 @@ For your convenience use the library scraper. Analysis results are written to di
 This mode is an extension to the match mode. It first detects library versions in the provided apps and conducts a library usage analysis (-u is implied). In addition, it requires library API compat data (via the -l switch) as generated in the <i>lib_api_analysis</i> mode . Based on the lib API usage in the app and the compat info, LibScout determines the highest version that is still compatible to the set of used lib APIs.<br>
 <b>Note:</b> The new implementation still lacks some features, e.g. the results are currently logged but not yet written to json. See the code comments for more information.
 
-<pre>java -jar LibScout.jar -o updatability [-a <i>android_sdk_jar</i>] [-j <i>json_dir</i>] -l <i>lib_api_data_dir</i> <i>path_to_app(s)<i></pre>
-
-### Database Generator (-o db)
-
-Generates a SQLite database from library profiles and serialized app stats:<br>
-<pre>java -jar LibScout.jar -o db -p <i>path_to_profiles</i> -s <i>path_to_app_stats</i> </pre>
+<pre>java -jar LibScout.jar -o updatability [-a <i>android_sdk_jar</i>] [-j <i>json_dir</i>] -l <i>lib_api_data_dir</i> <i>path_to_app(s)</i></pre>
 
 
 ## Scientific Publications

build.gradle

@@ -37,7 +37,6 @@ dependencies {
     provided 'org.smali:dexlib2:2.2.6'
 
     provided files('lib/android-xml.jar')  // axml relevant classes from Android SDK
-    provided files('lib/sqlite-jdbc-3.7.15-SNAPSHOT.jar')  // loaded via reflection, at that time, the only working version
 }
 
 jar {

lib/sqlite-jdbc-3.7.15-SNAPSHOT.jar

@@ -40,7 +40,6 @@
 import ch.qos.logback.classic.joran.JoranConfigurator;
 import ch.qos.logback.core.joran.spi.JoranException;
 import ch.qos.logback.core.util.StatusPrinter;
-import de.infsec.tpl.stats.SQLStats;
 import de.infsec.tpl.utils.Utils;
 import de.infsec.tpl.profile.LibProfile;
 
@@ -119,7 +118,7 @@ public static void main(String[] args) {
 			 * one time data loading
 			 */
 
-			if (LibScoutConfig.opMatch() || LibScoutConfig.opDB() || LibScoutConfig.opUpdatability())
+			if (LibScoutConfig.opMatch() || LibScoutConfig.opUpdatability())
 				profiles = Profile.loadLibraryProfiles(LibScoutConfig.profilesDir);
 
 			if (LibScoutConfig.opUpdatability())
@@ -135,12 +134,6 @@ public static void main(String[] args) {
 		 * choose mode of operation
 		 */
 
-		if (LibScoutConfig.opDB()) {
-			// generate SQLite DB from app stats
-			SQLStats.stats2DB(profiles);
-			System.exit(0);
-		}
-
 		// process input files, either library files or apps
 		for (File inputFile: inputFiles) {
 			try {
@@ -207,13 +200,13 @@ else if (cmd.hasOption(CliArgs.ARG_LOG_DIR)) {
 			}
 
 			// path to LibScout.toml
-			if (checkOptionalUse(cmd, CliArgs.ARG_CONFIG, LibScoutConfig.OpMode.PROFILE, LibScoutConfig.OpMode.MATCH, LibScoutConfig.OpMode.LIB_API_ANALYSIS, LibScoutConfig.OpMode.DB, LibScoutConfig.OpMode.UPDATABILITY)) {
+			if (checkOptionalUse(cmd, CliArgs.ARG_CONFIG, LibScoutConfig.OpMode.PROFILE, LibScoutConfig.OpMode.MATCH, LibScoutConfig.OpMode.LIB_API_ANALYSIS, LibScoutConfig.OpMode.UPDATABILITY)) {
 				LibScoutConfig.libScoutConfigFileName = cmd.getOptionValue(CliArgs.ARG_CONFIG);
 				LibScoutConfig.checkIfValidFile(LibScoutConfig.libScoutConfigFileName);
 			}
 
 			// profiles dir option, if provided without argument output is written to default dir
-			if (checkOptionalUse(cmd, CliArgs.ARG_PROFILES_DIR, LibScoutConfig.OpMode.PROFILE, LibScoutConfig.OpMode.MATCH, LibScoutConfig.OpMode.DB, LibScoutConfig.OpMode.UPDATABILITY)) {
+			if (checkOptionalUse(cmd, CliArgs.ARG_PROFILES_DIR, LibScoutConfig.OpMode.PROFILE, LibScoutConfig.OpMode.MATCH, LibScoutConfig.OpMode.UPDATABILITY)) {
 				File profilesDir = new File(cmd.getOptionValue(CliArgs.ARG_PROFILES_DIR));
 				if (profilesDir.exists() && !profilesDir.isDirectory())
 					throw new ParseException("Profiles directory " + profilesDir + " already exists and is not a directory");
@@ -252,7 +245,7 @@ else if (cmd.hasOption(CliArgs.ARG_LOG_DIR)) {
 			}
 
 			// enable/disable generation of stats with optional stats directory
-			if (checkOptionalUse(cmd, CliArgs.ARG_STATS_DIR, LibScoutConfig.OpMode.MATCH, LibScoutConfig.OpMode.DB)) {
+			if (checkOptionalUse(cmd, CliArgs.ARG_STATS_DIR, LibScoutConfig.OpMode.MATCH)) {
 				LibScoutConfig.generateStats = true;
 
 				if (cmd.getOptionValue(CliArgs.ARG_STATS_DIR) != null) {   // stats dir provided?
@@ -292,51 +285,50 @@ else if (cmd.hasOption(CliArgs.ARG_LOG_DIR)) {
 			 *  - in profile mode pass *one* library (since it is linked to lib description file)
 			 *  - in match mode pass one application file or one directory (including apks)
 			 */
-			if (!(LibScoutConfig.opDB())) {
-				inputFiles = new ArrayList<File>();
 
-				if (LibScoutConfig.opLibApiAnalysis()) {
-					// we require a directory including library packages/descriptions
-					for (String path: cmd.getArgs()) {
-						File dir = new File(path);
+			inputFiles = new ArrayList<File>();
 
-						if (dir.isDirectory())
-							inputFiles.add(dir);
-					}
+			if (LibScoutConfig.opLibApiAnalysis()) {
+				// we require a directory including library packages/descriptions
+				for (String path: cmd.getArgs()) {
+					File dir = new File(path);
 
-					if (inputFiles.isEmpty()) {
-						throw new ParseException("You have to provide at least one directory that includes a library package and description");
-					}
-				} else {
-					String[] fileExts = LibScoutConfig.opMatch() || LibScoutConfig.opUpdatability() ? new String[]{"apk"} : new String[]{"jar", "aar"};
-
-					for (String inputFile : cmd.getArgs()) {
-						File arg = new File(inputFile);
-
-						if (arg.isDirectory()) {
-							inputFiles.addAll(Utils.collectFiles(arg, fileExts));
-						} else if (arg.isFile()) {
-							if (arg.getName().endsWith("." + fileExts[0]))
-								inputFiles.add(arg);
-							else if (fileExts.length > 1 && arg.getName().endsWith("." + fileExts[1]))
-								inputFiles.add(arg);
-							else
-								throw new ParseException("File " + arg.getName() + " is no valid ." + Utils.join(Arrays.asList(fileExts), "/") + " file");
-						} else {
-							throw new ParseException("Argument " + inputFile + " is no valid file or directory!");
-						}
-					}
+					if (dir.isDirectory())
+						inputFiles.add(dir);
+				}
 
-					if (inputFiles.isEmpty()) {
-						if (LibScoutConfig.opProfile())
-							throw new ParseException("No libraries (jar|aar files) found to profile in "  + cmd.getArgList());
+				if (inputFiles.isEmpty()) {
+					throw new ParseException("You have to provide at least one directory that includes a library package and description");
+				}
+			} else {
+				String[] fileExts = LibScoutConfig.opMatch() || LibScoutConfig.opUpdatability() ? new String[]{"apk"} : new String[]{"jar", "aar"};
+
+				for (String inputFile : cmd.getArgs()) {
+					File arg = new File(inputFile);
+
+					if (arg.isDirectory()) {
+						inputFiles.addAll(Utils.collectFiles(arg, fileExts));
+					} else if (arg.isFile()) {
+						if (arg.getName().endsWith("." + fileExts[0]))
+							inputFiles.add(arg);
+						else if (fileExts.length > 1 && arg.getName().endsWith("." + fileExts[1]))
+							inputFiles.add(arg);
 						else
-							throw new ParseException("No apk files found in " + cmd.getArgList());
-					} else if (inputFiles.size() > 1 && LibScoutConfig.opProfile())
-						throw new ParseException("You have to provide a path to a single library file or a directory incl. a single lib file");
+							throw new ParseException("File " + arg.getName() + " is no valid ." + Utils.join(Arrays.asList(fileExts), "/") + " file");
+					} else {
+						throw new ParseException("Argument " + inputFile + " is no valid file or directory!");
+					}
 				}
+
+				if (inputFiles.isEmpty()) {
+					if (LibScoutConfig.opProfile())
+						throw new ParseException("No libraries (jar|aar files) found to profile in "  + cmd.getArgList());
+					else
+						throw new ParseException("No apk files found in " + cmd.getArgList());
+				} else if (inputFiles.size() > 1 && LibScoutConfig.opProfile())
+					throw new ParseException("You have to provide a path to a single library file or a directory incl. a single lib file");
 			}
-			
+
 		} catch (ParseException e) {
 			System.err.println("Command line parsing failed:\n  " + e.getMessage() + "\n");
 			usage();

src/de/infsec/tpl/TplCLI.java

@@ -30,9 +30,6 @@ public enum OpMode {
         // match lib profiles in provided apps
         MATCH("match", "[options] path_to_app(dir)"),
 
-        // build sqlite database from app stat files
-        DB("db", "-p path_to_profiles -s path_to_stats"),
-
         // analyzes library api stability (api additions, removals, changes)
         LIB_API_ANALYSIS("lib_api_analysis", "path_to_lib_sdks"),
 
@@ -63,7 +60,6 @@ public static String getOpModeString() {
     public static OpMode opmode = null;
     public static boolean opMatch() { return OpMode.MATCH.equals(opmode); }
     public static boolean opProfile() { return OpMode.PROFILE.equals(opmode); }
-    public static boolean opDB() { return OpMode.DB.equals(opmode); }
     public static boolean opLibApiAnalysis() { return OpMode.LIB_API_ANALYSIS.equals(opmode); }
     public static boolean opUpdatability() { return OpMode.UPDATABILITY.equals(opmode); }