New Library API analysis

- parses library jar|aar files directly
- improved public interface extractor
- json export of analysis results per lib

Author: reddr

logging/logback.xml

@@ -13,6 +13,9 @@
 	<logger name="de.infsec.tpl.LibCodeUsage" level="debug"/> 
 	<logger name="de.infsec.tpl.eval.LibraryApiAnalysis" level="info"/>
 
+	<logger name="de.infsec.tpl.modules.libapi" level="info"/>
+
+
 	<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
 		<encoder>
 			<pattern>%d{HH:mm:ss} %-5level %-25logger{0} : %msg%n</pattern>

src/de/infsec/tpl/TplCLI.java

@@ -20,6 +20,7 @@
 import java.util.Collections;
 import java.util.List;
 
+import de.infsec.tpl.modules.libapi.LibraryApiAnalysis;
 import org.apache.commons.cli.BasicParser;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.CommandLineParser;
@@ -36,7 +37,6 @@
 import ch.qos.logback.core.util.StatusPrinter;
 import de.infsec.tpl.stats.SQLStats;
 import de.infsec.tpl.utils.Utils;
-import de.infsec.tpl.eval.LibraryApiAnalysis;
 import de.infsec.tpl.profile.LibProfile;
 
 
@@ -49,7 +49,7 @@ public class TplCLI {
      *    -          PROFILE:  generate library profiles from original lib SDKs and descriptions
      *    -            MATCH:  match lib profiles in provided apps
      *    -               DB:  build sqlite database from app stat files
-     *    - LIB_API_ANALYSIS:  analyzes library api robustness (api additions, removals, changes)
+     *    - LIB_API_ANALYSIS:  analyzes library api stability (api additions, removals, changes)
      */
 	public static  enum OpMode {PROFILE, MATCH, DB, LIB_API_ANALYSIS};
 
@@ -110,7 +110,7 @@ public static class CliOptions {
 	private static final String USAGE_PROFILE = TOOLNAME + " --opmode profile -a <path-to-android.jar> -x <path-to-lib-desc> <options> $lib.[jar|aar]";
 	private static final String USAGE_MATCH = TOOLNAME + " --opmode match -a <path-to-android.jar> <options> $path-to-app(-dir)";
 	private static final String USAGE_DB = TOOLNAME + " --opmode db -p <path-to-profiles> -s <path-to-stats>";
-	private static final String USAGE_LIB_API_ANALYSIS = TOOLNAME + " --opmode lib_api_analysis -p <path-to-profiles> -j <output-dir>";
+	private static final String USAGE_LIB_API_ANALYSIS = TOOLNAME + " --opmode lib_api_analysis -a <path-to-android.jar> $path-to-lib-sdks";
 
 	private static ArrayList<File> inputFiles;
 	private static File libraryDescription = null;
@@ -124,7 +124,6 @@ public static void main(String[] args) {
 		// initialize logback
 		initLogging();
 
-
 		List<LibProfile> profiles = null;
 
 		// TODO MODE = LIB_UPDATABILITY
@@ -142,10 +141,6 @@ public static void main(String[] args) {
 				break;
 
 			case LIB_API_ANALYSIS:
-			    profiles = loadLibraryProfiles();
-				new LibraryApiAnalysis().run(profiles);
-				System.exit(0);
-
 			case PROFILE:
 		}
 
@@ -156,7 +151,10 @@ public static void main(String[] args) {
 					new LibraryIdentifier(inputFile).identifyLibraries(profiles);
 
 				} else if (CliOptions.opmode.equals(OpMode.PROFILE)) {
-					new LibraryProfiler(inputFile, libraryDescription).extractFingerPrints();   
+					new LibraryProfiler(inputFile, libraryDescription).extractFingerPrints();
+
+				} else if (CliOptions.opmode.equals(OpMode.LIB_API_ANALYSIS)) {
+					new LibraryApiAnalysis(inputFile);
 				}
 			} catch (Throwable t) {
 				logger.error("[FATAL " + (t instanceof Exception? "EXCEPTION" : "ERROR") + "] analysis aborted: " + t.getMessage());
@@ -229,13 +227,13 @@ else if (cmd.hasOption(CliArgs.ARG_LOG_DIR)) {
 			}
 
 			// path to Android SDK jar
-			if (checkRequiredUse(cmd, CliArgs.ARG_ANDROID_LIB, OpMode.PROFILE, OpMode.MATCH)) {
+			if (checkRequiredUse(cmd, CliArgs.ARG_ANDROID_LIB, OpMode.PROFILE, OpMode.MATCH, OpMode.LIB_API_ANALYSIS)) {
 				CliOptions.pathToAndroidJar = new File(cmd.getOptionValue(CliArgs.ARG_ANDROID_LIB));
 			}
 
 
 			// profiles dir option, if provided without argument output is written to default dir
-			if (checkOptionalUse(cmd, CliArgs.ARG_PROFILES_DIR, OpMode.PROFILE, OpMode.MATCH, OpMode.DB, OpMode.LIB_API_ANALYSIS)) {
+			if (checkOptionalUse(cmd, CliArgs.ARG_PROFILES_DIR, OpMode.PROFILE, OpMode.MATCH, OpMode.DB)) {
 				File profilesDir = new File(cmd.getOptionValue(CliArgs.ARG_PROFILES_DIR));
 				if (profilesDir.exists() && !profilesDir.isDirectory())
 					throw new ParseException("Profiles directory " + profilesDir + " already exists and is not a directory");
@@ -300,34 +298,49 @@ else if (cmd.hasOption(CliArgs.ARG_LOG_DIR)) {
 			 *  - in profile mode pass *one* library (since it is linked to lib description file)
 			 *  - in match mode pass one application file or one directory file (including apks)
 			 */
-			if (!(CliOptions.opmode.equals(OpMode.DB) || CliOptions.opmode.equals(OpMode.LIB_API_ANALYSIS))) {
+			if (!(CliOptions.opmode.equals(OpMode.DB))) {
 				inputFiles = new ArrayList<File>();
-				String[] fileExts = CliOptions.opmode.equals(OpMode.MATCH)? new String[]{"apk"} : new String[]{"jar", "aar"};
-
-				for (String apkFileName: cmd.getArgs()) {
-					File arg = new File(apkFileName);
-
-					if (arg.isDirectory()) {
-						inputFiles.addAll(Utils.collectFiles(arg, fileExts));
-					} else if (arg.isFile()) {
-						if (arg.getName().endsWith("." + fileExts[0]))
-							inputFiles.add(arg);
-						else if (fileExts.length > 1 && arg.getName().endsWith("." + fileExts[1]))
-							inputFiles.add(arg);
-						else
-							throw new ParseException("File " + arg.getName() + " is no valid ." + Utils.join(Arrays.asList(fileExts), "/")  + " file");
-					} else {
-						throw new ParseException("Argument is no valid file or directory!");
+
+				if (CliOptions.opmode.equals(OpMode.LIB_API_ANALYSIS)) {
+					// we require a directory including library packages/descriptions
+					for (String path: cmd.getArgs()) {
+						File dir = new File(path);
+
+						if (dir.isDirectory())
+							inputFiles.add(dir);
+					}
+
+					if (inputFiles.isEmpty()) {
+						throw new ParseException("You have to provide at least one directory that includes a library package and description");
 					}
+				} else {
+					String[] fileExts = CliOptions.opmode.equals(OpMode.MATCH) ? new String[]{"apk"} : new String[]{"jar", "aar"};
+
+					for (String apkFileName : cmd.getArgs()) {
+						File arg = new File(apkFileName);
+
+						if (arg.isDirectory()) {
+							inputFiles.addAll(Utils.collectFiles(arg, fileExts));
+						} else if (arg.isFile()) {
+							if (arg.getName().endsWith("." + fileExts[0]))
+								inputFiles.add(arg);
+							else if (fileExts.length > 1 && arg.getName().endsWith("." + fileExts[1]))
+								inputFiles.add(arg);
+							else
+								throw new ParseException("File " + arg.getName() + " is no valid ." + Utils.join(Arrays.asList(fileExts), "/") + " file");
+						} else {
+							throw new ParseException("Argument is no valid file or directory!");
+						}
+					}
+
+					if (inputFiles.isEmpty()) {
+						if (CliOptions.opmode.equals(OpMode.PROFILE))
+							throw new ParseException("You have to provide one library.jar to be processed");
+						else
+							throw new ParseException("You have to provide a path to a single application file or a directory");
+					} else if (inputFiles.size() > 1 && CliOptions.opmode.equals(OpMode.PROFILE))
+						throw new ParseException("You have to provide a path to a single library file or a directory incl. a single lib file");
 				}
-				
-				if (inputFiles.isEmpty()) {
-					if (CliOptions.opmode.equals(OpMode.PROFILE))
-						throw new ParseException("You have to provide one library.jar to be processed");
-					else
-						throw new ParseException("You have to provide a path to a single application file or a directory");
-				} else if (inputFiles.size() > 1 && CliOptions.opmode.equals(OpMode.PROFILE))
-					throw new ParseException("You have to provide a path to a single library file or a directory incl. a single lib file");
 			}
 
 		} catch (ParseException e) {

src/de/infsec/tpl/eval/LibApiRobustnessStats.java

@@ -32,6 +32,8 @@
  *   - # public APIs per version
  *   - APIs -> set of versions in which they are included
  *   - set of alternative APIs (if any), if API is no longer in some version
+ *
+ *   @deprecated
  */
 
 public class LibApiRobustnessStats implements Serializable {

src/de/infsec/tpl/eval/LibraryApiAnalysis.java

@@ -44,6 +44,8 @@
  * are public APIs removed, modified, and/or added
  * For each library a {@link LibApiRobustnessStats} is created that tracks APIs across versions
  * The results are written in the configured json directory and logged.
+ *
+ * @deprecated
  */
 
 public class LibraryApiAnalysis {

src/de/infsec/tpl/modules/libapi/JvmMethodAccessFlags.java

@@ -0,0 +1,150 @@
+package de.infsec.tpl.modules.libapi;
+
+import com.ibm.wala.classLoader.IMethod;
+import de.infsec.tpl.utils.Utils;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+/**
+ * Spec taken from https://docs.oracle.com/javase/specs/jvms/se7/html/jvms-4.html
+ */
+public enum JvmMethodAccessFlags {
+
+    NO_FLAG   (0x0000, "no-flag"),
+    PUBLIC    (0x0001, "public"),
+    PRIVATE   (0x0002, "private"),
+    PROTECTED (0x0004, "protected"),
+    STATIC    (0x0008, "static"),
+    FINAL     (0x0010, "final"),
+    ABSTRACT  (0x0400, "abstract"),
+    PACKAGE_PROTECTED (0x2000, "package-proteced");
+
+    private int value;
+    private String accessFlagName;
+
+    //cache the array of all AccessFlags, because .values() allocates a new array for every call
+    private final static JvmMethodAccessFlags[] allFlags;
+
+    private final static List<Integer> validFlagValues;
+
+    private static HashMap<String, JvmMethodAccessFlags> accessFlagsByName;
+
+    static {
+        allFlags = JvmMethodAccessFlags.values();
+
+        validFlagValues = new ArrayList<Integer>();
+        for (JvmMethodAccessFlags flag: allFlags)
+            validFlagValues.add(flag.getValue());
+
+        accessFlagsByName = new HashMap<String, JvmMethodAccessFlags>();
+        for (JvmMethodAccessFlags accessFlag: allFlags) {
+            accessFlagsByName.put(accessFlag.accessFlagName, accessFlag);
+        }
+    }
+
+    private JvmMethodAccessFlags(int value, String accessFlagName) {
+        this.value = value;
+        this.accessFlagName = accessFlagName;
+    }
+
+
+    private static String flags2Str(JvmMethodAccessFlags[] accessFlags) {
+        int size = 0;
+        for (JvmMethodAccessFlags accessFlag: accessFlags) {
+            size += accessFlag.toString().length() + 1;
+        }
+
+        StringBuilder sb = new StringBuilder(size);
+        for (JvmMethodAccessFlags accessFlag: accessFlags) {
+            sb.append(accessFlag.toString());
+            sb.append(" ");
+        }
+        if (accessFlags.length > 0) {
+            sb.delete(sb.length() - 1, sb.length());
+        }
+        return sb.toString();
+    }
+
+    public static boolean isValidFlag(int code) {
+        return validFlagValues.contains(code);
+    }
+
+
+    public static String flags2Str(int code) {
+        List<String> matchedFlags = new ArrayList<String>();
+
+        for (JvmMethodAccessFlags flag: allFlags) {
+            if ((code & flag.value) != 0x0) {
+                matchedFlags.add(flag.accessFlagName + "(" + flag.value + ")");
+            }
+        }
+
+        return Utils.join(matchedFlags, ",");
+    }
+
+    public static String flags2Str(IMethod m) {
+        return flags2Str(getMethodAccessCode(m));
+    }
+
+    public static JvmMethodAccessFlags getAccessFlag(String accessFlag) {
+        return accessFlagsByName.get(accessFlag);
+    }
+
+    public int getValue() {
+        return value;
+    }
+
+    public String toString() {
+        return accessFlagName;
+    }
+
+
+    public static int getAccessFlagFilter(JvmMethodAccessFlags... flags) {
+        int filter = NO_FLAG.getValue();
+
+        if (flags != null) {
+            for (JvmMethodAccessFlags flag: flags) {
+                if (!JvmMethodAccessFlags.isValidFlag(flag.getValue())) continue;
+
+                filter |= flag.getValue();
+            }
+        }
+
+        return filter;
+    }
+
+    public static int getPublicOnlyFilter() {
+        return getAccessFlagFilter(JvmMethodAccessFlags.PRIVATE, JvmMethodAccessFlags.PROTECTED, JvmMethodAccessFlags.STATIC,
+                JvmMethodAccessFlags.FINAL, JvmMethodAccessFlags.ABSTRACT, JvmMethodAccessFlags.PACKAGE_PROTECTED);
+    }
+
+
+    public static int getMethodAccessCode(IMethod m) {
+        int res = 0x0;
+
+        if (m == null)
+            return res;
+
+        if (m.isPublic()) {
+            res |= JvmMethodAccessFlags.PUBLIC.getValue();
+        } else if (m.isProtected()) {
+            res |= JvmMethodAccessFlags.PROTECTED.getValue();
+        } else if (m.isPrivate()) {
+            res |= JvmMethodAccessFlags.PRIVATE.getValue();
+        } else {
+            res |= JvmMethodAccessFlags.PACKAGE_PROTECTED.getValue();
+        }
+
+        if (m.isStatic())
+            res |= JvmMethodAccessFlags.STATIC.getValue();
+        if (m.isFinal())
+            res |= JvmMethodAccessFlags.FINAL.getValue();
+        if (m.isAbstract())
+            res |= JvmMethodAccessFlags.ABSTRACT.getValue();
+
+        return res;
+    }
+
+}