From 6164fb750b9a0cc9e70615e6d4eafd70f9f2f022 Mon Sep 17 00:00:00 2001 From: Andy Ayers Date: Thu, 23 May 2019 19:20:34 -0700 Subject: [PATCH 01/10] Update PMI to use the new System.CommandLine parser Replace PMI's home-grown command line parsing with the new System.CommandLine version. Syntax changes slightly as the "suffix" options (eg `PREPALL-QUIET`) now need to be separated out (`PREPALL --quiet`). Only real "breaking" change is that the method index for `PREPALL` and `PREPONE` is now a named option. With this new package we get built in help and (in some shells) command line completion. I plan to eventually migrate all the utilites over to this package as the one they are using is now orphaned and unsupported. See #193. --- src/pmi/PMIDriver.cs | 9 +- src/pmi/pmi.cs | 226 +++++++++++++++++++++++++------------------ src/pmi/pmi.csproj | 1 + 3 files changed, 140 insertions(+), 96 deletions(-) diff --git a/src/pmi/PMIDriver.cs b/src/pmi/PMIDriver.cs index 7299a2ff..26def320 100644 --- a/src/pmi/PMIDriver.cs +++ b/src/pmi/PMIDriver.cs @@ -166,16 +166,17 @@ private static PrepAllResult Compute(PrepAllInfo pi) p.StartInfo.RedirectStandardOutput = true; p.StartInfo.RedirectStandardError = true; - // Fetch our command line. Split off the arguments. + // Fetch our command line, and fix up the arguments. + + string newArgs = $"PREPALL --method {pi.methodToPrep}"; #if NETCOREAPP2_1 || NETCOREAPP2_2 || NETCOREAPP3_0 // For .Net Core the PMI assembly is an argument to dotnet. - string newCommandLine = Environment.CommandLine.Replace("DRIVEALL", "PREPALL"); + string newCommandLine = Environment.CommandLine.Replace("DRIVEALL", newArgs); #else // For .Net Framework the OS knows how to bootstrap .Net when // passed the PMI assembly as an executable. - string newCommandLine = "PREPALL \"" + pi.assemblyName + "\""; + string newCommandLine = newArgs + " \"" + pi.assemblyName + "\""; #endif - newCommandLine += " " + pi.methodToPrep; Process thisProcess = Process.GetCurrentProcess(); string driverName = thisProcess.MainModule.FileName; diff --git a/src/pmi/pmi.cs b/src/pmi/pmi.cs index 448de768..7faf843f 100644 --- a/src/pmi/pmi.cs +++ b/src/pmi/pmi.cs @@ -4,6 +4,8 @@ using System; using System.Collections.Generic; +using System.CommandLine; +using System.CommandLine.Invocation; using System.IO; using System.Linq; using System.Numerics; @@ -625,7 +627,7 @@ public override void AttemptMethod(Type type, MethodBase method) { Console.WriteLine($"PREPONE type# {typeCount} method# {methodCount} {type.FullName}::{method.Name}"); TimeSpan elapsedFunc = PrepareMethod(type, method); - Console.WriteLine($"Completed method {type.FullName}::{method.Name}"); + Console.Write($"Completed method {type.FullName}::{method.Name}"); if (elapsedFunc != TimeSpan.MinValue) { Console.WriteLine($", elapsed ms: {elapsedFunc.TotalMilliseconds:F2}"); @@ -973,10 +975,11 @@ bool Work(Type type) { visitor.StartMethod(type, methodBase); keepGoing = visitor.FinishMethod(type, methodBase); - if (!keepGoing) - { - break; - } + } + + if (!keepGoing) + { + break; } } } @@ -1315,121 +1318,160 @@ private static void EnsureTimerCallbackIsJitted() // >= 100 - failure public static int Main(string[] args) { - if (args.Length < 2) - { - return Usage(); - } - // Tracing infrastructure unconditionally creates a Timer and uses it for checking // whether tracing has been enabled. Since the Timer callback is called on a worker thread, // we may get corrupted disasm output. To prevent that, force jitting of Timer callback infrastructure // before we PMI any method. EnsureTimerCallbackIsJitted(); - string command = args[0].ToUpper(); - string assemblyName = args[1]; - int methodToPrep = -1; // For PREPONE, PREPALL. For PREPALL, this is the first method to prep. + // --- COUNT --- + Command countCommand = new Command("COUNT"); + { + countCommand.Description = "Count the number of types and methods in an assembly."; - Visitor v = null; + Argument assemblyArgument = new Argument(); + assemblyArgument.Arity = ArgumentArity.ExactlyOne; + assemblyArgument.Name = "assemblyName"; + assemblyArgument.Description = "Assembly file to process"; + countCommand.AddArgument(assemblyArgument); - int dashIndex = command.IndexOf('-'); - string rootCommand = dashIndex < 0 ? command : command.Substring(0, dashIndex); - switch (rootCommand) - { - case "DRIVEALL": - case "COUNT": - if (args.Length < 2) - { - Console.WriteLine("ERROR: too few arguments"); - return Usage(); - } - else if (args.Length > 2) + countCommand.Handler = CommandHandler.Create((assemblyName) => + { + if (File.Exists(assemblyName)) { - Console.WriteLine("ERROR: too many arguments"); - return Usage(); - } - if (rootCommand == "DRIVEALL") - { - return PMIDriver.PMIDriver.Drive(assemblyName); + Visitor v = new Counter(); + Worker w = new Worker(v, false); + return w.Work(assemblyName); } - v = new Counter(); - break; + Console.WriteLine($"Failed: assembly '{assemblyName}' does not exist"); + return 101; + }); + } - case "PREPALL": - case "PREPONE": - if (args.Length < 3) - { - methodToPrep = 0; - } - else if (args.Length > 3) - { - Console.WriteLine("ERROR: too many arguments"); - return Usage(); - } - else - { - try - { - methodToPrep = Convert.ToInt32(args[2]); - } - catch (System.FormatException) - { - Console.WriteLine("ERROR: illegal method number"); - return Usage(); - } - } + // --- PREPALL --- + Command prepallCommand = new Command("PREPALL"); + { + prepallCommand.Description = "JIT all the methods in an assembly or all assemblies in a directory tree."; - bool all = command.IndexOf("ALL") > 0; - bool verbose = !(command.IndexOf("QUIET") > 0); - bool time = verbose || command.IndexOf("TIME") > 0; + Argument assemblyArgument = new Argument(); + assemblyArgument.Arity = ArgumentArity.ExactlyOne; + assemblyArgument.Name = "assemblyName"; + assemblyArgument.Description = "Assembly file to process, or directory to recursively traverse"; + prepallCommand.AddArgument(assemblyArgument); - if (all) + Option methodOption = new Option("--method", "Index of the first method to jit", new Argument(0) { Arity = ArgumentArity.ExactlyOne }); + Option cctorOption = new Option("--cctors", "Try and invoke cctors first", new Argument()); + Option quietOption = new Option("--quiet", "Don't show progress messages", new Argument()); + Option timeOption = new Option("--time", "Show elapsed time information", new Argument()); + prepallCommand.AddOption(methodOption); + prepallCommand.AddOption(cctorOption); + prepallCommand.AddOption(quietOption); + prepallCommand.AddOption(timeOption); + + prepallCommand.Handler = CommandHandler.Create((assemblyName, method, cctors, quiet, time) => + { + if (File.Exists(assemblyName)) { - v = new PrepareAll(methodToPrep, verbose, time); + Visitor v = new PrepareAll(method, !quiet, time); + Worker w = new Worker(v, cctors); + return w.Work(assemblyName); } - else + +#if NETCOREAPP2_1 || NETCOREAPP2_2 || NETCOREAPP3_0 + else if (Directory.Exists(assemblyName)) { - v = new PrepareOne(methodToPrep, verbose, time); + EnumerationOptions options = new EnumerationOptions(); + options.RecurseSubdirectories = true; + IEnumerable exeFiles = Directory.EnumerateFiles(assemblyName, "*.exe", options); + IEnumerable dllFiles = Directory.EnumerateFiles(assemblyName, "*.dll", options); + IEnumerable allFiles = exeFiles.Concat(dllFiles); + Visitor v = new PrepareAll(method, !quiet, time); + Worker w = new Worker(v, cctors); + return w.Work(allFiles); } - break; +#endif - default: - Console.WriteLine("ERROR: Unknown command {0}", command); - return Usage(); + Console.WriteLine($"Failed: '{assemblyName}' does not exist"); + return 101; + }); } - bool runCctors = command.IndexOf("CCTORS") > 0; + // --- PREPONE --- + Command preponeCommand = new Command("PREPONE"); + { + preponeCommand.Description = "JIT exactly one method in an assembly."; - Worker w = new Worker(v, runCctors); - int result = 0; - string msg = "a file"; + Argument assemblyArgument = new Argument(); + assemblyArgument.Arity = ArgumentArity.ExactlyOne; + assemblyArgument.Name = "assemblyName"; + assemblyArgument.Description = "Assembly file to process"; + preponeCommand.AddArgument(assemblyArgument); -#if NETCOREAPP2_1 || NETCOREAPP2_2 || NETCOREAPP3_0 - msg += " or a directory"; - if (Directory.Exists(assemblyName)) - { - EnumerationOptions options = new EnumerationOptions(); - options.RecurseSubdirectories = true; - IEnumerable exeFiles = Directory.EnumerateFiles(assemblyName, "*.exe", options); - IEnumerable dllFiles = Directory.EnumerateFiles(assemblyName, "*.dll", options); - IEnumerable allFiles = exeFiles.Concat(dllFiles); - result = w.Work(allFiles); - } - else -#endif + Option methodOption = new Option("--method", "Index of the method to jit", new Argument(0) { Arity = ArgumentArity.ExactlyOne }); + Option cctorOption = new Option("--cctors", "Try and invoke cctors first", new Argument()); + Option quietOption = new Option("--quiet", "Don't show progress messages", new Argument()); + Option timeOption = new Option("--time", "Show elapsed time information", new Argument()); + preponeCommand.AddOption(methodOption); + preponeCommand.AddOption(cctorOption); + preponeCommand.AddOption(quietOption); + preponeCommand.AddOption(timeOption); - if (File.Exists(assemblyName)) - { - result = w.Work(assemblyName); + preponeCommand.Handler = CommandHandler.Create((assemblyName, method, cctors, quiet, time) => + { + if (File.Exists(assemblyName)) + { + Visitor v = new PrepareOne(method, !quiet, time); + Worker w = new Worker(v, cctors); + return w.Work(assemblyName); + } + + Console.WriteLine($"Failed: '{assemblyName}' does not exist"); + return 101; + }); } - else + + // --- DRIVEALL --- + Command driveallCommand = new Command("DRIVEALL"); { - Console.WriteLine($"ERROR: {assemblyName} is not {msg}"); - result = 101; + driveallCommand.Description = "JIT all the methods in an assembly robustly, skipping methods with asserts."; + + Argument assemblyArgument = new Argument(); + assemblyArgument.Arity = ArgumentArity.ExactlyOne; + assemblyArgument.Name = "assemblyName"; + assemblyArgument.Description = "Assembly file to process"; + driveallCommand.AddArgument(assemblyArgument); + + Option cctorOption = new Option("--cctors", "Try and invoke cctors first", new Argument()); + Option quietOption = new Option("--quiet", "Don't show progress messages", new Argument()); + Option timeOption = new Option("--time", "Show elapsed time information", new Argument()); + driveallCommand.AddOption(cctorOption); + driveallCommand.AddOption(quietOption); + driveallCommand.AddOption(timeOption); + + // Options are ignored here, as DRIVEALL will grab the command line for child invocations of PREPALL. + // By including them here we validate them early. + driveallCommand.Handler = CommandHandler.Create((assemblyName, cctors, quiet, time) => + { + if (File.Exists(assemblyName)) + { + return PMIDriver.PMIDriver.Drive(assemblyName); + } + + Console.WriteLine($"Failed: '{assemblyName}' does not exist"); + return 101; + }); } - return result; + // --- ROOT --- + RootCommand rootCommand = new RootCommand(); + rootCommand.AddCommand(countCommand); + rootCommand.AddCommand(prepallCommand); + rootCommand.AddCommand(preponeCommand); + rootCommand.AddCommand(driveallCommand); + + // Parse and execute + return rootCommand.InvokeAsync(args).Result; } } diff --git a/src/pmi/pmi.csproj b/src/pmi/pmi.csproj index 79d834da..e01c190f 100644 --- a/src/pmi/pmi.csproj +++ b/src/pmi/pmi.csproj @@ -7,6 +7,7 @@ + From 9a30e0338a8a21b070310c5f90586c1744f73b8c Mon Sep 17 00:00:00 2001 From: Andy Ayers Date: Fri, 24 May 2019 10:40:27 -0700 Subject: [PATCH 02/10] command lower case aliases --- src/pmi/pmi.cs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/pmi/pmi.cs b/src/pmi/pmi.cs index 7faf843f..7bed8d03 100644 --- a/src/pmi/pmi.cs +++ b/src/pmi/pmi.cs @@ -1328,6 +1328,7 @@ public static int Main(string[] args) Command countCommand = new Command("COUNT"); { countCommand.Description = "Count the number of types and methods in an assembly."; + countCommand.AddAlias("count"); Argument assemblyArgument = new Argument(); assemblyArgument.Arity = ArgumentArity.ExactlyOne; @@ -1354,6 +1355,7 @@ public static int Main(string[] args) Command prepallCommand = new Command("PREPALL"); { prepallCommand.Description = "JIT all the methods in an assembly or all assemblies in a directory tree."; + prepallCommand.AddAlias("prepall"); Argument assemblyArgument = new Argument(); assemblyArgument.Arity = ArgumentArity.ExactlyOne; @@ -1402,6 +1404,7 @@ public static int Main(string[] args) Command preponeCommand = new Command("PREPONE"); { preponeCommand.Description = "JIT exactly one method in an assembly."; + preponeCommand.AddAlias("prepone"); Argument assemblyArgument = new Argument(); assemblyArgument.Arity = ArgumentArity.ExactlyOne; @@ -1436,6 +1439,7 @@ public static int Main(string[] args) Command driveallCommand = new Command("DRIVEALL"); { driveallCommand.Description = "JIT all the methods in an assembly robustly, skipping methods with asserts."; + driveallCommand.AddAlias("driveall"); Argument assemblyArgument = new Argument(); assemblyArgument.Arity = ArgumentArity.ExactlyOne; @@ -1443,6 +1447,8 @@ public static int Main(string[] args) assemblyArgument.Description = "Assembly file to process"; driveallCommand.AddArgument(assemblyArgument); + // Thse options are ignored by the handler -- DRIVEALL will grab the command line for child invocations of PREPALL. + // But by including them here we validate them early. Option cctorOption = new Option("--cctors", "Try and invoke cctors first", new Argument()); Option quietOption = new Option("--quiet", "Don't show progress messages", new Argument()); Option timeOption = new Option("--time", "Show elapsed time information", new Argument()); @@ -1450,9 +1456,7 @@ public static int Main(string[] args) driveallCommand.AddOption(quietOption); driveallCommand.AddOption(timeOption); - // Options are ignored here, as DRIVEALL will grab the command line for child invocations of PREPALL. - // By including them here we validate them early. - driveallCommand.Handler = CommandHandler.Create((assemblyName, cctors, quiet, time) => + driveallCommand.Handler = CommandHandler.Create((assemblyName) => { if (File.Exists(assemblyName)) { From 4dbb6cc6099afa04a2fc9d06e6fde0fc8e0f4b90 Mon Sep 17 00:00:00 2001 From: Andy Ayers Date: Fri, 24 May 2019 15:17:58 -0700 Subject: [PATCH 03/10] Fix cijobs to use new command line parser Kind of pointless as it is referring to the old CI server, but perhaps it can serve as a template for something to retrieve things from AzDO. But it at least shows how to adapt the `Config` we see in the other projects over to the new format without too radical of changes. --- src/cijobs/cijobs.cs | 342 +++++++++++++++++++++--------------------- src/jit-diff/diff.cs | 2 +- src/jit-include.props | 4 +- 3 files changed, 177 insertions(+), 171 deletions(-) diff --git a/src/cijobs/cijobs.cs b/src/cijobs/cijobs.cs index fdda7851..0d2577c1 100755 --- a/src/cijobs/cijobs.cs +++ b/src/cijobs/cijobs.cs @@ -33,22 +33,14 @@ using System.Linq; using System.Text.RegularExpressions; using System.IO.Compression; -//using Microsoft.DotNet.Cli.Utils; -//using Microsoft.DotNet.Tools.Common; using Newtonsoft.Json; -//using Newtonsoft.Json.Linq; +using System.CommandLine.Invocation; namespace ManagedCodeGen { + using CommandResult = Microsoft.DotNet.Cli.Utils.CommandResult; internal class cijobs { - // Supported commands. List to view information from the CI system, and Copy to download artifacts. - public enum Command - { - List, - Copy - } - // List options control what level and level of detail to put out. // Jobs lists jobs under a product. // Builds lists build instances under a job. @@ -63,184 +55,113 @@ public enum ListOption // Define options to be parsed public class Config { - private ArgumentSyntax _syntaxResult; - private Command _command = Command.List; - private ListOption _listOption = ListOption.Invalid; - private string _server = "http://ci.dot.net/"; - private string _jobName; - private string _contentPath; - private string _repoName = "dotnet_coreclr"; - private int _number = 0; - private string _matchPattern = String.Empty; - private string _branchName = "master"; - private bool _lastSuccessful = false; - private string _commit; - private bool _unzip = false; - private string _outputPath; - private string _outputRoot; - private bool _artifacts = false; - - public Config(string[] args) - { - _syntaxResult = ArgumentSyntax.Parse(args, syntax => - { - // NOTE!!! - Commands and their options are ordered. Moving an option out of line - // could move it to another command. Take a careful look at how they're organized - // before changing. - syntax.DefineCommand("list", ref _command, Command.List, - "List jobs on dotnet-ci.cloudapp.net for the repo."); - syntax.DefineOption("s|server", ref _server, "Url of the server. Defaults to http://ci.dot.net/"); - syntax.DefineOption("j|job", ref _jobName, "Name of the job."); - syntax.DefineOption("b|branch", ref _branchName, - "Name of the branch (default is master)."); - syntax.DefineOption("r|repo", ref _repoName, - "Name of the repo (e.g. dotnet_corefx or dotnet_coreclr). Default is dotnet_coreclr."); - syntax.DefineOption("m|match", ref _matchPattern, - "Regex pattern used to select jobs output."); - syntax.DefineOption("n|number", ref _number, "Job number."); - syntax.DefineOption("l|last_successful", ref _lastSuccessful, - "List last successful build."); - syntax.DefineOption("c|commit", ref _commit, "List build at this commit."); - syntax.DefineOption("a|artifacts", ref _artifacts, "List job artifacts on server."); - - syntax.DefineCommand("copy", ref _command, Command.Copy, - "Copies job artifacts from dotnet-ci.cloudapp.net. " - + "This command copies a zip of artifacts from a repo (defaulted to dotnet_coreclr)." - + " The default location of the zips is the Product sub-directory, though " - + "that can be changed using the ContentPath(p) parameter"); - syntax.DefineOption("s|server", ref _server, "Url of the server. Defaults to http://ci.dot.net/"); - syntax.DefineOption("j|job", ref _jobName, "Name of the job."); - syntax.DefineOption("n|number", ref _number, "Job number."); - syntax.DefineOption("l|last_successful", ref _lastSuccessful, - "Copy last successful build."); - syntax.DefineOption("c|commit", ref _commit, "Copy this commit."); - syntax.DefineOption("b|branch", ref _branchName, - "Name of the branch (default is master)."); - syntax.DefineOption("r|repo", ref _repoName, - "Name of the repo (e.g. dotnet_corefx or dotnet_coreclr). Default is dotnet_coreclr."); - syntax.DefineOption("o|output", ref _outputPath, "The path where output will be placed."); - syntax.DefineOption("or|output_root", ref _outputRoot, - "The root directory where output will be placed. A subdirectory named by job and build number will be created within this to store the output."); - syntax.DefineOption("u|unzip", ref _unzip, "Unzip copied artifacts"); - syntax.DefineOption("p|ContentPath", ref _contentPath, - "Relative product zip path. Default is artifact/bin/Product/*zip*/Product.zip"); - }); - - // Run validation code on parsed input to ensure we have a sensible scenario. - validate(); - } - - private void validate() + public void ValidateCopy() { - if (!Uri.IsWellFormedUriString(_server, UriKind.Absolute)) + if (!Uri.IsWellFormedUriString(Server, UriKind.Absolute)) { - _syntaxResult.ReportError($"Invalid uri: {_server}."); + ReportError($"Invalid uri: {Server}."); } - switch (_command) - { - case Command.List: - { - validateList(); - } - break; - case Command.Copy: - { - validateCopy(); - } - break; - } - } - private void validateCopy() - { - if (_jobName == null) + if (Job == null) { - _syntaxResult.ReportError("Must have --job for copy."); + ReportError("Must have --job for copy."); } - if (_number == 0 && !_lastSuccessful && _commit == null) + if (Number == 0 && !LastSuccessful && Commit == null) { - _syntaxResult.ReportError("Must have --number , --last_successful, or --commit for copy."); + ReportError("Must have --number , --last_successful, or --commit for copy."); } - if (Convert.ToInt32(_number != 0) + Convert.ToInt32(_lastSuccessful) + Convert.ToInt32(_commit != null) > 1) + if (Convert.ToInt32(Number != 0) + Convert.ToInt32(LastSuccessful) + Convert.ToInt32(Commit != null) > 1) { - _syntaxResult.ReportError("Must have only one of --number , --last_successful, and --commit for copy."); + ReportError("Must have only one of --number , --last_successful, and --commit for copy."); } - if ((_outputPath == null) && (_outputRoot == null)) + if ((OutputPath == null) && (OutputRoot == null)) { - _syntaxResult.ReportError("Must specify either --output or --output_root for copy."); + ReportError("Must specify either --output or --output_root for copy."); } - if ((_outputPath != null) && (_outputRoot != null)) + if ((OutputPath != null) && (OutputRoot != null)) { - _syntaxResult.ReportError("Must specify only one of --output or --output_root ."); + ReportError("Must specify only one of --output or --output_root ."); } - if (_contentPath == null) + if (ContentPath == null) { - _contentPath = "artifact/bin/Product/*zip*/Product.zip"; + ContentPath = "artifact/bin/Product/*zip*/Product.zip"; } } - private void validateList() + public void ValidateList() { - if (_jobName != null) + if (!Uri.IsWellFormedUriString(Server, UriKind.Absolute)) { - _listOption = ListOption.Builds; + ReportError($"Invalid uri: {Server}."); + } - if (Convert.ToInt32(_number != 0) + Convert.ToInt32(_lastSuccessful) + Convert.ToInt32(_commit != null) > 1) + if (Job != null) + { + DoListOption = ListOption.Builds; + + if (Convert.ToInt32(Number != 0) + Convert.ToInt32(LastSuccessful) + Convert.ToInt32(Commit != null) > 1) { - _syntaxResult.ReportError("Must have at most one of --number , --last_successful, and --commit for list."); + ReportError("Must have at most one of --number , --last_successful, and --commit for list."); } - if (_matchPattern != String.Empty) + if (Match != String.Empty) { - _syntaxResult.ReportError("Match pattern not valid with --job"); + ReportError("Match pattern not valid with --job"); } } else { - _listOption = ListOption.Jobs; + DoListOption = ListOption.Jobs; - if (_number != 0) + if (Number != 0) { - _syntaxResult.ReportError("Must select --job to specify --number ."); + ReportError("Must select --job to specify --number ."); } - if (_lastSuccessful) + if (LastSuccessful) { - _syntaxResult.ReportError("Must select --job to specify --last_successful."); + ReportError("Must select --job to specify --last_successful."); } - if (_commit != null) + if (Commit != null) { - _syntaxResult.ReportError("Must select --job to specify --commit ."); + ReportError("Must select --job to specify --commit ."); } - if (_artifacts) + if (Artifacts) { - _syntaxResult.ReportError("Must select --job to specify --artifacts."); + ReportError("Must select --job to specify --artifacts."); } } } - public Command DoCommand { get { return _command; } } - public ListOption DoListOption { get { return _listOption; } } - public string JobName { get { return _jobName; } } - public string Server { get { return _server; } } - public string ContentPath { get { return _contentPath; } } - public int Number { get { return _number; } set { this._number = value; } } - public string MatchPattern { get { return _matchPattern; } } - public string BranchName { get { return _branchName; } } - public string RepoName { get { return _repoName; } } - public bool LastSuccessful { get { return _lastSuccessful; } } - public string Commit { get { return _commit; } } - public bool DoUnzip { get { return _unzip; } } - public string OutputPath { get { return _outputPath; } } - public string OutputRoot { get { return _outputRoot; } } - public bool Artifacts { get { return _artifacts; } } + void ReportError(string message) + { + Console.WriteLine(message); + Error = true; + } + + public Command DoCommand { get; set; } + public ListOption DoListOption { get; set; } + public string Job { get; set; } + public string Server { get; set; } + public string ContentPath { get; set; } + public int Number { get; set; } + public string Match { get; set; } + public string Branch { get; set; } + public string Repo { get; set; } + public bool LastSuccessful { get; set; } + public string Commit { get; set; } + public bool DoUnzip { get; set; } + public string OutputPath { get; set; } + public string OutputRoot { get; set; } + public bool Artifacts { get; set; } + public bool Error { get; set; } } // The following block of simple structs maps to the data extracted from the CI system as json. @@ -302,33 +223,118 @@ private struct JobBuilds // and switch on the command to invoke underlying logic. private static int Main(string[] args) { - Config config = new Config(args); - int error = 0; - - CIClient cic = new CIClient(config); - - Command currentCommand = config.DoCommand; - switch (currentCommand) + Command listCommand = new Command("list"); { - case Command.List: - { - ListCommand.List(cic, config).Wait(); - break; - } - case Command.Copy: + listCommand.Description = "List jobs on dotnet-ci.cloudapp.net for the repo."; + + Option serverOption = new Option("--server", "Url of the server. Defaults to http://ci.dot.net/", new Argument("http://ci.dot.net/")); + serverOption.AddAlias("-s"); + Option jobOption = new Option("--job", "Name of the job.", new Argument()); + jobOption.AddAlias("-j"); + Option branchOption = new Option("--branch", "Name of the branch (default is master).", new Argument("master")); + branchOption.AddAlias("-b"); + Option repoOption = new Option("--repo", "Name of the repo(e.g.dotnet_corefx or dotnet_coreclr). Default is dotnet_coreclr.", new Argument("dotnet_coreclr")); + repoOption.AddAlias("-r"); + Option matchOption = new Option("--match", "Regex pattern used to select jobs output.", new Argument(String.Empty)); + matchOption.AddAlias("-m"); + Option numberOption = new Option("--number", "Job number.", new Argument()); + numberOption.AddAlias("-n"); + Option lastOption = new Option("--last-successful", "List last successful build.", new Argument()); + lastOption.AddAlias("-l"); + Option commitOption = new Option("--commit", "List build at this commit.", new Argument()); + commitOption.AddAlias("-c"); + Option artifactsOption = new Option("--artifacts", "List job artifacts on server.", new Argument()); + artifactsOption.AddAlias("-a"); + + listCommand.AddOption(serverOption); + listCommand.AddOption(jobOption); + listCommand.AddOption(branchOption); + listCommand.AddOption(repoOption); + listCommand.AddOption(matchOption); + listCommand.AddOption(numberOption); + listCommand.AddOption(lastOption); + listCommand.AddOption(commitOption); + listCommand.AddOption(artifactsOption); + + listCommand.Handler = CommandHandler.Create((config) => + { + config.ValidateList(); + + if (config.Error) { - CopyCommand.Copy(cic, config).Wait(); - break; + return -1; } - default: + + CIClient cic = new CIClient(config); + ListCommand.List(cic, config).Wait(); + + return 0; + }); + } + + Command copyCommand = new Command("copy"); + { + copyCommand.Description = "Copies job artifacts from dotnet-ci.cloudapp.net. " + + "This command copies a zip of artifacts from a repo (defaulted to dotnet_coreclr). " + + "The default location of the zips is the Product sub-directory, though " + + "that can be changed using the ContentPath(p) parameter"; + + Option serverOption = new Option("--server", "Url of the server. Defaults to http://ci.dot.net/", new Argument("http://ci.dot.net/")); + serverOption.AddAlias("-s"); + Option jobOption = new Option("--job", "Name of the job.", new Argument()); + jobOption.AddAlias("-j"); + Option branchOption = new Option("--branch", "Name of the branch (default is master).", new Argument("master")); + branchOption.AddAlias("-b"); + Option repoOption = new Option("--repo", "Name of the repo(e.g.dotnet_corefx or dotnet_coreclr). Default is dotnet_coreclr.", new Argument("dotnet_coreclr")); + repoOption.AddAlias("-r"); + Option numberOption = new Option("--number", "Job number.", new Argument()); + numberOption.AddAlias("-n"); + Option lastOption = new Option("--last-successful", "List last successful build.", new Argument()); + lastOption.AddAlias("-l"); + Option commitOption = new Option("--commit", "List build at this commit.", new Argument()); + commitOption.AddAlias("-c"); + Option outputOption = new Option("--output", "The path where output will be placed.", new Argument()); + outputOption.AddAlias("-o"); + Option outputRootOption = new Option("--output-root", "The root directory where output will be placed. A subdirectory named by job and build number will be created within this to store the output.", new Argument()); + outputRootOption.AddAlias("-or"); + Option unzipOption = new Option("--unzip", "Unzip copied artifacts", new Argument()); + unzipOption.AddAlias("-u"); + Option contentOption = new Option("--content-path", "Relative product zip path. Default is artifact/bin/Product/*zip*/Product.zip", new Argument()); + contentOption.AddAlias("-p"); + + copyCommand.AddOption(serverOption); + copyCommand.AddOption(jobOption); + copyCommand.AddOption(branchOption); + copyCommand.AddOption(repoOption); + copyCommand.AddOption(numberOption); + copyCommand.AddOption(lastOption); + copyCommand.AddOption(commitOption); + copyCommand.AddOption(outputOption); + copyCommand.AddOption(outputRootOption); + copyCommand.AddOption(unzipOption); + copyCommand.AddOption(contentOption); + + copyCommand.Handler = CommandHandler.Create((config) => + { + config.ValidateCopy(); + + if (config.Error) { - Console.Error.WriteLine("super bad! why no command!"); - error = 1; - break; + return -1; } + + CIClient cic = new CIClient(config); + CopyCommand.Copy(cic, config).Wait(); + + return 0; + }); } - return error; + RootCommand rootCommand = new RootCommand(); + rootCommand.AddCommand(copyCommand); + rootCommand.AddCommand(listCommand); + + return rootCommand.InvokeAsync(args).Result; } // Wrap CI httpClient with focused APIs for product, job, and build. @@ -350,7 +356,7 @@ public async Task DownloadProduct(Config config, string outputPath, string { string messageString = String.Format("job/{0}/job/{1}/job/{2}/{3}/{4}", - config.RepoName, config.BranchName, config.JobName, config.Number, contentPath); + config.Repo, config.Branch, config.Job, config.Number, contentPath); Console.WriteLine("Downloading: {0}", messageString); @@ -521,11 +527,11 @@ public static async Task List(CIClient cic, Config config) { case ListOption.Jobs: { - var jobs = await cic.GetProductJobs(config.RepoName, config.BranchName); + var jobs = await cic.GetProductJobs(config.Repo, config.Branch); - if (config.MatchPattern != null) + if (config.Match != null) { - var pattern = new Regex(config.MatchPattern); + var pattern = new Regex(config.Match); PrettyJobs(jobs.Where(x => pattern.IsMatch(x.name))); } else @@ -536,8 +542,8 @@ public static async Task List(CIClient cic, Config config) break; case ListOption.Builds: { - var builds = await cic.GetJobBuilds(config.RepoName, config.BranchName, - config.JobName, config.LastSuccessful, + var builds = await cic.GetJobBuilds(config.Repo, config.Branch, + config.Job, config.LastSuccessful, config.Number, config.Commit); if (config.LastSuccessful && builds.Any()) @@ -613,7 +619,7 @@ public static async Task Copy(CIClient cic, Config config) if (config.LastSuccessful) { // Query last successful build and extract the number. - var builds = await cic.GetJobBuilds(config.RepoName, config.BranchName, config.JobName, true, 0, null); + var builds = await cic.GetJobBuilds(config.Repo, config.Branch, config.Job, true, 0, null); if (!builds.Any()) { @@ -626,7 +632,7 @@ public static async Task Copy(CIClient cic, Config config) } else if (config.Commit != null) { - var builds = await cic.GetJobBuilds(config.RepoName, config.BranchName, config.JobName, false, 0, config.Commit); + var builds = await cic.GetJobBuilds(config.Repo, config.Branch, config.Job, false, 0, config.Commit); if (!builds.Any()) { @@ -646,7 +652,7 @@ public static async Task Copy(CIClient cic, Config config) } else { - string tag = String.Format("{0}-{1}", config.JobName, config.Number); + string tag = String.Format("{0}-{1}", config.Job, config.Number); outputPath = Path.Combine(config.OutputRoot, tag); } diff --git a/src/jit-diff/diff.cs b/src/jit-diff/diff.cs index 9736fd6b..d2130629 100644 --- a/src/jit-diff/diff.cs +++ b/src/jit-diff/diff.cs @@ -7,11 +7,11 @@ using System.IO; using System.Collections.Generic; using System.Linq; -using Microsoft.DotNet.Cli.Utils; using System.Threading.Tasks; namespace ManagedCodeGen { + using CommandResult = Microsoft.DotNet.Cli.Utils.CommandResult; using DasmWorkTask = Task<(DasmWorkKind kind, int errorCount)>; public enum DasmWorkKind diff --git a/src/jit-include.props b/src/jit-include.props index aee352dd..a63fdddb 100644 --- a/src/jit-include.props +++ b/src/jit-include.props @@ -9,12 +9,12 @@ - + - + From f96b911f83590c3984616d8f309f8b6a45153762 Mon Sep 17 00:00:00 2001 From: Andy Ayers Date: Fri, 24 May 2019 15:54:41 -0700 Subject: [PATCH 04/10] update jit-analyze --- src/jit-analyze/jit-analyze.cs | 214 +++++++++++++++++---------------- 1 file changed, 110 insertions(+), 104 deletions(-) diff --git a/src/jit-analyze/jit-analyze.cs b/src/jit-analyze/jit-analyze.cs index a17ff4e2..57fb1722 100644 --- a/src/jit-analyze/jit-analyze.cs +++ b/src/jit-analyze/jit-analyze.cs @@ -3,15 +3,16 @@ // See the LICENSE file in the project root for more information. using System; -using System.Diagnostics; using System.CommandLine; using System.IO; using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; using Microsoft.DotNet.Cli.Utils; -using Microsoft.DotNet.Tools.Common; using Newtonsoft.Json; +using System.CommandLine.Invocation; +using Command = Microsoft.DotNet.Cli.Utils.Command; +using CommandResult = Microsoft.DotNet.Cli.Utils.CommandResult; namespace ManagedCodeGen { @@ -19,76 +20,40 @@ public class jitanalyze { public class Config { - private ArgumentSyntax _syntaxResult; - private string _basePath = null; - private string _diffPath = null; - private bool _recursive = false; - private bool _full = false; - private bool _warn = false; - private int _count = 5; - private string _json; - private string _tsv; - private bool _noreconcile = false; - private string _note; - private string _filter; - - public Config(string[] args) + void ReportError(string message) { - _syntaxResult = ArgumentSyntax.Parse(args, syntax => - { - syntax.DefineOption("b|base", ref _basePath, "Base file or directory."); - syntax.DefineOption("d|diff", ref _diffPath, "Diff file or directory."); - syntax.DefineOption("r|recursive", ref _recursive, "Search directories recursively."); - syntax.DefineOption("c|count", ref _count, - "Count of files and methods (at most) to output in the summary." - + " (count) improvements and (count) regressions of each will be included." - + " (default 5)"); - syntax.DefineOption("w|warn", ref _warn, - "Generate warning output for files/methods that only " - + "exists in one dataset or the other (only in base or only in diff)."); - syntax.DefineOption("note", ref _note, - "Descriptive note to add to summary output"); - syntax.DefineOption("noreconcile", ref _noreconcile, - "Do not reconcile unique methods in base/diff"); - syntax.DefineOption("json", ref _json, - "Dump analysis data to specified file in JSON format."); - syntax.DefineOption("tsv", ref _tsv, - "Dump analysis data to specified file in tab-separated format."); - syntax.DefineOption("filter", ref _filter, - "Only consider assembly files whose names match the filter"); - }); - - // Run validation code on parsed input to ensure we have a sensible scenario. - validate(); + Console.WriteLine(message); + Error = true; } - private void validate() + public void validate() { - if (_basePath == null) + if (Base == null) { - _syntaxResult.ReportError("Base path (--base) is required."); + ReportError("Base path (--base) is required."); } - if (_diffPath == null) + if (Diff == null) { - _syntaxResult.ReportError("Diff path (--diff) is required."); + ReportError("Diff path (--diff) is required."); } } - public string BasePath { get { return _basePath; } } - public string DiffPath { get { return _diffPath; } } - public bool Recursive { get { return _recursive; } } - public bool Full { get { return _full; } } - public bool Warn { get { return _warn; } } - public int Count { get { return _count; } } - public string TSVFileName { get { return _tsv; } } - public string JsonFileName { get { return _json; } } - public bool DoGenerateJson { get { return _json != null; } } - public bool DoGenerateTSV { get { return _tsv != null; } } - public bool Reconcile { get { return !_noreconcile; } } - public string Note { get { return _note; } } - - public string Filter { get { return _filter; } } + public string Base { get; set; } + public string Diff { get; set; } + public bool Recursive { get; set; } + public bool Full { get; set; } + public bool Warn { get; set; } + public int Count { get; set; } + public string TsvFileName { get; set; } + public string JsonFileName { get; set; } + public bool DoGenerateJson { get { return JsonFileName != null; } } + public bool DoGenerateTSV { get { return TsvFileName != null; } } + public bool NoReconcile { get; set; } + public bool Reconcile { get { return !NoReconcile; } } + public string Note { get; set; } + public string Filter { get; set; } + public bool Error { get; set; } } public class FileInfo @@ -688,61 +653,102 @@ public static Dictionary DiffInText(string diffPath, string basePat public static int Main(string[] args) { - // Parse incoming arguments - Config config = new Config(args); - - Dictionary diffCounts = DiffInText(config.DiffPath, config.BasePath); - - // Early out if no textual diffs found. - if (diffCounts == null) - { - Console.WriteLine("No diffs found."); - return 0; - } - - try + RootCommand rootCommand = new RootCommand(); + + Option basePathOption = new Option("--base", "Base file or directory.", new Argument()); + basePathOption.AddAlias("-b"); + Option diffPathOption = new Option("--diff", "Diff file or directory.", new Argument()); + diffPathOption.AddAlias("-d"); + Option recursiveOption = new Option("--recursive", "Search directories recursively.", new Argument()); + recursiveOption.AddAlias("-r"); + Option countOption = new Option("--count", "Count of files and methods (at most) to output in the summary." + + " (count) improvements and (count) regressions of each will be included." + + " (default 5)", new Argument(5)); + countOption.AddAlias("-c"); + Option warnOption = new Option("--warn", "Generate warning output for files/methods that only " + + "exists in one dataset or the other (only in base or only in diff).", new Argument()); + warnOption.AddAlias("-w"); + Option noReconcileOption = new Option("--noreconcile", "Do not reconcile unique methods in base/diff", new Argument()); + Option noteOption = new Option("--note", "Descriptive note to add to summary output", new Argument()); + Option jsonOption = new Option("--jsonFileName", "Dump analysis data to specified file in JSON format.", new Argument()); + Option tsvOption = new Option("--tsvFileName", "Dump analysis data to specified file in tab-separated format.", new Argument()); + Option filterOption = new Option("--filter", "Only consider assembly files whose names match the filter", new Argument()); + + rootCommand.AddOption(basePathOption); + rootCommand.AddOption(diffPathOption); + rootCommand.AddOption(recursiveOption); + rootCommand.AddOption(countOption); + rootCommand.AddOption(warnOption); + rootCommand.AddOption(noReconcileOption); + rootCommand.AddOption(noteOption); + rootCommand.AddOption(jsonOption); + rootCommand.AddOption(tsvOption); + rootCommand.AddOption(filterOption); + + rootCommand.Handler = CommandHandler.Create((config) => { - // Extract method info from base and diff directory or file. - var baseList = ExtractFileInfo(config.BasePath, config.Filter, config.Recursive); - var diffList = ExtractFileInfo(config.DiffPath, config.Filter, config.Recursive); + config.validate(); - // Compare the method info for each file and generate a list of - // non-zero deltas. The lists that include files in one but not - // the other are used as the comparator function only compares where it - // has both sides. + if (config.Error) + { + return -1; + } - var compareList = Comparator(baseList, diffList, config); + Dictionary diffCounts = DiffInText(config.Diff, config.Base); - // Generate warning lists if requested. - if (config.Warn) + // Early out if no textual diffs found. + if (diffCounts == null) { - WarnFiles(diffList, baseList); - WarnMethods(compareList); + Console.WriteLine("No diffs found."); + return 0; } - if (config.DoGenerateTSV) + try { - GenerateTSV(compareList, config.TSVFileName); - } + // Extract method info from base and diff directory or file. + var baseList = ExtractFileInfo(config.Base, config.Filter, config.Recursive); + var diffList = ExtractFileInfo(config.Diff, config.Filter, config.Recursive); + + // Compare the method info for each file and generate a list of + // non-zero deltas. The lists that include files in one but not + // the other are used as the comparator function only compares where it + // has both sides. + + var compareList = Comparator(baseList, diffList, config); - if (config.DoGenerateJson) + // Generate warning lists if requested. + if (config.Warn) + { + WarnFiles(diffList, baseList); + WarnMethods(compareList); + } + + if (config.DoGenerateTSV) + { + GenerateTSV(compareList, config.TsvFileName); + } + + if (config.DoGenerateJson) + { + GenerateJson(compareList, config.JsonFileName); + } + + return Summarize(compareList, config, diffCounts); + + } + catch (DirectoryNotFoundException e) { - GenerateJson(compareList, config.JsonFileName); + Console.WriteLine("Error: {0}", e.Message); + return 0; } + catch (FileNotFoundException e) + { + Console.WriteLine("Error: {0}", e.Message); + return 0; + } + }); - return Summarize(compareList, config, diffCounts); - - } - catch (System.IO.DirectoryNotFoundException e) - { - Console.WriteLine("Error: {0}", e.Message); - return 0; - } - catch (System.IO.FileNotFoundException e) - { - Console.WriteLine("Error: {0}", e.Message); - return 0; - } + return rootCommand.InvokeAsync(args).Result; } } } From c7b6f7122e887d50a133fb84f964f86e6b7ff835 Mon Sep 17 00:00:00 2001 From: Andy Ayers Date: Fri, 24 May 2019 17:06:59 -0700 Subject: [PATCH 05/10] update jit-dasm --- src/jit-dasm/jit-dasm.cs | 221 +++++++++++++++++++-------------------- 1 file changed, 108 insertions(+), 113 deletions(-) diff --git a/src/jit-dasm/jit-dasm.cs b/src/jit-dasm/jit-dasm.cs index b9b9b2a3..5d16c6d7 100644 --- a/src/jit-dasm/jit-dasm.cs +++ b/src/jit-dasm/jit-dasm.cs @@ -29,129 +29,94 @@ using System.Linq; using Microsoft.DotNet.Cli.Utils; using Microsoft.DotNet.Tools.Common; +using System.CommandLine.Invocation; +using Process = System.Diagnostics.Process; +using Command = Microsoft.DotNet.Cli.Utils.Command; +using CommandResult = Microsoft.DotNet.Cli.Utils.CommandResult; namespace ManagedCodeGen { // Define options to be parsed public class Config { - private ArgumentSyntax _syntaxResult; - private string _altjit = null; - private string _crossgenExe = null; - private string _jitPath = null; - private string _rootPath = null; - private string _fileName = null; - private IReadOnlyList _assemblyList = Array.Empty(); - private bool _wait = false; - private bool _recursive = false; - private IReadOnlyList _methods = Array.Empty(); - private IReadOnlyList _platformPaths = Array.Empty(); - private bool _dumpGCInfo = false; - private bool _dumpDebugInfo = false; - private bool _verbose = false; - - public Config(string[] args) + void ReportError(string message) { - _syntaxResult = ArgumentSyntax.Parse(args, syntax => - { - syntax.DefineOption("altjit", ref _altjit, "If set, the name of the altjit to use (e.g., protononjit.dll)."); - syntax.DefineOption("c|crossgen", ref _crossgenExe, "The crossgen compiler exe."); - syntax.DefineOption("j|jit", ref _jitPath, "The full path to the jit library."); - syntax.DefineOption("o|output", ref _rootPath, "The output path."); - syntax.DefineOption("f|file", ref _fileName, "Name of file to take list of assemblies from. Both a file and assembly list can be used."); - syntax.DefineOption("gcinfo", ref _dumpGCInfo, "Add GC info to the disasm output."); - syntax.DefineOption("debuginfo", ref _dumpDebugInfo, "Add Debug info to the disasm output."); - syntax.DefineOption("v|verbose", ref _verbose, "Enable verbose output."); - var waitArg = syntax.DefineOption("w|wait", ref _wait, "Wait for debugger to attach."); - waitArg.IsHidden = true; - - syntax.DefineOption("r|recursive", ref _recursive, "Scan directories recursively."); - syntax.DefineOptionList("p|platform", ref _platformPaths, "Path to platform assemblies"); - var methodsArg = syntax.DefineOptionList("m|methods", ref _methods, - "List of methods to disasm."); - methodsArg.IsHidden = true; - - // Warning!! - Parameters must occur after options to preserve parsing semantics. - - syntax.DefineParameterList("assembly", ref _assemblyList, "The list of assemblies or directories to scan for assemblies."); - }); - - // Run validation code on parsed input to ensure we have a sensible scenario. - - Validate(); + Console.WriteLine(message); + Error = true; } // Validate arguments // // Pass a single tool as --crossgen. Optionally specify a jit for crossgen to use. // - private void Validate() + public void Validate() { - if (_crossgenExe == null) + if (CrossgenExe == null) { - _syntaxResult.ReportError("Specify --crossgen."); + ReportError("Specify --crossgenExe."); } - if ((_fileName == null) && (_assemblyList.Count == 0)) + if ((FileName == null) && (AssemblyList.Count == 0)) { - _syntaxResult.ReportError("No input: Specify --file or list input assemblies."); + ReportError("No input: Specify --fileName or list input assemblies."); } // Check that we can find the crossgenExe - if (_crossgenExe != null) + if (CrossgenExe != null) { - if (!File.Exists(_crossgenExe)) + if (!File.Exists(CrossgenExe)) { - _syntaxResult.ReportError("Can't find --crossgen tool."); + ReportError("Can't find --crossgen tool."); } else { // Set to full path for command resolution logic. - string fullCrossgenPath = Path.GetFullPath(_crossgenExe); - _crossgenExe = fullCrossgenPath; + string fullCrossgenPath = Path.GetFullPath(CrossgenExe); + CrossgenExe = fullCrossgenPath; } } // Check that we can find the jit library. - if (_jitPath != null) + if (JitPath != null) { - if (!File.Exists(_jitPath)) + if (!File.Exists(JitPath)) { - _syntaxResult.ReportError("Can't find --jit library."); + ReportError("Can't find --jit library."); } else { // Set to full path for command resolution logic. - string fullJitPath = Path.GetFullPath(_jitPath); - _jitPath = fullJitPath; + string fullJitPath = Path.GetFullPath(JitPath); + JitPath = fullJitPath; } } - if (_fileName != null) + if (FileName != null) { - if (!File.Exists(_fileName)) + if (!File.Exists(FileName)) { - var message = String.Format("Error reading input file {0}, file not found.", _fileName); - _syntaxResult.ReportError(message); + var message = String.Format("Error reading input file {0}, file not found.", FileName); + ReportError(message); } } } public bool HasUserAssemblies { get { return AssemblyList.Count > 0; } } - public bool WaitForDebugger { get { return _wait; } } - public bool UseJitPath { get { return (_jitPath != null); } } - public bool Recursive { get { return _recursive; } } - public bool UseFileName { get { return (_fileName != null); } } - public bool DumpGCInfo { get { return _dumpGCInfo; } } - public bool DumpDebugInfo { get { return _dumpDebugInfo; } } - public bool DoVerboseOutput { get { return _verbose; } } - public string CrossgenExecutable { get { return _crossgenExe; } } - public string JitPath { get { return _jitPath; } } - public string AltJit { get { return _altjit; } } - public string RootPath { get { return _rootPath; } } - public IReadOnlyList PlatformPaths { get { return _platformPaths; } } - public string FileName { get { return _fileName; } } - public IReadOnlyList AssemblyList { get { return _assemblyList; } } + public bool Wait { get; set; } + public bool UseJitPath { get { return (JitPath != null); } } + public bool Recursive { get; set; } + public bool UseFileName { get { return (FileName != null); } } + public bool DumpGCInfo { get; set; } + public bool DumpDebugInfo { get; set; } + public bool Verbose { get; set; } + public string CrossgenExe { get; set; } + public string JitPath { get; set; } + public string AltJit { get; set; } + public string RootPath { get; set; } + public IReadOnlyList Platform { get; set; } + public string FileName { get; set; } + public IReadOnlyList AssemblyList { get; set; } + public bool Error { get; set; } } public class AssemblyInfo @@ -168,49 +133,79 @@ public class jitdasm { public static int Main(string[] args) { - // Error count will be returned. Start at 0 - this will be incremented - // based on the error counts derived from the DisasmEngine executions. - int errorCount = 0; + RootCommand rootCommand = new RootCommand(); + + Option altJitOption = new Option("--altjit", "If set, the name of the altjit to use (e.g., protononjit.dll).", new Argument()); + Option crossgenExeOption = new Option("--crossgenExe", "The crossgen compiler exe.", new Argument()); + crossgenExeOption.AddAlias("-c"); + Option jitPathOption = new Option("--jitPath", "The full path to the jit library.", new Argument()); + jitPathOption.AddAlias("-j"); + Option outputOption = new Option("--output", "The output path.", new Argument()); + outputOption.AddAlias("-o"); + Option fileNameOption = new Option("--fileName", "Name of file to take list of assemblies from. Both a file and assembly list can be used.", new Argument()); + fileNameOption.AddAlias("-f"); + Option gcInfoOption = new Option("--gcinfo", "Add GC info to the disasm output.", new Argument()); + Option debugInfoOption = new Option("--debuginfo", "Add Debug info to the disasm output.", new Argument()); + Option verboseOption = new Option("--verbose", "Enable verbose output.", new Argument()); + verboseOption.AddAlias("-v"); + Option recursiveOption = new Option("--recursive", "Scan directories recursively", new Argument()); + recursiveOption.AddAlias("-r"); + Option platformOption = new Option("--platform", "Path to platform assemblies", new Argument() { Arity = ArgumentArity.OneOrMore }); + platformOption.AddAlias("-p"); + + Argument assemblies = new Argument() { Arity = ArgumentArity.OneOrMore }; + assemblies.Name = "assemblyList"; + + rootCommand.AddOption(altJitOption); + rootCommand.AddOption(crossgenExeOption); + rootCommand.AddOption(jitPathOption); + rootCommand.AddOption(outputOption); + rootCommand.AddOption(fileNameOption); + rootCommand.AddOption(gcInfoOption); + rootCommand.AddOption(debugInfoOption); + rootCommand.AddOption(verboseOption); + rootCommand.AddOption(recursiveOption); + rootCommand.AddOption(platformOption); + + rootCommand.AddArgument(assemblies); + + rootCommand.Handler = CommandHandler.Create((config) => + { + config.Validate(); - // Parse and store comand line options. - var config = new Config(args); + if (config.Error) + { + return -1; + } - // Stop to attach a debugger if desired. - if (config.WaitForDebugger) - { - WaitForDebugger(); - } + // Builds assemblyInfoList on jitdasm - // Builds assemblyInfoList on jitdasm + List assemblyWorkList = GenerateAssemblyWorklist(config); - List assemblyWorkList = GenerateAssemblyWorklist(config); - - // The disasm engine encapsulates a particular set of diffs. An engine is - // produced with a given code generator and assembly list, which then produces - // a set of disasm outputs. + // The disasm engine encapsulates a particular set of diffs. An engine is + // produced with a given code generator and assembly list, which then produces + // a set of disasm outputs. - DisasmEngine crossgenDisasm = new DisasmEngine(config.CrossgenExecutable, config, config.RootPath, assemblyWorkList); - crossgenDisasm.GenerateAsm(); - - if (crossgenDisasm.ErrorCount > 0) - { - Console.Error.WriteLine("{0} errors compiling set.", crossgenDisasm.ErrorCount); - errorCount += crossgenDisasm.ErrorCount; - } + DisasmEngine crossgenDisasm = new DisasmEngine(config.CrossgenExe, config, config.RootPath, assemblyWorkList); + crossgenDisasm.GenerateAsm(); - return errorCount; - } + int errorCount = 0; - private static void WaitForDebugger() - { - Console.WriteLine("Wait for a debugger to attach. Press ENTER to continue"); - Console.WriteLine($"Process ID: {Process.GetCurrentProcess().Id}"); - Console.ReadLine(); + if (crossgenDisasm.ErrorCount > 0) + { + Console.Error.WriteLine("{0} errors compiling set.", crossgenDisasm.ErrorCount); + errorCount += crossgenDisasm.ErrorCount; + } + + return errorCount; + }); + + return rootCommand.InvokeAsync(args).Result; } public static List GenerateAssemblyWorklist(Config config) { - bool verbose = config.DoVerboseOutput; + bool verbose = config.Verbose; List assemblyList = new List(); List assemblyInfoList = new List(); @@ -309,7 +304,7 @@ private static List IdentifyAssemblies(string rootPath, Config con foreach (var filePath in subFiles) { - if (config.DoVerboseOutput) + if (config.Verbose) { Console.WriteLine("Scanning: {0}", filePath); } @@ -360,14 +355,14 @@ public DisasmEngine(string executable, Config config, string outputPath, _config = config; _executablePath = executable; _rootPath = outputPath; - _platformPaths = config.PlatformPaths; + _platformPaths = config.Platform; _jitPath = config.JitPath; _altjit = config.AltJit; _assemblyInfoList = assemblyInfoList; this.doGCDump = config.DumpGCInfo; this.doDebugDump = config.DumpDebugInfo; - this.verbose = config.DoVerboseOutput; + this.verbose = config.Verbose; } class ScriptResolverPolicyWrapper : ICommandResolverPolicy @@ -380,7 +375,7 @@ public void GenerateAsm() // Build a command per assembly to generate the asm output. foreach (var assembly in _assemblyInfoList) { - if (_config.DoVerboseOutput) + if (_config.Verbose) { Console.WriteLine("assembly name: " + assembly.Name); } @@ -412,7 +407,7 @@ public void GenerateAsm() } // Set platform assembly path if it's defined. - if (_platformPaths.Count > 0) + if ((_platformPaths != null) && (_platformPaths.Count > 0)) { commandArgs.Insert(0, "/Platform_Assemblies_Paths"); commandArgs.Insert(1, String.Join(" ", _platformPaths)); From 9c5ddd5d3476a671ede71e052916666dace08b48 Mon Sep 17 00:00:00 2001 From: Andy Ayers Date: Fri, 24 May 2019 17:47:20 -0700 Subject: [PATCH 06/10] update jit-dasm-pmi --- src/jit-dasm-pmi/jit-dasm-pmi.cs | 269 +++++++++++++++---------------- 1 file changed, 130 insertions(+), 139 deletions(-) diff --git a/src/jit-dasm-pmi/jit-dasm-pmi.cs b/src/jit-dasm-pmi/jit-dasm-pmi.cs index 7157e6ff..a370b2de 100644 --- a/src/jit-dasm-pmi/jit-dasm-pmi.cs +++ b/src/jit-dasm-pmi/jit-dasm-pmi.cs @@ -14,148 +14,98 @@ // using System; -using System.Diagnostics; using System.CommandLine; +using System.CommandLine.Invocation; using System.IO; using System.Collections.Generic; using System.Reflection; using System.Linq; using Microsoft.DotNet.Cli.Utils; using Microsoft.DotNet.Tools.Common; +using Command = Microsoft.DotNet.Cli.Utils.Command; +using CommandResult = Microsoft.DotNet.Cli.Utils.CommandResult; namespace ManagedCodeGen { - // Define options to be parsed public class Config { - private ArgumentSyntax _syntaxResult; - private string _altjit = null; - private string _corerunExe = null; - private string _jitPath = null; - private string _rootPath = null; - private string _fileName = null; - private IReadOnlyList _assemblyList = Array.Empty(); - private bool _wait = false; - private bool _recursive = false; - private IReadOnlyList _methods = Array.Empty(); - private IReadOnlyList _platformPaths = Array.Empty(); - private bool _dumpGCInfo = false; - private bool _dumpDebugInfo = false; - private bool _noCopyJit = false; - private bool _verbose = false; - private bool _tiering = false; - private bool _cctors = false; - - public Config(string[] args) + void ReportError(string message) { - _syntaxResult = ArgumentSyntax.Parse(args, syntax => - { - syntax.DefineOption("altjit", ref _altjit, "If set, the name of the altjit to use (e.g., protononjit.dll)."); - syntax.DefineOption("c|corerun", ref _corerunExe, "The corerun compiler exe."); - syntax.DefineOption("j|jit", ref _jitPath, "The full path to the jit library."); - syntax.DefineOption("o|output", ref _rootPath, "The output path."); - syntax.DefineOption("f|file", ref _fileName, "Name of file to take list of assemblies from. Both a file and assembly list can be used."); - syntax.DefineOption("gcinfo", ref _dumpGCInfo, "Add GC info to the disasm output."); - syntax.DefineOption("debuginfo", ref _dumpDebugInfo, "Add Debug info to the disasm output."); - syntax.DefineOption("v|verbose", ref _verbose, "Enable verbose output."); - syntax.DefineOption("t|tiering", ref _tiering, "Enable tiered jitting"); - syntax.DefineOption("cctors", ref _cctors, "Jit and run cctors before jitting other methods"); - syntax.DefineOption("r|recursive", ref _recursive, "Scan directories recursively."); - syntax.DefineOptionList("p|platform", ref _platformPaths, "Path to platform assemblies"); - - var waitArg = syntax.DefineOption("w|wait", ref _wait, "Wait for debugger to attach."); - waitArg.IsHidden = true; - - var methodsArg = syntax.DefineOptionList("m|methods", ref _methods, "List of methods to disasm."); - methodsArg.IsHidden = true; - - var noCopyArg = syntax.DefineOption("nocopy", ref _noCopyJit, "Correct jit has already been copied into the corerun directory"); - noCopyArg.IsHidden = true; - - // Warning!! - Parameters must occur after options to preserve parsing semantics. - - syntax.DefineParameterList("assembly", ref _assemblyList, "The list of assemblies or directories to scan for assemblies."); - }); - - // Run validation code on parsed input to ensure we have a sensible scenario. - - Validate(); + Console.WriteLine(message); + Error = true; } // Validate arguments // // Pass a single tool as --corerun. Optionally specify a jit for corerun to use. // - private void Validate() + public void Validate() { - if (_corerunExe == null) + // Corerun must be specified and exist + if (Corerun == null) { - _syntaxResult.ReportError("Specify --corerun."); + ReportError("You must specify --corerun."); } - - if ((_fileName == null) && (_assemblyList.Count == 0)) + else if (!File.Exists(Corerun)) { - _syntaxResult.ReportError("No input: Specify --file or list input assemblies."); + ReportError("Can't find --corerun tool."); + } + else + { + // Set to full path for command resolution logic. + string fullCorerunPath = Path.GetFullPath(Corerun); + Corerun = fullCorerunPath; } - // Check that we can find the corerunExe - if (_corerunExe != null) + // JitPath must be specified and exist. + if (JitPath == null) { - if (!File.Exists(_corerunExe)) - { - _syntaxResult.ReportError("Can't find --corerun tool."); - } - else - { - // Set to full path for command resolution logic. - string fullCorerunPath = Path.GetFullPath(_corerunExe); - _corerunExe = fullCorerunPath; - } + ReportError("You must specify --jitPath."); + } + else if (!File.Exists(JitPath)) + { + ReportError("Can't find --jitPath library."); + } + else + { + // Set to full path for command resolution logic. + string fullJitPath = Path.GetFullPath(JitPath); + JitPath = fullJitPath; } - // Check that we can find the jit library. - if (_jitPath != null) + if ((FileName == null) && (AssemblyList.Count == 0)) { - if (!File.Exists(_jitPath)) - { - _syntaxResult.ReportError("Can't find --jit library."); - } - else - { - // Set to full path for command resolution logic. - string fullJitPath = Path.GetFullPath(_jitPath); - _jitPath = fullJitPath; - } + ReportError("No input: Specify --fileName or list input assemblies."); } - if (_fileName != null) + if (FileName != null) { - if (!File.Exists(_fileName)) + if (!File.Exists(FileName)) { - var message = String.Format("Error reading input file {0}, file not found.", _fileName); - _syntaxResult.ReportError(message); + var message = String.Format("Error reading input file {0}, file not found.", FileName); + ReportError(message); } } } public bool HasUserAssemblies { get { return AssemblyList.Count > 0; } } - public bool WaitForDebugger { get { return _wait; } } - public bool UseJitPath { get { return (_jitPath != null); } } - public bool Recursive { get { return _recursive; } } - public bool UseFileName { get { return (_fileName != null); } } - public bool DumpGCInfo { get { return _dumpGCInfo; } } - public bool DumpDebugInfo { get { return _dumpDebugInfo; } } - public bool DoVerboseOutput { get { return _verbose; } } - public bool CopyJit { get { return !_noCopyJit; } } - public string CorerunExecutable { get { return _corerunExe; } } - public string JitPath { get { return _jitPath; } } - public string AltJit { get { return _altjit; } } - public string RootPath { get { return _rootPath; } } - public IReadOnlyList PlatformPaths { get { return _platformPaths; } } - public string FileName { get { return _fileName; } } - public IReadOnlyList AssemblyList { get { return _assemblyList; } } - public bool Tiering => _tiering; - public bool Cctors => _cctors; + public bool Recursive { get; set; } + public bool UseFileName { get { return (FileName != null); } } + public bool DumpGCInfo { get; set; } + public bool DumpDebugInfo { get; set; } + public bool Verbose { get; set; } + public bool NoCopy { get; set; } + public bool CopyJit { get { return !NoCopy; } } + public string Corerun { get; set; } + public string JitPath { get; set; } + public string AltJit { get; set; } + public string RootPath { get; set; } + public IReadOnlyList Platform { get; set; } + public string FileName { get; set; } + public IReadOnlyList AssemblyList { get; set; } + public bool Tiering { get; set; } + public bool Cctors { get; set; } + public bool Error { get; set; } } public class AssemblyInfo @@ -172,49 +122,90 @@ public class jitdasmpmi { public static int Main(string[] args) { - // Error count will be returned. Start at 0 - this will be incremented - // based on the error counts derived from the DisasmEngine executions. - int errorCount = 0; + RootCommand rootCommand = new RootCommand(); + + Option altJitOption = new Option("--altjit", "If set, the name of the altjit to use (e.g., protononjit.dll).", new Argument()); + Option corerunOption = new Option("--corerun", "Path to corerun.", new Argument()); + corerunOption.AddAlias("-c"); + Option jitPathOption = new Option("--jitPath", "The full path to the jit library.", new Argument()); + jitPathOption.AddAlias("-j"); + Option outputOption = new Option("--output", "The output path.", new Argument()); + outputOption.AddAlias("-o"); + Option fileNameOption = new Option("--fileName", "Name of file to take list of assemblies from. Both a file and assembly list can be used.", new Argument()); + fileNameOption.AddAlias("-f"); + Option gcInfoOption = new Option("--gcinfo", "Add GC info to the disasm output.", new Argument()); + Option debugInfoOption = new Option("--debuginfo", "Add Debug info to the disasm output.", new Argument()); + Option verboseOption = new Option("--verbose", "Enable verbose output.", new Argument()); + verboseOption.AddAlias("-v"); + Option recursiveOption = new Option("--recursive", "Scan directories recursively", new Argument()); + recursiveOption.AddAlias("-r"); + Option platformOption = new Option("--platform", "Path to platform assemblies", new Argument() { Arity = ArgumentArity.OneOrMore }); + platformOption.AddAlias("-p"); + Option tieringOption = new Option("--tiering", "Enable tiered jitting", new Argument()); + tieringOption.AddAlias("-t"); + Option cctorOption = new Option("--cctors", "Jit and run cctors before jitting other methods", new Argument()); + Option methodsOption = new Option("--methods", "List of methods to disasm.", new Argument() { Arity = ArgumentArity.OneOrMore }); + methodsOption.AddAlias("-m"); + Option noCopyOption = new Option("--nocopy", "Correct jit has already been copied into the corerun directory", new Argument()); + noCopyOption.IsHidden = true; + + Argument assemblies = new Argument() { Arity = ArgumentArity.OneOrMore }; + assemblies.Name = "assemblyList"; + + rootCommand.AddOption(altJitOption); + rootCommand.AddOption(corerunOption); + rootCommand.AddOption(jitPathOption); + rootCommand.AddOption(outputOption); + rootCommand.AddOption(fileNameOption); + rootCommand.AddOption(gcInfoOption); + rootCommand.AddOption(debugInfoOption); + rootCommand.AddOption(verboseOption); + rootCommand.AddOption(recursiveOption); + rootCommand.AddOption(platformOption); + rootCommand.AddOption(tieringOption); + rootCommand.AddOption(cctorOption); + rootCommand.AddOption(methodsOption); + rootCommand.AddOption(noCopyOption); + + rootCommand.AddArgument(assemblies); + + rootCommand.Handler = CommandHandler.Create((config) => + { + config.Validate(); - // Parse and store comand line options. - var config = new Config(args); + if (config.Error) + { + return -1; + } - // Stop to attach a debugger if desired. - if (config.WaitForDebugger) - { - WaitForDebugger(); - } + int errorCount = 0; - // Builds assemblyInfoList on jitdasm + // Builds assemblyInfoList on jitdasm - List assemblyWorkList = GenerateAssemblyWorklist(config); + List assemblyWorkList = GenerateAssemblyWorklist(config); - // The disasm engine encapsulates a particular set of diffs. An engine is - // produced with a given code generator and assembly list, which then produces - // a set of disasm outputs. + // The disasm engine encapsulates a particular set of diffs. An engine is + // produced with a given code generator and assembly list, which then produces + // a set of disasm outputs. - DisasmEnginePmi corerunDisasm = new DisasmEnginePmi(config.CorerunExecutable, config, config.RootPath, assemblyWorkList); - corerunDisasm.GenerateAsm(); + DisasmEnginePmi corerunDisasm = new DisasmEnginePmi(config.Corerun, config, config.RootPath, assemblyWorkList); + corerunDisasm.GenerateAsm(); - if (corerunDisasm.ErrorCount > 0) - { - Console.Error.WriteLine("{0} errors compiling set.", corerunDisasm.ErrorCount); - errorCount += corerunDisasm.ErrorCount; - } + if (corerunDisasm.ErrorCount > 0) + { + Console.Error.WriteLine("{0} errors compiling set.", corerunDisasm.ErrorCount); + errorCount += corerunDisasm.ErrorCount; + } - return errorCount; - } + return errorCount; + }); - private static void WaitForDebugger() - { - Console.WriteLine("Wait for a debugger to attach. Press ENTER to continue"); - Console.WriteLine($"Process ID: {Process.GetCurrentProcess().Id}"); - Console.ReadLine(); + return rootCommand.InvokeAsync(args).Result; } public static List GenerateAssemblyWorklist(Config config) { - bool verbose = config.DoVerboseOutput; + bool verbose = config.Verbose; List assemblyList = new List(); List assemblyInfoList = new List(); @@ -313,7 +304,7 @@ private static List IdentifyAssemblies(string rootPath, Config con foreach (var filePath in subFiles) { - if (config.DoVerboseOutput) + if (config.Verbose) { Console.WriteLine("Scanning: {0}", filePath); } @@ -371,14 +362,14 @@ public DisasmEnginePmi(string executable, Config config, string outputPath, _config = config; _executablePath = executable; _rootPath = outputPath; - _platformPaths = config.PlatformPaths; + _platformPaths = config.Platform; _jitPath = config.JitPath; _altjit = config.AltJit; _assemblyInfoList = assemblyInfoList; this.doGCDump = config.DumpGCInfo; this.doDebugDump = config.DumpDebugInfo; - this.verbose = config.DoVerboseOutput; + this.verbose = config.Verbose; } class ScriptResolverPolicyWrapper : ICommandResolverPolicy @@ -388,7 +379,7 @@ class ScriptResolverPolicyWrapper : ICommandResolverPolicy public void GenerateAsm() { - string testOverlayDir = Path.GetDirectoryName(_config.CorerunExecutable); + string testOverlayDir = Path.GetDirectoryName(_config.Corerun); string jitDir = Path.GetDirectoryName(_jitPath); string realJitPath = Path.Combine(testOverlayDir, GetPmiJitLibraryName()); string tempJitPath = Path.Combine(testOverlayDir, GetPmiJitLibraryName("-backup")); @@ -434,7 +425,7 @@ void GenerateAsmInternal() // Build a command per assembly to generate the asm output. foreach (var assembly in _assemblyInfoList) { - if (_config.DoVerboseOutput) + if (_config.Verbose) { Console.WriteLine("assembly name: " + assembly.Name); } From 47c7b1dc3fbde0318dda79c1b96e37fa1617304f Mon Sep 17 00:00:00 2001 From: Andy Ayers Date: Fri, 24 May 2019 21:10:43 -0700 Subject: [PATCH 07/10] some progress on jit-diff --- src/jit-diff/diff.cs | 74 ++-- src/jit-diff/install.cs | 7 +- src/jit-diff/jit-diff.cs | 762 +++++++++++++++++++++------------------ 3 files changed, 452 insertions(+), 391 deletions(-) diff --git a/src/jit-diff/diff.cs b/src/jit-diff/diff.cs index d2130629..cfaa4fde 100644 --- a/src/jit-diff/diff.cs +++ b/src/jit-diff/diff.cs @@ -125,7 +125,7 @@ protected static DiffTool NewDiffTool(Config config) { DiffTool result = null; - switch (config.DoCommand) + switch (config.Command) { case Commands.Diff: result = new CrossgenDiffTool(config); @@ -134,7 +134,7 @@ protected static DiffTool NewDiffTool(Config config) result = new PmiDiffTool(config); break; default: - Console.WriteLine($"Unexpected command for diff: {config.DoCommand}"); + Console.WriteLine($"Unexpected command for diff: {config.Command}"); break; } @@ -182,7 +182,7 @@ protected void StartDasmWorkOne(DasmWorkKind kind, List commandArgs, str { List args = ConstructArgs(commandArgs, clrPath); - string outputPath = Path.Combine(m_config.OutputPath, tagBaseDiff, assemblyInfo.OutputPath); + string outputPath = Path.Combine(m_config.Output, tagBaseDiff, assemblyInfo.OutputPath); args.Add("--output"); args.Add(outputPath); @@ -197,11 +197,11 @@ protected virtual void StartDasmWorkBaseDiff(List commandArgs, List commandArgs = new List(); commandArgs.Add("--platform"); - commandArgs.Add(config.CoreRoot); + commandArgs.Add(config.Core_Root); - if (config.GenerateGCInfo) + if (config.GcInfo) { commandArgs.Add("--gcinfo"); } - if (config.GenerateDebugInfo) + if (config.DebugInfo) { commandArgs.Add("--debuginfo"); } @@ -317,7 +317,7 @@ public static int DiffCommand(Config config) commandArgs.Add(config.AltJit); } - if ((config.DoCommand == Commands.PmiDiff) && config.Cctors) + if ((config.Command == Commands.PmiDiff) && config.Cctors) { commandArgs.Add("--cctors"); diffString += " [invoking .cctors]"; @@ -327,7 +327,7 @@ public static int DiffCommand(Config config) List assemblyWorkList = GenerateAssemblyWorklist(config); DasmResult dasmResult = diffTool.RunDasmTool(commandArgs, assemblyWorkList); Console.WriteLine($"Completed {diffString} in {(DateTime.Now - startTime).TotalSeconds:F2}s"); - Console.WriteLine($"Diffs (if any) can be viewed by comparing: {Path.Combine(config.OutputPath, "base")} {Path.Combine(config.OutputPath, "diff")}"); + Console.WriteLine($"Diffs (if any) can be viewed by comparing: {Path.Combine(config.Output, "base")} {Path.Combine(config.Output, "diff")}"); // Analyze completed run. @@ -336,19 +336,19 @@ public static int DiffCommand(Config config) List analysisArgs = new List(); analysisArgs.Add("--base"); - analysisArgs.Add(Path.Combine(config.OutputPath, "base")); + analysisArgs.Add(Path.Combine(config.Output, "base")); analysisArgs.Add("--diff"); - analysisArgs.Add(Path.Combine(config.OutputPath, "diff")); + analysisArgs.Add(Path.Combine(config.Output, "diff")); analysisArgs.Add("--recursive"); analysisArgs.Add("--note"); string jitName = config.AltJit ?? "default jit"; analysisArgs.Add($"{diffString} for {config.Arch} {jitName}"); - if (config.tsv) + if (config.Tsv) { analysisArgs.Add("--tsv"); - analysisArgs.Add(Path.Combine(config.OutputPath, "diffs.tsv")); + analysisArgs.Add(Path.Combine(config.Output, "diffs.tsv")); } if (config.Verbose) @@ -384,7 +384,7 @@ public static List GenerateAssemblyWorklist(Config config) { List assemblyInfoList = new List(); - foreach (string assembly in config.AssemblyList) + foreach (string assembly in config.Assembly) { if (Directory.Exists(assembly)) { @@ -411,11 +411,11 @@ public static List GenerateAssemblyWorklist(Config config) // These files will all be put in the same output directory. This // works because they all have unique names, and live in the same // source directory already. - if (config.CoreLib || config.DoFrameworks) + if (config.CoreLib || config.Frameworks) { foreach (var assembly in config.CoreLib ? s_CoreLibAssembly : s_frameworkAssemblies) { - string fullPathAssembly = Path.Combine(config.CoreRoot, assembly); + string fullPathAssembly = Path.Combine(config.Core_Root, assembly); if (!File.Exists(fullPathAssembly)) { @@ -435,9 +435,9 @@ public static List GenerateAssemblyWorklist(Config config) // The tests are in a tree hierarchy of directories. We will output these to a tree // structure matching their source tree structure, to avoid name conflicts. - if (config.Benchmarks || config.DoTestTree) + if (config.Benchmarks || config.Tests) { - string basepath = config.Benchmarks ? Utility.CombinePath(config.TestRoot, s_benchmarksPath) : config.TestRoot; + string basepath = config.Benchmarks ? Utility.CombinePath(config.Test_Root, s_benchmarksPath) : config.Test_Root; foreach (var dir in config.Benchmarks ? s_benchmarkDirectories : s_testDirectories) { string fullPathDir = Path.Combine(basepath, dir); @@ -516,11 +516,11 @@ protected override List ConstructArgs(List commandArgs, string c dasmArgs.Add("--crossgen"); if (m_config.HasCrossgenExe) { - dasmArgs.Add(m_config.CrossgenExe); + dasmArgs.Add(m_config.Crossgen); } else { - var crossgenPath = Path.Combine(m_config.CoreRoot, GetCrossgenExecutableName(m_config.PlatformMoniker)); + var crossgenPath = Path.Combine(m_config.Core_Root, GetCrossgenExecutableName(m_config.PlatformMoniker)); if (!File.Exists(crossgenPath)) { Console.Error.WriteLine("crossgen not found at {0}", crossgenPath); @@ -564,7 +564,7 @@ public PmiDiffTool(Config config) : base(config) { m_name = "PMI"; m_commandName = s_asmToolJit; - m_corerunPath = Path.Combine(m_config.CoreRoot, GetCorerunExecutableName(m_config.PlatformMoniker)); + m_corerunPath = Path.Combine(m_config.Core_Root, GetCorerunExecutableName(m_config.PlatformMoniker)); m_defaultJitName = GetJitLibraryName(m_config.PlatformMoniker); if (m_config.AltJit != null) { @@ -607,9 +607,9 @@ protected override List ConstructArgs(List commandArgs, string c void InstallBaseJit() { - string existingJitPath = Path.Combine(m_config.CoreRoot, m_testJitName); - string backupJitPath = Path.Combine(m_config.CoreRoot, "backup-" + m_testJitName); - string testJitPath = Path.Combine(m_config.BasePath, m_testJitName); + string existingJitPath = Path.Combine(m_config.Core_Root, m_testJitName); + string backupJitPath = Path.Combine(m_config.Core_Root, "backup-" + m_testJitName); + string testJitPath = Path.Combine(m_config.Base, m_testJitName); if (File.Exists(existingJitPath)) { if (m_config.Verbose) @@ -627,9 +627,9 @@ void InstallBaseJit() void InstallDiffJit() { - string exitingJitPath = Path.Combine(m_config.CoreRoot, m_testJitName); - string backupJitPath = Path.Combine(m_config.CoreRoot, "backup-" + m_testJitName); - string testJitPath = Path.Combine(m_config.DiffPath, m_testJitName); + string exitingJitPath = Path.Combine(m_config.Core_Root, m_testJitName); + string backupJitPath = Path.Combine(m_config.Core_Root, "backup-" + m_testJitName); + string testJitPath = Path.Combine(m_config.Diff, m_testJitName); if (File.Exists(exitingJitPath)) { if (m_config.Verbose) @@ -647,8 +647,8 @@ void InstallDiffJit() void RestoreDefaultJit() { - string existingJitPath = Path.Combine(m_config.CoreRoot, m_testJitName); - string backupJitPath = Path.Combine(m_config.CoreRoot, "backup-" + m_testJitName); + string existingJitPath = Path.Combine(m_config.Core_Root, m_testJitName); + string backupJitPath = Path.Combine(m_config.Core_Root, "backup-" + m_testJitName); if (File.Exists(backupJitPath)) { if (m_config.Verbose) @@ -672,7 +672,7 @@ protected override void StartDasmWorkBaseDiff(List commandArgs, List commandArgs, List (string)x["tag"] == tag).Any()) @@ -70,7 +71,7 @@ public static int InstallCommand(Config config) cijobsArgs.Add(config.BranchName); } - if (config.DoLastSucessful) + if (config.LastSuccessful) { cijobsArgs.Add("--last_successful"); } @@ -106,7 +107,7 @@ public static int InstallCommand(Config config) // // However, if we passed "--last_successful", we don't know that number! So, figure it out. - if (config.DoLastSucessful) + if (config.LastSuccessful) { // Find the largest numbered build with this job name. int maxBuildNum = -1; diff --git a/src/jit-diff/jit-diff.cs b/src/jit-diff/jit-diff.cs index 741c1ddd..b7708b6f 100755 --- a/src/jit-diff/jit-diff.cs +++ b/src/jit-diff/jit-diff.cs @@ -3,7 +3,6 @@ // See the LICENSE file in the project root for more information. using System; -//using System.Diagnostics; using System.CommandLine; using System.IO; using System.Collections.Generic; @@ -11,11 +10,11 @@ using System.Linq; using Microsoft.DotNet.Cli.Utils; using Microsoft.DotNet.Tools.Common; -//using Newtonsoft.Json; using Newtonsoft.Json.Linq; -//using System.Collections.Concurrent; -using System.Threading.Tasks; using System.Runtime.InteropServices; +using CommandResult = Microsoft.DotNet.Cli.Utils.CommandResult; +using Command = System.CommandLine.Command; +using System.CommandLine.Invocation; namespace ManagedCodeGen { @@ -112,134 +111,14 @@ private static string GetBuildOS(string platformMoniker) public class Config { - private ArgumentSyntax _syntaxResult; - private Commands _command = Commands.Diff; - private bool _baseSpecified = false; // True if user specified "--base" or "--base " or "--base " - private bool _diffSpecified = false; // True if user specified "--diff" or "--diff " or "--diff " - private string _basePath = null; // Non-null if user specified "--base " or "--base " - private string _diffPath = null; // Non-null if user specified "--diff " or "--diff " - private string _crossgenExe = null; - private string _outputPath = null; - private bool _noanalyze = false; - private string _tag = null; - private bool _sequential = false; - private string _platformPath = null; - private string _testPath = null; - private string _baseRoot = null; - private string _diffRoot = null; - private string _arch = null; - private string _build = null; - private string _altjit = null; - private bool _corelib = false; - private bool _frameworks = false; - private bool _benchmarks = false; - private bool _tests = false; - private bool _gcinfo = false; - private bool _debuginfo = false; - private bool _verbose = false; - private string _jobName = null; - private string _number = null; - private bool _lastSuccessful = false; - private string _jitUtilsRoot = null; - private string _rid = null; - private string _platformName = null; - private string _branchName = null; - private bool _pmi = false; - private IReadOnlyList _assemblyList = Array.Empty(); - private bool _tsv; - private bool _cctors; - + public static string s_DefaultBaseDiff = "^^^"; + public bool BaseSpecified = false; // True if user specified "--base" or "--base " or "--base " + public bool DiffSpecified = false; // True if user specified "--diff" or "--diff " or "--diff " private JObject _jObj; private bool _configFileLoaded = false; private bool _noJitUtilsRoot = false; - private bool _validationError = false; - - public Config(string[] args) - { - // Get configuration values from JIT_UTILS_ROOT/config.json - LoadFileConfig(); - - _syntaxResult = ArgumentSyntax.Parse(args, syntax => - { - // Diff command section. - syntax.DefineCommand("diff", ref _command, Commands.Diff, "Run asm diffs via crossgen."); - - var baseOption = syntax.DefineOption("b|base", ref _basePath, false, - "The base compiler directory or tag. Will use crossgen, corerun, or clrjit from this directory."); - var diffOption = syntax.DefineOption("d|diff", ref _diffPath, false, - "The diff compiler directory or tag. Will use crossgen, corerun, or clrjit from this directory."); - syntax.DefineOption("crossgen", ref _crossgenExe, - "The crossgen compiler exe. When this is specified, will use clrjit from the --base and " + - "--diff directories with this crossgen."); - syntax.DefineOption("o|output", ref _outputPath, "The output path."); - syntax.DefineOption("noanalyze", ref _noanalyze, "Do not analyze resulting base, diff dasm directories. (By default, the directories are analyzed for diffs.)"); - syntax.DefineOption("s|sequential", ref _sequential, "Run sequentially; don't do parallel compiles."); - syntax.DefineOption("t|tag", ref _tag, "Name of root in output directory. Allows for many sets of output."); - syntax.DefineOption("c|corelib", ref _corelib, "Diff System.Private.CoreLib.dll."); - syntax.DefineOption("f|frameworks", ref _frameworks, "Diff frameworks."); - syntax.DefineOption("benchmarks", ref _benchmarks, "Diff core benchmarks."); - syntax.DefineOption("tests", ref _tests, "Diff all tests."); - syntax.DefineOption("gcinfo", ref _gcinfo, "Add GC info to the disasm output."); - syntax.DefineOption("debuginfo", ref _debuginfo, "Add Debug info to the disasm output."); - syntax.DefineOption("v|verbose", ref _verbose, "Enable verbose output."); - syntax.DefineOption("core_root", ref _platformPath, "Path to test CORE_ROOT."); - syntax.DefineOption("test_root", ref _testPath, "Path to test tree. Use with --benchmarks or --tests."); - syntax.DefineOption("base_root", ref _baseRoot, "Path to root of base dotnet/coreclr repo."); - syntax.DefineOption("diff_root", ref _diffRoot, "Path to root of diff dotnet/coreclr repo."); - syntax.DefineOption("arch", ref _arch, "Architecture to diff (x86, x64)."); - syntax.DefineOption("build", ref _build, "Build flavor to diff (Checked, Debug)."); - syntax.DefineOption("altjit", ref _altjit, "If set, the name of the altjit to use (e.g., protononjit.dll)."); - var pmiOption = syntax.DefineOption("pmi", ref _pmi, "Run asm diffs via pmi."); - syntax.DefineOption("cctors", ref _cctors, "With --pmi, jit and run cctors before jitting other methods"); - syntax.DefineOptionList("assembly", ref _assemblyList, "Run asm diffs on a given set of assemblies. An individual item can be an assembly or a directory tree containing assemblies."); - syntax.DefineOption("tsv", ref _tsv, "Dump analysis data to diffs.tsv in output directory."); - // List command section. - syntax.DefineCommand("list", ref _command, Commands.List, - "List defaults and available tools in " + s_configFileName + "."); - syntax.DefineOption("v|verbose", ref _verbose, "Enable verbose output."); - - // Install command section. - syntax.DefineCommand("install", ref _command, Commands.Install, "Install tool in " + s_configFileName + "."); - syntax.DefineOption("j|job", ref _jobName, "Name of the job."); - syntax.DefineOption("n|number", ref _number, "Job number."); - syntax.DefineOption("l|last_successful", ref _lastSuccessful, "Last successful build."); - syntax.DefineOption("b|branch", ref _branchName, "Name of branch."); - syntax.DefineOption("v|verbose", ref _verbose, "Enable verbose output."); - - // Uninstall command section.s - syntax.DefineCommand("uninstall", ref _command, Commands.Uninstall, "Uninstall tool from " + s_configFileName + "."); - syntax.DefineOption("t|tag", ref _tag, "Name of tool tag in config file."); - - _baseSpecified = baseOption.IsSpecified; - _diffSpecified = diffOption.IsSpecified; - - if (pmiOption.IsSpecified) - { - _command = Commands.PmiDiff; - } - }); - - SetRID(); - - ExpandToolTags(); - - SetDefaults(); - - Validate(); - - if (_command == Commands.Diff || _command == Commands.PmiDiff) - { - // Do additional initialization relevant for just the "diff" command. - - DeriveOutputTag(); - - // Now that output path and tag are guaranteed to be set, update - // the output path to included the tag. - _outputPath = Path.Combine(_outputPath, _tag); - } - } - private void SetRID() + public void SetRID() { // Extract system RID from dotnet cli List commandArgs = new List { "--info" }; @@ -259,7 +138,7 @@ private void SetRID() Match ridMatch = ridPattern.Match(line); if (ridMatch.Success) { - _rid = ridMatch.Groups[1].Value; + Rid = ridMatch.Groups[1].Value; continue; } @@ -267,7 +146,7 @@ private void SetRID() Match platMatch = platPattern.Match(line); if (platMatch.Success) { - _platformName = platMatch.Groups[1].Value; + PlatformName = platMatch.Groups[1].Value; continue; } } @@ -277,7 +156,7 @@ public string PlatformMoniker { get { - switch (_platformName) + switch (PlatformName) { case "Windows": return "Windows"; @@ -286,30 +165,16 @@ public string PlatformMoniker case "Darwin": return "OSX"; default: - Console.Error.WriteLine("No platform mapping! (Platform name = {0})", _platformName); + Console.Error.WriteLine("No platform mapping! (Platform name = {0})", PlatformName); return null; } } } - private void SetDefaults() + public void SetDiffDefaults() { - // Figure out what we need to set from defaults. - - switch (_command) - { - case Commands.Diff: - case Commands.PmiDiff: - break; - case Commands.Install: - case Commands.Uninstall: - case Commands.List: - // Don't need any defaults. - return; - } - - bool needOutputPath = (_outputPath == null); // We need to find --output - bool needCoreRoot = (_platformPath == null); // We need to find --core_root + bool needOutputPath = (Output == null); // We need to find --output + bool needCoreRoot = (Core_Root == null); // We need to find --core_root // It's not clear we should find a default for crossgen: in the current code, if crossgen is specified, // then we always use that. If not specified, we find crossgen in core_root. That seems appropriate to @@ -317,11 +182,11 @@ private void SetDefaults() // bool needCrossgen = (_crossgenExe == null); // We need to find --crossgen bool needCrossgen = false; - bool needBasePath = _baseSpecified && (_basePath == null); // We need to find --base - bool needDiffPath = _diffSpecified && (_diffPath == null); // We need to find --diff - bool needTestTree = (Benchmarks || DoTestTree) && (_testPath == null); // We need to find --test_root + bool needBasePath = BaseSpecified && (Base == null); // We need to find --base + bool needDiffPath = DiffSpecified && (Diff == null); // We need to find --diff + bool needTestTree = (Benchmarks || Tests) && (TestPath == null); // We need to find --test_root - bool needDiffRoot = (_diffRoot == null) && + bool needDiffRoot = (Diff_Root == null) && (needOutputPath || needCoreRoot || needCrossgen || needDiffPath || needTestTree); // If --diff_root wasn't specified, see if we can figure it out from the current directory @@ -329,15 +194,15 @@ private void SetDefaults() if (needDiffRoot) { - _diffRoot = Utility.GetRepoRoot(Verbose); + Diff_Root = Utility.GetRepoRoot(Verbose); } - if (needOutputPath && (_diffRoot != null)) + if (needOutputPath && (Diff_Root != null)) { - _outputPath = Utility.CombinePath(_diffRoot, s_defaultDiffDirectoryPath); - PathUtility.EnsureDirectoryExists(_outputPath); + Output = Utility.CombinePath(Diff_Root, s_defaultDiffDirectoryPath); + PathUtility.EnsureDirectoryExists(Output); - Console.WriteLine("Using --output {0}", _outputPath); + Console.WriteLine("Using --output {0}", Output); } if (needCoreRoot || needCrossgen || needBasePath || needDiffPath || needTestTree) @@ -380,7 +245,7 @@ private void SetDefaults() List archList; List buildList; - if (_arch == null) + if (Arch == null) { Architecture arch = RuntimeInformation.ProcessArchitecture; switch (arch) @@ -405,18 +270,18 @@ private void SetDefaults() } else { - archList = new List { _arch }; + archList = new List { Arch }; } foreach (var arch in archList) { - if (_build == null) + if (Build == null) { buildList = new List { "Checked", "Debug" }; } else { - buildList = new List { _build }; + buildList = new List { Build }; } foreach (var build in buildList) @@ -424,18 +289,18 @@ private void SetDefaults() var buildDirName = GetBuildOS(PlatformMoniker) + "." + arch + "." + build; string tryBasePath = null, tryDiffPath = null; - if (needBasePath && (_baseRoot != null)) + if (needBasePath && (Base_Root != null)) { - tryBasePath = Path.Combine(_baseRoot, "bin", "Product", buildDirName); + tryBasePath = Path.Combine(Base_Root, "bin", "Product", buildDirName); if (!Directory.Exists(tryBasePath)) { continue; } } - if (needDiffPath && (_diffRoot != null)) + if (needDiffPath && (Diff_Root != null)) { - tryDiffPath = Path.Combine(_diffRoot, "bin", "Product", buildDirName); + tryDiffPath = Path.Combine(Diff_Root, "bin", "Product", buildDirName); if (!Directory.Exists(tryDiffPath)) { continue; @@ -448,33 +313,33 @@ private void SetDefaults() if (tryBasePath != null) { - _basePath = tryBasePath; + Base= tryBasePath; needBasePath = false; - Console.WriteLine($"Using --base {_basePath}"); + Console.WriteLine($"Using --base {Base}"); } if (tryDiffPath != null) { - _diffPath = tryDiffPath; + Diff = tryDiffPath; needDiffPath = false; - Console.WriteLine($"Using --diff {_diffPath}"); + Console.WriteLine($"Using --diff {Diff}"); } - if (_arch == null) + if (Arch == null) { - _arch = arch; - Console.WriteLine($"Using --arch {_arch}"); + Arch = arch; + Console.WriteLine($"Using --arch {Arch}"); } break; } - if (_build == null) + if (Build == null) { buildList = new List { "Release", "Checked", "Debug" }; } else { - buildList = new List { _build }; + buildList = new List { Build }; } foreach (var build in buildList) @@ -482,27 +347,27 @@ private void SetDefaults() var buildDirName = GetBuildOS(PlatformMoniker) + "." + arch + "." + build; string tryPlatformPath = null, tryCrossgen = null, tryTestPath = null; - if (needCoreRoot && (_diffRoot != null)) + if (needCoreRoot && (Diff_Root != null)) { - tryPlatformPath = Path.Combine(_diffRoot, "bin", "tests", buildDirName, "Tests", "Core_Root"); + tryPlatformPath = Path.Combine(Diff_Root, "bin", "tests", buildDirName, "Tests", "Core_Root"); if (!Directory.Exists(tryPlatformPath)) { continue; } } - if (needCrossgen && (_diffRoot != null)) + if (needCrossgen && (Diff_Root != null)) { - tryCrossgen = Path.Combine(_diffRoot, "bin", "Product", buildDirName, GetCrossgenExecutableName(PlatformMoniker)); + tryCrossgen = Path.Combine(Diff_Root, "bin", "Product", buildDirName, GetCrossgenExecutableName(PlatformMoniker)); if (!File.Exists(tryCrossgen)) { continue; } } - if (needTestTree && (_diffRoot != null)) + if (needTestTree && (Diff_Root != null)) { - tryTestPath = Path.Combine(_diffRoot, "bin", "tests", buildDirName); + tryTestPath = Path.Combine(Diff_Root, "bin", "tests", buildDirName); if (!Directory.Exists(tryTestPath)) { continue; @@ -515,21 +380,21 @@ private void SetDefaults() if (tryPlatformPath != null) { - _platformPath = tryPlatformPath; + Core_Root = tryPlatformPath; needCoreRoot = false; - Console.WriteLine("Using --core_root {0}", _platformPath); + Console.WriteLine("Using --core_root {0}", Core_Root); } if (tryCrossgen != null) { - _crossgenExe = tryCrossgen; + Crossgen = tryCrossgen; needCrossgen = false; - Console.WriteLine("Using --crossgen {0}", _crossgenExe); + Console.WriteLine("Using --crossgen {0}", Crossgen); } if (tryTestPath != null) { - _testPath = tryTestPath; + TestPath = tryTestPath; needTestTree = false; - Console.WriteLine("Using --test_root {0}", _testPath); + Console.WriteLine("Using --test_root {0}", TestPath); } if (build != "Release") @@ -570,65 +435,65 @@ private void SetDefaults() } } - if (!_corelib && !_frameworks && !_benchmarks && !_tests && (_assemblyList.Count == 0)) + if (!CoreLib && !Frameworks && !Benchmarks && !Tests && (Assembly.Count == 0)) { // Setting --corelib as the default Console.WriteLine("No assemblies specified; defaulting to corelib"); - _corelib = true; + CoreLib = true; } if (Verbose) { Console.WriteLine("After setting defaults:"); - if (_baseRoot != null) + if (Base_Root != null) { - Console.WriteLine("--base_root {0}", _baseRoot); + Console.WriteLine("--base_root {0}", Base_Root); } - if (_diffRoot != null) + if (Diff_Root != null) { - Console.WriteLine("--diff_root {0}", _diffRoot); + Console.WriteLine("--diff_root {0}", Diff_Root); } - if (_platformPath != null) + if (Core_Root != null) { - Console.WriteLine("--core_root {0}", _platformPath); + Console.WriteLine("--core_root {0}", Core_Root); } - if (_crossgenExe != null) + if (Crossgen != null) { - Console.WriteLine("--crossgen {0}", _crossgenExe); + Console.WriteLine("--crossgen {0}", Crossgen); } - if (_arch != null) + if (Arch != null) { - Console.WriteLine("--arch {0}", _arch); + Console.WriteLine("--arch {0}", Arch); } - if (_build != null) + if (Build != null) { - Console.WriteLine("--build {0}", _build); + Console.WriteLine("--build {0}", Build); } - if (_outputPath != null) + if (Output != null) { - Console.WriteLine("--output {0}", _outputPath); + Console.WriteLine("--output {0}", Output); } - if (_basePath != null) + if (Base != null) { - Console.WriteLine("--base {0}", _basePath); + Console.WriteLine("--base {0}", Base); } - if (_diffPath != null) + if (Diff != null) { - Console.WriteLine("--diff {0}", _diffPath); + Console.WriteLine("--diff {0}", Diff); } - if (_testPath != null) + if (TestPath != null) { - Console.WriteLine("--test_root {0}", _testPath); + Console.WriteLine("--test_root {0}", TestPath); } } } - private void DeriveOutputTag() + public void DeriveOutputTag() { - if (_tag == null) + if (Tag == null) { int currentCount = 0; - foreach (var dir in Directory.EnumerateDirectories(_outputPath)) + foreach (var dir in Directory.EnumerateDirectories(Output)) { var name = Path.GetFileName(dir); Regex pattern = new Regex(@"dasmset_([0-9]{1,})"); @@ -644,11 +509,11 @@ private void DeriveOutputTag() } currentCount++; - _tag = String.Format("dasmset_{0}", currentCount); + Tag = String.Format("dasmset_{0}", currentCount); } } - private void ExpandToolTags() + public void ExpandToolTags() { if (!_configFileLoaded) { @@ -673,56 +538,51 @@ private void ExpandToolTags() var tag = (string)tool["tag"]; var path = (string)tool["path"]; - if (_basePath == tag) + if (Base == tag) { // passed base tag matches installed tool, reset path. - _basePath = path; + Base = path; } - if (_diffPath == tag) + if (Diff == tag) { // passed diff tag matches installed tool, reset path. - _diffPath = path; + Diff = path; } } } - private void Validate() + //private void Validate() + //{ + // _validationError = false; + + // switch (_command) + // { + // case Commands.Diff: + // case Commands.PmiDiff: + // ValidateDiff(); + // break; + // case Commands.Install: + // ValidateInstall(); + // break; + // case Commands.Uninstall: + // ValidateUninstall(); + // break; + // case Commands.List: + // ValidateList(); + // break; + // } + + // if (_validationError) + // { + // DisplayDiffUsageMessage(); + // Environment.Exit(-1); + // } + //} + + public void DisplayDiffUsageMessage() { - _validationError = false; - - switch (_command) - { - case Commands.Diff: - case Commands.PmiDiff: - ValidateDiff(); - break; - case Commands.Install: - ValidateInstall(); - break; - case Commands.Uninstall: - ValidateUninstall(); - break; - case Commands.List: - ValidateList(); - break; - } - - if (_validationError) - { - DisplayUsageMessage(); - Environment.Exit(-1); - } - } - - private void DisplayUsageMessage() - { - Console.Error.WriteLine(""); - Console.Error.Write(_syntaxResult.GetHelpText(100)); - - if (_command == Commands.Diff) - { - string[] diffExampleText = { + string[] diffExampleText = { @"Examples:", @"", @" jit-diff diff --output c:\diffs --corelib --core_root c:\coreclr\bin\tests\Windows_NT.x64.Release\Tests\Core_Root --base c:\coreclr_base\bin\Product\Windows_NT.x64.Checked --diff c:\coreclr\bin\Product\Windows_NT.x86.Checked", @@ -761,71 +621,70 @@ private void DisplayUsageMessage() @" jit-diff diff --diff --build Debug", @" Generate diffs, but using a Debug build, even if there is a Checked build available." }; - foreach (var line in diffExampleText) - { - Console.Error.WriteLine(line); - } + foreach (var line in diffExampleText) + { + Console.Error.WriteLine(line); } } private void DisplayErrorMessage(string error) { Console.Error.WriteLine("error: {0}", error); - _validationError = true; + Error = true; } - private void ValidateDiff() + public void ValidateDiff() { - if (_platformPath == null) + if (Core_Root == null) { DisplayErrorMessage("Specify --core_root "); } - if (_corelib && _frameworks) + if (CoreLib && Frameworks) { DisplayErrorMessage("Specify only one of --corelib or --frameworks"); } - if (_benchmarks && _tests) + if (Benchmarks && Tests) { DisplayErrorMessage("Specify only one of --benchmarks or --tests"); } - if (_benchmarks && (_testPath == null)) + if (Benchmarks && (TestPath == null)) { DisplayErrorMessage("--benchmarks requires specifying --test_root or --diff_root or running in the coreclr tree"); } - if (_tests && (_testPath == null)) + if (Tests && (TestPath == null)) { DisplayErrorMessage("--tests requires specifying --test_root or --diff_root or running in the coreclr tree"); } - if (_outputPath == null) + if (Output == null) { DisplayErrorMessage("Specify --output "); } - else if (!Directory.Exists(_outputPath)) + else if (!Directory.Exists(Output)) { DisplayErrorMessage("Can't find --output path."); } - if ((_basePath == null) && (_diffPath == null)) + if ((Base == null) && (Diff == null)) { DisplayErrorMessage("Specify either --base or --diff or both."); } - if (_basePath != null && !Directory.Exists(_basePath)) + if (Base != null && !Directory.Exists(Base)) { DisplayErrorMessage("Can't find --base directory."); } - if (_diffPath != null && !Directory.Exists(_diffPath)) + if (Diff != null && !Directory.Exists(Diff)) { DisplayErrorMessage("Can't find --diff directory."); } - if (_crossgenExe != null && !File.Exists(_crossgenExe)) + if (Crossgen != null && !File.Exists(Crossgen)) { DisplayErrorMessage("Can't find --crossgen executable."); } @@ -833,12 +692,12 @@ private void ValidateDiff() private void ValidateInstall() { - if (_jobName == null) + if (JobName == null) { DisplayErrorMessage("Specify --job "); } - if ((_number == null) && !_lastSuccessful) + if ((Number == null) && !LastSuccessful) { DisplayErrorMessage("Specify --number or --last_successful to identify build to install."); } @@ -859,7 +718,7 @@ private void ValidateInstall() private void ValidateUninstall() { - if (_tag == null) + if (Tag == null) { DisplayErrorMessage("Specify --tag "); } @@ -954,16 +813,16 @@ public T ExtractDefault(string name, out bool found) return default(T); } - private void LoadFileConfig() + public void LoadFileConfig() { - _jitUtilsRoot = Environment.GetEnvironmentVariable("JIT_UTILS_ROOT"); - if (_jitUtilsRoot == null) + JitUtilsRoot = Environment.GetEnvironmentVariable("JIT_UTILS_ROOT"); + if (JitUtilsRoot == null) { _noJitUtilsRoot = true; return; } - string configFilePath = Path.Combine(_jitUtilsRoot, s_configFileName); + string configFilePath = Path.Combine(JitUtilsRoot, s_configFileName); if (!File.Exists(configFilePath)) { Console.Error.WriteLine("Can't find {0}", configFilePath); @@ -1009,7 +868,7 @@ private void LoadFileConfig() } else { - _basePath = basePath; + Base = basePath; } } @@ -1023,7 +882,7 @@ private void LoadFileConfig() } else { - _diffPath = diffPath; + Diff = diffPath; } } @@ -1031,75 +890,75 @@ private void LoadFileConfig() var crossgenExe = GetToolPath("crossgen", out found); if (found) { - if (!File.Exists(_crossgenExe)) + if (!File.Exists(Crossgen)) { Console.WriteLine("Default crossgen file {0} not found! Investigate config file entry.", crossgenExe); } else { - _crossgenExe = crossgenExe; + Crossgen = crossgenExe; } } // Set up output var outputPath = ExtractDefault("output", out found); - _outputPath = (found) ? outputPath : _outputPath; + Output = (found) ? outputPath : Output; // Setup platform path (core_root). var platformPath = ExtractDefault("core_root", out found); - _platformPath = (found) ? platformPath : _platformPath; + Core_Root = (found) ? platformPath : Core_Root; // Set up test path (test_root). var testPath = ExtractDefault("test_root", out found); - _testPath = (found) ? testPath : _testPath; + TestPath = (found) ? testPath : TestPath; var baseRoot = ExtractDefault("base_root", out found); - _baseRoot = (found) ? baseRoot : _baseRoot; + Base_Root = (found) ? baseRoot : Base_Root; var diffRoot = ExtractDefault("diff_root", out found); - _diffRoot = (found) ? diffRoot : _diffRoot; + Diff_Root = (found) ? diffRoot : Diff_Root; var arch = ExtractDefault("arch", out found); - _arch = (found) ? arch : _arch; + Arch = (found) ? arch : Arch; var build = ExtractDefault("build", out found); - _build = (found) ? build : _build; + Build = (found) ? build : Build; // Set flag from default for analyze. var noanalyze = ExtractDefault("noanalyze", out found); - _noanalyze = (found) ? noanalyze : _noanalyze; + NoAnalyze = (found) ? noanalyze : NoAnalyze; // Set flag from default for corelib. var corelib = ExtractDefault("corelib", out found); - _corelib = (found) ? corelib : _corelib; + CoreLib = (found) ? corelib : CoreLib; // Set flag from default for frameworks. var frameworks = ExtractDefault("frameworks", out found); - _frameworks = (found) ? frameworks : _frameworks; + Frameworks = (found) ? frameworks : Frameworks; // Set flag from default for benchmarks. var benchmarks = ExtractDefault("benchmarks", out found); - _benchmarks = (found) ? benchmarks : _benchmarks; + Benchmarks = (found) ? benchmarks : Benchmarks; // Set flag from default for tests. var tests = ExtractDefault("tests", out found); - _tests = (found) ? tests : _tests; + Tests = (found) ? tests : Tests; // Set flag from default for gcinfo. var gcinfo = ExtractDefault("gcinfo", out found); - _gcinfo = (found) ? gcinfo : _gcinfo; + GcInfo = (found) ? gcinfo : GcInfo; // Set flag from default for debuginfo. var debuginfo = ExtractDefault("debuginfo", out found); - _debuginfo = (found) ? debuginfo : _debuginfo; + DebugInfo = (found) ? debuginfo : DebugInfo; // Set flag from default for tag. var tag = ExtractDefault("tag", out found); - _tag = (found) ? tag : _tag; + Tag = (found) ? tag : Tag; // Set flag from default for verbose. var verbose = ExtractDefault("verbose", out found); - _verbose = (found) ? verbose : _verbose; + Verbose = (found) ? verbose : Verbose; } } @@ -1143,7 +1002,7 @@ void PrintDefault(string parameter, DefaultType dt) public int ListCommand() { - string configPath = Path.Combine(_jitUtilsRoot, s_configFileName); + string configPath = Path.Combine(JitUtilsRoot, s_configFileName); Console.WriteLine("Listing {0} key in {1}", s_configFileRootKey, configPath); Console.WriteLine(); @@ -1193,7 +1052,7 @@ public int ListCommand() { string tag = (string)tool["tag"]; string path = (string)tool["path"]; - if (_verbose) + if (Verbose) { Console.WriteLine("\t{0}: {1}", tag, path); } @@ -1213,39 +1072,47 @@ public int ListCommand() return 0; } - public bool DoDiffCompiles { get { return _diffSpecified; } } - public bool DoBaseCompiles { get { return _baseSpecified; } } - public string CoreRoot { get { return _platformPath; } } - public string TestRoot { get { return _testPath; } } - public string BasePath { get { return _basePath; } } - public string DiffPath { get { return _diffPath; } } - public string CrossgenExe { get { return _crossgenExe; } } - public bool HasCrossgenExe { get { return (_crossgenExe != null); } } - public string OutputPath { get { return _outputPath; } } - public bool Sequential { get { return _sequential; } } - public string Tag { get { return _tag; } } - public bool HasTag { get { return (_tag != null); } } - public bool CoreLib { get { return _corelib; } } - public bool DoFrameworks { get { return _frameworks; } } - public bool Benchmarks { get { return _benchmarks; } } - public bool DoTestTree { get { return _tests; } } - public bool GenerateGCInfo { get { return _gcinfo; } } - public bool GenerateDebugInfo { get { return _debuginfo; } } - public bool Verbose { get { return _verbose; } } - public bool DoAnalyze { get { return !_noanalyze; } } - public Commands DoCommand { get { return _command; } } - public string JobName { get { return _jobName; } } - public bool DoLastSucessful { get { return _lastSuccessful; } } - public string JitUtilsRoot { get { return _jitUtilsRoot; } } - public bool HasJitUtilsRoot { get { return (_jitUtilsRoot != null); } } - public string RID { get { return _rid; } } - public string Number { get { return _number; } } - public string BranchName { get { return _branchName; } } - public string AltJit { get { return _altjit; } } - public string Arch { get { return _arch; } } - public IReadOnlyList AssemblyList => _assemblyList; - public bool tsv { get { return _tsv; } } - public bool Cctors => _cctors; + public Commands Command { get; set; } + public bool DoDiffCompiles { get { return DiffSpecified; } } + public bool DoBaseCompiles { get { return BaseSpecified; } } + public string Core_Root { get; set; } + public string Test_Root { get; set; } + public string Base { get; set; } + public string Diff { get; set; } + public string Crossgen { get; set; } + public bool HasCrossgenExe { get { return (Crossgen != null); } } + public string Output { get; set; } + public bool Sequential { get; set; } + public string TestPath { get; set; } + public string Base_Root { get; set; } + public string Diff_Root { get; set; } + public string Tag { get; set; } + public bool HasTag { get { return (Tag != null); } } + public bool CoreLib { get; set; } + public bool Frameworks { get; set; } + public bool Benchmarks { get; set; } + public bool Tests { get; set; } + public bool GcInfo { get; set; } + public bool DebugInfo { get; set; } + public bool Verbose { get; set; } + public bool NoAnalyze { get; set; } + public bool DoAnalyze { get { return !NoAnalyze; } } + public string JobName { get; set; } + public bool LastSuccessful { get; set; } + public string JitUtilsRoot { get; set; } + public bool HasJitUtilsRoot { get { return (JitUtilsRoot != null); } } + public string Rid { get; set; } + public string Number { get; set; } + public string BranchName { get; set; } + public string PlatformName { get; set; } + public string AltJit { get; set; } + public string Arch { get; set; } + public string Build { get; set; } + public IReadOnlyList Assembly { get; set; } + public bool Tsv { get; set; } + public bool Pmi { get; set; } + public bool Cctors { get; set; } + public bool Error { get; set; } } private static string[] s_testDirectories = @@ -1420,36 +1287,229 @@ public int ListCommand() public static int Main(string[] args) { - Config config = new Config(args); - int ret = 0; - - switch (config.DoCommand) + Command diffCommand = new Command("diff", "Run asm diffs via crossgen or pmi"); { - case Commands.Diff: - case Commands.PmiDiff: + // how to hanle the dir/tag/aspect..? + Option baseOption = new Option("--base", "The base compiler directory or tag. Will use crossgen, corerun, or clrjit from this directory.", + new Argument(Config.s_DefaultBaseDiff) { Arity = ArgumentArity.ZeroOrOne }); + baseOption.AddAlias("-b"); + Option baseRootOption = new Option("--base_root", "Path to root of base dotnet/coreclr repo.", new Argument()); + Option diffOption = new Option("--diff", "The diff compiler directory or tag. Will use crossgen, corerun, or clrjit from this directory.", + new Argument(Config.s_DefaultBaseDiff) { Arity = ArgumentArity.ZeroOrOne }); + diffOption.AddAlias("-d"); + Option diffRootOption = new Option("--diff_root", "Path to root of diff dotnet/coreclr repo.", new Argument()); + Option crossgenOption = new Option("--crossgen", "The crossgen compiler exe. When this is specified, will use clrjit from the --base and " + + "--diff directories with this crossgen.", new Argument()); + Option outputOption = new Option("--output", "The output path.", new Argument()); + outputOption.AddAlias("-o"); + Option noAnalyzeOption = new Option("--noanalyze", "Do not analyze resulting base, diff dasm directories. (By default, the directories are analyzed for diffs.)", new Argument()); + Option sequentialOption = new Option("--sequential", "Run sequentially; don't do parallel compiles.", new Argument()); + sequentialOption.AddAlias("-s"); + Option tagOption = new Option("--tag", "Name of root in output directory. Allows for many sets of output.", new Argument()); + tagOption.AddAlias("-t"); + Option corelibOption = new Option("--corelib", "Diff System.Private.CoreLib.dll.", new Argument()); + corelibOption.AddAlias("-c"); + Option frameworksOption = new Option("--frameworks", "Diff frameworks.", new Argument()); + frameworksOption.AddAlias("-f"); + Option benchmarksOption = new Option("--benchmarks", "Diff core benchmarks.", new Argument()); + Option testsOption = new Option("--tests", "Diff all tests.", new Argument()); + Option gcInfoOption = new Option("--gcinfo", "Add GC info to the disasm output.", new Argument()); + Option debugInfoOption = new Option("--debuginfo", "Add Debug info to the disasm output.", new Argument()); + Option verboseOption = new Option("--verbose", "Enable verbose output.", new Argument()); + verboseOption.AddAlias("-v"); + Option coreRootOption = new Option("--core_root", "Path to test CORE_ROOT.", new Argument()); + Option testRootOption = new Option("--test_root", "Path to test tree.Use with--benchmarks or--tests.", new Argument()); + Option archOption = new Option("--arch", "Architecture to diff (x86, x64).", new Argument()); + Option buildOption = new Option("--build", "Build flavor to diff (Checked, Debug).", new Argument()); + Option altJitOption = new Option("--altjit", "If set, the name of the altjit to use (e.g., protononjit.dll).", new Argument()); + Option pmiOption = new Option("--pmi", "Run asm diffs via pmi.", new Argument()); + Option cctorsOption = new Option("--cctors", "With --pmi, jit and run cctors before jitting other methods", new Argument()); + Option tsvOption = new Option("--tsv", "Dump analysis data to diffs.tsv in output directory.", new Argument()); + Option assemblies = new Option("--assembly", "Run asm diffs on a given set of assemblies. An individual item can be an assembly or a directory tree containing assemblies.", + new Argument() { Arity = ArgumentArity.OneOrMore }); + + diffCommand.AddOption(baseOption); + diffCommand.AddOption(baseRootOption); + diffCommand.AddOption(diffOption); + diffCommand.AddOption(diffRootOption); + diffCommand.AddOption(crossgenOption); + diffCommand.AddOption(outputOption); + diffCommand.AddOption(noAnalyzeOption); + diffCommand.AddOption(sequentialOption); + diffCommand.AddOption(tagOption); + diffCommand.AddOption(corelibOption); + diffCommand.AddOption(frameworksOption); + diffCommand.AddOption(benchmarksOption); + diffCommand.AddOption(testsOption); + diffCommand.AddOption(gcInfoOption); + diffCommand.AddOption(debugInfoOption); + diffCommand.AddOption(verboseOption); + diffCommand.AddOption(coreRootOption); + diffCommand.AddOption(testRootOption); + diffCommand.AddOption(archOption); + diffCommand.AddOption(buildOption); + diffCommand.AddOption(altJitOption); + diffCommand.AddOption(pmiOption); + diffCommand.AddOption(cctorsOption); + diffCommand.AddOption(tsvOption); + diffCommand.AddOption(assemblies); + + diffCommand.Handler = CommandHandler.Create((config) => + { + // --base and --diff are tricky to model since they have 3 behaviors: + // not specified, specified with no arg, and specified with arg. + // Try and sort that here... + if (config.Base != Config.s_DefaultBaseDiff) + { + config.BaseSpecified = true; + + if (config.Base == String.Empty) + { + config.Base = null; + } + } + else { - ret = DiffTool.DiffCommand(config); + Console.WriteLine("Base NOT specified!"); + config.Base = null; } - break; - case Commands.List: + + if (config.Diff != Config.s_DefaultBaseDiff) { - // List command: list loaded configuration - ret = config.ListCommand(); + config.DiffSpecified = true; + + if (config.Diff == String.Empty) + { + config.Diff = null; + } } - break; - case Commands.Install: + else { - ret = InstallCommand(config); + config.Diff = null; } - break; - case Commands.Uninstall: + + if (config.Pmi) { - ret = UninstallCommand(config); + config.Command = Commands.PmiDiff; } - break; + else + { + config.Command = Commands.Diff; + } + + config.LoadFileConfig(); + config.SetRID(); + config.ExpandToolTags(); + config.SetDiffDefaults(); + config.ValidateDiff(); + + if (config.Error) + { + config.DisplayDiffUsageMessage(); + return -1; + } + + config.DeriveOutputTag(); + + // Elswhere we expect this to always be set to something. + if (config.Assembly == null) + { + config.Assembly = new List(); + } + + return DiffTool.DiffCommand(config); + }); + } + + Command listCommand = new Command("list", "List defaults and available tools in " + s_configFileName + "."); + { + } + + Command installCommand = new Command("install", "Install tool in " + s_configFileName + "."); + { } - return ret; + Command uninstallCommand = new Command("uninstall", "Uninstall tool from " + s_configFileName + "."); + + RootCommand rootCommand = new RootCommand(); + rootCommand.AddCommand(diffCommand); + rootCommand.AddCommand(listCommand); + rootCommand.AddCommand(installCommand); + rootCommand.AddCommand(uninstallCommand); + + //// Get configuration values from JIT_UTILS_ROOT/config.json + //LoadFileConfig(); + + // // List command section. + // syntax.DefineCommand("list", ref _command, Commands.List, + // "List defaults and available tools in " + s_configFileName + "."); + // syntax.DefineOption("v|verbose", ref Verbose, "Enable verbose output."); + + // // Install command section. + // syntax.DefineCommand("install", ref _command, Commands.Install, "Install tool in " + s_configFileName + "."); + // syntax.DefineOption("j|job", ref JobName, "Name of the job."); + // syntax.DefineOption("n|number", ref Number, "Job number."); + // syntax.DefineOption("l|last_successful", ref LastSuccessful, "Last successful build."); + // syntax.DefineOption("b|branch", ref BranchName, "Name of branch."); + // syntax.DefineOption("v|verbose", ref Verbose, "Enable verbose output."); + + // // Uninstall command section.s + // syntax.DefineCommand("uninstall", ref _command, Commands.Uninstall, "Uninstall tool from " + s_configFileName + "."); + // syntax.DefineOption("t|tag", ref Tag, "Name of tool tag in config file."); + + // _baseSpecified = baseOption.IsSpecified; + // _diffSpecified = diffOption.IsSpecified; + + //}); + + //SetRID(); + + //ExpandToolTags(); + + //SetDefaults(); + + //Validate(); + + //if (_command == Commands.Diff || _command == Commands.PmiDiff) + //{ + // // Do additional initialization relevant for just the "diff" command. + + // DeriveOutputTag(); + + // // Now that output path and tag are guaranteed to be set, update + // // the output path to included the tag. + // OutputPath = Path.Combine(OutputPath, Tag); + //} + // Config config = new Config(args); + // int ret = 0; + + //switch (config.DoCommand) + //{ + // case Commands.Diff: + // case Commands.PmiDiff: + // { + // ret = DiffTool.DiffCommand(config); + // } + // break; + // case Commands.List: + // { + // // List command: list loaded configuration + // ret = config.ListCommand(); + // } + // break; + // case Commands.Install: + // { + // ret = InstallCommand(config); + // } + // break; + // case Commands.Uninstall: + // { + // ret = UninstallCommand(config); + // } + // break; + //} + + return rootCommand.InvokeAsync(args).Result; } } } From de34d44c171201cb9cd27185b95ef0a732b5f6fa Mon Sep 17 00:00:00 2001 From: Andy Ayers Date: Sat, 25 May 2019 11:59:10 -0700 Subject: [PATCH 08/10] update jit-format --- src/jit-format/jit-format.cs | 277 +++++++++++++++++++---------------- 1 file changed, 152 insertions(+), 125 deletions(-) diff --git a/src/jit-format/jit-format.cs b/src/jit-format/jit-format.cs index dd3bd04f..b4b656f8 100644 --- a/src/jit-format/jit-format.cs +++ b/src/jit-format/jit-format.cs @@ -21,6 +21,9 @@ using Microsoft.DotNet.Tools.Common; using Newtonsoft.Json; using Newtonsoft.Json.Linq; +using CommandResult = Microsoft.DotNet.Cli.Utils.CommandResult; +using System.CommandLine.Invocation; +using Process = System.Diagnostics.Process; namespace ManagedCodeGen { @@ -31,50 +34,14 @@ public class Config { private static string s_configFileName = "config.json"; private static string s_configFileRootKey = "format"; - - private ArgumentSyntax _syntaxResult; - private string _arch = null; - private string _os = null; - private string _build = null; - private string _rootPath = null; - private IReadOnlyList _filenames = Array.Empty(); - private IReadOnlyList _projects = Array.Empty(); - private string _srcDirectory = null; - private bool _untidy = false; - private bool _noformat = false; - private bool _fix = false; - private bool _verbose = false; - private bool _ignoreErrors = false; - private string _compileCommands = null; private bool _rewriteCompileCommands = false; - private JObject _jObj; private string _jitUtilsRoot = null; - public Config(string[] args) + void ReportError(string message) { - LoadFileConfig(); - - _syntaxResult = ArgumentSyntax.Parse(args, syntax => - { - syntax.DefineOption("a|arch", ref _arch, "The architecture of the build (options: x64, x86)"); - syntax.DefineOption("o|os", ref _os, "The operating system of the build (options: Windows, OSX, Ubuntu, Fedora, etc.)"); - syntax.DefineOption("b|build", ref _build, "The build type of the build (options: Release, Checked, Debug)"); - syntax.DefineOption("c|coreclr", ref _rootPath, "Full path to base coreclr directory"); - syntax.DefineOption("compile-commands", ref _compileCommands, "Full path to compile_commands.json"); - syntax.DefineOption("v|verbose", ref _verbose, "Enable verbose output."); - syntax.DefineOption("untidy", ref _untidy, "Do not run clang-tidy"); - syntax.DefineOption("noformat", ref _noformat, "Do not run clang-format"); - syntax.DefineOption("f|fix", ref _fix, "Fix formatting errors discovered by clang-format and clang-tidy."); - syntax.DefineOption("i|ignore-errors", ref _ignoreErrors, "Ignore clang-tidy errors"); - syntax.DefineOptionList("projects", ref _projects, "List of build projects clang-tidy should consider (e.g. dll, standalone, protojit, etc.). Default: dll"); - - syntax.DefineParameterList("filenames", ref _filenames, "Optional list of files that should be formatted."); - }); - - // Run validation code on parsed input to ensure we have a sensible scenario. - - validate(); + Console.WriteLine(message); + Error = true; } private void SetPlatform() @@ -85,7 +52,7 @@ private void SetPlatform() if (result.ExitCode != 0) { - Console.Error.WriteLine("dotnet --info returned non-zero"); + ReportError("dotnet --info returned non-zero"); } var lines = result.StdOut.Split(new[] { Environment.NewLine }, StringSplitOptions.None); @@ -98,140 +65,143 @@ private void SetPlatform() { if (match.Groups[1].Value.Trim() == "Windows") { - _os = "Windows_NT"; + OS = "Windows_NT"; } else if (match.Groups[1].Value.Trim() == "Darwin") { - _os = "OSX"; + OS = "OSX"; } else if (match.Groups[1].Value.Trim() == "Linux") { // Assuming anything other than Windows or OSX is a Linux flavor - _os = "Linux"; + OS = "Linux"; } else { - Console.WriteLine("Unknown operating system. Please specify with --os"); - Environment.Exit(-1); + ReportError("Unknown operating system. Please specify with --os"); } } } } - private void validate() + public void Validate() { - if (_arch == null) + if (Arch == null) { - if (_verbose) + if (Verbose) { Console.WriteLine("Defaulting architecture to x64."); } - _arch = "x64"; + Arch = "x64"; } - if (_build == null) + if (Build == null) { - if (_verbose) + if (Verbose) { Console.WriteLine("Defaulting build to Debug."); } - _build = "Debug"; + Build = "Debug"; } - if (_os == null) + if (OS == null) { - if (_verbose) + if (Verbose) { Console.WriteLine("Discovering operating system."); } SetPlatform(); - if (_verbose) + if (Error) + { + return; + } + + if (Verbose) { - Console.WriteLine("Operating system is {0}", _os); + Console.WriteLine("Operating system is {0}", OS); } } - if (_os == "Windows") + if (OS == "Windows") { - _os = "Windows_NT"; + OS = "Windows_NT"; } - if (_srcDirectory == null) + if (SourceDirectory == null) { - if (_verbose) + if (Verbose) { Console.WriteLine("Formatting jit directory."); } - _srcDirectory = "jit"; + SourceDirectory = "jit"; } - if (_projects.Count == 0 && _verbose) + if (Projects.Count == 0 && Verbose) { Console.WriteLine("Formatting dll project."); } - if (!_untidy && ( (_arch == null) || (_os == null) || (_build == null))) + if (!Untidy && ( (Arch == null) || (OS == null) || (Build == null))) { - _syntaxResult.ReportError("Specify --arch, --plaform, and --build for clang-tidy run."); + ReportError("Specify --arch, --plaform, and --build for clang-tidy run."); } - if (_rootPath == null) + if (CoreCLR == null) { - if (_verbose) + if (Verbose) { Console.WriteLine("Discovering --coreclr."); } - _rootPath = Utility.GetRepoRoot(_verbose); - if (_rootPath == null) + CoreCLR = Utility.GetRepoRoot(Verbose); + if (CoreCLR == null) { - _syntaxResult.ReportError("Specify --coreclr"); + ReportError("Specify --coreclr"); } else { - Console.WriteLine("Using --coreclr={0}", _rootPath); + Console.WriteLine("Using --coreclr={0}", CoreCLR); } } - if (!Directory.Exists(_rootPath)) + if (!Directory.Exists(CoreCLR)) { // If _rootPath doesn't exist, it is an invalid path - _syntaxResult.ReportError("Invalid path to coreclr directory. Specify with --coreclr"); + ReportError("Invalid path to coreclr directory. Specify with --coreclr"); } - else if (!File.Exists(Path.Combine(_rootPath, "build.cmd")) || !File.Exists(Path.Combine(_rootPath, "build.sh"))) + else if (!File.Exists(Path.Combine(CoreCLR, "build.cmd")) || !File.Exists(Path.Combine(CoreCLR, "build.sh"))) { // If _rootPath\build.cmd or _rootPath\build.sh do not exist, it is an invalid path to a coreclr repo - _syntaxResult.ReportError("Invalid path to coreclr directory. Specify with --coreclr"); + ReportError("Invalid path to coreclr directory. Specify with --coreclr"); } // Check that we can find compile_commands.json on windows - if (_os == "Windows_NT") + if (OS == "Windows_NT") { // If the user didn't specify a compile_commands.json, we need to see if one exists, and if not, create it. - if (!_untidy && _compileCommands == null) + if (!Untidy && CompileCommands == null) { - string[] compileCommandsPath = { _rootPath, "bin", "nmakeobj", "Windows_NT." + _arch + "." + _build, "compile_commands.json" }; - _compileCommands = Path.Combine(compileCommandsPath); + string[] compileCommandsPath = { CoreCLR, "bin", "nmakeobj", "Windows_NT." + Arch + "." + Build, "compile_commands.json" }; + CompileCommands = Path.Combine(compileCommandsPath); _rewriteCompileCommands = true; - if (!File.Exists(_compileCommands)) + if (!File.Exists(CompileCommands)) { // We haven't done a build, so we need to do one. - if (_verbose) + if (Verbose) { - Console.WriteLine("Neither compile_commands.json exists, nor is there a build log. Running CMake to generate compile_commands.json."); + ReportError("Neither compile_commands.json exists, nor is there a build log. Running CMake to generate compile_commands.json."); } - string[] commandArgs = { _arch, _build, "usenmakemakefiles" }; - string buildPath = Path.Combine(_rootPath, "build.cmd"); - CommandResult result = Utility.TryCommand(buildPath, commandArgs, !_verbose, _rootPath); + string[] commandArgs = { Arch, Build, "usenmakemakefiles" }; + string buildPath = Path.Combine(CoreCLR, "build.cmd"); + CommandResult result = Utility.TryCommand(buildPath, commandArgs, !Verbose, CoreCLR); if (result.ExitCode != 0) { - Console.WriteLine("There was an error running CMake to generate compile_commands.json. Please do a full build to generate a build log."); - Environment.Exit(-1); + ReportError("There was an error running CMake to generate compile_commands.json. Please do a full build to generate a build log."); } } } @@ -241,30 +211,29 @@ private void validate() else { // If the user didn't specify a compile_commands.json, we need to see if one exists, and if not, create it. - if (!_untidy && _compileCommands == null) + if (!Untidy && CompileCommands == null) { - string[] compileCommandsPath = { _rootPath, "bin", "obj", _os + "." + _arch + "." + _build, "compile_commands.json" }; - _compileCommands = Path.Combine(compileCommandsPath); + string[] compileCommandsPath = { CoreCLR, "bin", "obj", OS + "." + Arch + "." + Build, "compile_commands.json" }; + CompileCommands = Path.Combine(compileCommandsPath); _rewriteCompileCommands = true; if (!File.Exists(Path.Combine(compileCommandsPath))) { Console.WriteLine("Can't find compile_commands.json file. Running configure."); - string[] commandArgs = { _arch, _build, "configureonly", "-cmakeargs", "-DCMAKE_EXPORT_COMPILE_COMMANDS=1" }; - string buildPath = Path.Combine(_rootPath, "build.sh"); - CommandResult result = Utility.TryCommand(buildPath, commandArgs, true, _rootPath); + string[] commandArgs = { Arch, Build, "configureonly", "-cmakeargs", "-DCMAKE_EXPORT_COMPILE_COMMANDS=1" }; + string buildPath = Path.Combine(CoreCLR, "build.sh"); + CommandResult result = Utility.TryCommand(buildPath, commandArgs, true, CoreCLR); if (result.ExitCode != 0) { - Console.WriteLine("There was an error running CMake to generate compile_commands.json. Please run build.sh configureonly"); - Environment.Exit(-1); + ReportError("There was an error running CMake to generate compile_commands.json. Please run build.sh configureonly"); } } } } } - private void LoadFileConfig() + public void LoadFileConfig() { _jitUtilsRoot = Environment.GetEnvironmentVariable("JIT_UTILS_ROOT"); @@ -285,39 +254,39 @@ private void LoadFileConfig() // Set up arch var arch = ExtractDefault("arch", out found); - _arch = (found) ? arch : _arch; + Arch = (found) ? arch : Arch; // Set up build var build = ExtractDefault("build", out found); - _build = (found) ? build : _build; + Build = (found) ? build : Build; // Set up os var os = ExtractDefault("os", out found); - _os = (found) ? os : _os; + OS = (found) ? os : OS; // Set up core_root. var rootPath = ExtractDefault("coreclr", out found); - _rootPath = (found) ? rootPath : _rootPath; + CoreCLR = (found) ? rootPath : CoreCLR; // Set up compileCommands var compileCommands = ExtractDefault("compile-commands", out found); - _compileCommands = (found) ? compileCommands : _compileCommands; + CompileCommands = (found) ? compileCommands : CompileCommands; // Set flag from default for verbose. var verbose = ExtractDefault("verbose", out found); - _verbose = (found) ? verbose : _verbose; + Verbose = (found) ? verbose : Verbose; // Set up untidy var untidy = ExtractDefault("untidy", out found); - _untidy = (found) ? untidy : _untidy; + Untidy = (found) ? untidy : Untidy; // Set up noformat var noformat = ExtractDefault("noformat", out found); - _noformat = (found) ? noformat : _noformat; + NoFormat = (found) ? noformat : NoFormat; // Set up fix var fix = ExtractDefault("fix", out found); - _fix = (found) ? fix : _fix; + Fix = (found) ? fix : Fix; } Console.WriteLine("Environment variable JIT_UTILS_ROOT found - configuration loaded."); @@ -351,21 +320,24 @@ public T ExtractDefault(string name, out bool found) return default(T); } - public bool IsWindows { get { return (_os == "Windows_NT"); } } - public bool DoVerboseOutput { get { return _verbose; } } - public bool DoClangTidy { get { return !_untidy; } } - public bool DoClangFormat { get { return !_noformat; } } - public bool Fix { get { return _fix; } } - public bool IgnoreErrors { get { return _ignoreErrors; } } + public bool IsWindows { get { return (OS == "Windows_NT"); } } + public bool Verbose { get; set; } + public bool Untidy { get; set; } + public bool DoClangTidy { get { return !Untidy; } } + public bool NoFormat { get; set; } + public bool DoClangFormat { get { return !NoFormat; } } + public bool Fix { get; set; } + public bool IgnoreErrors { get; set; } public bool RewriteCompileCommands { get { return _rewriteCompileCommands; } } - public string CoreCLRRoot { get { return _rootPath; } } - public string Arch { get { return _arch; } } - public string OS { get { return _os; } } - public string Build { get { return _build; } } - public string CompileCommands { get { return _compileCommands; } } - public IReadOnlyList Filenames { get { return _filenames; } } - public IReadOnlyList Projects { get { return _projects.Count == 0 ? new List{"dll"} : _projects; } } - public string SourceDirectory { get { return _srcDirectory; } } + public string CoreCLR { get; set; } + public string Arch { get; set; } + public string OS { get; set; } + public string Build { get; set; } + public string CompileCommands { get; set; } + public IReadOnlyList Filenames { get; set; } + public IReadOnlyList Projects { get; set; } + public string SourceDirectory { get; set; } + public bool Error { get; set; } } private class CompileCommand @@ -384,11 +356,66 @@ public CompileCommand(string dir, string cmd, string filename) public static int Main(string[] args) { - // Parse and store comand line options. - var config = new Config(args); + RootCommand rootCommand = new RootCommand(); + + Option archOption = new Option("--arch", "The architecture of the build (options: x64, x86)", new Argument()); + archOption.AddAlias("-a"); + Option osOption = new Option("--os", "The operating system of the build (options: Windows, OSX, Ubuntu, Fedora, etc.)", new Argument()); + osOption.AddAlias("-o"); + Option buildOption = new Option("--build", "The build type of the build (options: Release, Checked, Debug)", new Argument()); + buildOption.AddAlias("-b"); + Option coreclrOption = new Option("--coreclr", "Full path to base coreclr directory", new Argument()); + coreclrOption.AddAlias("-c"); + Option compileCommandsOption = new Option("--compile-commands", "Full path to compile_commands.json", new Argument()); + Option verboseOption = new Option("--verbose", "Enable verbose output.", new Argument()); + verboseOption.AddAlias("-v"); + Option untidyOption = new Option("--untidy", "Do not run clang-tidy", new Argument()); + Option noFormatOption = new Option("--noformat", "Do not run clang-format", new Argument()); + Option fixOption = new Option("--fix", "Fix formatting errors discovered by clang-format and clang-tidy.", new Argument()); + fixOption.AddAlias("-f"); + Option ignoreOption = new Option("--ignore-errors", "Ignore clang-tidy errors", new Argument()); + ignoreOption.AddAlias("-i"); + Option projectsOption = new Option("--projects", "List of build projects clang-tidy should consider (e.g. dll, standalone, protojit, etc.). Default: dll", + new Argument() { Arity = ArgumentArity.OneOrMore }); + + Argument fileNameList = new Argument() { Arity = ArgumentArity.OneOrMore }; + fileNameList.Name = "filenames"; + fileNameList.Description = "Optional list of files that should be formatted."; + + rootCommand.AddOption(archOption); + rootCommand.AddOption(osOption); + rootCommand.AddOption(buildOption); + rootCommand.AddOption(coreclrOption); + rootCommand.AddOption(compileCommandsOption); + rootCommand.AddOption(verboseOption); + rootCommand.AddOption(untidyOption); + rootCommand.AddOption(noFormatOption); + rootCommand.AddOption(fixOption); + rootCommand.AddOption(ignoreOption); + rootCommand.AddOption(projectsOption); + + rootCommand.AddArgument(fileNameList); + + rootCommand.Handler = CommandHandler.Create((config) => + { + config.LoadFileConfig(); + config.Validate(); + + if (config.Error) + { + return -1; + } + + return Process(config); + }); + + return rootCommand.InvokeAsync(args).Result; + } + public static int Process(Config config) + { int returncode = 0; - bool verbose = config.DoVerboseOutput; + bool verbose = config.Verbose; List filenames = new List(); @@ -396,7 +423,7 @@ public static int Main(string[] args) if (config.Filenames.Count() == 0) { // add all files to a list of files - foreach (string filename in Directory.GetFiles(Path.Combine(config.CoreCLRRoot, "src", config.SourceDirectory))) + foreach (string filename in Directory.GetFiles(Path.Combine(config.CoreCLR, "src", config.SourceDirectory))) { // if it's not a directory, add it to our list if (!Directory.Exists(filename) && (filename.EndsWith(".cpp") || filename.EndsWith(".h") || filename.EndsWith(".hpp"))) @@ -410,9 +437,9 @@ public static int Main(string[] args) foreach (string filename in config.Filenames) { string prefix = ""; - if (!filename.Contains(config.CoreCLRRoot)) + if (!filename.Contains(config.CoreCLR)) { - prefix = Path.Combine(config.CoreCLRRoot, "src", config.SourceDirectory); + prefix = Path.Combine(config.CoreCLR, "src", config.SourceDirectory); } if (File.Exists(Path.Combine(prefix, filename))) @@ -429,7 +456,7 @@ public static int Main(string[] args) if (config.DoClangTidy) { - string[] newCompileCommandsDirPath = { config.CoreCLRRoot, "bin", "obj", config.OS + "." + config.Arch + "." + config.Build }; + string[] newCompileCommandsDirPath = { config.CoreCLR, "bin", "obj", config.OS + "." + config.Arch + "." + config.Build }; string compileCommands = config.CompileCommands; string newCompileCommandsDir = Path.Combine(newCompileCommandsDirPath); string newCompileCommands = Path.Combine(newCompileCommandsDir, "compile_commands_full.json"); From dc4453cd197158359edfef3a408c79786bd8980d Mon Sep 17 00:00:00 2001 From: Andy Ayers Date: Sat, 25 May 2019 13:06:04 -0700 Subject: [PATCH 09/10] fixes for jit-format --- src/jit-format/jit-format.cs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/jit-format/jit-format.cs b/src/jit-format/jit-format.cs index b4b656f8..c5316ce0 100644 --- a/src/jit-format/jit-format.cs +++ b/src/jit-format/jit-format.cs @@ -37,6 +37,8 @@ public class Config private bool _rewriteCompileCommands = false; private JObject _jObj; private string _jitUtilsRoot = null; + private IReadOnlyList _filenames = Array.Empty(); + private IReadOnlyList _projects = Array.Empty(); void ReportError(string message) { @@ -334,8 +336,16 @@ public T ExtractDefault(string name, out bool found) public string OS { get; set; } public string Build { get; set; } public string CompileCommands { get; set; } - public IReadOnlyList Filenames { get; set; } - public IReadOnlyList Projects { get; set; } + public IReadOnlyList Filenames + { + get { return _filenames; } + set { _filenames = value; } + } + public IReadOnlyList Projects + { + get { return _projects.Count == 0 ? new List { "dll" } : _projects; } + set { _projects = value; } + } public string SourceDirectory { get; set; } public bool Error { get; set; } } @@ -378,7 +388,7 @@ public static int Main(string[] args) Option projectsOption = new Option("--projects", "List of build projects clang-tidy should consider (e.g. dll, standalone, protojit, etc.). Default: dll", new Argument() { Arity = ArgumentArity.OneOrMore }); - Argument fileNameList = new Argument() { Arity = ArgumentArity.OneOrMore }; + Argument fileNameList = new Argument() { Arity = ArgumentArity.ZeroOrMore }; fileNameList.Name = "filenames"; fileNameList.Description = "Optional list of files that should be formatted."; From ec1781933ab1c3a8e1561b42c34b96746d47e5e9 Mon Sep 17 00:00:00 2001 From: Andy Ayers Date: Thu, 30 May 2019 14:51:09 -0700 Subject: [PATCH 10/10] fix some binding issues --- src/jit-dasm-pmi/jit-dasm-pmi.cs | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/jit-dasm-pmi/jit-dasm-pmi.cs b/src/jit-dasm-pmi/jit-dasm-pmi.cs index a370b2de..ce3bb597 100644 --- a/src/jit-dasm-pmi/jit-dasm-pmi.cs +++ b/src/jit-dasm-pmi/jit-dasm-pmi.cs @@ -58,19 +58,19 @@ public void Validate() } // JitPath must be specified and exist. - if (JitPath == null) + if (Jit == null) { ReportError("You must specify --jitPath."); } - else if (!File.Exists(JitPath)) + else if (!File.Exists(Jit)) { ReportError("Can't find --jitPath library."); } else { // Set to full path for command resolution logic. - string fullJitPath = Path.GetFullPath(JitPath); - JitPath = fullJitPath; + string fullJitPath = Path.GetFullPath(Jit); + Jit = fullJitPath; } if ((FileName == null) && (AssemblyList.Count == 0)) @@ -97,9 +97,10 @@ public void Validate() public bool NoCopy { get; set; } public bool CopyJit { get { return !NoCopy; } } public string Corerun { get; set; } - public string JitPath { get; set; } + public string Jit { get; set; } public string AltJit { get; set; } - public string RootPath { get; set; } + public string Output { get; set; } + public string RootPath { get { return Output; } } public IReadOnlyList Platform { get; set; } public string FileName { get; set; } public IReadOnlyList AssemblyList { get; set; } @@ -127,8 +128,8 @@ public static int Main(string[] args) Option altJitOption = new Option("--altjit", "If set, the name of the altjit to use (e.g., protononjit.dll).", new Argument()); Option corerunOption = new Option("--corerun", "Path to corerun.", new Argument()); corerunOption.AddAlias("-c"); - Option jitPathOption = new Option("--jitPath", "The full path to the jit library.", new Argument()); - jitPathOption.AddAlias("-j"); + Option jitOption = new Option("--jit", "The full path to the jit library.", new Argument()); + jitOption.AddAlias("-j"); Option outputOption = new Option("--output", "The output path.", new Argument()); outputOption.AddAlias("-o"); Option fileNameOption = new Option("--fileName", "Name of file to take list of assemblies from. Both a file and assembly list can be used.", new Argument()); @@ -139,7 +140,7 @@ public static int Main(string[] args) verboseOption.AddAlias("-v"); Option recursiveOption = new Option("--recursive", "Scan directories recursively", new Argument()); recursiveOption.AddAlias("-r"); - Option platformOption = new Option("--platform", "Path to platform assemblies", new Argument() { Arity = ArgumentArity.OneOrMore }); + Option platformOption = new Option("--platform", "Path to platform assemblies", new Argument() { Arity = ArgumentArity.ExactlyOne }); platformOption.AddAlias("-p"); Option tieringOption = new Option("--tiering", "Enable tiered jitting", new Argument()); tieringOption.AddAlias("-t"); @@ -154,7 +155,7 @@ public static int Main(string[] args) rootCommand.AddOption(altJitOption); rootCommand.AddOption(corerunOption); - rootCommand.AddOption(jitPathOption); + rootCommand.AddOption(jitOption); rootCommand.AddOption(outputOption); rootCommand.AddOption(fileNameOption); rootCommand.AddOption(gcInfoOption); @@ -363,7 +364,7 @@ public DisasmEnginePmi(string executable, Config config, string outputPath, _executablePath = executable; _rootPath = outputPath; _platformPaths = config.Platform; - _jitPath = config.JitPath; + _jitPath = config.Jit; _altjit = config.AltJit; _assemblyInfoList = assemblyInfoList;