changed the directory structure for generated project (dotnet#243)

srsaggam · web-flow · commit 20d45db5b2eb · 2019-02-28T18:50:41.000-08:00
* changed the directory structure for generated project

* changed test

* upgraded commandline package
diff --git a/src/mlnet.Test/ApprovalTests/ConsoleCodeGeneratorTests.GeneratedTrainCodeTest.approved.txt b/src/mlnet.Test/ApprovalTests/ConsoleCodeGeneratorTests.GeneratedTrainCodeTest.approved.txt
@@ -16,7 +16,7 @@ namespace MyNamespace
     {
         private static string TrainDataPath = @"x:\dummypath\dummy_train.csv";
         private static string TestDataPath = @"x:\dummypath\dummy_test.csv";
-        private static string ModelPath = @"..\..\..\model.zip";
+        private static string ModelPath = @"x:\models\model.zip";
 
         // Set this flag to enable the training process.
         private static bool EnableTraining = false;
diff --git a/src/mlnet.Test/ApprovalTests/ConsoleCodeGeneratorTests.cs b/src/mlnet.Test/ApprovalTests/ConsoleCodeGeneratorTests.cs
@@ -35,7 +35,8 @@ public void GeneratedTrainCodeTest()
                 OutputName = "MyNamespace",
                 TrainDataset = new FileInfo("x:\\dummypath\\dummy_train.csv"),
                 TestDataset = new FileInfo("x:\\dummypath\\dummy_test.csv"),
-                LabelName = "Label"
+                LabelName = "Label",
+                ModelPath = new FileInfo("x:\\models\\model.zip")
             });
 
             (string trainCode, string projectCode, string helperCode) = consoleCodeGen.GenerateCode();
@@ -58,7 +59,8 @@ public void GeneratedProjectCodeTest()
                 OutputName = "MyNamespace",
                 TrainDataset = new FileInfo("x:\\dummypath\\dummy_train.csv"),
                 TestDataset = new FileInfo("x:\\dummypath\\dummy_test.csv"),
-                LabelName = "Label"
+                LabelName = "Label",
+                ModelPath = new FileInfo("x:\\models\\model.zip")
             });
 
             (string trainCode, string projectCode, string helperCode) = consoleCodeGen.GenerateCode();
@@ -81,7 +83,8 @@ public void GeneratedHelperCodeTest()
                 OutputName = "MyNamespace",
                 TrainDataset = new FileInfo("x:\\dummypath\\dummy_train.csv"),
                 TestDataset = new FileInfo("x:\\dummypath\\dummy_test.csv"),
-                LabelName = "Label"
+                LabelName = "Label",
+                ModelPath = new FileInfo("x:\\models\\model.zip")
             });
 
             (string trainCode, string projectCode, string helperCode) = consoleCodeGen.GenerateCode();
diff --git a/src/mlnet/CodeGenerator/CSharp/CodeGenerator.cs b/src/mlnet/CodeGenerator/CSharp/CodeGenerator.cs
@@ -109,7 +109,8 @@ internal string GenerateTrainCode(string usings, string trainer, List<string> tr
                 TestPath = settings.TestDataset?.FullName,
                 TaskType = settings.MlTask.ToString(),
                 Namespace = namespaceValue,
-                LabelName = settings.LabelName
+                LabelName = settings.LabelName,
+                ModelPath = settings.ModelPath.FullName
             };
 
             return trainingAndScoringCodeGen.TransformText();
diff --git a/src/mlnet/CodeGenerator/CSharp/CodeGeneratorSettings.cs b/src/mlnet/CodeGenerator/CSharp/CodeGeneratorSettings.cs
@@ -5,7 +5,10 @@ namespace Microsoft.ML.CLI.CodeGenerator.CSharp
 {
     internal class CodeGeneratorSettings
     {
-        public string LabelName { get; internal set; }
+        internal string LabelName { get; set; }
+
+        internal FileInfo ModelPath { get; set; }
+
         internal string OutputName { get; set; }
 
         internal string OutputBaseDir { get; set; }
diff --git a/src/mlnet/Commands/CommandDefinitions.cs b/src/mlnet/Commands/CommandDefinitions.cs
@@ -89,7 +89,7 @@ Option Verbosity() =>
 
             Option Name() =>
               new Option(new List<string>() { "--name" }, "Name for the output project or solution to create. ",
-                         new Argument<string>(defaultValue: "Sample"));
+                         new Argument<string>());
 
             Option OutputPath() =>
               new Option(new List<string>() { "--output-path" }, "Location folder to place the generated output. The default is the current directory.",
diff --git a/src/mlnet/Commands/New/NewCommandHandler.cs b/src/mlnet/Commands/New/NewCommandHandler.cs
@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information.
 
 using System;
+using System.IO;
 using Microsoft.Data.DataView;
 using Microsoft.ML.Auto;
 using Microsoft.ML.CLI.CodeGenerator.CSharp;
@@ -75,10 +76,11 @@ public void Execute()
 
             // Save the model
             logger.Log(LogLevel.Info, Strings.SavingBestModel);
-            Utils.SaveModel(model, settings.OutputPath.FullName, $"model.zip", context);
+            var modelPath = new FileInfo(Path.Combine(settings.OutputPath.FullName, $"model.zip"));
+            Utils.SaveModel(model, modelPath, context);
 
             // Generate the Project
-            GenerateProject(columnInference, pipeline, sanitized_Label_Name);
+            GenerateProject(columnInference, pipeline, sanitized_Label_Name, modelPath);
         }
 
         internal ColumnInferenceResults InferColumns(MLContext context)
@@ -99,7 +101,7 @@ internal ColumnInferenceResults InferColumns(MLContext context)
             return columnInference;
         }
 
-        internal void GenerateProject(ColumnInferenceResults columnInference, Pipeline pipeline, string labelName)
+        internal void GenerateProject(ColumnInferenceResults columnInference, Pipeline pipeline, string labelName, FileInfo modelPath)
         {
             //Generate code
             logger.Log(LogLevel.Info, $"{Strings.GenerateProject} : {settings.OutputPath.FullName}");
@@ -113,7 +115,8 @@ internal void GenerateProject(ColumnInferenceResults columnInference, Pipeline p
                     TestDataset = settings.TestDataset,
                     OutputName = settings.Name,
                     OutputBaseDir = settings.OutputPath.FullName,
-                    LabelName = labelName
+                    LabelName = labelName,
+                    ModelPath = modelPath
                 });
             codeGenerator.GenerateOutput();
         }
diff --git a/src/mlnet/Program.cs b/src/mlnet/Program.cs
@@ -5,6 +5,7 @@
 using System;
 using System.CommandLine.Builder;
 using System.CommandLine.Invocation;
+using System.IO;
 using Microsoft.ML.CLI.Commands;
 using Microsoft.ML.CLI.Commands.New;
 using Microsoft.ML.CLI.Data;
@@ -25,13 +26,29 @@ public static void Main(string[] args)
                  // Map the verbosity to internal levels
                  var verbosity = Utils.GetVerbosity(options.Verbosity);
 
+                 // Build the output path
+                 string outputBaseDir = string.Empty;
+                 if (options.Name == null)
+                 {
+                     var datasetName = Path.GetFileNameWithoutExtension(options.Dataset.FullName);
+                     options.Name = Utils.Sanitize(datasetName) + "_" + Utils.GetTaskKind(options.MlTask).ToString();
+                     outputBaseDir = Path.Combine(options.OutputPath.FullName, options.Name);
+                 }
+                 else
+                 {
+                     outputBaseDir = Path.Combine(options.OutputPath.FullName, options.Name);
+                 }
+
+                 // Override the output path
+                 options.OutputPath = new DirectoryInfo(outputBaseDir);
+
                  // Instantiate the command
                  var command = new NewCommand(options);
 
                  // Override the Logger Configuration
                  var logconsole = LogManager.Configuration.FindTargetByName("logconsole");
                  var logfile = (FileTarget)LogManager.Configuration.FindTargetByName("logfile");
-                 logfile.FileName = $"{options.OutputPath.FullName}/debug_log.txt";
+                 logfile.FileName = $"{outputBaseDir}/logs/debug_log.txt";
                  var config = LogManager.Configuration;
                  config.AddRule(verbosity, LogLevel.Fatal, logconsole);
 
diff --git a/src/mlnet/Templates/Console/MLCodeGen.cs b/src/mlnet/Templates/Console/MLCodeGen.cs
@@ -41,27 +41,28 @@ public virtual string TransformText()
             this.Write(this.ToStringHelper.ToStringWithCulture(TestPath));
             this.Write("\";\r\n");
  } 
-            this.Write("        private static string ModelPath = @\"..\\..\\..\\model.zip\";\r\n\r\n        // Se" +
-                    "t this flag to enable the training process.\r\n        private static bool EnableT" +
-                    "raining = false;\r\n\r\n        static void Main(string[] args)\r\n        {\r\n        " +
-                    "    // Create MLContext to be shared across the model creation workflow objects " +
-                    "\r\n            // Set a random seed for repeatable/deterministic results across m" +
-                    "ultiple trainings.\r\n            var mlContext = new MLContext(seed: 1);\r\n\r\n     " +
-                    "       if (EnableTraining)\r\n            {\r\n                // Create, Train, Eva" +
-                    "luate and Save a model\r\n                BuildTrainEvaluateAndSaveModel(mlContext" +
-                    ");\r\n                ConsoleHelper.ConsoleWriteHeader(\"=============== End of tra" +
-                    "ining process ===============\");\r\n            }\r\n            else\r\n            {" +
-                    "\r\n                ConsoleHelper.ConsoleWriteHeader(\"Skipping the training proces" +
-                    "s. Please set the flag : \'EnableTraining\' to \'true\' to enable the training proce" +
-                    "ss.\");\r\n            }\r\n\r\n            // Make a single test prediction loading th" +
-                    "e model from .ZIP file\r\n            TestSinglePrediction(mlContext);\r\n\r\n        " +
-                    "    ConsoleHelper.ConsoleWriteHeader(\"=============== End of process, hit any ke" +
-                    "y to finish ===============\");\r\n            Console.ReadKey();\r\n\r\n        }\r\n\r\n " +
-                    "       private static ITransformer BuildTrainEvaluateAndSaveModel(MLContext mlCo" +
-                    "ntext)\r\n        {\r\n            // Data loading\r\n            IDataView trainingDa" +
-                    "taView = mlContext.Data.ReadFromTextFile<SampleObservation>(\r\n                  " +
-                    "                          path: TrainDataPath,\r\n                                " +
-                    "            hasHeader : ");
+            this.Write("        private static string ModelPath = @\"");
+            this.Write(this.ToStringHelper.ToStringWithCulture(ModelPath));
+            this.Write("\";\r\n\r\n        // Set this flag to enable the training process.\r\n        private s" +
+                    "tatic bool EnableTraining = false;\r\n\r\n        static void Main(string[] args)\r\n " +
+                    "       {\r\n            // Create MLContext to be shared across the model creation" +
+                    " workflow objects \r\n            // Set a random seed for repeatable/deterministi" +
+                    "c results across multiple trainings.\r\n            var mlContext = new MLContext(" +
+                    "seed: 1);\r\n\r\n            if (EnableTraining)\r\n            {\r\n                // " +
+                    "Create, Train, Evaluate and Save a model\r\n                BuildTrainEvaluateAndS" +
+                    "aveModel(mlContext);\r\n                ConsoleHelper.ConsoleWriteHeader(\"========" +
+                    "======= End of training process ===============\");\r\n            }\r\n            e" +
+                    "lse\r\n            {\r\n                ConsoleHelper.ConsoleWriteHeader(\"Skipping t" +
+                    "he training process. Please set the flag : \'EnableTraining\' to \'true\' to enable " +
+                    "the training process.\");\r\n            }\r\n\r\n            // Make a single test pre" +
+                    "diction loading the model from .ZIP file\r\n            TestSinglePrediction(mlCon" +
+                    "text);\r\n\r\n            ConsoleHelper.ConsoleWriteHeader(\"=============== End of p" +
+                    "rocess, hit any key to finish ===============\");\r\n            Console.ReadKey();" +
+                    "\r\n\r\n        }\r\n\r\n        private static ITransformer BuildTrainEvaluateAndSaveMo" +
+                    "del(MLContext mlContext)\r\n        {\r\n            // Data loading\r\n            ID" +
+                    "ataView trainingDataView = mlContext.Data.ReadFromTextFile<SampleObservation>(\r\n" +
+                    "                                            path: TrainDataPath,\r\n              " +
+                    "                              hasHeader : ");
             this.Write(this.ToStringHelper.ToStringWithCulture(HasHeader.ToString().ToLowerInvariant()));
             this.Write(",\r\n                                            separatorChar : \'");
             this.Write(this.ToStringHelper.ToStringWithCulture(Regex.Escape(Separator.ToString())));
@@ -260,6 +261,7 @@ private static void TestSinglePrediction(MLContext mlContext)
 public int Kfolds {get;set;} = 5;
 public string Namespace {get;set;}
 public string LabelName {get;set;}
+public string ModelPath {get;set;}
 
     }
     #region Base class
diff --git a/src/mlnet/Templates/Console/MLCodeGen.tt b/src/mlnet/Templates/Console/MLCodeGen.tt
@@ -24,7 +24,7 @@ namespace <#= Namespace #>
 <#if(!string.IsNullOrEmpty(TestPath)){ #>
         private static string TestDataPath = @"<#= TestPath #>";
 <# } #>
-        private static string ModelPath = @"..\..\..\model.zip";
+        private static string ModelPath = @"<#= ModelPath #>";
 
         // Set this flag to enable the training process.
         private static bool EnableTraining = false;
@@ -215,4 +215,5 @@ public bool TrimWhiteSpace {get;set;}
 public int Kfolds {get;set;} = 5;
 public string Namespace {get;set;}
 public string LabelName {get;set;}
+public string ModelPath {get;set;}
 #>
diff --git a/src/mlnet/Utilities/Utils.cs b/src/mlnet/Utilities/Utils.cs
@@ -29,14 +29,15 @@ internal static LogLevel GetVerbosity(string verbosity)
         }
 
 
-        internal static void SaveModel(ITransformer model, string ModelPath, string modelName, MLContext mlContext)
+        internal static void SaveModel(ITransformer model, FileInfo modelPath, MLContext mlContext)
         {
-            if (!Directory.Exists(ModelPath))
+
+            if (!Directory.Exists(modelPath.Directory.FullName))
             {
-                Directory.CreateDirectory(ModelPath);
+                Directory.CreateDirectory(modelPath.Directory.FullName);
             }
-            ModelPath = Path.Combine(ModelPath, modelName);
-            using (var fs = File.Create(ModelPath))
+
+            using (var fs = File.Create(modelPath.FullName))
                 model.SaveTo(mlContext, fs);
         }
 
diff --git a/src/mlnet/mlnet.csproj b/src/mlnet/mlnet.csproj
@@ -13,7 +13,7 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.CodeAnalysis" Version="2.10.0" />
     <PackageReference Include="NLog" Version="4.5.11" />
-    <PackageReference Include="System.CommandLine.Experimental" Version="0.1.0-alpha-63825-02" />
+    <PackageReference Include="System.CommandLine.Experimental" Version="0.1.0-alpha-63828-02" />
   </ItemGroup>
 
   <ItemGroup>

Original file line number	Diff line number	Diff line change
`@@ -16,7 +16,7 @@ namespace MyNamespace`
`16`	`16`	`{`
`17`	`17`	`private static string TrainDataPath = @"x:\dummypath\dummy_train.csv";`
`18`	`18`	`private static string TestDataPath = @"x:\dummypath\dummy_test.csv";`
`19`		`- private static string ModelPath = @"..\..\..\model.zip";`
	`19`	`+ private static string ModelPath = @"x:\models\model.zip";`
`20`	`20`
`21`	`21`	`// Set this flag to enable the training process.`
`22`	`22`	`private static bool EnableTraining = false;`
Original file line number	Diff line number	Diff line change
`@@ -5,7 +5,10 @@ namespace Microsoft.ML.CLI.CodeGenerator.CSharp`
`5`	`5`	`{`
`6`	`6`	`internal class CodeGeneratorSettings`
`7`	`7`	`{`
`8`		`- public string LabelName { get; internal set; }`
	`8`	`+ internal string LabelName { get; set; }`
	`9`	`+`
	`10`	`+ internal FileInfo ModelPath { get; set; }`
	`11`	`+`
`9`	`12`	`internal string OutputName { get; set; }`
`10`	`13`
`11`	`14`	`internal string OutputBaseDir { get; set; }`
Original file line number	Diff line number	Diff line change
`@@ -29,14 +29,15 @@ internal static LogLevel GetVerbosity(string verbosity)`
`29`	`29`	`}`
`30`	`30`
`31`	`31`
`32`		`- internal static void SaveModel(ITransformer model, string ModelPath, string modelName, MLContext mlContext)`
	`32`	`+ internal static void SaveModel(ITransformer model, FileInfo modelPath, MLContext mlContext)`
`33`	`33`	`{`
`34`		`- if (!Directory.Exists(ModelPath))`
	`34`	`+`
	`35`	`+ if (!Directory.Exists(modelPath.Directory.FullName))`
`35`	`36`	`{`
`36`		`- Directory.CreateDirectory(ModelPath);`
	`37`	`+ Directory.CreateDirectory(modelPath.Directory.FullName);`
`37`	`38`	`}`
`38`		`- ModelPath = Path.Combine(ModelPath, modelName);`
`39`		`- using (var fs = File.Create(ModelPath))`
	`39`	`+`
	`40`	`+ using (var fs = File.Create(modelPath.FullName))`
`40`	`41`	`model.SaveTo(mlContext, fs);`
`41`	`42`	`}`
`42`	`43`