Ungroup Columns in Column Inference (dotnet#40)

* Added sequential grouping of columns

* added ungrouping of column option

* reverted the file

Author: srsaggam

src/AutoML/API/MLContextDataExtensions.cs

@@ -13,19 +13,19 @@ public static class DataExtensions
     {
         // Delimiter, header, column datatype inference
         public static ColumnInferenceResult InferColumns(this DataOperations catalog, string path, string label,
-            bool hasHeader = false, char? separatorChar = null, bool? allowQuotedStrings = null, bool? supportSparse = null, bool trimWhitespace = false)
+            bool hasHeader = false, char? separatorChar = null, bool? allowQuotedStrings = null, bool? supportSparse = null, bool trimWhitespace = false, bool groupColumns = true)
         {
             UserInputValidationUtil.ValidateInferColumnsArgs(path, label);
             var mlContext = new MLContext();
-            return ColumnInferenceApi.InferColumns(mlContext, path, label, hasHeader, separatorChar, allowQuotedStrings, supportSparse, trimWhitespace);
+            return ColumnInferenceApi.InferColumns(mlContext, path, label, hasHeader, separatorChar, allowQuotedStrings, supportSparse, trimWhitespace, groupColumns);
         }
 
         public static IDataView AutoRead(this DataOperations catalog, string path, string label,
-            bool hasHeader = false, char? separatorChar = null, bool? allowQuotedStrings = null, bool? supportSparse = null, bool trimWhitespace = false)
+            bool hasHeader = false, char? separatorChar = null, bool? allowQuotedStrings = null, bool? supportSparse = null, bool trimWhitespace = false, bool groupColumns = true)
         {
             UserInputValidationUtil.ValidateAutoReadArgs(path, label);
             var mlContext = new MLContext();
-            var columnInferenceResult = ColumnInferenceApi.InferColumns(mlContext, path, label, hasHeader, separatorChar, allowQuotedStrings, supportSparse, trimWhitespace);
+            var columnInferenceResult = ColumnInferenceApi.InferColumns(mlContext, path, label, hasHeader, separatorChar, allowQuotedStrings, supportSparse, trimWhitespace, groupColumns);
             var textLoader = columnInferenceResult.BuildTextLoader();
             return textLoader.Read(path);
         }

src/AutoML/ColumnInference/ColumnInferenceApi.cs

@@ -9,15 +9,16 @@ namespace Microsoft.ML.Auto
 {
     internal static class ColumnInferenceApi
     {
-        public static ColumnInferenceResult InferColumns(MLContext context, string path, string label, 
-            bool hasHeader, char? separatorChar, bool? allowQuotedStrings, bool? supportSparse, bool trimWhitespace)
+        public static ColumnInferenceResult InferColumns(MLContext context, string path, string label,
+            bool hasHeader, char? separatorChar, bool? allowQuotedStrings, bool? supportSparse, bool trimWhitespace, bool groupColumns)
         {
             var sample = TextFileSample.CreateFromFullFile(path);
             var splitInference = InferSplit(sample, separatorChar, allowQuotedStrings, supportSparse);
             var typeInference = InferColumnTypes(context, sample, splitInference, hasHeader);
+            var loaderColumns = ColumnTypeInference.GenerateLoaderColumns(typeInference.Columns);
             var typedLoaderArgs = new TextLoader.Arguments
             {
-                Column = ColumnTypeInference.GenerateLoaderColumns(typeInference.Columns),
+                Column = loaderColumns,
                 Separator = splitInference.Separator,
                 AllowSparse = splitInference.AllowSparse,
                 AllowQuoting = splitInference.AllowQuote,
@@ -29,12 +30,24 @@ public static ColumnInferenceResult InferColumns(MLContext context, string path,
 
             var purposeInferenceResult = PurposeInference.InferPurposes(context, dataView, label);
 
+            (TextLoader.Column, ColumnPurpose Purpose)[] inferredColumns = null;
             // infer column grouping and generate column names
-            var groupingResult = ColumnGroupingInference.InferGroupingAndNames(context, hasHeader,
-                typeInference.Columns, purposeInferenceResult);
+            if (groupColumns)
+            {
+                var groupingResult = ColumnGroupingInference.InferGroupingAndNames(context, hasHeader,
+                    typeInference.Columns, purposeInferenceResult);
 
-            // build result objects & return
-            var inferredColumns = groupingResult.Select(c => (c.GenerateTextLoaderColumn(), c.Purpose)).ToArray();
+                // build result objects & return
+                inferredColumns = groupingResult.Select(c => (c.GenerateTextLoaderColumn(), c.Purpose)).ToArray();
+            }
+            else
+            {
+                inferredColumns = new (TextLoader.Column, ColumnPurpose Purpose)[loaderColumns.Length];
+                for (int i = 0; i < loaderColumns.Length; i++)
+                {
+                    inferredColumns[i] = (loaderColumns[i], purposeInferenceResult[i].Purpose);
+                }
+            }
             return new ColumnInferenceResult(inferredColumns, splitInference.AllowQuote, splitInference.AllowSparse, splitInference.Separator, hasHeader, trimWhitespace);
         }
 
@@ -44,15 +57,15 @@ private static TextFileContents.ColumnSplitResult InferSplit(TextFileSample samp
             var splitInference = TextFileContents.TrySplitColumns(sample, separatorCandidates);
 
             // respect passed-in overrides
-            if(allowQuotedStrings != null)
+            if (allowQuotedStrings != null)
             {
                 splitInference.AllowQuote = allowQuotedStrings.Value;
             }
-            if(supportSparse != null)
+            if (supportSparse != null)
             {
                 splitInference.AllowSparse = supportSparse.Value;
             }
-            
+
             if (!splitInference.IsSuccess)
             {
                 throw new InferenceException(InferenceType.ColumnSplit, "Unable to split the file provided into multiple, consistent columns.");

src/Test/ColumnInferenceTests.cs

@@ -0,0 +1,24 @@
+using System.Linq;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace Microsoft.ML.Auto.Test
+{
+    [TestClass]
+    public class ColumnInferenceTests
+    {
+        [TestMethod]
+        public void UnGroupColumnsTest()
+        {
+            var dataPath = DatasetUtil.DownloadUciAdultDataset();
+            var context = new MLContext();
+            var columnInferenceWithoutGrouping = context.Data.InferColumns(dataPath, DatasetUtil.UciAdultLabel, true, groupColumns: false);
+            foreach (var col in columnInferenceWithoutGrouping.Columns)
+            {
+                Assert.IsFalse(col.Item1.Source.Length > 1 || col.Item1.Source[0].Min != col.Item1.Source[0].Max);
+            }
+
+            var columnInferenceWithGrouping = context.Data.InferColumns(dataPath, DatasetUtil.UciAdultLabel, true, groupColumns: true);
+            Assert.IsTrue(columnInferenceWithGrouping.Columns.Count() < columnInferenceWithoutGrouping.Columns.Count());
+        }
+    }
+}