Add PartitionedFileLoader

Samples/Partitioned/Named/Year=2017/Month=01/data1.csv

@@ -0,0 +1,2 @@
+col1, col2
+0, 1

Samples/Partitioned/Named/Year=2017/Month=01/data2.csv

@@ -0,0 +1,3 @@
+col1, col2
+4, 5
+6, 7

Samples/Partitioned/Named/Year=2017/Month=01/dataEmpty.csv

@@ -0,0 +1 @@
+col1, col2

Samples/Partitioned/Named/Year=2017/Month=02/data1.csv

@@ -0,0 +1,4 @@
+col1, col2
+21, 22
+23, 24
+25, 26

Samples/Partitioned/Named/Year=2017/TestBadDir/data1.csv

@@ -0,0 +1,4 @@
+col1, col2
+21, 22
+23, 24
+25, 26

Samples/Partitioned/Unnamed/2017/01/data1.csv

@@ -0,0 +1,2 @@
+col1, col2
+0, 1

Samples/Partitioned/Unnamed/2017/01/data2.csv

@@ -0,0 +1,3 @@
+col1, col2
+4, 5
+6, 7

Samples/Partitioned/Unnamed/2017/01/dataBadSchema.csv

@@ -0,0 +1,3 @@
+col1
+11
+12

Samples/Partitioned/Unnamed/2017/02/data1.csv

@@ -0,0 +1,4 @@
+col1, col2
+21, 22
+23, 24
+25, 26

ZBaselines/SingleDebug/SavePipe/PartitionedNamedDirectories-Data.txt

@@ -0,0 +1,14 @@
+#@ TextLoader{
+#@   header+
+#@   sep=tab
+#@   col=L0:TX:0
+#@   col=Year:TX:1
+#@   col=Month:TX:2
+#@ }
+L0	Year	Month
+0	2017	01
+4	2017	01
+6	2017	01
+21	2017	02
+23	2017	02
+25	2017	02

ZBaselines/SingleDebug/SavePipe/PartitionedNamedDirectories-Schema.txt

@@ -0,0 +1,5 @@
+---- PartitionedFileLoader ----
+3 columns:
+  L0: Text
+  Year: Text
+  Month: Text

ZBaselines/SingleDebug/SavePipe/PartitionedUnnamedDirectories-Data.txt

@@ -0,0 +1,16 @@
+#@ TextLoader{
+#@   header+
+#@   sep=tab
+#@   col=L0:I4:0
+#@   col=Month:I4:1
+#@   col=Path:TX:2
+#@ }
+L0	Month	Path
+1	1	2017\01\data1.csv
+5	1	2017\01\data2.csv
+7	1	2017\01\data2.csv
+0	1	2017\01\dataBadSchema.csv
+0	1	2017\01\dataBadSchema.csv
+22	2	2017\02\data1.csv
+24	2	2017\02\data1.csv
+26	2	2017\02\data1.csv

ZBaselines/SingleDebug/SavePipe/PartitionedUnnamedDirectories-Schema.txt

@@ -0,0 +1,5 @@
+---- PartitionedFileLoader ----
+3 columns:
+  L0: I4
+  Month: I4
+  Path: Text

ZBaselines/SingleRelease/PartitionedLoader/PartitionedNamedDirectories-Data.txt

@@ -0,0 +1,14 @@
+#@ TextLoader{
+#@   header+
+#@   sep=tab
+#@   col=L0:TX:0
+#@   col=Year:TX:1
+#@   col=Month:TX:2
+#@ }
+L0	Year	Month
+0	2017	01
+4	2017	01
+6	2017	01
+21	2017	02
+23	2017	02
+25	2017	02

ZBaselines/SingleRelease/PartitionedLoader/PartitionedNamedDirectories-Schema.txt

@@ -0,0 +1,5 @@
+---- PartitionedFileLoader ----
+3 columns:
+  L0: Text
+  Year: Text
+  Month: Text

ZBaselines/SingleRelease/PartitionedLoader/PartitionedUnnamedDirectories-Data.txt

@@ -0,0 +1,16 @@
+#@ TextLoader{
+#@   header+
+#@   sep=tab
+#@   col=L0:I4:0
+#@   col=Month:I4:1
+#@   col=Path:TX:2
+#@ }
+L0	Month	Path
+1	1	2017\01\data1.csv
+5	1	2017\01\data2.csv
+7	1	2017\01\data2.csv
+0	1	2017\01\dataBadSchema.csv
+0	1	2017\01\dataBadSchema.csv
+22	2	2017\02\data1.csv
+24	2	2017\02\data1.csv
+26	2	2017\02\data1.csv

ZBaselines/SingleRelease/PartitionedLoader/PartitionedUnnamedDirectories-Schema.txt

@@ -0,0 +1,5 @@
+---- PartitionedFileLoader ----
+3 columns:
+  L0: I4
+  Month: I4
+  Path: Text

src/Microsoft.ML.Core/Utilities/PathUtils.cs

@@ -1,9 +1,11 @@
-// Licensed to the .NET Foundation under one or more agreements.
+// Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
 using System;
+using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Threading;
 
 namespace Microsoft.ML.Runtime.Internal.Utilities
@@ -67,13 +69,13 @@ public static string FindExistentFileOrNull(string fileName, string folderPrefix
             // 1. Search in customSearchDir.
             if (!string.IsNullOrWhiteSpace(customSearchDir)
                 && TryFindFile(fileName, folderPrefix, customSearchDir, out candidate))
-                    return candidate;
+                return candidate;
 
             // 2. Search in the path specified by the environment variable.
             var envDir = Environment.GetEnvironmentVariable(CustomSearchDirEnvVariable);
             if (!string.IsNullOrWhiteSpace(envDir)
                 && TryFindFile(fileName, folderPrefix, envDir, out candidate))
-                    return candidate;
+                return candidate;
 
             // 3. Search in the path specified by the assemblyForBasePath.
             if (assemblyForBasePath != null)
@@ -139,5 +141,63 @@ public static string CreateFolderIfNotExists(string folder)
 
             return null;
         }
+
+        /// <summary>
+        /// Make a full path realtive to a base path.
+        /// </summary>
+        /// <param name="basepath">The base path, assumed to be a directory.</param>
+        /// <param name="path">The full path.</param>
+        /// <returns>The relative path.</returns>
+        /// <exception cref="ArgumentException">If the paths are not relative.</exception>
+        public static string MakePathRelative(string basepath, string path)
+        {
+            Contracts.AssertNonEmpty(basepath);
+            Contracts.AssertNonEmpty(path);
+
+            Uri baseUri = new Uri(basepath);
+            Uri uri = new Uri(path);
+
+            if (baseUri.Scheme != uri.Scheme)
+            {
+                throw new ArgumentException("Paths cannot be made relative as they are of different schemas.");
+            }
+
+            string relativePath;
+            try
+            {
+                if (!baseUri.AbsoluteUri.EndsWith("/"))
+                {
+                    baseUri = new Uri(baseUri.AbsoluteUri + "/");
+                }
+
+                relativePath = baseUri.MakeRelativeUri(uri).ToString();
+            }
+            catch (ArgumentNullException e)
+            {
+                throw new ArgumentException("Paths could not be made relative.", e);
+            }
+            catch (InvalidOperationException e)
+            {
+                throw new ArgumentException("Paths could not be made relative.", e);
+            }
+
+            if (uri.Scheme.Equals("file", StringComparison.InvariantCultureIgnoreCase))
+            {
+                relativePath = relativePath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar);
+            }
+
+            return relativePath;
+        }
+
+        /// <summary>
+        /// Split a path string into an enumerable list of the directories.
+        /// </summary>
+        /// <param name="path">The path string to split.</param>
+        /// <returns>An enumerable list of all non-empty directories.</returns>
+        public static IEnumerable<string> SplitDirectories(string path)
+        {
+            var cleanPath = path.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar);
+            return cleanPath.Split(new char[] { Path.DirectorySeparatorChar }, StringSplitOptions.RemoveEmptyEntries);
+        }
     }
 }